diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractDiskRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractDiskRegion.java index 30768dc96..571f8d5b6 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractDiskRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractDiskRegion.java @@ -850,7 +850,8 @@ public final boolean isBackup() { } @Override - public void updateMemoryStats(Object oldValue, Object newValue) { + public void updateMemoryStats(Object oldValue, Object newValue, + AbstractRegionEntry re) { // only used by BucketRegion as of now } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java index ddad7def7..bf3beedb1 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java @@ -1441,14 +1441,14 @@ final void _setValue(RegionEntryContext context, @Unretained final Object val) { if (rawOldVal instanceof SerializedDiskBuffer) { synchronized (rawOldVal) { setValueField(val); - if (context != null) context.updateMemoryStats(rawOldVal, val); + if (context != null) context.updateMemoryStats(rawOldVal, val, this); ((SerializedDiskBuffer)rawOldVal).release(); } return; } else if (val instanceof SerializedDiskBuffer) { synchronized (val) { setValueField(val); - if (context != null) context.updateMemoryStats(rawOldVal, val); + if (context != null) context.updateMemoryStats(rawOldVal, val, this); } return; } @@ -1463,7 +1463,7 @@ final void _setValue(RegionEntryContext context, @Unretained final Object val) { || (Token.isRemoved(val) && getValueAsToken() != Token.NOT_A_TOKEN)) { setValueField(val); if (!isOffHeap && context != null) { - context.updateMemoryStats(rawOldVal, val); + context.updateMemoryStats(rawOldVal, val, this); } } else { @@ -1501,7 +1501,7 @@ final void _setValue(RegionEntryContext context, @Unretained final Object val) { setContainerInfo(null, val); } if (!isOffHeap && context != null) { - context.updateMemoryStats(rawOldVal, val); + context.updateMemoryStats(rawOldVal, val, this); } return; } catch (IllegalAccessException e) { @@ -1739,10 +1739,10 @@ public final void setOwner(LocalRegion owner, Object previousOwner) { // set the context into the value if required initContextForDiskBuffer(owner, val); // add for new owner - if (owner != null) owner.updateMemoryStats(null, val); + if (owner != null) owner.updateMemoryStats(null, val, this); // reduce from previous owner if (previousOwner instanceof RegionEntryContext) { - ((RegionEntryContext)previousOwner).updateMemoryStats(val, null); + ((RegionEntryContext)previousOwner).updateMemoryStats(val, null, this); } } final StaticSystemCallbacks sysCb = diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java index 08da0b1a1..04cc87d16 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java @@ -38,7 +38,12 @@ import java.io.DataOutput; import java.io.IOException; import java.io.InputStream; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -72,6 +77,7 @@ import com.gemstone.gemfire.internal.cache.locks.LockingPolicy.ReadEntryUnderLock; import com.gemstone.gemfire.internal.cache.locks.ReentrantReadWriteWriteShareLock; import com.gemstone.gemfire.internal.cache.partitioned.*; +import com.gemstone.gemfire.internal.cache.store.ColumnBatchKey; import com.gemstone.gemfire.internal.cache.store.SerializedDiskBuffer; import com.gemstone.gemfire.internal.cache.tier.sockets.CacheClientNotifier; import com.gemstone.gemfire.internal.cache.tier.sockets.ClientTombstoneMessage; @@ -126,6 +132,7 @@ public class BucketRegion extends DistributedRegion implements Bucket { * Contains size in bytes of the direct byte buffers stored in memory. */ private final AtomicLong directBufferBytesInMemory = new AtomicLong(); + private final AtomicLong numRowsInColumnTable = new AtomicLong(); private final AtomicLong inProgressSize = new AtomicLong(); public static final ReadEntryUnderLock READ_SER_VALUE = new ReadEntryUnderLock() { @@ -2765,6 +2772,7 @@ else if (GemFireCacheImpl.gfxdSystem()) { prDs.updateMemoryStats(-oldMemValue); } this.directBufferBytesInMemory.set(0); + this.numRowsInColumnTable.set(0); // explicitly clear overflow counters if no diskRegion is present // (for latter the counters are cleared by DiskRegion.statsClear) if (getDiskRegion() == null) { @@ -2982,9 +2990,16 @@ protected void closeCallbacksExceptListener() { closeCacheCallback(getEvictionController()); } - public long getSizeInMemory() { - return Math.max(this.bytesInMemory.get(), 0L) + - this.directBufferBytesInMemory.get(); + public final long getSizeInMemory() { + return Math.max(this.bytesInMemory.get(), 0L) + getDirectBytesSizeInMemory(); + } + + public final long getDirectBytesSizeInMemory() { + return this.directBufferBytesInMemory.get(); + } + + public final long getNumRowsInColumnTable() { + return this.numRowsInColumnTable.get(); } public long getInProgressSize() { @@ -3102,22 +3117,44 @@ void updateBucket2Size(int oldSize, int newSize, } @Override - public void updateMemoryStats(final Object oldValue, final Object newValue) { + public void updateMemoryStats(final Object oldValue, final Object newValue, + final AbstractRegionEntry re) { if (newValue != oldValue) { int oldValueSize = calcMemSize(oldValue); int newValueSize = calcMemSize(newValue); updateBucketMemoryStats(newValueSize - oldValueSize); - if (this.cache.getMemorySize() > 0) { - int directBufferDelta = 0; - if (oldValue instanceof SerializedDiskBuffer) { - directBufferDelta -= ((SerializedDiskBuffer)oldValue).getOffHeapSizeInBytes(); - } - if (newValue instanceof SerializedDiskBuffer) { - directBufferDelta += ((SerializedDiskBuffer)newValue).getOffHeapSizeInBytes(); - } - if (directBufferDelta != 0) { - this.directBufferBytesInMemory.getAndAdd(directBufferDelta); + // update number of rows in table and off-heap size if applicable + if (re != null) { + int numColumns = this.partitionedRegion.getNumColumns(); + if (numColumns > 0) { + Object key = re.getRawKey(); + final boolean hasNewOffHeap = this.cache.getMemorySize() > 0; + if (key instanceof ColumnBatchKey) { + ColumnBatchKey batchKey = (ColumnBatchKey)key; + int directBufferDelta = 0; + int numRowsDelta = 0; + if (oldValue instanceof SerializedDiskBuffer) { + SerializedDiskBuffer oldBuffer = (SerializedDiskBuffer)oldValue; + if (hasNewOffHeap) { + directBufferDelta -= oldBuffer.getOffHeapSizeInBytes(); + } + numRowsDelta -= batchKey.getColumnBatchRowCount(this, oldBuffer); + } + if (newValue instanceof SerializedDiskBuffer) { + SerializedDiskBuffer newBuffer = (SerializedDiskBuffer)newValue; + if (hasNewOffHeap) { + directBufferDelta += newBuffer.getOffHeapSizeInBytes(); + } + numRowsDelta += batchKey.getColumnBatchRowCount(this, newBuffer); + } + if (directBufferDelta != 0) { + this.directBufferBytesInMemory.getAndAdd(directBufferDelta); + } + if (numRowsDelta != 0) { + this.numRowsInColumnTable.getAndAdd(numRowsDelta); + } + } } } } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java index 6742290ba..f6878ea89 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java @@ -16,6 +16,7 @@ */ package com.gemstone.gemfire.internal.cache; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -25,6 +26,7 @@ public ExternalTableMetaData(String entityName, Object schema, String tableType, Object externalStore, + int numColumns, int columnBatchSize, int columnMaxDeltaRows, String compressionCodec, @@ -37,6 +39,7 @@ public ExternalTableMetaData(String entityName, this.schema = schema; this.tableType = tableType; this.externalStore = externalStore; + this.numColumns = numColumns; this.columnBatchSize = columnBatchSize; this.columnMaxDeltaRows = columnMaxDeltaRows; this.compressionCodec = compressionCodec; @@ -45,23 +48,25 @@ public ExternalTableMetaData(String entityName, this.dependents = dependents; this.dataSourcePath = dataSourcePath; this.driverClass = driverClass; + this.columns = Collections.emptyList(); } - public String entityName; - public Object schema; - public String tableType; + public final String entityName; + public final Object schema; + public final String tableType; // No type specified as the class is in snappy core - public Object externalStore; - public int columnBatchSize; - public int columnMaxDeltaRows; - public String compressionCodec; - public String baseTable; - public String dml; - public String[] dependents; + public final Object externalStore; + public final int numColumns; + public final int columnBatchSize; + public final int columnMaxDeltaRows; + public final String compressionCodec; + public final String baseTable; + public final String dml; + public final String[] dependents; public String provider; public String shortProvider; - public String dataSourcePath; - public String driverClass; + public final String dataSourcePath; + public final String driverClass; public String viewText; // columns for metadata queries public List columns; diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java index 296bae864..317ee3bcf 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java @@ -6587,7 +6587,7 @@ public void beforeReturningOffHeapMemoryToAllocator(long address, /** * Fetches hive meta data for Snappy tables. */ - public ExternalTableMetaData fetchSnappyTablesHiveMetaData(PartitionedRegion region); + public ExternalTableMetaData fetchTableMetaData(PartitionedRegion region); } /** diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java index aecbea582..3f5c4590b 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java @@ -150,6 +150,7 @@ import com.gemstone.gemfire.internal.offheap.annotations.Retained; import com.gemstone.gemfire.internal.offheap.annotations.Unretained; import com.gemstone.gemfire.internal.sequencelog.EntryLogger; +import com.gemstone.gemfire.internal.shared.ClientSharedUtils; import com.gemstone.gemfire.internal.shared.SystemProperties; import com.gemstone.gemfire.internal.shared.Version; import com.gemstone.gemfire.internal.size.ReflectionObjectSizer; @@ -680,10 +681,10 @@ protected LocalRegion(String regionName, RegionAttributes attrs, Assert.assertTrue(regionName != null, "regionName must not be null"); this.sharedDataView = buildDataView(); this.regionName = regionName; - this.isInternalColumnTable = regionName.toUpperCase(Locale.ENGLISH) - .endsWith(StoreCallbacks.SHADOW_TABLE_SUFFIX); this.parentRegion = parentRegion; this.fullPath = calcFullPath(regionName, parentRegion); + this.isInternalColumnTable = ClientSharedUtils.isColumnTable( + this.fullPath.toUpperCase(Locale.ENGLISH)); // cannot support patterns like "..._/..." due to ambiguity in encoding // of bucket regions if (this.fullPath.contains("_/")) { @@ -3186,7 +3187,8 @@ public boolean isBackup() { } @Override - public void updateMemoryStats(Object oldValue, Object newValue) { + public void updateMemoryStats(Object oldValue, Object newValue, + AbstractRegionEntry re) { // only used by BucketRegion as of now } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java index 0c45aff1e..d5752a9a8 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java @@ -394,17 +394,20 @@ public class PartitionedRegion extends LocalRegion implements private volatile int shutDownAllStatus = RUNNING_MODE; + /** Number of columns if this region is a store for a column table. */ + private volatile int numColumns = -1; + /** Maximum size in bytes for ColumnBatches. */ - private int columnBatchSize = -1; + private volatile int columnBatchSize = -1; /** Maximum rows to keep in the delta buffer. */ - private int columnMaxDeltaRows = -1; + private volatile int columnMaxDeltaRows = -1; /** Minimum size for ColumnBatches. */ - private int columnMinDeltaRows = SystemProperties.SNAPPY_MIN_COLUMN_DELTA_ROWS; + private volatile int columnMinDeltaRows = SystemProperties.SNAPPY_MIN_COLUMN_DELTA_ROWS; /** default compression used by the column store */ - private String columnCompressionCodec; + private volatile String columnCompressionCodec; public void setColumnBatchSizes(int size, int maxDeltaRows, int minDeltaRows) { @@ -417,19 +420,27 @@ private void initFromHiveMetaData() { final GemFireCacheImpl.StaticSystemCallbacks sysCb = GemFireCacheImpl .getInternalProductCallbacks(); if (sysCb != null) { - ExternalTableMetaData metadata = sysCb.fetchSnappyTablesHiveMetaData(this); - if (this.columnBatchSize == -1) { - this.columnBatchSize = metadata.columnBatchSize; - } - if (this.columnMaxDeltaRows == -1) { - this.columnMaxDeltaRows = metadata.columnMaxDeltaRows; - } - if (this.columnCompressionCodec == null) { - this.columnCompressionCodec = metadata.compressionCodec; + synchronized (this) { + ExternalTableMetaData metadata = sysCb.fetchTableMetaData(this); + if (this.numColumns == -1) { + this.numColumns = metadata.numColumns; + this.columnBatchSize = metadata.columnBatchSize; + this.columnMaxDeltaRows = metadata.columnMaxDeltaRows; + this.columnCompressionCodec = metadata.compressionCodec; + } } } } + public int getNumColumns() { + int numColumns = this.numColumns; + if (numColumns == -1) { + initFromHiveMetaData(); + return this.numColumns; + } + return numColumns; + } + public int getColumnBatchSize() { int columnBatchSize = this.columnBatchSize; if (columnBatchSize == -1) { diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java index 3bf474c21..83a21b370 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java @@ -54,5 +54,6 @@ public interface RegionEntryContext extends HasCachePerfStats { */ public boolean isBackup(); - public void updateMemoryStats(Object oldValue, Object newValue); + public void updateMemoryStats(Object oldValue, Object newValue, + AbstractRegionEntry re); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ColumnBatchKey.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/store/ColumnBatchKey.java similarity index 68% rename from gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ColumnBatchKey.java rename to gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/store/ColumnBatchKey.java index c1d7758fd..120a08c49 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ColumnBatchKey.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/store/ColumnBatchKey.java @@ -14,27 +14,23 @@ * permissions and limitations under the License. See accompanying * LICENSE file. */ -package com.pivotal.gemfirexd.internal.snappy; +package com.gemstone.gemfire.internal.cache.store; import com.gemstone.gemfire.internal.cache.AbstractRegionEntry; import com.gemstone.gemfire.internal.cache.BucketRegion; import com.gemstone.gemfire.internal.cache.lru.Sizeable; -import com.gemstone.gemfire.internal.cache.partitioned.PREntriesIterator; /** * Interface for a key object in the column store. */ -public interface ColumnBatchKey extends Sizeable { - - /** - * Get the number of columns defined for the given - * column table (qualified name). - */ - int getNumColumnsInTable(String columnTableName); +public abstract class ColumnBatchKey implements Sizeable { /** * Get the number of rows in this column batch. + * This will return a non-zero result only for the STATS keys while + * for a key of DELETE bitmask it will return negative value + * indicating the delete count. */ - int getColumnBatchRowCount(BucketRegion bucketRegion, AbstractRegionEntry re, - int numColumnsInTable); + public abstract int getColumnBatchRowCount(BucketRegion bucketRegion, + SerializedDiskBuffer value); } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java index 0eb3bae41..91b80b4fa 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java @@ -54,11 +54,6 @@ public List getInternalTableSchemas() { return Collections.emptyList(); } - @Override - public boolean isColumnTable(String qualifiedName) { - return false; - } - @Override public boolean skipEvictionForEntry(LRUEntry entry) { return false; diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java index 492639b2a..8299cea15 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java @@ -44,8 +44,6 @@ Set createColumnBatch(BucketRegion region, long batchID, List getInternalTableSchemas(); - boolean isColumnTable(String qualifiedName); - boolean skipEvictionForEntry(LRUEntry entry); int getHashCodeSnappy(Object dvd, int numPartitions); diff --git a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/ClientSharedUtils.java b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/ClientSharedUtils.java index e3fb04e2d..512aaa974 100644 --- a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/ClientSharedUtils.java +++ b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/ClientSharedUtils.java @@ -53,10 +53,8 @@ import java.nio.channels.Channel; import java.nio.channels.SocketChannel; import java.nio.file.Files; -import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.CodeSource; import java.security.PrivilegedAction; import java.util.*; import java.util.concurrent.locks.LockSupport; @@ -1358,27 +1356,9 @@ public static RuntimeException newRuntimeException(String message, return cre.newRunTimeException(message, cause); } - public static Path getProductJarsDirectory(Class c) throws IOException { - CodeSource cs = c.getProtectionDomain().getCodeSource(); - URL jarURL = cs != null ? cs.getLocation() : null; - if (jarURL != null) { - return Paths.get(URLDecoder.decode(jarURL.getFile(), "UTF-8")) - .getParent().toRealPath(LinkOption.NOFOLLOW_LINKS); - } else { - // try in SNAPPY_HOME and SPARK_HOME - String productHome = System.getenv("SNAPPY_HOME"); - if (productHome == null) { - productHome = System.getenv("SPARK_HOME"); - } - if (productHome != null) { - return Paths.get(productHome, "jars") - .toRealPath(LinkOption.NOFOLLOW_LINKS); - } else { - throw new IllegalStateException("Unable to locate product install " + - "location. Set SNAPPY_HOME or SPARK_HOME explicitly if not using " + - "the standard scripts."); - } - } + public static boolean isColumnTable(String fullName) { + return fullName.endsWith(SystemProperties.SHADOW_TABLE_SUFFIX) && + fullName.contains(SystemProperties.SHADOW_SCHEMA_NAME); } // Convert log4j.Level to java.util.logging.Level diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java index 7b155d108..42ccebb83 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java @@ -56,13 +56,18 @@ Licensed to the Apache Software Foundation (ASF) under one or more import com.gemstone.gemfire.CancelException; import com.gemstone.gemfire.LogWriter; -import com.gemstone.gemfire.cache.*; +import com.gemstone.gemfire.cache.DataPolicy; +import com.gemstone.gemfire.cache.EvictionAction; +import com.gemstone.gemfire.cache.EvictionAttributes; +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.Scope; import com.gemstone.gemfire.distributed.internal.DistributionManager; import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem; import com.gemstone.gemfire.internal.ClassPathLoader; import com.gemstone.gemfire.internal.GFToSlf4jBridge; import com.gemstone.gemfire.internal.LogWriterImpl; import com.gemstone.gemfire.internal.cache.*; +import com.gemstone.gemfire.internal.shared.ClientSharedUtils; import com.gemstone.gemfire.internal.shared.SystemProperties; import com.gemstone.gemfire.internal.util.ArrayUtils; import com.gemstone.gnu.trove.THashMap; @@ -81,7 +86,10 @@ Licensed to the Apache Software Foundation (ASF) under one or more import com.pivotal.gemfirexd.internal.engine.access.GemFireTransaction; import com.pivotal.gemfirexd.internal.engine.access.index.GfxdIndexManager; import com.pivotal.gemfirexd.internal.engine.access.index.MemIndex; -import com.pivotal.gemfirexd.internal.engine.ddl.*; +import com.pivotal.gemfirexd.internal.engine.ddl.DDLConflatable; +import com.pivotal.gemfirexd.internal.engine.ddl.GfxdDDLQueueEntry; +import com.pivotal.gemfirexd.internal.engine.ddl.GfxdDDLRegionQueue; +import com.pivotal.gemfirexd.internal.engine.ddl.ReplayableConflatable; import com.pivotal.gemfirexd.internal.engine.ddl.catalog.messages.GfxdSystemProcedureMessage; import com.pivotal.gemfirexd.internal.engine.ddl.wan.messages.AbstractGfxdReplayableMessage; import com.pivotal.gemfirexd.internal.engine.distributed.GfxdMessage; @@ -697,7 +705,7 @@ public static void checkSnappyCatalogConsistency(EmbedConnection embedConn, List internalColumnTablesListPerSchema = new LinkedList<>(); for (Map.Entry> e : gfDBTablesMap.entrySet()) { for (String t : e.getValue()) { - if (CallbackFactoryProvider.getStoreCallbacks().isColumnTable(e.getKey() + "." + t)) { + if (ClientSharedUtils.isColumnTable(e.getKey() + "." + t)) { internalColumnTablesListPerSchema.add(t); } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java index bd3896715..e7a0837a4 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java @@ -1615,7 +1615,7 @@ public Properties getSecurityPropertiesForReconnect() { } } - public ExternalTableMetaData fetchSnappyTablesHiveMetaData( + public ExternalTableMetaData fetchTableMetaData( PartitionedRegion region) { GemFireContainer container = (GemFireContainer)region.getUserAttribute(); if (container != null) { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java index 435f3cd3d..15875250b 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java @@ -40,6 +40,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more package com.pivotal.gemfirexd.internal.impl.sql.execute; +import com.gemstone.gemfire.internal.shared.ClientSharedUtils; import com.pivotal.gemfirexd.internal.catalog.ExternalCatalog; import com.pivotal.gemfirexd.internal.engine.GfxdConstants; import com.pivotal.gemfirexd.internal.engine.Misc; @@ -232,7 +233,7 @@ public void executeGrantRevoke( Activation activation, GemFireStore ms = Misc.getMemStore(); boolean isSnappyStoreWithSecurityEnabled = ms.isSnappyStore() && Misc.isSecurityEnabled(); if (isSnappyStoreWithSecurityEnabled && - CallbackFactoryProvider.getStoreCallbacks().isColumnTable(Misc.getFullTableName(td.getSchemaName(), + ClientSharedUtils.isColumnTable(Misc.getFullTableName(td.getSchemaName(), td.getName(), activation.getLanguageConnectionContext()))) { // do nothing for columm batch tables, they will be handled during the corresponding // main table handling diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java index c42926219..c49f425a5 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java @@ -22,18 +22,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.sql.Connection; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingDeque; @@ -54,6 +43,8 @@ import com.gemstone.gemfire.internal.cache.lru.Sizeable; import com.gemstone.gemfire.internal.cache.partitioned.Bucket; import com.gemstone.gemfire.internal.cache.partitioned.PREntriesIterator; +import com.gemstone.gemfire.internal.cache.store.ColumnBatchKey; +import com.gemstone.gemfire.internal.cache.store.SerializedDiskBuffer; import com.gemstone.gemfire.internal.cache.wan.AbstractGatewaySender; import com.gemstone.gemfire.internal.cache.wan.GatewaySenderEventImpl; import com.gemstone.gemfire.internal.cache.wan.parallel.ConcurrentParallelGatewaySenderQueue; @@ -76,7 +67,6 @@ import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer; import com.pivotal.gemfirexd.internal.engine.store.GemFireStore; import com.pivotal.gemfirexd.internal.engine.store.RegionEntryUtils; -import com.pivotal.gemfirexd.internal.engine.store.RowFormatter; import com.pivotal.gemfirexd.internal.engine.store.offheap.OffHeapRow; import com.pivotal.gemfirexd.internal.engine.store.offheap.OffHeapRowWithLobs; import com.pivotal.gemfirexd.internal.iapi.error.StandardException; @@ -84,12 +74,10 @@ import com.pivotal.gemfirexd.internal.iapi.services.context.ContextManager; import com.pivotal.gemfirexd.internal.iapi.services.io.FormatableHashtable; import com.pivotal.gemfirexd.internal.iapi.services.sanity.SanityManager; -import com.pivotal.gemfirexd.internal.iapi.sql.dictionary.ColumnDescriptor; import com.pivotal.gemfirexd.internal.iapi.types.DataValueDescriptor; import com.pivotal.gemfirexd.internal.iapi.types.RowLocation; import com.pivotal.gemfirexd.internal.impl.sql.catalog.GfxdDataDictionary; import com.pivotal.gemfirexd.internal.shared.common.reference.SQLState; -import com.pivotal.gemfirexd.internal.snappy.ColumnBatchKey; /** * This class helps in finding out memory footprint per Region and divides @@ -638,7 +626,6 @@ private void iterateRegionEntries(final GemFireContainer c, final boolean agentAttached = SIZE_OF_UTIL.isAgentAttached(); boolean isValueTypeEvaluated = false; final boolean isColumnTable = c.isColumnStore(); - int numColumnsInColumnTable = -1; long dvdArraySize = 0; boolean gatewayEntries = false; boolean offHeapEntries = false; @@ -755,8 +742,6 @@ else if (!isSingleEntrySizeComputed) { if (value != null) { if (!isValueTypeEvaluated) { if (isColumnTable) { - numColumnsInColumnTable = batchKey.getNumColumnsInTable( - c.getQualifiedTableName()); } else if (value instanceof DataValueDescriptor[]) { dvdArraySize = SIZE_OF_UTIL.sizeof(value); } else if (value instanceof GatewaySenderEventImpl) { @@ -768,8 +753,10 @@ else if (!isSingleEntrySizeComputed) { } if (isColumnTable) { int valueSize = ((Sizeable)value).getSizeInBytes(); - columnRowCount += batchKey.getColumnBatchRowCount(prEntryIter.getHostedBucketRegion(), - re, numColumnsInColumnTable); + if (value instanceof SerializedDiskBuffer) { + columnRowCount += batchKey.getColumnBatchRowCount( + prEntryIter.getHostedBucketRegion(), (SerializedDiskBuffer)value); + } valInMemoryCount++; valInMemorySize += valueSize; if (valueSize > maxSize) { @@ -840,11 +827,6 @@ else if (!isSingleEntrySizeComputed) { break OUTER; } else if (valClass == byte[][].class) { - - if (c.isColumnStore()) { - columnRowCount += getRowCountFromColumnTable(c, (byte[][])value); - } - valInMemoryCount++; byte[][] values = ((byte[][])value); // add in size of Object[] long len = SIZE_OF_UTIL.sizeof(value); @@ -993,16 +975,6 @@ else if (GatewaySenderEventImpl.class.isAssignableFrom(valClass)) { } - private int getRowCountFromColumnTable(GemFireContainer c, byte[][] value) throws StandardException { - int rowCount = 0; - RowFormatter rf = c.getRowFormatter(value); - ColumnDescriptor cd = c.getTableDescriptor().getColumnDescriptor("NUMROWS"); - if (cd != null) - rowCount = rf.getColumn(cd.getPosition(), value).getInt(); - - return rowCount; - } - private static void createDistributionInfo(StringBuilder miscInfo, String msg, TreeMap bucketDist, int maxKeyLength, int maxValueLength) {