From d8cb3eeb614c6289ad904ee2596c19ed63b1aaa0 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 31 Jul 2019 15:46:43 +0530 Subject: [PATCH 1/7] change critical-heap-percentage to leave out max 200MB (upper limit of 99% for critical-heap) --- .../gemfire/internal/shared/LauncherBase.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/LauncherBase.java b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/LauncherBase.java index e3d6ab829..14133ae9a 100644 --- a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/LauncherBase.java +++ b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/LauncherBase.java @@ -300,21 +300,22 @@ protected void setDefaultVMArgs(Map map, boolean hostData, if (maxHeapStr != null && maxHeapStr.equals(this.initialHeapSize)) { String criticalHeapStr = (String)map.get(CRITICAL_HEAP_PERCENTAGE); if (criticalHeapStr == null) { - // for larger heaps, keep critical as 95% and 90% for smaller ones; + // for larger heaps, keep critical as 95-99% and 90% for smaller ones; // also limit memory remaining beyond critical to 4GB double heapSize = ClientSharedUtils.parseMemorySize(maxHeapStr, 0L, 0); if (heapSize > (40.0 * 1024.0 * 1024.0 * 1024.0)) { - // calculate percent that will leave out at max 4GB - criticalPercent = (float)(100.0 * (1.0 - twoGB / heapSize)); - // don't exceed 98% - if (criticalPercent > 98.0f) criticalPercent = 98.0f; + // calculate percent that will leave out at max 1GB + criticalPercent = (float)(100.0 * (1.0 - oneGB / heapSize)); } else if (heapSize >= twoGB) { - criticalPercent = 95.0f; + // leave out max 200MB + criticalPercent = (float)(100.0 * (1.0 - (200.0 * 1024.0 * 1024.0) / heapSize)); } else { criticalPercent = 90.0f; } + // don't exceed 99% + if (criticalPercent > 99.0f) criticalPercent = 99.0f; map.put(CRITICAL_HEAP_PERCENTAGE, "-" + CRITICAL_HEAP_PERCENTAGE + - '=' + criticalPercent); + '=' + String.format("%.2f", criticalPercent)); } else { criticalPercent = Float.parseFloat(criticalHeapStr.substring( criticalHeapStr.indexOf('=') + 1).trim()); From 43f5fefa1214e4518ab0fe23292e3a3bea1e22c9 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 31 Jul 2019 20:59:59 +0530 Subject: [PATCH 2/7] fix occasional failures in nano timer tests Tests checking for native timer being loaded occasionally fail due to some previous tests initializing system in a different way. Now re-initialize the native timer for such a case. --- .../gemstone/gemfire/internal/NanoTimer.java | 6 ++- .../gemfire/internal/shared/NativeCalls.java | 8 +++- .../internal/shared/jna/LinuxNativeCalls.java | 48 +++++++++++++------ .../java/com/pivotal/gemfirexd/TestUtil.java | 9 +++- 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/NanoTimer.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/NanoTimer.java index c27bf442d..423655ce2 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/NanoTimer.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/NanoTimer.java @@ -55,7 +55,7 @@ public final class NanoTimer { private static final NativeCalls nativeCall = NativeCalls.getInstance(); - public static final int CLOCKID_BEST; + public static int CLOCKID_BEST; public static boolean CLOCKID_USE_SYSNANOTIME; public final static String NATIVETIMER_TYPE_PROPERTY = @@ -64,6 +64,10 @@ public final class NanoTimer { public static int nativeTimerType; static { + init(); + } + + public static void init() { /* * currently _nanoTime(..) isn't implemented in gemfire lib. * for gemfirexd, its implemented only for Linux/Solaris as of now. diff --git a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/NativeCalls.java b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/NativeCalls.java index 960309ccc..0193a26c4 100644 --- a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/NativeCalls.java +++ b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/NativeCalls.java @@ -501,6 +501,12 @@ public static interface RehashServerOnSIGHUP { public boolean loadNativeLibrary() { return false; } + + /** + * Try to reinitialize native timer if available. + */ + public void reInitNativeTimer() { + } /** * whether o/s supports high resolution clock or equivalent @@ -511,7 +517,7 @@ public boolean loadNativeLibrary() { public boolean isNativeTimerEnabled() { return false; } - + /** * This is fall back for jni based library implementation of NanoTimer which * is more efficient than current impl through jna. diff --git a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/jna/LinuxNativeCalls.java b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/jna/LinuxNativeCalls.java index 2ab35f446..c31f924e9 100644 --- a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/jna/LinuxNativeCalls.java +++ b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/jna/LinuxNativeCalls.java @@ -126,10 +126,18 @@ protected boolean isNoProtocolOptionCode(int errno) { private static boolean isJNATimerEnabled = false; public static class TimeSpec extends Structure { - public int tv_sec; - public int tv_nsec; + int tv_sec; + int tv_nsec; static { + init(); + } + + static void loadClass() { + // just to ensure class is loaded + } + + static void init() { try { Native.register("rt"); TimeSpec res = new TimeSpec(); @@ -146,10 +154,6 @@ public static class TimeSpec extends Structure { } } - static void init() { - // just invoke the static block - } - public static native int clock_getres(int clkId, TimeSpec time) throws LastErrorException; @@ -163,10 +167,18 @@ protected List getFieldOrder() { } public static class TimeSpec64 extends Structure { - public long tv_sec; - public long tv_nsec; + long tv_sec; + long tv_nsec; static { + init(); + } + + static void loadClass() { + // just to ensure class is loaded + } + + static void init() { try { Native.register("rt"); TimeSpec64 res = new TimeSpec64(); @@ -183,10 +195,6 @@ public static class TimeSpec64 extends Structure { } } - static void init() { - // just invoke the static block - } - public static native int clock_getres(int clkId, TimeSpec64 time) throws LastErrorException; @@ -207,13 +215,25 @@ public boolean loadNativeLibrary() { * {@inheritDoc} */ @Override - public boolean isNativeTimerEnabled() { - // initialize static blocks + public void reInitNativeTimer() { if (NativeCallsJNAImpl.is64BitPlatform) { TimeSpec64.init(); } else { TimeSpec.init(); } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isNativeTimerEnabled() { + // initialization already done in static blocks + if (NativeCallsJNAImpl.is64BitPlatform) { + TimeSpec64.loadClass(); + } else { + TimeSpec.loadClass(); + } return isJNATimerEnabled; } diff --git a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/TestUtil.java b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/TestUtil.java index 27a8b28ba..1fc99b74f 100644 --- a/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/TestUtil.java +++ b/gemfirexd/tools/src/test/java/com/pivotal/gemfirexd/TestUtil.java @@ -56,8 +56,8 @@ import com.gemstone.gemfire.internal.cache.xmlcache.RegionCreation; import com.gemstone.gemfire.internal.shared.ClientSharedUtils; import com.gemstone.gemfire.internal.shared.NativeCalls; -import com.gemstone.gemfire.internal.shared.jna.OSType; import com.gemstone.gemfire.internal.shared.StringPrintWriter; +import com.gemstone.gemfire.internal.shared.jna.OSType; import com.pivotal.gemfirexd.internal.engine.GemFireXDQueryObserver; import com.pivotal.gemfirexd.internal.engine.GemFireXDQueryObserverAdapter; import com.pivotal.gemfirexd.internal.engine.GfxdConstants; @@ -2231,8 +2231,13 @@ public static Object getField(Class clazz, T instance, } public static void assertTimerLibraryLoaded() { - final OSType ostype = NativeCalls.getInstance().getOSType(); + NativeCalls nc = NativeCalls.getInstance(); + final OSType ostype = nc.getOSType(); if (ostype == OSType.LINUX) { + if (!NanoTimer.isJNINativeTimerEnabled()) { + NanoTimer.init(); + nc.reInitNativeTimer(); + } assertTrue("Couldn't initialize jni native timer for " + ostype, NanoTimer.isJNINativeTimerEnabled()); assertTrue("Couldn't initialize the native timer for " + ostype, From df6279ea847dd6c5ec6bcdeea40d27db758008f3 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 31 Jul 2019 22:13:36 +0530 Subject: [PATCH 3/7] Track off-heap size in bucket - Track off-heap size in BucketRegion and add to getSizeInMemory() and getTotalBytes(). This fixes the callers of getTotalBytes() including rebalancing and determination of smallest bucket in SD layer. - Update SnappyRegionStatsCollectorFunction to avoid separate collection of off-heap size. --- .../gemfire/internal/cache/BucketRegion.java | 29 +++++++++++++++++-- .../SnappyRegionStatsCollectorFunction.java | 9 +++--- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java index 7ff10014c..bae21edb5 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java @@ -72,6 +72,7 @@ import com.gemstone.gemfire.internal.cache.locks.LockingPolicy.ReadEntryUnderLock; import com.gemstone.gemfire.internal.cache.locks.ReentrantReadWriteWriteShareLock; import com.gemstone.gemfire.internal.cache.partitioned.*; +import com.gemstone.gemfire.internal.cache.store.SerializedDiskBuffer; import com.gemstone.gemfire.internal.cache.tier.sockets.CacheClientNotifier; import com.gemstone.gemfire.internal.cache.tier.sockets.ClientTombstoneMessage; import com.gemstone.gemfire.internal.cache.tier.sockets.ClientUpdateMessage; @@ -120,8 +121,11 @@ public class BucketRegion extends DistributedRegion implements Bucket { * in theRealMap. Sizes are tallied during put and remove operations. */ private final AtomicLongWithTerminalState bytesInMemory = - new AtomicLongWithTerminalState(); - + new AtomicLongWithTerminalState(); + /** + * Contains size in bytes of the direct byte buffers stored in memory. + */ + private final AtomicLong directBufferBytesInMemory = new AtomicLong(); private final AtomicLong inProgressSize = new AtomicLong(); public static final ReadEntryUnderLock READ_SER_VALUE = new ReadEntryUnderLock() { @@ -2760,6 +2764,7 @@ else if (GemFireCacheImpl.gfxdSystem()) { this.partitionedRegion.getPrStats().incDataStoreEntryCount(-sizeBeforeClear); prDs.updateMemoryStats(-oldMemValue); } + this.directBufferBytesInMemory.set(0); // explicitly clear overflow counters if no diskRegion is present // (for latter the counters are cleared by DiskRegion.statsClear) if (getDiskRegion() == null) { @@ -2981,6 +2986,10 @@ public long getSizeInMemory() { return Math.max(this.bytesInMemory.get(), 0L); } + public long getDirectBufferSizeInMemory() { + return this.directBufferBytesInMemory.get(); + } + public long getInProgressSize() { return inProgressSize.get(); } @@ -2991,9 +3000,10 @@ public void updateInProgressSize(long delta) { public long getTotalBytes() { long result = this.bytesInMemory.get(); - if(result == BUCKET_DESTROYED) { + if (result == BUCKET_DESTROYED) { return 0; } + result += this.directBufferBytesInMemory.get(); result += getNumOverflowBytesOnDisk(); return result; } @@ -3100,6 +3110,19 @@ public void updateMemoryStats(final Object oldValue, final Object newValue) { int oldValueSize = calcMemSize(oldValue); int newValueSize = calcMemSize(newValue); updateBucketMemoryStats(newValueSize - oldValueSize); + + if (this.cache.getMemorySize() > 0) { + int directBufferDelta = 0; + if (oldValue instanceof SerializedDiskBuffer) { + directBufferDelta -= ((SerializedDiskBuffer)oldValue).getOffHeapSizeInBytes(); + } + if (newValue instanceof SerializedDiskBuffer) { + directBufferDelta += ((SerializedDiskBuffer)newValue).getOffHeapSizeInBytes(); + } + if (directBufferDelta != 0) { + this.directBufferBytesInMemory.getAndAdd(directBufferDelta); + } + } } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java index 610214203..c0696fc6a 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java @@ -228,7 +228,6 @@ private SnappyRegionStats collectDataFromBeanImpl(LocalRegion lr, RegionMXBean b PartitionedRegionDataStore datastore = pr.getDataStore(); long sizeInMemory = 0L; long sizeOfRegion = 0L; - long offHeapBytes = 0L; long entryOverhead = 0L; long entryCount = 0L; if (datastore != null) { @@ -243,18 +242,18 @@ private SnappyRegionStats collectDataFromBeanImpl(LocalRegion lr, RegionMXBean b entryOverhead = getEntryOverhead(re, sizer); } } - sizeInMemory += constantOverhead + br.getSizeInMemory(); + sizeInMemory += constantOverhead + br.getSizeInMemory() + + br.getDirectBufferSizeInMemory(); sizeOfRegion += constantOverhead + br.getTotalBytes(); entryCount += br.entryCount(); } - offHeapBytes = pr.getPrStats().getOffHeapSizeInBytes(); } if (entryOverhead > 0) { entryOverhead *= entryCount; } - tableStats.setSizeInMemory(sizeInMemory + offHeapBytes + entryOverhead); - tableStats.setTotalSize(sizeOfRegion + offHeapBytes + entryOverhead); + tableStats.setSizeInMemory(sizeInMemory + entryOverhead); + tableStats.setTotalSize(sizeOfRegion + entryOverhead); tableStats.setSizeSpillToDisk(tableStats.getTotalSize() - tableStats.getSizeInMemory()); } return tableStats; From bf69b0d26f3060c1aff669a05c022bdb1fcf66e7 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Wed, 31 Jul 2019 22:50:04 +0530 Subject: [PATCH 4/7] cleanup size accounting a bit --- .../com/gemstone/gemfire/internal/cache/BucketRegion.java | 7 ++----- .../engine/ui/SnappyRegionStatsCollectorFunction.java | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java index bae21edb5..08da0b1a1 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java @@ -2983,11 +2983,8 @@ protected void closeCallbacksExceptListener() { } public long getSizeInMemory() { - return Math.max(this.bytesInMemory.get(), 0L); - } - - public long getDirectBufferSizeInMemory() { - return this.directBufferBytesInMemory.get(); + return Math.max(this.bytesInMemory.get(), 0L) + + this.directBufferBytesInMemory.get(); } public long getInProgressSize() { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java index c0696fc6a..f66513222 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java @@ -242,8 +242,7 @@ private SnappyRegionStats collectDataFromBeanImpl(LocalRegion lr, RegionMXBean b entryOverhead = getEntryOverhead(re, sizer); } } - sizeInMemory += constantOverhead + br.getSizeInMemory() + - br.getDirectBufferSizeInMemory(); + sizeInMemory += constantOverhead + br.getSizeInMemory(); sizeOfRegion += constantOverhead + br.getTotalBytes(); entryCount += br.entryCount(); } From cbe91ae161a3ad4a8814b874aab64d78657829ce Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 1 Aug 2019 02:00:42 +0530 Subject: [PATCH 5/7] maintain column table stats in BucketRegion other cleanups --- .../internal/cache/AbstractDiskRegion.java | 3 +- .../internal/cache/AbstractRegionEntry.java | 12 ++-- .../gemfire/internal/cache/BucketRegion.java | 67 ++++++++++++++----- .../internal/cache/ExternalTableMetaData.java | 29 ++++---- .../internal/cache/GemFireCacheImpl.java | 2 +- .../gemfire/internal/cache/LocalRegion.java | 7 +- .../internal/cache/PartitionedRegion.java | 37 ++++++---- .../internal/cache/RegionEntryContext.java | 3 +- .../internal/cache/store}/ColumnBatchKey.java | 18 ++--- .../snappy/CallbackFactoryProvider.java | 5 -- .../internal/snappy/StoreCallbacks.java | 2 - .../internal/shared/SystemProperties.java | 5 ++ .../internal/engine/db/FabricDatabase.java | 13 +++- .../engine/store/RegionEntryUtils.java | 2 +- .../impl/sql/execute/TablePrivilegeInfo.java | 3 +- .../gemfirexd/tools/sizer/ObjectSizer.java | 42 ++---------- 16 files changed, 140 insertions(+), 110 deletions(-) rename {gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy => gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/store}/ColumnBatchKey.java (68%) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractDiskRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractDiskRegion.java index 30768dc96..571f8d5b6 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractDiskRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractDiskRegion.java @@ -850,7 +850,8 @@ public final boolean isBackup() { } @Override - public void updateMemoryStats(Object oldValue, Object newValue) { + public void updateMemoryStats(Object oldValue, Object newValue, + AbstractRegionEntry re) { // only used by BucketRegion as of now } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java index ddad7def7..bf3beedb1 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java @@ -1441,14 +1441,14 @@ final void _setValue(RegionEntryContext context, @Unretained final Object val) { if (rawOldVal instanceof SerializedDiskBuffer) { synchronized (rawOldVal) { setValueField(val); - if (context != null) context.updateMemoryStats(rawOldVal, val); + if (context != null) context.updateMemoryStats(rawOldVal, val, this); ((SerializedDiskBuffer)rawOldVal).release(); } return; } else if (val instanceof SerializedDiskBuffer) { synchronized (val) { setValueField(val); - if (context != null) context.updateMemoryStats(rawOldVal, val); + if (context != null) context.updateMemoryStats(rawOldVal, val, this); } return; } @@ -1463,7 +1463,7 @@ final void _setValue(RegionEntryContext context, @Unretained final Object val) { || (Token.isRemoved(val) && getValueAsToken() != Token.NOT_A_TOKEN)) { setValueField(val); if (!isOffHeap && context != null) { - context.updateMemoryStats(rawOldVal, val); + context.updateMemoryStats(rawOldVal, val, this); } } else { @@ -1501,7 +1501,7 @@ final void _setValue(RegionEntryContext context, @Unretained final Object val) { setContainerInfo(null, val); } if (!isOffHeap && context != null) { - context.updateMemoryStats(rawOldVal, val); + context.updateMemoryStats(rawOldVal, val, this); } return; } catch (IllegalAccessException e) { @@ -1739,10 +1739,10 @@ public final void setOwner(LocalRegion owner, Object previousOwner) { // set the context into the value if required initContextForDiskBuffer(owner, val); // add for new owner - if (owner != null) owner.updateMemoryStats(null, val); + if (owner != null) owner.updateMemoryStats(null, val, this); // reduce from previous owner if (previousOwner instanceof RegionEntryContext) { - ((RegionEntryContext)previousOwner).updateMemoryStats(val, null); + ((RegionEntryContext)previousOwner).updateMemoryStats(val, null, this); } } final StaticSystemCallbacks sysCb = diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java index 08da0b1a1..5c8e12396 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java @@ -38,7 +38,12 @@ import java.io.DataOutput; import java.io.IOException; import java.io.InputStream; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -72,6 +77,7 @@ import com.gemstone.gemfire.internal.cache.locks.LockingPolicy.ReadEntryUnderLock; import com.gemstone.gemfire.internal.cache.locks.ReentrantReadWriteWriteShareLock; import com.gemstone.gemfire.internal.cache.partitioned.*; +import com.gemstone.gemfire.internal.cache.store.ColumnBatchKey; import com.gemstone.gemfire.internal.cache.store.SerializedDiskBuffer; import com.gemstone.gemfire.internal.cache.tier.sockets.CacheClientNotifier; import com.gemstone.gemfire.internal.cache.tier.sockets.ClientTombstoneMessage; @@ -126,6 +132,7 @@ public class BucketRegion extends DistributedRegion implements Bucket { * Contains size in bytes of the direct byte buffers stored in memory. */ private final AtomicLong directBufferBytesInMemory = new AtomicLong(); + private final AtomicLong numRowsInColumnTable = new AtomicLong(); private final AtomicLong inProgressSize = new AtomicLong(); public static final ReadEntryUnderLock READ_SER_VALUE = new ReadEntryUnderLock() { @@ -2765,6 +2772,7 @@ else if (GemFireCacheImpl.gfxdSystem()) { prDs.updateMemoryStats(-oldMemValue); } this.directBufferBytesInMemory.set(0); + this.numRowsInColumnTable.set(0); // explicitly clear overflow counters if no diskRegion is present // (for latter the counters are cleared by DiskRegion.statsClear) if (getDiskRegion() == null) { @@ -2982,9 +2990,16 @@ protected void closeCallbacksExceptListener() { closeCacheCallback(getEvictionController()); } - public long getSizeInMemory() { - return Math.max(this.bytesInMemory.get(), 0L) + - this.directBufferBytesInMemory.get(); + public final long getSizeInMemory() { + return Math.max(this.bytesInMemory.get(), 0L) + getDirectBytesSizeInMemory(); + } + + public final long getDirectBytesSizeInMemory() { + return this.directBufferBytesInMemory.get(); + } + + public final long getNumRowsInColumnTable() { + return this.numRowsInColumnTable.get(); } public long getInProgressSize() { @@ -3102,22 +3117,44 @@ void updateBucket2Size(int oldSize, int newSize, } @Override - public void updateMemoryStats(final Object oldValue, final Object newValue) { + public void updateMemoryStats(final Object oldValue, final Object newValue, + final AbstractRegionEntry re) { if (newValue != oldValue) { int oldValueSize = calcMemSize(oldValue); int newValueSize = calcMemSize(newValue); updateBucketMemoryStats(newValueSize - oldValueSize); - if (this.cache.getMemorySize() > 0) { - int directBufferDelta = 0; - if (oldValue instanceof SerializedDiskBuffer) { - directBufferDelta -= ((SerializedDiskBuffer)oldValue).getOffHeapSizeInBytes(); - } - if (newValue instanceof SerializedDiskBuffer) { - directBufferDelta += ((SerializedDiskBuffer)newValue).getOffHeapSizeInBytes(); - } - if (directBufferDelta != 0) { - this.directBufferBytesInMemory.getAndAdd(directBufferDelta); + // update number of rows in table and off-heap size if applicable + if (re != null) { + int numColumns = this.partitionedRegion.getNumColumns(); + if (numColumns > 0) { + Object key = re.getRawKey(); + final boolean hasNewOffHeap = this.cache.getMemorySize() > 0; + if (key instanceof ColumnBatchKey) { + ColumnBatchKey batchKey = (ColumnBatchKey)key; + int directBufferDelta = 0; + int numRowsDelta = 0; + if (oldValue instanceof SerializedDiskBuffer) { + SerializedDiskBuffer oldBuffer = (SerializedDiskBuffer)oldValue; + if (hasNewOffHeap) { + directBufferDelta -= oldBuffer.getOffHeapSizeInBytes(); + } + numRowsDelta -= batchKey.getColumnBatchRowCount(this, oldBuffer); + } + if (newValue instanceof SerializedDiskBuffer) { + SerializedDiskBuffer newBuffer = (SerializedDiskBuffer)newValue; + if (hasNewOffHeap) { + directBufferDelta -= newBuffer.getOffHeapSizeInBytes(); + } + numRowsDelta += batchKey.getColumnBatchRowCount(this, newBuffer); + } + if (directBufferDelta != 0) { + this.directBufferBytesInMemory.getAndAdd(directBufferDelta); + } + if (numRowsDelta != 0) { + this.numRowsInColumnTable.getAndAdd(numRowsDelta); + } + } } } } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java index 6742290ba..f6878ea89 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExternalTableMetaData.java @@ -16,6 +16,7 @@ */ package com.gemstone.gemfire.internal.cache; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -25,6 +26,7 @@ public ExternalTableMetaData(String entityName, Object schema, String tableType, Object externalStore, + int numColumns, int columnBatchSize, int columnMaxDeltaRows, String compressionCodec, @@ -37,6 +39,7 @@ public ExternalTableMetaData(String entityName, this.schema = schema; this.tableType = tableType; this.externalStore = externalStore; + this.numColumns = numColumns; this.columnBatchSize = columnBatchSize; this.columnMaxDeltaRows = columnMaxDeltaRows; this.compressionCodec = compressionCodec; @@ -45,23 +48,25 @@ public ExternalTableMetaData(String entityName, this.dependents = dependents; this.dataSourcePath = dataSourcePath; this.driverClass = driverClass; + this.columns = Collections.emptyList(); } - public String entityName; - public Object schema; - public String tableType; + public final String entityName; + public final Object schema; + public final String tableType; // No type specified as the class is in snappy core - public Object externalStore; - public int columnBatchSize; - public int columnMaxDeltaRows; - public String compressionCodec; - public String baseTable; - public String dml; - public String[] dependents; + public final Object externalStore; + public final int numColumns; + public final int columnBatchSize; + public final int columnMaxDeltaRows; + public final String compressionCodec; + public final String baseTable; + public final String dml; + public final String[] dependents; public String provider; public String shortProvider; - public String dataSourcePath; - public String driverClass; + public final String dataSourcePath; + public final String driverClass; public String viewText; // columns for metadata queries public List columns; diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java index 296bae864..317ee3bcf 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java @@ -6587,7 +6587,7 @@ public void beforeReturningOffHeapMemoryToAllocator(long address, /** * Fetches hive meta data for Snappy tables. */ - public ExternalTableMetaData fetchSnappyTablesHiveMetaData(PartitionedRegion region); + public ExternalTableMetaData fetchTableMetaData(PartitionedRegion region); } /** diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java index aecbea582..658d5a3f3 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java @@ -680,10 +680,10 @@ protected LocalRegion(String regionName, RegionAttributes attrs, Assert.assertTrue(regionName != null, "regionName must not be null"); this.sharedDataView = buildDataView(); this.regionName = regionName; - this.isInternalColumnTable = regionName.toUpperCase(Locale.ENGLISH) - .endsWith(StoreCallbacks.SHADOW_TABLE_SUFFIX); this.parentRegion = parentRegion; this.fullPath = calcFullPath(regionName, parentRegion); + this.isInternalColumnTable = SystemProperties.isColumnTable( + this.fullPath.toUpperCase(Locale.ENGLISH)); // cannot support patterns like "..._/..." due to ambiguity in encoding // of bucket regions if (this.fullPath.contains("_/")) { @@ -3186,7 +3186,8 @@ public boolean isBackup() { } @Override - public void updateMemoryStats(Object oldValue, Object newValue) { + public void updateMemoryStats(Object oldValue, Object newValue, + AbstractRegionEntry re) { // only used by BucketRegion as of now } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java index 0c45aff1e..d5752a9a8 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java @@ -394,17 +394,20 @@ public class PartitionedRegion extends LocalRegion implements private volatile int shutDownAllStatus = RUNNING_MODE; + /** Number of columns if this region is a store for a column table. */ + private volatile int numColumns = -1; + /** Maximum size in bytes for ColumnBatches. */ - private int columnBatchSize = -1; + private volatile int columnBatchSize = -1; /** Maximum rows to keep in the delta buffer. */ - private int columnMaxDeltaRows = -1; + private volatile int columnMaxDeltaRows = -1; /** Minimum size for ColumnBatches. */ - private int columnMinDeltaRows = SystemProperties.SNAPPY_MIN_COLUMN_DELTA_ROWS; + private volatile int columnMinDeltaRows = SystemProperties.SNAPPY_MIN_COLUMN_DELTA_ROWS; /** default compression used by the column store */ - private String columnCompressionCodec; + private volatile String columnCompressionCodec; public void setColumnBatchSizes(int size, int maxDeltaRows, int minDeltaRows) { @@ -417,19 +420,27 @@ private void initFromHiveMetaData() { final GemFireCacheImpl.StaticSystemCallbacks sysCb = GemFireCacheImpl .getInternalProductCallbacks(); if (sysCb != null) { - ExternalTableMetaData metadata = sysCb.fetchSnappyTablesHiveMetaData(this); - if (this.columnBatchSize == -1) { - this.columnBatchSize = metadata.columnBatchSize; - } - if (this.columnMaxDeltaRows == -1) { - this.columnMaxDeltaRows = metadata.columnMaxDeltaRows; - } - if (this.columnCompressionCodec == null) { - this.columnCompressionCodec = metadata.compressionCodec; + synchronized (this) { + ExternalTableMetaData metadata = sysCb.fetchTableMetaData(this); + if (this.numColumns == -1) { + this.numColumns = metadata.numColumns; + this.columnBatchSize = metadata.columnBatchSize; + this.columnMaxDeltaRows = metadata.columnMaxDeltaRows; + this.columnCompressionCodec = metadata.compressionCodec; + } } } } + public int getNumColumns() { + int numColumns = this.numColumns; + if (numColumns == -1) { + initFromHiveMetaData(); + return this.numColumns; + } + return numColumns; + } + public int getColumnBatchSize() { int columnBatchSize = this.columnBatchSize; if (columnBatchSize == -1) { diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java index 3bf474c21..83a21b370 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java @@ -54,5 +54,6 @@ public interface RegionEntryContext extends HasCachePerfStats { */ public boolean isBackup(); - public void updateMemoryStats(Object oldValue, Object newValue); + public void updateMemoryStats(Object oldValue, Object newValue, + AbstractRegionEntry re); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ColumnBatchKey.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/store/ColumnBatchKey.java similarity index 68% rename from gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ColumnBatchKey.java rename to gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/store/ColumnBatchKey.java index c1d7758fd..120a08c49 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ColumnBatchKey.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/store/ColumnBatchKey.java @@ -14,27 +14,23 @@ * permissions and limitations under the License. See accompanying * LICENSE file. */ -package com.pivotal.gemfirexd.internal.snappy; +package com.gemstone.gemfire.internal.cache.store; import com.gemstone.gemfire.internal.cache.AbstractRegionEntry; import com.gemstone.gemfire.internal.cache.BucketRegion; import com.gemstone.gemfire.internal.cache.lru.Sizeable; -import com.gemstone.gemfire.internal.cache.partitioned.PREntriesIterator; /** * Interface for a key object in the column store. */ -public interface ColumnBatchKey extends Sizeable { - - /** - * Get the number of columns defined for the given - * column table (qualified name). - */ - int getNumColumnsInTable(String columnTableName); +public abstract class ColumnBatchKey implements Sizeable { /** * Get the number of rows in this column batch. + * This will return a non-zero result only for the STATS keys while + * for a key of DELETE bitmask it will return negative value + * indicating the delete count. */ - int getColumnBatchRowCount(BucketRegion bucketRegion, AbstractRegionEntry re, - int numColumnsInTable); + public abstract int getColumnBatchRowCount(BucketRegion bucketRegion, + SerializedDiskBuffer value); } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java index 0eb3bae41..91b80b4fa 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java @@ -54,11 +54,6 @@ public List getInternalTableSchemas() { return Collections.emptyList(); } - @Override - public boolean isColumnTable(String qualifiedName) { - return false; - } - @Override public boolean skipEvictionForEntry(LRUEntry entry) { return false; diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java index 492639b2a..8299cea15 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java @@ -44,8 +44,6 @@ Set createColumnBatch(BucketRegion region, long batchID, List getInternalTableSchemas(); - boolean isColumnTable(String qualifiedName); - boolean skipEvictionForEntry(LRUEntry entry); int getHashCodeSnappy(Object dvd, int numPartitions); diff --git a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/SystemProperties.java b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/SystemProperties.java index 1a9b591de..b391b7f7c 100644 --- a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/SystemProperties.java +++ b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/SystemProperties.java @@ -254,6 +254,11 @@ public static Callbacks getGFXDServerCallbacks() { } } + public static boolean isColumnTable(String fullName) { + return fullName.endsWith(SHADOW_TABLE_SUFFIX) && + fullName.contains(SHADOW_SCHEMA_NAME); + } + public synchronized void setCallbacks(Callbacks cb) { this.callbacks = cb; } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java index 7b155d108..98f961ee4 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java @@ -56,7 +56,11 @@ Licensed to the Apache Software Foundation (ASF) under one or more import com.gemstone.gemfire.CancelException; import com.gemstone.gemfire.LogWriter; -import com.gemstone.gemfire.cache.*; +import com.gemstone.gemfire.cache.DataPolicy; +import com.gemstone.gemfire.cache.EvictionAction; +import com.gemstone.gemfire.cache.EvictionAttributes; +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.Scope; import com.gemstone.gemfire.distributed.internal.DistributionManager; import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem; import com.gemstone.gemfire.internal.ClassPathLoader; @@ -81,7 +85,10 @@ Licensed to the Apache Software Foundation (ASF) under one or more import com.pivotal.gemfirexd.internal.engine.access.GemFireTransaction; import com.pivotal.gemfirexd.internal.engine.access.index.GfxdIndexManager; import com.pivotal.gemfirexd.internal.engine.access.index.MemIndex; -import com.pivotal.gemfirexd.internal.engine.ddl.*; +import com.pivotal.gemfirexd.internal.engine.ddl.DDLConflatable; +import com.pivotal.gemfirexd.internal.engine.ddl.GfxdDDLQueueEntry; +import com.pivotal.gemfirexd.internal.engine.ddl.GfxdDDLRegionQueue; +import com.pivotal.gemfirexd.internal.engine.ddl.ReplayableConflatable; import com.pivotal.gemfirexd.internal.engine.ddl.catalog.messages.GfxdSystemProcedureMessage; import com.pivotal.gemfirexd.internal.engine.ddl.wan.messages.AbstractGfxdReplayableMessage; import com.pivotal.gemfirexd.internal.engine.distributed.GfxdMessage; @@ -697,7 +704,7 @@ public static void checkSnappyCatalogConsistency(EmbedConnection embedConn, List internalColumnTablesListPerSchema = new LinkedList<>(); for (Map.Entry> e : gfDBTablesMap.entrySet()) { for (String t : e.getValue()) { - if (CallbackFactoryProvider.getStoreCallbacks().isColumnTable(e.getKey() + "." + t)) { + if (SystemProperties.isColumnTable(e.getKey() + "." + t)) { internalColumnTablesListPerSchema.add(t); } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java index bd3896715..e7a0837a4 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java @@ -1615,7 +1615,7 @@ public Properties getSecurityPropertiesForReconnect() { } } - public ExternalTableMetaData fetchSnappyTablesHiveMetaData( + public ExternalTableMetaData fetchTableMetaData( PartitionedRegion region) { GemFireContainer container = (GemFireContainer)region.getUserAttribute(); if (container != null) { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java index 435f3cd3d..d03c41777 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java @@ -40,6 +40,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more package com.pivotal.gemfirexd.internal.impl.sql.execute; +import com.gemstone.gemfire.internal.shared.SystemProperties; import com.pivotal.gemfirexd.internal.catalog.ExternalCatalog; import com.pivotal.gemfirexd.internal.engine.GfxdConstants; import com.pivotal.gemfirexd.internal.engine.Misc; @@ -232,7 +233,7 @@ public void executeGrantRevoke( Activation activation, GemFireStore ms = Misc.getMemStore(); boolean isSnappyStoreWithSecurityEnabled = ms.isSnappyStore() && Misc.isSecurityEnabled(); if (isSnappyStoreWithSecurityEnabled && - CallbackFactoryProvider.getStoreCallbacks().isColumnTable(Misc.getFullTableName(td.getSchemaName(), + SystemProperties.isColumnTable(Misc.getFullTableName(td.getSchemaName(), td.getName(), activation.getLanguageConnectionContext()))) { // do nothing for columm batch tables, they will be handled during the corresponding // main table handling diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java index c42926219..c49f425a5 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java @@ -22,18 +22,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.sql.Connection; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingDeque; @@ -54,6 +43,8 @@ import com.gemstone.gemfire.internal.cache.lru.Sizeable; import com.gemstone.gemfire.internal.cache.partitioned.Bucket; import com.gemstone.gemfire.internal.cache.partitioned.PREntriesIterator; +import com.gemstone.gemfire.internal.cache.store.ColumnBatchKey; +import com.gemstone.gemfire.internal.cache.store.SerializedDiskBuffer; import com.gemstone.gemfire.internal.cache.wan.AbstractGatewaySender; import com.gemstone.gemfire.internal.cache.wan.GatewaySenderEventImpl; import com.gemstone.gemfire.internal.cache.wan.parallel.ConcurrentParallelGatewaySenderQueue; @@ -76,7 +67,6 @@ import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer; import com.pivotal.gemfirexd.internal.engine.store.GemFireStore; import com.pivotal.gemfirexd.internal.engine.store.RegionEntryUtils; -import com.pivotal.gemfirexd.internal.engine.store.RowFormatter; import com.pivotal.gemfirexd.internal.engine.store.offheap.OffHeapRow; import com.pivotal.gemfirexd.internal.engine.store.offheap.OffHeapRowWithLobs; import com.pivotal.gemfirexd.internal.iapi.error.StandardException; @@ -84,12 +74,10 @@ import com.pivotal.gemfirexd.internal.iapi.services.context.ContextManager; import com.pivotal.gemfirexd.internal.iapi.services.io.FormatableHashtable; import com.pivotal.gemfirexd.internal.iapi.services.sanity.SanityManager; -import com.pivotal.gemfirexd.internal.iapi.sql.dictionary.ColumnDescriptor; import com.pivotal.gemfirexd.internal.iapi.types.DataValueDescriptor; import com.pivotal.gemfirexd.internal.iapi.types.RowLocation; import com.pivotal.gemfirexd.internal.impl.sql.catalog.GfxdDataDictionary; import com.pivotal.gemfirexd.internal.shared.common.reference.SQLState; -import com.pivotal.gemfirexd.internal.snappy.ColumnBatchKey; /** * This class helps in finding out memory footprint per Region and divides @@ -638,7 +626,6 @@ private void iterateRegionEntries(final GemFireContainer c, final boolean agentAttached = SIZE_OF_UTIL.isAgentAttached(); boolean isValueTypeEvaluated = false; final boolean isColumnTable = c.isColumnStore(); - int numColumnsInColumnTable = -1; long dvdArraySize = 0; boolean gatewayEntries = false; boolean offHeapEntries = false; @@ -755,8 +742,6 @@ else if (!isSingleEntrySizeComputed) { if (value != null) { if (!isValueTypeEvaluated) { if (isColumnTable) { - numColumnsInColumnTable = batchKey.getNumColumnsInTable( - c.getQualifiedTableName()); } else if (value instanceof DataValueDescriptor[]) { dvdArraySize = SIZE_OF_UTIL.sizeof(value); } else if (value instanceof GatewaySenderEventImpl) { @@ -768,8 +753,10 @@ else if (!isSingleEntrySizeComputed) { } if (isColumnTable) { int valueSize = ((Sizeable)value).getSizeInBytes(); - columnRowCount += batchKey.getColumnBatchRowCount(prEntryIter.getHostedBucketRegion(), - re, numColumnsInColumnTable); + if (value instanceof SerializedDiskBuffer) { + columnRowCount += batchKey.getColumnBatchRowCount( + prEntryIter.getHostedBucketRegion(), (SerializedDiskBuffer)value); + } valInMemoryCount++; valInMemorySize += valueSize; if (valueSize > maxSize) { @@ -840,11 +827,6 @@ else if (!isSingleEntrySizeComputed) { break OUTER; } else if (valClass == byte[][].class) { - - if (c.isColumnStore()) { - columnRowCount += getRowCountFromColumnTable(c, (byte[][])value); - } - valInMemoryCount++; byte[][] values = ((byte[][])value); // add in size of Object[] long len = SIZE_OF_UTIL.sizeof(value); @@ -993,16 +975,6 @@ else if (GatewaySenderEventImpl.class.isAssignableFrom(valClass)) { } - private int getRowCountFromColumnTable(GemFireContainer c, byte[][] value) throws StandardException { - int rowCount = 0; - RowFormatter rf = c.getRowFormatter(value); - ColumnDescriptor cd = c.getTableDescriptor().getColumnDescriptor("NUMROWS"); - if (cd != null) - rowCount = rf.getColumn(cd.getPosition(), value).getInt(); - - return rowCount; - } - private static void createDistributionInfo(StringBuilder miscInfo, String msg, TreeMap bucketDist, int maxKeyLength, int maxValueLength) { From 02dede4bfed64720822488324cc9b90f7c20a1d9 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Thu, 1 Aug 2019 11:11:06 +0530 Subject: [PATCH 6/7] moved isColumnTable from SystemProperties to ClientSharedUtils --- .../gemfire/internal/cache/LocalRegion.java | 3 ++- .../internal/shared/ClientSharedUtils.java | 26 +++---------------- .../internal/shared/SystemProperties.java | 5 ---- .../internal/engine/db/FabricDatabase.java | 3 ++- .../impl/sql/execute/TablePrivilegeInfo.java | 4 +-- 5 files changed, 9 insertions(+), 32 deletions(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java index 658d5a3f3..3f5c4590b 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java @@ -150,6 +150,7 @@ import com.gemstone.gemfire.internal.offheap.annotations.Retained; import com.gemstone.gemfire.internal.offheap.annotations.Unretained; import com.gemstone.gemfire.internal.sequencelog.EntryLogger; +import com.gemstone.gemfire.internal.shared.ClientSharedUtils; import com.gemstone.gemfire.internal.shared.SystemProperties; import com.gemstone.gemfire.internal.shared.Version; import com.gemstone.gemfire.internal.size.ReflectionObjectSizer; @@ -682,7 +683,7 @@ protected LocalRegion(String regionName, RegionAttributes attrs, this.regionName = regionName; this.parentRegion = parentRegion; this.fullPath = calcFullPath(regionName, parentRegion); - this.isInternalColumnTable = SystemProperties.isColumnTable( + this.isInternalColumnTable = ClientSharedUtils.isColumnTable( this.fullPath.toUpperCase(Locale.ENGLISH)); // cannot support patterns like "..._/..." due to ambiguity in encoding // of bucket regions diff --git a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/ClientSharedUtils.java b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/ClientSharedUtils.java index e3fb04e2d..512aaa974 100644 --- a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/ClientSharedUtils.java +++ b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/ClientSharedUtils.java @@ -53,10 +53,8 @@ import java.nio.channels.Channel; import java.nio.channels.SocketChannel; import java.nio.file.Files; -import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.CodeSource; import java.security.PrivilegedAction; import java.util.*; import java.util.concurrent.locks.LockSupport; @@ -1358,27 +1356,9 @@ public static RuntimeException newRuntimeException(String message, return cre.newRunTimeException(message, cause); } - public static Path getProductJarsDirectory(Class c) throws IOException { - CodeSource cs = c.getProtectionDomain().getCodeSource(); - URL jarURL = cs != null ? cs.getLocation() : null; - if (jarURL != null) { - return Paths.get(URLDecoder.decode(jarURL.getFile(), "UTF-8")) - .getParent().toRealPath(LinkOption.NOFOLLOW_LINKS); - } else { - // try in SNAPPY_HOME and SPARK_HOME - String productHome = System.getenv("SNAPPY_HOME"); - if (productHome == null) { - productHome = System.getenv("SPARK_HOME"); - } - if (productHome != null) { - return Paths.get(productHome, "jars") - .toRealPath(LinkOption.NOFOLLOW_LINKS); - } else { - throw new IllegalStateException("Unable to locate product install " + - "location. Set SNAPPY_HOME or SPARK_HOME explicitly if not using " + - "the standard scripts."); - } - } + public static boolean isColumnTable(String fullName) { + return fullName.endsWith(SystemProperties.SHADOW_TABLE_SUFFIX) && + fullName.contains(SystemProperties.SHADOW_SCHEMA_NAME); } // Convert log4j.Level to java.util.logging.Level diff --git a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/SystemProperties.java b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/SystemProperties.java index b391b7f7c..1a9b591de 100644 --- a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/SystemProperties.java +++ b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/SystemProperties.java @@ -254,11 +254,6 @@ public static Callbacks getGFXDServerCallbacks() { } } - public static boolean isColumnTable(String fullName) { - return fullName.endsWith(SHADOW_TABLE_SUFFIX) && - fullName.contains(SHADOW_SCHEMA_NAME); - } - public synchronized void setCallbacks(Callbacks cb) { this.callbacks = cb; } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java index 98f961ee4..42ccebb83 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/db/FabricDatabase.java @@ -67,6 +67,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more import com.gemstone.gemfire.internal.GFToSlf4jBridge; import com.gemstone.gemfire.internal.LogWriterImpl; import com.gemstone.gemfire.internal.cache.*; +import com.gemstone.gemfire.internal.shared.ClientSharedUtils; import com.gemstone.gemfire.internal.shared.SystemProperties; import com.gemstone.gemfire.internal.util.ArrayUtils; import com.gemstone.gnu.trove.THashMap; @@ -704,7 +705,7 @@ public static void checkSnappyCatalogConsistency(EmbedConnection embedConn, List internalColumnTablesListPerSchema = new LinkedList<>(); for (Map.Entry> e : gfDBTablesMap.entrySet()) { for (String t : e.getValue()) { - if (SystemProperties.isColumnTable(e.getKey() + "." + t)) { + if (ClientSharedUtils.isColumnTable(e.getKey() + "." + t)) { internalColumnTablesListPerSchema.add(t); } } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java index d03c41777..15875250b 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/execute/TablePrivilegeInfo.java @@ -40,7 +40,7 @@ Licensed to the Apache Software Foundation (ASF) under one or more package com.pivotal.gemfirexd.internal.impl.sql.execute; -import com.gemstone.gemfire.internal.shared.SystemProperties; +import com.gemstone.gemfire.internal.shared.ClientSharedUtils; import com.pivotal.gemfirexd.internal.catalog.ExternalCatalog; import com.pivotal.gemfirexd.internal.engine.GfxdConstants; import com.pivotal.gemfirexd.internal.engine.Misc; @@ -233,7 +233,7 @@ public void executeGrantRevoke( Activation activation, GemFireStore ms = Misc.getMemStore(); boolean isSnappyStoreWithSecurityEnabled = ms.isSnappyStore() && Misc.isSecurityEnabled(); if (isSnappyStoreWithSecurityEnabled && - SystemProperties.isColumnTable(Misc.getFullTableName(td.getSchemaName(), + ClientSharedUtils.isColumnTable(Misc.getFullTableName(td.getSchemaName(), td.getName(), activation.getLanguageConnectionContext()))) { // do nothing for columm batch tables, they will be handled during the corresponding // main table handling From 16db6c5fcdd4ad22fefa38985f9cf8ef19e98486 Mon Sep 17 00:00:00 2001 From: Sumedh Wale Date: Tue, 13 Aug 2019 13:17:36 +0530 Subject: [PATCH 7/7] fix off-heap accounting --- .../java/com/gemstone/gemfire/internal/cache/BucketRegion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java index 5c8e12396..04cc87d16 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java @@ -3144,7 +3144,7 @@ public void updateMemoryStats(final Object oldValue, final Object newValue, if (newValue instanceof SerializedDiskBuffer) { SerializedDiskBuffer newBuffer = (SerializedDiskBuffer)newValue; if (hasNewOffHeap) { - directBufferDelta -= newBuffer.getOffHeapSizeInBytes(); + directBufferDelta += newBuffer.getOffHeapSizeInBytes(); } numRowsDelta += batchKey.getColumnBatchRowCount(this, newBuffer); }