diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegion.java index 0742fcdf9..d11a5eea2 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegion.java @@ -95,6 +95,7 @@ import com.gemstone.gemfire.internal.cache.lru.LRUAlgorithm; import com.gemstone.gemfire.internal.cache.snapshot.RegionSnapshotServiceImpl; import com.gemstone.gemfire.internal.i18n.LocalizedStrings; +import com.gemstone.gemfire.internal.shared.SystemProperties; import com.gemstone.gemfire.internal.util.ArrayUtils; import com.gemstone.gemfire.pdx.internal.PeerTypeRegistration; import com.google.common.util.concurrent.Service.State; @@ -1888,6 +1889,13 @@ private void setAttributes(RegionAttributes attrs,String regionName, InternalReg } this.compressor = attrs.getCompressor(); + if (this.compressor != null && !RegionEntryContext.COMPRESSION_ENABLED) { + throw new IllegalStateException("Compressor set on region " + + getFullPath() + " with global compression disabled. " + + "First enable compression with '" + + SystemProperties.getServerInstance().getSystemPropertyNamePrefix() + + "compression.enable=true'."); + } // enable concurrency checks for persistent regions if(!attrs.getConcurrencyChecksEnabled() && attrs.getDataPolicy().withPersistence() diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java index ddad7def7..ff2967b81 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java @@ -2709,8 +2709,9 @@ public void changeEventValue(Object v) { } static boolean isCompressible(RegionEntryContext context, Object value) { - return (context != null && context.getCompressor() != null && - value != null && !Token.isInvalidOrRemoved(value)); + return (RegionEntryContext.COMPRESSION_ENABLED && context != null && + context.getCompressor() != null && value != null && + !Token.isInvalidOrRemoved(value)); } /* subclasses supporting versions must override this */ diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java index 80f2e02f8..cb61f0240 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java @@ -2306,7 +2306,7 @@ else if (cbEvent != null && owner.getConcurrencyChecksEnabled() catch (RegionClearedException rce) { clearOccured = true; } - owner.txApplyDestroyPart2(re, key, inTokenMode, + owner.txApplyDestroyPart2(txState, re, key, inTokenMode, clearOccured /* Clear Conflciting with the operation */); if (cbEvent != null) { if (pendingCallbacks == null) { @@ -2376,12 +2376,12 @@ else if (inTokenMode || owner.concurrencyChecksEnabled) { EntryLogger.logTXDestroy(_getOwnerObject(), key); } owner.updateSizeOnRemove(key, oldSize); - owner.txApplyDestroyPart2(re, key, inTokenMode, + owner.txApplyDestroyPart2(txState, re, key, inTokenMode, false /* Clear Conflicting with the operation */); lruEntryDestroy(re); } catch (RegionClearedException rce) { - owner.txApplyDestroyPart2(re, key, inTokenMode, + owner.txApplyDestroyPart2(txState, re, key, inTokenMode, true /* Clear Conflicting with the operation */); } @@ -3249,7 +3249,8 @@ public final void txApplyInvalidate(final RegionEntry re, } catch (RegionClearedException rce) { clearOccured = true; } - owner.txApplyInvalidatePart2(re, key, didDestroy, true, clearOccured); + owner.txApplyInvalidatePart2(txState, re, key, didDestroy, + true, clearOccured); // didInvalidate = true; if (cbEvent != null) { if (pendingCallbacks == null) { diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketAdvisor.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketAdvisor.java index 51bc30e9d..0681b9afe 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketAdvisor.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketAdvisor.java @@ -50,12 +50,16 @@ import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember; import com.gemstone.gemfire.i18n.LogWriterI18n; import com.gemstone.gemfire.internal.Assert; +import com.gemstone.gemfire.internal.cache.locks.ExclusiveSharedSynchronizer; +import com.gemstone.gemfire.internal.cache.locks.LockMode; +import com.gemstone.gemfire.internal.cache.locks.ReentrantReadWriteWriteShareLock; import com.gemstone.gemfire.internal.cache.partitioned.Bucket; import com.gemstone.gemfire.internal.cache.partitioned.BucketListener; import com.gemstone.gemfire.internal.cache.partitioned.BucketProfileUpdateMessage; import com.gemstone.gemfire.internal.cache.partitioned.DeposePrimaryBucketMessage; import com.gemstone.gemfire.internal.cache.partitioned.DeposePrimaryBucketMessage.DeposePrimaryBucketResponse; import com.gemstone.gemfire.internal.cache.partitioned.RegionAdvisor; +import com.gemstone.gemfire.internal.concurrent.ConcurrentHashSet; import com.gemstone.gemfire.internal.i18n.LocalizedStrings; import com.gemstone.gemfire.internal.util.StopWatch; @@ -92,6 +96,12 @@ public final class BucketAdvisor extends CacheDistributionAdvisor { /** Local lock used by PRHARedundancyProvider */ protected final ReentrantLock redundancyLock; + /** lock for maintenance operations */ + private final ReentrantReadWriteWriteShareLock maintenanceLock = + new ReentrantReadWriteWriteShareLock(); + private final ConcurrentHashSet maintenanceLockReaders = + new ConcurrentHashSet<>(8); + //private static final byte MASK_HOSTING = 1; // 0001 //private static final byte MASK_VOLUNTEERING = 2; // 0010 //private static final byte MASK_OTHER_PRIMARY = 4; // 0100 @@ -293,6 +303,86 @@ public void tryLockIfPrimary() { } } + boolean isLockededForMaintenance() { + return ExclusiveSharedSynchronizer.isExclusive(maintenanceLock.getState()); + } + + private BucketRegion getRowBuffer() { + PartitionedRegion pr = getPartitionedRegion(); + assert pr.isInternalColumnTable(); + PartitionedRegion rowBuffer = ColocationHelper.getColocatedRegion(pr); + assert rowBuffer != null; + return rowBuffer.getDataStore().getLocalBucketById(getProxyBucketRegion().getId()); + } + + boolean lockForMaintenance(boolean forWrite, long msecs, Object owner) { + if (getPartitionedRegion().isInternalColumnTable()) { + return getRowBuffer().getBucketAdvisor().lockForMaintenance( + forWrite, msecs, owner); + } + boolean locked; + if (forWrite) { + locked = maintenanceLock.attemptLock(LockMode.EX, msecs, owner); + } else { + locked = maintenanceLock.attemptLock(LockMode.SH, msecs, owner); + // avoid re-entry by same owner + if (locked && !maintenanceLockReaders.add(owner)) { + maintenanceLock.releaseLock(LockMode.SH, false, owner); + locked = false; + } + } + getLogWriter().convertToLogWriter().info("SW:0: locked " + getProxyBucketRegion().getFullPath() + " forWrite=" + forWrite + " locked=" + locked); + return locked; + } + + void unlockAfterMaintenance(boolean forWrite, Object owner) { + if (getPartitionedRegion().isInternalColumnTable()) { + getRowBuffer().getBucketAdvisor().unlockAfterMaintenance(forWrite, owner); + return; + } + if (forWrite) { + maintenanceLock.releaseLock(LockMode.EX, false, owner); + } else if (maintenanceLockReaders.remove(owner)) { + maintenanceLock.releaseLock(LockMode.SH, false, owner); + } else { + throw new IllegalMonitorStateException("Bucket " + getProxyBucketRegion() + .getFullPath() + " not read-locked for maintenance by " + owner); + } + getLogWriter().convertToLogWriter().info("SW:0: unlocked " + getProxyBucketRegion().getFullPath() + " forWrite=" + forWrite); + } + + void unlockAllAfterMaintenance(boolean forWrite) { + if (getPartitionedRegion().isInternalColumnTable()) { + getRowBuffer().getBucketAdvisor().unlockAllAfterMaintenance(forWrite); + return; + } + if (forWrite) { + Object writer = maintenanceLock.getOwnerId(null); + if (writer != null) { + maintenanceLock.releaseLock(LockMode.EX, false, writer); + } + } else { + for (Object reader : maintenanceLockReaders) { + try { + maintenanceLock.releaseLock(LockMode.SH, false, reader); + } catch (IllegalMonitorStateException ignored) { + } + } + } + getLogWriter().convertToLogWriter().info("SW:0: unlocked all " + getProxyBucketRegion().getFullPath() + " forWrite=" + forWrite); + } + + boolean hasMaintenanceLock(boolean forWrite, Object owner) { + if (getPartitionedRegion().isInternalColumnTable()) { + return getRowBuffer().getBucketAdvisor().hasMaintenanceLock(forWrite, owner); + } + if (forWrite) { + return Objects.equals(owner, maintenanceLock.getOwnerId(null)); + } else { + return owner != null && maintenanceLockReaders.contains(owner); + } + } + /** * Makes this BucketAdvisor give up being a primary and become * a secondary. Does nothing if not currently the primary. @@ -1230,6 +1320,8 @@ public boolean becomePrimary(boolean isRebalance) { long startTime = getPartitionedRegionStats().startPrimaryTransfer(isRebalance); + // acquire maintenance lock + lockForMaintenance(true, Long.MAX_VALUE, this); try { long waitTime = 2000; // time each iteration will wait while (!isPrimary()) { @@ -1326,6 +1418,7 @@ public boolean becomePrimary(boolean isRebalance) { Thread.currentThread().interrupt(); } finally { + unlockAfterMaintenance(true, this); getPartitionedRegionStats().endPrimaryTransfer( startTime, isPrimary(), isRebalance); } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java index 7ff10014c..04fea8b50 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/BucketRegion.java @@ -42,7 +42,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Predicate; import com.gemstone.gemfire.*; import com.gemstone.gemfire.cache.*; @@ -268,10 +268,6 @@ public static RawValueFactory getFactory() { public static final long INVALID_UUID = VMIdAdvisor.INVALID_ID; - public final ReentrantReadWriteLock columnBatchFlushLock = - new ReentrantReadWriteLock(); - - /** * A read/write lock to prevent writing to the bucket when GII from this bucket is in progress */ @@ -761,19 +757,53 @@ public final boolean checkForColumnBatchCreation(TXStateInterface tx) { || getTotalBytes() >= pr.getColumnBatchSize()); } + private static final Predicate TRUE_CHECK = v -> true; + + @SuppressWarnings("unchecked") + public static Predicate TRUE_PREDICATE() { + return (Predicate)TRUE_CHECK; + } + + public boolean isLockededForMaintenance() { + return getBucketAdvisor().isLockededForMaintenance(); + } + + public boolean lockForMaintenance(boolean forWrite, long msecs, Object owner) { + return getBucketAdvisor().lockForMaintenance(forWrite, msecs, owner); + } + + public void unlockAfterMaintenance(boolean forWrite, Object owner) { + getBucketAdvisor().unlockAfterMaintenance(forWrite, owner); + } + + public void unlockAllAfterMaintenance(boolean forWrite) { + getBucketAdvisor().unlockAllAfterMaintenance(forWrite); + } + + public boolean hasMaintenanceLock(boolean forWrite, Object owner) { + return getBucketAdvisor().hasMaintenanceLock(forWrite, owner); + } + + @SuppressWarnings("unchecked") public final boolean createAndInsertColumnBatch(TXStateInterface tx, boolean forceFlush) { - // do nothing if a flush is already in progress - if (this.columnBatchFlushLock.isWriteLocked()) { + return createAndInsertColumnBatch(tx, forceFlush, 0, TRUE_PREDICATE()); + } + + public final boolean createAndInsertColumnBatch(TXStateInterface tx, + boolean forceFlush, long msecs, Predicate checkFlushInLock) { + // first acquire the maintenance lock to prevent concurrent + // updates/deletes to change entries being rolled over + Object owner = tx != null ? tx.getTransactionId() : null; + if (owner == null) owner = Thread.currentThread(); + if (!lockForMaintenance(true, msecs, owner)) { return false; } - final ReentrantReadWriteLock.WriteLock sync = - this.columnBatchFlushLock.writeLock(); - sync.lock(); try { - return internalCreateAndInsertColumnBatch(tx, forceFlush); + return checkFlushInLock.test(this) && + internalCreateAndInsertColumnBatch(tx, forceFlush); } finally { - sync.unlock(); + unlockAfterMaintenance(true, owner); } } @@ -864,7 +894,8 @@ private Set createColumnBatchAndPutInColumnTable(long key) { // This destroy is under a lock which makes sure that there is no put into the region // No need to take the lock on key private void destroyAllEntries(Set keysToDestroy, long batchKey) { - for(Object key : keysToDestroy) { + TXStateInterface txState = getTXState(); + for (Object key : keysToDestroy) { if (getCache().getLoggerI18n().fineEnabled()) { getCache() .getLoggerI18n() @@ -878,8 +909,8 @@ private void destroyAllEntries(Set keysToDestroy, long batchKey) { event.setKey(key); event.setBucketId(this.getId()); + event.setTXState(txState); - TXStateInterface txState = event.getTXState(this); if (txState != null) { event.setRegion(this); txState.destroyExistingEntry(event, true, null); diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegion.java index 53ebda228..0d3ed048d 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskRegion.java @@ -724,7 +724,7 @@ public void releaseReadLock() { getDiskStore().releaseReadLock(this); } void basicAcquireReadLock() { - this.rwLock.readLock().lock(); + this.rwLock.readLock().lockDelayCancel(); // basicAcquireLock(); } void basicReleaseReadLock() { diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskStoreImpl.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskStoreImpl.java index d67ae3e23..d967f168d 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskStoreImpl.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DiskStoreImpl.java @@ -3972,10 +3972,10 @@ public boolean contains(long id) { public int size() { return this.ints.size() + this.longs.size(); } - + public void addAll(OplogEntryIdSet toAdd) { - this.ints.addAll(toAdd.ints.toArray()); - this.longs.addAll(toAdd.longs.toArray()); + this.ints.addAll(toAdd.ints); + this.longs.addAll(toAdd.longs); } } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DistributedRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DistributedRegion.java index 895e8d3dc..74c8bdbbc 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DistributedRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DistributedRegion.java @@ -117,7 +117,6 @@ import com.gemstone.gemfire.internal.util.ArraySortedCollection; import com.gemstone.gemfire.internal.util.concurrent.StoppableCountDownLatch; import com.gemstone.gnu.trove.TObjectIntProcedure; -import com.gemstone.gnu.trove.TObjectObjectProcedure; import com.gemstone.org.jgroups.util.StringId; import org.eclipse.collections.impl.map.mutable.UnifiedMap; @@ -1796,7 +1795,7 @@ private void cleanUpDestroyedTokensAndMarkGIIComplete(GIIStatus giiStatus) final TXManagerImpl txMgr = getCache().getCacheTransactionManager(); for (TXRegionState txrs : orderedTXRegionState) { TXState txState = txrs.getTXState(); - int txOrder = 0; + long txOrder = 0; getLogWriterI18n().info(LocalizedStrings.DEBUG, "Locking txState = " + txState); txState.lockTXState(); if (txState.isInProgress() && (txOrder = is.getFinishedTXOrder( @@ -1825,8 +1824,8 @@ private void cleanUpDestroyedTokensAndMarkGIIComplete(GIIStatus giiStatus) Arrays.sort(finishedTXRS, new Comparator() { @Override public int compare(TXRegionState t1, TXRegionState t2) { - final int o1 = t1.getFinishOrder(); - final int o2 = t2.getFinishOrder(); + final long o1 = t1.getFinishOrder(); + final long o2 = t2.getFinishOrder(); return (o1 < o2) ? -1 : ((o1 == o2) ? 0 : 1); } }); @@ -3656,7 +3655,8 @@ private RegionEntry moveNext() { } } RegionEntry re = (RegionEntry)this.it.next(); - if (re.isOverflowedToDisk(this.region, dp, false)) { + if (re.isValueNull() && + re.isOverflowedToDisk(this.region, dp, false)) { // add dp to sorted list // avoid create DiskPage everytime for lookup // TODO: SW: Can reduce the intermediate memory and boost diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java index ddd777db9..ca73290e1 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/GemFireCacheImpl.java @@ -127,7 +127,6 @@ import com.gemstone.gemfire.internal.cache.tier.sockets.ClientHealthMonitor; import com.gemstone.gemfire.internal.cache.tier.sockets.ClientProxyMembershipID; import com.gemstone.gemfire.internal.cache.versions.RegionVersionHolder; -import com.gemstone.gemfire.internal.cache.versions.RegionVersionVector; import com.gemstone.gemfire.internal.cache.versions.VersionSource; import com.gemstone.gemfire.internal.cache.versions.VersionTag; import com.gemstone.gemfire.internal.cache.wan.AbstractGatewaySender; @@ -451,9 +450,6 @@ public class GemFireCacheImpl implements InternalCache, ClientCache, HasCachePer private TombstoneService tombstoneService; - private Map snapshotRVV = new ConcurrentHashMap(); - - private final ReentrantReadWriteLock snapshotLock = new ReentrantReadWriteLock(); private final ReentrantReadWriteLock lockForSnapshotRvv = new ReentrantReadWriteLock(); private volatile RvvSnapshotTestHook testHook; @@ -1776,6 +1772,7 @@ public Map getSnapshotRVV() { region.getVersionVector().getSnapShotOfMemberVersion()); } }); + // getLogger().info("SW:1: acquired snapshot " + snapshot); return snapshot; } finally { lockForSnapshotRvv.readLock().unlock(); @@ -1875,14 +1872,6 @@ public void releaseWriteLockOnSnapshotRvv() { lockForSnapshotRvv.writeLock().unlock(); } - public void lockForSnapshot() { - this.snapshotLock.writeLock().lock(); - } - - public void releaseSnapshotLocks() { - this.snapshotLock.writeLock().unlock(); - } - protected final class Stopper extends CancelCriterion { /* @@ -6013,7 +6002,7 @@ public static final StaticSystemCallbacks getInternalProductCallbacks() { */ public static interface StaticSystemCallbacks extends SystemProperties.Callbacks { - + /** * Log in-memory until the buffer is full, instead of file io * synchronization. This is mainly used in the GemFire code that invokes diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java index 3c7112037..2c461b564 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ImageState.java @@ -165,7 +165,7 @@ public interface ImageState /* extends Lock */ { * transaction has not been recorded as finished, and a negative of the order * if it was recorded as having been rolled back. */ - public int getFinishedTXOrder(TXId txId); + public long getFinishedTXOrder(TXId txId); /** * Adjust the orders returned by getPendingTXOrder using the ordering from diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java index 788b5d1b1..0099c1720 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java @@ -3140,12 +3140,7 @@ public int getRegionSize() { if (!includeHDFSResults) { result -= this.txLockCreateCount.get(); } - if (result > 0) { - return result; - } - else { - return 0; - } + return Math.max(result, 0); } private int getRegionSizeNoLock(boolean includeHDFSResults) { @@ -6336,7 +6331,7 @@ && getScope().isDistributed() && getDataPolicy().withReplication() void basicInvalidatePart2(RegionEntry re, EntryEventImpl event, boolean conflictwithClear, boolean invokeCallbacks) { - updateStatsForInvalidate(); + updateStatsForInvalidate(event.getEventTime(0L)); if (invokeCallbacks) { try { @@ -6356,8 +6351,9 @@ void basicInvalidatePart2(RegionEntry re, EntryEventImpl event, /** * Update stats */ - private void updateStatsForInvalidate() { + private void updateStatsForInvalidate(long eventTime) { getCachePerfStats().incInvalidates(); + setLastModifiedTime(eventTime); } void basicInvalidatePart3(RegionEntry re, EntryEventImpl event, @@ -6417,14 +6413,13 @@ final void txApplyInvalidate(final RegionEntry re, * Called by lower levels, while still holding the write sync lock, and the * low level has completed its part of the basic destroy */ - final void txApplyInvalidatePart2(RegionEntry re, Object key, - boolean didDestroy, boolean didInvalidate, boolean clearConflict) - { + final void txApplyInvalidatePart2(TXStateInterface tx, RegionEntry re, Object key, + boolean didDestroy, boolean didInvalidate, boolean clearConflict) { if (this.testCallable != null) { this.testCallable.call(this, Operation.INVALIDATE, re); } if (didInvalidate) { - updateStatsForInvalidate(); + updateStatsForInvalidate(tx.getCommitTime()); // Bug 40842: clearing index of the old value // performed in AbstractRegionMap } @@ -8483,7 +8478,7 @@ void basicDestroyPart3(RegionEntry re, EntryEventImpl event, boolean inTokenMode, boolean duringRI, boolean invokeCallbacks, Object expectedOldValue) { if (!inTokenMode || duringRI) { - updateStatsForDestroy(); + updateStatsForDestroy(event.getEventTime(0L)); } if (this.entryUserAttributes != null) { @@ -8494,8 +8489,9 @@ void basicDestroyPart3(RegionEntry re, EntryEventImpl event, /** * Update stats */ - private void updateStatsForDestroy() { + private void updateStatsForDestroy(long eventTime) { getCachePerfStats().incDestroys(); + setLastModifiedTime(eventTime); } // Asif : This method will clear the tranxnl entries @@ -8606,8 +8602,8 @@ final void txApplyDestroy(final RegionEntry re, * Called by lower levels, while still holding the write sync lock, and the * low level has completed its part of the basic destroy */ - void txApplyDestroyPart2(RegionEntry re, Object key, boolean inTokenMode, boolean clearConflict) - { + void txApplyDestroyPart2(TXStateInterface tx, RegionEntry re, Object key, + boolean inTokenMode, boolean clearConflict) { if (this.testCallable != null) { this.testCallable.call(this, Operation.DESTROY, re); } @@ -8615,7 +8611,7 @@ void txApplyDestroyPart2(RegionEntry re, Object key, boolean inTokenMode, boolea getImageState().addDestroyedEntry(key); } else { - updateStatsForDestroy(); + updateStatsForDestroy(tx.getCommitTime()); } if (this.entryUserAttributes != null) { this.entryUserAttributes.remove(key); diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/Oplog.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/Oplog.java index 993f2cec9..14b8813cf 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/Oplog.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/Oplog.java @@ -755,7 +755,8 @@ public final class Oplog implements CompactableOplog { if (ex instanceof DiskAccessException) { throw (DiskAccessException) ex; } - throw new DiskAccessException(LocalizedStrings.Oplog_FAILED_CREATING_OPERATION_LOG_BECAUSE_0.toLocalizedString(ex), getParent()); + throw new DiskAccessException(LocalizedStrings.Oplog_FAILED_CREATING_OPERATION_LOG_BECAUSE_0 + .toLocalizedString(ex), ex, getParent()); } } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java index 3f41ccb3a..11e29d332 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java @@ -7201,7 +7201,7 @@ else if ((ds = dataStore) == null) { } - PRLocalScanIterator(final Set bucketIds, final TXState tx, + public PRLocalScanIterator(final Set bucketIds, final TXState tx, final boolean forUpdate, final boolean includeValues, final boolean fetchRemote) { this(bucketIds, tx, DEFAULT_ITERATOR_CREATOR, diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionDataStore.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionDataStore.java index e7443201a..1925f196e 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionDataStore.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegionDataStore.java @@ -74,8 +74,8 @@ import com.gemstone.gemfire.internal.util.concurrent.StoppableReentrantReadWriteLock.StoppableReadLock; import com.gemstone.gemfire.internal.util.concurrent.StoppableReentrantReadWriteLock.StoppableWriteLock; import com.gemstone.gnu.trove.THashSet; -import com.gemstone.gnu.trove.TIntArrayList; import com.gemstone.org.jgroups.util.StringId; +import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList; import org.eclipse.collections.impl.set.mutable.UnifiedSet; /** @@ -3110,8 +3110,8 @@ public Set getAllLocalPrimaryBucketIds() { } @SuppressWarnings("unchecked") - public TIntArrayList getAllLocalPrimaryBucketIdArray() { - TIntArrayList bucketIds = new TIntArrayList(); + public IntArrayList getAllLocalPrimaryBucketIdArray() { + IntArrayList bucketIds = new IntArrayList(); for (BucketRegion bucket : localBucket2RegionMap.values()) { if (bucket.getBucketAdvisor().isPrimary()) { bucketIds.add(bucket.getId()); @@ -3120,19 +3120,6 @@ public TIntArrayList getAllLocalPrimaryBucketIdArray() { return bucketIds; } - public Set getAllLocalPrimaryBucketIdsBetweenProvidedIds(int low, - int high) { - Set bucketIds = new HashSet(); - for (Map.Entry bucketEntry : getAllLocalBuckets()) { - BucketRegion bucket = bucketEntry.getValue(); - if (bucket.getBucketAdvisor().isPrimary() && (bucket.getId() >= low) - && (bucket.getId() < high)) { - bucketIds.add(Integer.valueOf(bucket.getId())); - } - } - return bucketIds; - } - /** a fast estimate of total bucket size */ public long getEstimatedLocalBucketSize(boolean primaryOnly) { long size = 0; diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ProxyBucketRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ProxyBucketRegion.java index 40de4725c..29cff8a16 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ProxyBucketRegion.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ProxyBucketRegion.java @@ -52,7 +52,7 @@ import com.gemstone.gemfire.internal.cache.persistence.PersistentMemberManager; import com.gemstone.gemfire.internal.cache.persistence.PersistentMembershipView; import com.gemstone.gemfire.internal.i18n.LocalizedStrings; -import com.gemstone.gnu.trove.THashSet; +import org.eclipse.collections.impl.set.mutable.UnifiedSet; /** * Empty shell for {@link BucketRegion} which exists only to maintain metadata @@ -367,11 +367,10 @@ public ProxyBucketRegion initialize() return this; } - @SuppressWarnings("unchecked") public Set getBucketOwners() { Set s = getOtherBucketOwners(); - if (s == Collections. emptySet()) { - s = new THashSet(); + if (s == Collections.emptySet()) { + s = new UnifiedSet<>(1); } if (isHosting()) { s.add(this.partitionedRegion.getDistributionManager().getId()); diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ProxyRegionMap.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ProxyRegionMap.java index 8f82c7ccd..fc3592b0b 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ProxyRegionMap.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ProxyRegionMap.java @@ -307,7 +307,7 @@ public void txApplyDestroy(RegionEntry re, TXStateInterface txState, FilterRoutingInfo filterRoutingInfo, ClientProxyMembershipID bridgeContext, VersionTag versionTag, long tailKey, TXRegionState txr, EntryEventImpl cbEvent) { - this.owner.txApplyDestroyPart2(markerEntry, key, inTokenMode, + this.owner.txApplyDestroyPart2(txState, markerEntry, key, inTokenMode, false /*Clear conflict occured */); if (!inTokenMode) { /* @@ -349,8 +349,8 @@ public void txApplyInvalidate(RegionEntry re, TXStateInterface txState, FilterRoutingInfo filterRoutingInfo, ClientProxyMembershipID bridgeContext, VersionTag versionTag, long tailKey, TXRegionState txr, EntryEventImpl cbEvent) { - this.owner.txApplyInvalidatePart2(markerEntry, key, didDestroy, true, - false /*Clear conflic occured */); + this.owner.txApplyInvalidatePart2(txState, markerEntry, key, didDestroy, + true, false /* clear conflict occurred */); if (this.owner.isInitialized()) { /* if (txEvent != null) { diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java index 3bf474c21..e79ab025b 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/RegionEntryContext.java @@ -18,6 +18,7 @@ package com.gemstone.gemfire.internal.cache; import com.gemstone.gemfire.compression.Compressor; +import com.gemstone.gemfire.internal.shared.SystemProperties; /** * Provides important contextual information that allows a {@link RegionEntry} to manage its state. @@ -25,8 +26,14 @@ * @since 7.5 */ public interface RegionEntryContext extends HasCachePerfStats { - public static final String DEFAULT_COMPRESSION_PROVIDER="com.gemstone.gemfire.compression.SnappyCompressor"; - + + String DEFAULT_COMPRESSION_PROVIDER = + "com.gemstone.gemfire.compression.SnappyCompressor"; + + boolean COMPRESSION_ENABLED = SystemProperties.getServerInstance().getBoolean( + "compression.enable", !GemFireCacheImpl.gfxdSystem() && + !SystemProperties.isUsingGemFireXDEntryPoint()); + /** * Returns the compressor to be used by this region entry when storing the * entry value. diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXCleanupEntryMessage.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXCleanupEntryMessage.java index 0b104725d..345c5fd64 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXCleanupEntryMessage.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXCleanupEntryMessage.java @@ -23,11 +23,11 @@ import java.util.Set; import com.gemstone.gemfire.DataSerializer; -import com.gemstone.gemfire.distributed.DistributedMember; import com.gemstone.gemfire.distributed.internal.DM; import com.gemstone.gemfire.distributed.internal.DistributionManager; import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem; import com.gemstone.gemfire.distributed.internal.ReplyProcessor21; +import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember; import com.gemstone.gemfire.internal.cache.delta.Delta; /** @@ -74,7 +74,7 @@ private TXCleanupEntryMessage(final TXStateInterface tx, public static ReplyProcessor21 send(final InternalDistributedSystem system, final DM dm, final TXStateInterface txState, - final Set recipients, final LocalRegion dataRegion, + final Set recipients, final LocalRegion dataRegion, final Object regionKey, final byte originalOp, final byte originalDestroy, final boolean originalBulkOp, final Object originalValue, final Delta originalDelta) { diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXManagerImpl.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXManagerImpl.java index 21d9e8044..530835574 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXManagerImpl.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXManagerImpl.java @@ -54,8 +54,8 @@ import com.gemstone.gemfire.internal.i18n.LocalizedStrings; import com.gemstone.gemfire.internal.shared.ClientSharedUtils; import com.gemstone.gemfire.internal.shared.SystemProperties; -import com.gemstone.gnu.trove.TObjectIntHashMap; import org.eclipse.collections.impl.map.mutable.UnifiedMap; +import org.eclipse.collections.impl.map.mutable.primitive.ObjectLongHashMap; /** *

@@ -378,7 +378,7 @@ else if (numRemoved == 0) { */ Object isTXCommitted(TXId txId) { TXFinished result = this.finishedMap.get(txId); - return result != null ? Boolean.valueOf(result.isCommit) : null; + return result != null ? result.isCommit : null; } /** @@ -386,9 +386,9 @@ Object isTXCommitted(TXId txId) { * transaction has been rolled back then it returns negative of the ordering * to indicate so. */ - TObjectIntHashMap getTXCommitOrders(Collection txIds) { + ObjectLongHashMap getTXCommitOrders(Collection txIds) { final HashMap txIdSet = new HashMap(txIds.size()); - final TObjectIntHashMap txIdOrders = new TObjectIntHashMap(); + final ObjectLongHashMap txIdOrders = new ObjectLongHashMap<>(16); for (TXId txId : txIds) { txIdSet.put(txId, txId); } @@ -398,7 +398,7 @@ TObjectIntHashMap getTXCommitOrders(Collection txIds) { final TXFinished head = this.tail.next; TXFinished next = head; TXId txId; - int pos = 1; + long pos = 1; while ((next = next.next) != head) { lookupId.memberId = next.txMemberId; lookupId.uniqId = next.txUniqId; diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXNewGIINode.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXNewGIINode.java index dc83c477e..0ec9a28f3 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXNewGIINode.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXNewGIINode.java @@ -21,6 +21,7 @@ import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; +import java.util.List; import com.gemstone.gemfire.cache.TransactionException; import com.gemstone.gemfire.distributed.internal.DM; @@ -61,7 +62,7 @@ private TXNewGIINode(final TXStateInterface tx, @SuppressWarnings({ "unchecked", "rawtypes" }) public static void send(final InternalDistributedSystem system, final DM dm, final TXStateInterface tx, final InternalDistributedMember recipient, - final ArrayList memberData, final boolean forCommit) { + final List memberData, final boolean forCommit) { final int dataLen; if (memberData == null || (dataLen = memberData.size()) == 0) { return; diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java index 3834d4312..e9b2baefd 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXRegionState.java @@ -104,7 +104,7 @@ public final class TXRegionState extends ReentrantLock { */ private boolean isValid; /** a sort order for commit/rollback if GII was not done */ - int finishOrder; + long finishOrder; // temporary transient variables private transient int tmpEntryCount; @@ -746,7 +746,7 @@ public final void value(final Object obj) { return this.tmpEx; } - public final int getFinishOrder() { + public final long getFinishOrder() { return this.finishOrder; } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXState.java index f0f422a07..8c5129bed 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXState.java @@ -124,7 +124,8 @@ public int computeHashCode(Object o) { Map> snapshot; - private final Map writeRegions = new ConcurrentHashMap<>(); + private final ConcurrentHashMap writeRegions = + new ConcurrentHashMap<>(); /* private TXLockRequest locks = null; @@ -1286,11 +1287,11 @@ protected void cleanup(final boolean commit, pendingReadLocksCleanup(lockPolicy, null, null); } - writeRegions.keySet().stream().filter(region -> - region instanceof BucketRegion - ).forEach(region -> - ((BucketRegion)region).releaseSnapshotGIIReadLock() - ); + writeRegions.keySet().forEach(region -> { + if (region instanceof BucketRegion) { + ((BucketRegion)region).releaseSnapshotGIIReadLock(); + } + }); } finally { if (this.txLocked.compareAndSet(true, false)) { @@ -3909,16 +3910,7 @@ public void postPutAll(DistributedPutAllOperation putAllOp, } } - /** - * Return either the {@link TXEntryState} if the given entry is in TXState - * else return the provided region entry itself. - */ - public final Object getLocalEntry(final LocalRegion region, - LocalRegion dataRegion, final int bucketId, final AbstractRegionEntry re, boolean isWrite) { - - // for local/distributed regions, the key is the RegionEntry itself - // getDataRegion will work correctly neverthless - + private boolean checkTX(LocalRegion region, AbstractRegionEntry re) { // need to check in TXState only if the entry has been locked by a TX final boolean checkTX = getLockingPolicy().lockedForWrite(re, null, null); if (TXStateProxy.LOG_FINE) { @@ -3927,7 +3919,32 @@ public final Object getLocalEntry(final LocalRegion region, + region.getFullPath() + " RegionEntry(" + re + ") checkTX=" + checkTX); } - if (checkTX) { + return checkTX; + } + + /** + * Return either the {@link TXEntryState} if the given entry is in TXState + * else return the provided region entry itself. + */ + public final Object getLocalEntry(final LocalRegion region, + LocalRegion dataRegion, final int bucketId, final AbstractRegionEntry re, + boolean isWrite) { + + // for local/distributed regions, the key is the RegionEntry itself + // getDataRegion will work correctly nevertheless + + if (!isWrite && shouldGetOldEntry(dataRegion)) { + final Object key = re.getKey(); + if (dataRegion == null) { + dataRegion = region.getDataRegionForRead(key, null, bucketId, + Operation.GET_ENTRY); + } + if (dataRegion.getVersionVector() != null) { + if (!checkEntryInSnapshot(this, dataRegion, re)) { + return getOldVersionedEntry(this, dataRegion, re.getKeyCopy(), re); + } + } + } else if (checkTX(region, re)) { final Object key = re.getKey(); if (dataRegion == null) { dataRegion = region.getDataRegionForRead(key, null, bucketId, @@ -3967,17 +3984,6 @@ public final Object getLocalEntry(final LocalRegion region, txr.unlock(); } } - } else if (!isWrite && shouldGetOldEntry(dataRegion)) { - final Object key = re.getKeyCopy(); - if (dataRegion == null) { - dataRegion = region.getDataRegionForRead(key, null, bucketId, - Operation.GET_ENTRY); - } - if (dataRegion.getVersionVector() != null) { - if (!checkEntryInSnapshot(this, dataRegion, re)) { - return getOldVersionedEntry(this, dataRegion, key, re); - } - } } return re; } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateProxy.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateProxy.java index c7db2e1a3..f8d59bc49 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateProxy.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TXStateProxy.java @@ -30,15 +30,9 @@ import com.gemstone.gemfire.cache.*; import com.gemstone.gemfire.cache.execute.ResultCollector; import com.gemstone.gemfire.cache.query.internal.IndexUpdater; -import com.gemstone.gemfire.distributed.internal.DM; -import com.gemstone.gemfire.distributed.internal.DirectReplyProcessor; -import com.gemstone.gemfire.distributed.internal.DistributionAdvisor; +import com.gemstone.gemfire.distributed.internal.*; import com.gemstone.gemfire.distributed.internal.DistributionAdvisor.Profile; import com.gemstone.gemfire.distributed.internal.DistributionAdvisor.ProfileVisitor; -import com.gemstone.gemfire.distributed.internal.DistributionManager; -import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem; -import com.gemstone.gemfire.distributed.internal.ReplyException; -import com.gemstone.gemfire.distributed.internal.ReplyProcessor21; import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember; import com.gemstone.gemfire.distributed.internal.membership.MembershipManager; import com.gemstone.gemfire.i18n.LogWriterI18n; @@ -66,6 +60,7 @@ import com.gemstone.gemfire.internal.cache.partitioned.RegionAdvisor.PartitionProfile; import com.gemstone.gemfire.internal.cache.tier.sockets.ClientProxyMembershipID; import com.gemstone.gemfire.internal.cache.tier.sockets.VersionedObjectList; +import com.gemstone.gemfire.internal.cache.versions.RegionVersionVector; import com.gemstone.gemfire.internal.concurrent.MapCallback; import com.gemstone.gemfire.internal.concurrent.MapCallbackAdapter; import com.gemstone.gemfire.internal.i18n.LocalizedStrings; @@ -73,11 +68,12 @@ import com.gemstone.gemfire.internal.util.concurrent.StoppableReentrantReadWriteLock; import com.gemstone.gnu.trove.THash; import com.gemstone.gnu.trove.THashMap; -import com.gemstone.gnu.trove.THashSet; -import com.gemstone.gnu.trove.TObjectLongProcedure; import org.eclipse.collections.api.block.function.Function; import org.eclipse.collections.api.block.procedure.Procedure2; +import org.eclipse.collections.api.block.procedure.primitive.ObjectLongProcedure; import org.eclipse.collections.impl.map.mutable.UnifiedMap; +import org.eclipse.collections.impl.map.mutable.primitive.ObjectLongHashMap; +import org.eclipse.collections.impl.set.mutable.UnifiedSet; /** * TXStateProxy lives on the source node when we are remoting a transaction. It @@ -326,7 +322,7 @@ public boolean visit(DistributionAdvisor advisor, assert profile instanceof BucketProfile; BucketProfile bp = (BucketProfile)profile; if (!bp.inRecovery && bp.cachedOrAllEventsWithListener()) { - ArrayList uninitializedRegions = recipients.members.getIfAbsentPutWith( + List uninitializedRegions = recipients.members.getIfAbsentPutWith( bp.getDistributedMember(), recipients, bp); if (uninitializedRegions != null) { final ProxyBucketRegion pbr = ((BucketAdvisor)advisor) @@ -376,7 +372,7 @@ public boolean visit(DistributionAdvisor advisor, if (!(profile instanceof PartitionProfile)) { final CacheProfile cp = (CacheProfile)profile; if (!cp.inRecovery && cp.cachedOrAllEventsWithListener()) { - ArrayList uninitializedRegions = recipients.members.getIfAbsentPutWith( + List uninitializedRegions = recipients.members.getIfAbsentPutWith( cp.getDistributedMember(), recipients, cp); if (uninitializedRegions != null) { if (!cp.regionInitialized) { @@ -403,12 +399,13 @@ public boolean visit(DistributionAdvisor advisor, }; public static final class MemberToGIIRegions - implements Function> { - final UnifiedMap> members; + implements Function> { + final UnifiedMap> members; final THashMap eventsToBePublished; long[] viewVersions; DistributionAdvisor[] viewAdvisors; boolean hasUninitialized; + private transient boolean regionInitialized; /** * If this is the transaction coordinator, this gets a consistent DiskStoreID * (primary for PRs and anyone for RRs) to be used for region versions by all @@ -446,10 +443,6 @@ public MemberToGIIRegions(int numRegions, THashMap eventsToBePublished) { this.hasUninitialized = false; } - public final UnifiedMap> getMembers() { - return this.members; - } - public final void endOperationSend(TXStateProxy proxy) { final long[] viewVersions = this.viewVersions; final DistributionAdvisor[] advisors; @@ -464,13 +457,22 @@ public final void endOperationSend(TXStateProxy proxy) { while (--numOps >= 0) { viewVersion = viewVersions[numOps]; if (viewVersion != -1) { - advisors[numOps].endOperation(viewVersion); + DistributionAdvisor advisor = advisors[numOps]; + advisor.endOperation(viewVersion); if (logger != null) { + DistributionAdvisee advisee = advisor.getAdvisee(); + String logVersion = ""; + if (advisee instanceof LocalRegion) { + RegionVersionVector rvv = ((LocalRegion)advisee).getVersionVector(); + if (rvv != null && rvv.getSnapShotOfMemberVersion() != null) { + logVersion = " RVV = " + rvv.getSnapShotOfMemberVersion(); + } + } logger.info(LocalizedStrings.DEBUG, "TXCommit: " + proxy.getTransactionId().shortToString() + " done dispatching operation for " - + advisors[numOps].getAdvisee().getFullPath() - + " in view version " + viewVersion); + + advisor.getAdvisee().getFullPath() + + " in view version " + viewVersion + logVersion); } } } @@ -484,10 +486,10 @@ public final boolean hasUninitialized() { } @Override - public ArrayList valueOf(CacheProfile cp) { + public List valueOf(CacheProfile cp) { if (cp.regionInitialized) { if (this.eventsToBePublished == null) { - return null; + return Collections.emptyList(); } else { // put an empty placeHolder expecting some events for the member @@ -760,7 +762,7 @@ final TXManagerImpl.TXContext commitPhase1(final Object callbackArg) // get the set of recipients for commit using the list of affected regions final MemberToGIIRegions finishRecipients = getFinishMessageRecipients(true); - final Set recipients = finishRecipients.members.keySet(); + final Set recipients = finishRecipients.members.keySet(); final TXManagerImpl.TXContext context = TXManagerImpl.currentTXContext(); context.setCommitRecipients(finishRecipients); @@ -814,7 +816,9 @@ final TXManagerImpl.TXContext commitPhase1(final Object callbackArg) // need to send the phase1 message separately to remaining // recipients if (numRecipients > batchResponses.size()) { - final THashSet remainingRecipients = new THashSet(recipients); + // make a copy since original is required later + final UnifiedSet remainingRecipients = + new UnifiedSet<>(recipients); for (TXBatchResponse batchResponse : batchResponses) { remainingRecipients.remove(batchResponse.recipient); } @@ -1052,7 +1056,7 @@ final void commitPhase2(final TXManagerImpl.TXContext context, // final commit message checkAllCopiesDown(dm); - final UnifiedMap> recipientsData = + final UnifiedMap> recipientsData = finishRecipients.members; // set the commit time only once if (this.commitTime == 0) { @@ -1134,7 +1138,7 @@ final void commitPhase2(final TXManagerImpl.TXContext context, // mark operation end for state flush finishRecipients.endOperationSend(this); if (thr == null) { - cleanup(); + cleanup(false); } else { getCache().getCancelCriterion().checkCancelInProgress(thr); @@ -1235,7 +1239,7 @@ protected final void commit(final TXState localState, if (txState != null) { txMgr.commit(txState, callbackArg, TXManagerImpl.FULL_COMMIT, null, true/*remote to coordinator*/); - cleanup(); + cleanup(false); } } else { @@ -1339,12 +1343,10 @@ private final void signalLocalTXCommit() { } } - private static final TObjectLongProcedure endBatchSend = - new TObjectLongProcedure() { - @Override - public boolean execute(Object o, long viewVersion) { + private static final ObjectLongProcedure + endBatchSend = (advisor, viewVersion) -> { + { if (viewVersion != -1) { - DistributionAdvisor advisor = (DistributionAdvisor)o; advisor.endOperation(viewVersion); if (LOG_VERSIONS) { LogWriterI18n logger = InternalDistributedSystem.getLoggerI18n(); @@ -1356,7 +1358,6 @@ public boolean execute(Object o, long viewVersion) { } } } - return true; } }; /** @@ -1378,13 +1379,13 @@ public final void flushPendingOps(DM dm) throws TransactionException { dm = this.txManager.getDM(); } - TObjectLongHashMapWithIndex versions = new TObjectLongHashMapWithIndex(); + ObjectLongHashMap versions = new ObjectLongHashMap(16); this.lock.lock(); try { batchResponses = sendPendingOps(dm, localState, versions, null, false); } finally { - if (!versions.isEmpty()) { - versions.forEachEntry(endBatchSend); + if (versions.size() > 0) { + versions.forEachKeyValue(endBatchSend); } this.lock.unlock(); } @@ -1412,7 +1413,8 @@ private final ArrayList sendPendingOpsBeforeCommit( if (LOG_FINE) { if (batchResponses != null) { final LogWriterI18n logger = dm.getLoggerI18n(); - final THashSet batchRecipients = new THashSet(); + final UnifiedSet batchRecipients = + new UnifiedSet<>(batchResponses.size()); for (TXBatchResponse br : batchResponses) { batchRecipients.add(br.recipient); } @@ -1442,7 +1444,7 @@ private final void waitForPendingOpsBeforeCommit( @SuppressWarnings("unchecked") private final ArrayList sendPendingOps(final DM dm, - final TXState localState, final TObjectLongHashMapWithIndex versions, + final TXState localState, final ObjectLongHashMap versions, final List postMessages, final boolean conflictWithEX) throws TransactionException { int numPending; @@ -1472,19 +1474,23 @@ private final ArrayList sendPendingOps(final DM dm, // put in the map only once (and also avoid calling startOperation // more than once for a region) DistributionAdvisor advisor = dataRegion.getDistributionAdvisor(); - int insertionIndex = versions.getInsertionIndex(advisor); - if (insertionIndex >= 0) { - long viewVersion = -1; - viewVersion = advisor.startOperation(); - if (viewVersion != -1) { - versions.putAtIndex(advisor, viewVersion, insertionIndex); - if (LOG_VERSIONS) { - getTxMgr().getLogger().info(LocalizedStrings.DEBUG, - "TXBatch: dispatching operation for " - + dataRegion.getFullPath() + " in view version " - + viewVersion); + try { + versions.getIfAbsentPut(advisor, () -> { + long viewVersion = advisor.startOperation(); + if (viewVersion != -1) { + if (LOG_VERSIONS) { + getTxMgr().getLogger().info(LocalizedStrings.DEBUG, + "TXBatch: dispatching operation for " + + dataRegion.getFullPath() + " in view version " + + viewVersion); + } + return viewVersion; + } else { + // abort the insert + throw new IllegalStateException(); } - } + }); + } catch (IllegalStateException ignored) { } } // find the recipients for this entry @@ -1574,7 +1580,7 @@ private final void waitForPendingOps( protected final void sendNewGIINodeMessages( final MemberToGIIRegions finishRecipients, final DM dm, final boolean forCommit) { - UnifiedMap> members; + UnifiedMap> members; if (finishRecipients.hasUninitialized && !(members = finishRecipients.members).isEmpty()) { members.forEachKeyValue((m, l) -> { @@ -1589,7 +1595,7 @@ protected final void sendNewGIINodeMessages( * This should be invoked by TXState when non-null else from commit/rollback * here. */ - protected void cleanup() { + protected void cleanup(boolean rollback) { this.hasCohorts = false; this.isDirty = false; this.hasReadOps = false; @@ -1993,7 +1999,6 @@ public List getEvents() { * * ONLY FOR TESTS. */ - @SuppressWarnings("unchecked") public final Collection getRegions() { final Collection allRegions; // coordinator has the record of all regions @@ -2011,7 +2016,7 @@ public final Collection getRegions() { allRegions = Collections.emptySet(); } } - final THashSet regs = new THashSet(allRegions.size()); + final UnifiedSet regs = new UnifiedSet<>(allRegions.size()); for (Object owner : allRegions) { LocalRegion r; if (owner instanceof Bucket) { @@ -2335,7 +2340,7 @@ private final void rollback(final Object callbackArg, mlock.readLock().unlock(); // remove from hosted txState list neverthless this.txManager.removeHostedTXState(this.txId, Boolean.FALSE); - cleanup(); + cleanup(true); } if (TRACE_EXECUTE) { @@ -2394,7 +2399,7 @@ protected final void rollback(final TXState localState, } onRollback(null, callbackArg); } finally { - cleanup(); + cleanup(true); } } else { @@ -2413,7 +2418,7 @@ protected final void rollback(final TXState localState, /** only for testing */ public Set getFinishMessageRecipientsForTest() { - return getFinishMessageRecipients(false).getMembers().keySet(); + return getFinishMessageRecipients(false).members.keySet(); } private boolean checkIfRegionSetHasBucketRegion() { @@ -3624,13 +3629,13 @@ else if (cacheWrite && dataRegion != null) { return null; } - @SuppressWarnings("unchecked") - private final GemFireException revertFailedOp(final LocalRegion dataRegion, + private GemFireException revertFailedOp(final LocalRegion dataRegion, final Object regionKey, GemFireException failedEx, final Set originalRecipients, final Map exceptions, final boolean revertSelf) { - final THashSet revertMembers = new THashSet(originalRecipients); + final UnifiedSet revertMembers = + new UnifiedSet<>(originalRecipients); final Class failedExClass = failedEx.getClass(); Throwable cause; if (exceptions != null) { diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TombstoneService.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TombstoneService.java index e7a01e928..3a541a1a1 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TombstoneService.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/TombstoneService.java @@ -539,7 +539,7 @@ private static class Tombstone extends CompactVersionHolder { public int getSize() { return Tombstone.PER_TOMBSTONE_OVERHEAD // includes per-entry overhead - + ObjectSizer.DEFAULT.sizeof(entry.getRawKey()); + + CachedDeserializableFactory.calcMemSize(entry.getRawKey(), ObjectSizer.DEFAULT, true); } @Override diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java index e51aba7b8..d7894cfb5 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/UnsharedImageState.java @@ -17,11 +17,12 @@ package com.gemstone.gemfire.internal.cache; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import com.gemstone.gemfire.CancelCriterion; import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem; @@ -35,9 +36,9 @@ import com.gemstone.gemfire.internal.i18n.LocalizedStrings; import com.gemstone.gemfire.internal.util.concurrent.StoppableNonReentrantLock; import com.gemstone.gemfire.internal.util.concurrent.StoppableReentrantReadWriteLock; -import com.gemstone.gnu.trove.TObjectIntHashMap; import com.gemstone.org.jgroups.util.StringId; import org.eclipse.collections.impl.map.mutable.UnifiedMap; +import org.eclipse.collections.impl.map.mutable.primitive.ObjectLongHashMap; /** * Used on distributed replicated regions to track GII and various state. @@ -78,8 +79,8 @@ public class UnsharedImageState implements ImageState { private volatile UnifiedMap pendingTXRegionStates; private final NonReentrantReadWriteLock pendingTXRegionStatesLock; private volatile Thread pendingTXRegionStatesLockOwner; - private final AtomicInteger pendingTXOrder; - private volatile TObjectIntHashMap finishedTXIdOrders; + private final AtomicLong pendingTXOrder; + private volatile ObjectLongHashMap finishedTXIdOrders; UnsharedImageState(final boolean isClient, final boolean isReplicate, @@ -103,7 +104,7 @@ public class UnsharedImageState implements ImageState { this.pendingTXRegionStates = isLocal ? null : new UnifiedMap<>(); this.pendingTXRegionStatesLock = isLocal ? null : new NonReentrantReadWriteLock(stopper); - this.pendingTXOrder = new AtomicInteger(0); + this.pendingTXOrder = new AtomicLong(0L); } private void initDestroyedKeysMap() { @@ -350,14 +351,16 @@ public void writeUnlockRI() { */ @Override public boolean addPendingTXRegionState(TXRegionState txrs) { - if (this.pendingTXRegionStates != null) { + final UnifiedMap pendingTXRegionStates = + this.pendingTXRegionStates; + if (pendingTXRegionStates != null) { // don't add if the advisor has not been initialized yet, that is the // initial CreateRegionMessage replies are still on the wire so all // operations on TXRegionState are essentially ignored final LocalRegion region = txrs.region; if (region.isProfileExchanged()) { Object old; - if ((old = this.pendingTXRegionStates.getIfAbsentPut(txrs.getTXState() + if ((old = pendingTXRegionStates.getIfAbsentPut(txrs.getTXState() .getTransactionId(), txrs)) != txrs) { Assert.fail("ImageState#addPendingTXRegionState: failed to add " + txrs + ", existing=" + old); @@ -379,10 +382,11 @@ public boolean addPendingTXRegionState(TXRegionState txrs) { */ @Override public void removePendingTXRegionState(TXId txId) { - if (this.pendingTXRegionStates != null) { + final UnifiedMap pendingTXRegionStates = + this.pendingTXRegionStates; + if (pendingTXRegionStates != null) { TXRegionState txrs; - if ((txrs = (TXRegionState)this.pendingTXRegionStates.remove( - txId)) != null) { + if ((txrs = pendingTXRegionStates.remove(txId)) != null) { if (TXStateProxy.LOG_FINE) { final LogWriterI18n logger = txrs.region.getLogWriterI18n(); logger.info(LocalizedStrings.DEBUG, @@ -410,8 +414,10 @@ public TXRegionState getPendingTXRegionState(TXId txId, boolean lock) { } } try { - if (this.pendingTXRegionStates != null) { - txrs = this.pendingTXRegionStates.get(txId); + final UnifiedMap pendingTXRegionStates = + this.pendingTXRegionStates; + if (pendingTXRegionStates != null) { + txrs = pendingTXRegionStates.get(txId); if (TXStateProxy.LOG_FINE) { if (txrs != null) { final LogWriterI18n logger = txrs.region.getLogWriterI18n(); @@ -504,10 +510,11 @@ public void unlockPendingTXRegionStates(final boolean forWrite) { */ @Override public Collection getPendingTXRegionStates() { - if (this.pendingTXRegionStates != null) { - @SuppressWarnings("unchecked") - final Collection result = this.pendingTXRegionStates - .values(); + final UnifiedMap pendingTXRegionStates = + this.pendingTXRegionStates; + if (pendingTXRegionStates != null && !pendingTXRegionStates.isEmpty()) { + final Collection result = + new ArrayList<>(pendingTXRegionStates.values()); if (TXStateProxy.LOG_FINE) { final LogWriterI18n logger = InternalDistributedSystem.getLoggerI18n(); if (logger != null) { @@ -528,11 +535,11 @@ public Collection getPendingTXRegionStates() { @Override public void setTXOrderForFinish(TXRegionState txrs) { if (this.pendingTXRegionStatesLock != null) { - TObjectIntHashMap finishedOrders; + final ObjectLongHashMap finishedOrders; // assume read lock on pendingTXRegionStates is already held Assert.assertTrue(this.pendingTXRegionStatesLock.numReaders() > 0); if ((finishedOrders = this.finishedTXIdOrders) != null) { - int order = finishedOrders.get(txrs.getTXState().getTransactionId()); + long order = finishedOrders.get(txrs.getTXState().getTransactionId()); if (order != 0) { txrs.finishOrder = Math.abs(order); return; @@ -546,8 +553,8 @@ public void setTXOrderForFinish(TXRegionState txrs) { * {@inheritDoc} */ @Override - public int getFinishedTXOrder(TXId txId) { - TObjectIntHashMap finishedOrders; + public long getFinishedTXOrder(TXId txId) { + final ObjectLongHashMap finishedOrders; if ((finishedOrders = this.finishedTXIdOrders) != null) { return finishedOrders.get(txId); } @@ -570,7 +577,7 @@ public void mergeFinishedTXOrders(final LocalRegion region, // this is deliberately invoked under the lock to sync against // any concurrent getPendingTXOrder call final TXManagerImpl txMgr = region.getCache().getTxManager(); - TObjectIntHashMap txIdOrders = txMgr.finishedTXStates + final ObjectLongHashMap txIdOrders = txMgr.finishedTXStates .getTXCommitOrders(txIds); // loop through existing TXRegionStates and reset the order to that // passed in txIdOrders @@ -586,7 +593,7 @@ public void mergeFinishedTXOrders(final LocalRegion region, if (txrs.finishOrder == 0) { continue; } - int order = txIdOrders.get(txrs.getTXState().getTransactionId()); + long order = txIdOrders.get(txrs.getTXState().getTransactionId()); if (order != 0) { txrs.finishOrder = Math.abs(order); } @@ -614,7 +621,7 @@ public void clearPendingTXRegionStates(boolean reset) { if (txrs != null) { txrs.clear(); } - final TObjectIntHashMap txIds = this.finishedTXIdOrders; + final ObjectLongHashMap txIds = this.finishedTXIdOrders; if (txIds != null) { txIds.clear(); } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/locks/ReentrantReadWriteWriteShareLock.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/locks/ReentrantReadWriteWriteShareLock.java index 4b520f924..7e803739b 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/locks/ReentrantReadWriteWriteShareLock.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/locks/ReentrantReadWriteWriteShareLock.java @@ -51,7 +51,7 @@ public final class ReentrantReadWriteWriteShareLock extends private final QueuedSynchronizer sync; - private Object ownerId; + private volatile Object ownerId; public ReentrantReadWriteWriteShareLock() { this.sync = new QueuedSynchronizer(); diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/store/SerializedDiskBuffer.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/store/SerializedDiskBuffer.java index 4a190264f..a28050094 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/store/SerializedDiskBuffer.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/store/SerializedDiskBuffer.java @@ -152,12 +152,19 @@ public SerializedDiskBuffer getValueRetain(FetchRequest fetchRequest) { protected abstract void releaseBuffer(); /** - * For buffers which are stored in region, set its DiskId. + * For buffers which are stored in region, set DiskId and RegionEntryContext. */ public void setDiskEntry(AbstractOplogDiskRegionEntry entry, RegionEntryContext context) { } + /** + * Return the RegionEntryContext set by {@link #setDiskEntry} for this buffer. + */ + public RegionEntryContext getRegionContext() { + return null; + } + /** * Write the underlying data in the buffer fully to the channel. * The serialized data is expected to be exactly the same data as diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionHolder.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionHolder.java index 00262b28e..5d988dd98 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionHolder.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionHolder.java @@ -27,9 +27,7 @@ import java.util.List; import com.gemstone.gemfire.DataSerializable; -import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem; import com.gemstone.gemfire.i18n.LogWriterI18n; -import com.gemstone.gemfire.internal.Assert; import com.gemstone.gemfire.internal.InternalDataSerializer; import com.gemstone.gemfire.internal.cache.versions.RVVException.ReceivedVersionsIterator; import com.gemstone.gemfire.internal.i18n.LocalizedStrings; @@ -164,7 +162,7 @@ public synchronized RegionVersionHolder clone() { if (this.bitSet != null) { clone.bitSet = (BitSet)this.bitSet.clone(); clone.bitSetVersion = this.bitSetVersion; - clone.mergeBitSet(); + clone.mergeBitSetNoLock(); } }/*else { if (this.bitSet != null) { @@ -183,6 +181,7 @@ public synchronized RegionVersionHolder clone() { public synchronized String toString() { // mergeBitSet(); StringBuilder sb = new StringBuilder(); + sb.append('@').append(System.identityHashCode(this)); sb.append("{rv").append(this.version) .append(" bsv").append(this.bitSetVersion) .append(" bs=["); @@ -267,15 +266,14 @@ void flushBitSetDuringRecording(long version, LogWriterI18n logger) { } } - - private synchronized void mergeBitSetWithoutException() { - if (this.bitSet != null && this.bitSetVersion < this.version) { - addBitSet((int)(this.version-this.bitSetVersion), this.version, null); - } + /** merge bit-set exceptions into the regular exceptions list */ + private void mergeBitSet() { + assert Thread.holdsLock(this); + mergeBitSetNoLock(); } - /** merge bit-set exceptions into the regular exceptions list */ - private synchronized void mergeBitSet() { + /** merge bit-set exceptions into the regular exceptions list without requiring sync */ + private void mergeBitSetNoLock() { if (this.bitSet != null && this.bitSetVersion < this.version) { addBitSetExceptions((int)(this.version-this.bitSetVersion), this.version, null); } @@ -343,57 +341,6 @@ private void addBitSetExceptions(int numBits, long newVersion, LogWriterI18n log } } - private void addBitSet(int numBits, long newVersion, LogWriterI18n logger) { - int lastSetIndex = -1; - - if (RegionVersionVector.DEBUG && logger != null) { - logger.info(LocalizedStrings.DEBUG, "addBitSetExceptions("+numBits+","+newVersion+")"); - } - - for (int idx = 0; idx < numBits; ) { - int nextMissingIndex = this.bitSet.nextClearBit(idx); - if (nextMissingIndex < 0) { - break; - } - - lastSetIndex = nextMissingIndex-1; - - int nextReceivedIndex = this.bitSet.nextSetBit(nextMissingIndex+1); - long nextReceivedVersion = -1; - if (nextReceivedIndex > 0) { - lastSetIndex = nextReceivedIndex; - nextReceivedVersion = (long)(nextReceivedIndex) + this.bitSetVersion; - idx = nextReceivedIndex+1; - if (RegionVersionVector.DEBUG && logger != null) { - logger.info(LocalizedStrings.DEBUG, "found gap in bitSet: missing bit at index="+nextMissingIndex+"; next set index="+nextReceivedIndex); - } - } else { - // We can't flush any more bits from the bit set because there - //are no more received versions - if (RegionVersionVector.DEBUG && logger != null) { - logger.info(LocalizedStrings.DEBUG, "terminating flush at bit " + lastSetIndex + " because of missing entries"); - } - this.bitSetVersion += lastSetIndex; - this.bitSet.clear(); - if(lastSetIndex != -1) { - this.bitSet.set(0); - } - return; - } - long nextMissingVersion = Math.max(1, nextMissingIndex+this.bitSetVersion); - if (nextReceivedVersion > nextMissingVersion) { - //addException(nextMissingVersion-1, nextReceivedVersion); - if (RegionVersionVector.DEBUG && logger != null) { - logger.info(LocalizedStrings.DEBUG, "added rvv exception e{rv" + (nextMissingVersion-1) + " - rv" + nextReceivedVersion + "}"); - } - } - } - this.bitSet = this.bitSet.get(lastSetIndex, Math.max(lastSetIndex+1, bitSet.size())); - if (lastSetIndex > 0) { - this.bitSetVersion = this.bitSetVersion + (long)lastSetIndex; - } - } - synchronized void recordVersion(long version, LogWriterI18n logger) { updateVersion(version, logger); } @@ -509,7 +456,7 @@ public synchronized void initializeFrom(RegionVersionHolder source) { mergeBitSet(); RegionVersionHolder other = source.clone(); - other.mergeBitSet(); + other.mergeBitSetNoLock(); //Get a copy of the local version and exceptions long myVersion = this.version; @@ -657,7 +604,7 @@ public synchronized boolean isNewerThanOrCanFillExceptionsFor(RegionVersionHolde // we can make one pass over both sets to see if there are overlapping // exceptions or exceptions I don't have that the other does mergeBitSet(); // dump the bit-set exceptions into the regular exceptions list - other.mergeBitSet(); + other.mergeBitSetNoLock(); List mine = canonicalExceptions(this.exceptions); Iterator myIterator = mine.iterator(); List his = canonicalExceptions(other.exceptions); diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionVector.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionVector.java index a5ce08768..af5d10345 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionVector.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/versions/RegionVersionVector.java @@ -82,14 +82,13 @@ public abstract class RegionVersionVector> implements /** map of member to version h older. This is the actual version "vector" */ - //private ConcurrentHashMap> memberToVersion; - private final Map> memberToVersion; - private final Map> memberToVersionSnapshot; + private final ConcurrentHashMap> memberToVersion; + private final CopyOnWriteHashMap> memberToVersionSnapshot; /** current version in the local region for generating next version */ - private AtomicLong localVersion = new AtomicLong(0); - private AtomicLong localVersionForSnapshot = new AtomicLong(0); - private AtomicLong localVersionForWrite = new AtomicLong(0); + private final AtomicLong localVersion = new AtomicLong(0); + // private AtomicLong localVersionForSnapshot = new AtomicLong(0); + // private AtomicLong localVersionForWrite = new AtomicLong(0); /** * The list of exceptions for the local member. The version held * in this RegionVersionHolder may not be accurate, but the exception list @@ -102,39 +101,37 @@ public abstract class RegionVersionVector> implements * the version of the local exceptions under lock. */ private RegionVersionHolder localExceptions; - + /** highest reaped tombstone region-version for this member */ - private AtomicLong localGCVersion = new AtomicLong(0); - + private final AtomicLong localGCVersion = new AtomicLong(0); + /** the member that this version vector applies to */ private T myId; - - /** * a flag stating whether this vector contains only the version information * for a single member. This is used when a member crashed to transmit only * the version information for that member. */ private boolean singleMember; - + /** a flag to prevent accidental serialization of a live member */ private transient boolean isLiveVector; - + private final ConcurrentHashMap memberToGCVersion; - + /** map of canonical IDs for this RVV that are not in the memberToVersion map */ private transient ConcurrentHashMap canonicalIds = new ConcurrentHashMap<>(); /** a log writer used in debugging */ private transient LogWriterI18n log; - + /** is recording disabled? */ private transient boolean recordingDisabled; - + /** is this a vector in a client cache? */ private transient boolean clientVector; - + /** * this read/write lock is used to stop generation of new versions by the vector * while a region-level operation is underway. The locking scheme assumes that @@ -152,16 +149,16 @@ public abstract class RegionVersionVector> implements private transient final Object clearLockSync = new Object(); // sync for coordinating thread startup and lockOwner setting - + /** create a live version vector for a region */ public RegionVersionVector(T ownerId) { this.myId = ownerId; this.isLiveVector = true; - this.localExceptions = new RegionVersionHolder(0); - this.memberToVersionSnapshot = new CopyOnWriteHashMap>(); - this.memberToVersion = new ConcurrentHashMap>(INITIAL_CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); - this.memberToGCVersion = new ConcurrentHashMap (INITIAL_CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); + this.localExceptions = new RegionVersionHolder<>(0); + this.memberToVersionSnapshot = new CopyOnWriteHashMap<>(); + this.memberToVersion = new ConcurrentHashMap<>(INITIAL_CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); + this.memberToGCVersion = new ConcurrentHashMap<>(INITIAL_CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); } // this has to make sure that no other thread is modifying the memberToSnapshotVersion @@ -220,8 +217,7 @@ public RegionVersionVector getCloneForTransmission() { * if it wants consistent view of snapshot across cache */ public Map> getSnapShotOfMemberVersion() { - - return ((CopyOnWriteHashMap)memberToVersionSnapshot).getInnerMap(); + return memberToVersionSnapshot.getInnerMap(); } protected abstract RegionVersionVector createCopy(T ownerId, @@ -259,7 +255,7 @@ public RegionVersionVector getCloneForTransmission(T mbr) { * Retrieve a collection of tombstone GC region-versions */ public Map getTombstoneGCVector() { - Map result; + UnifiedMap result; synchronized(memberToGCVersion) { result = new UnifiedMap<>(this.memberToGCVersion); } @@ -1289,7 +1285,7 @@ protected RegionVersionVector(T ownerId, RegionVersionHolder localExceptions) { this.myId = ownerId; this.memberToVersion = vector; - this.memberToVersionSnapshot = new CopyOnWriteHashMap(vector); + this.memberToVersionSnapshot = new CopyOnWriteHashMap<>(vector); this.memberToGCVersion = gcVersions; this.localGCVersion.set(gcVersion); this.localVersion.set(version); @@ -1300,9 +1296,9 @@ protected RegionVersionVector(T ownerId, /** deserialize a cloned vector */ public RegionVersionVector() { - this.memberToVersionSnapshot = new CopyOnWriteHashMap>(); - this.memberToVersion = new ConcurrentHashMap>(INITIAL_CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); - this.memberToGCVersion = new ConcurrentHashMap(INITIAL_CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); + this.memberToVersionSnapshot = new CopyOnWriteHashMap<>(); + this.memberToVersion = new ConcurrentHashMap<>(INITIAL_CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); + this.memberToGCVersion = new ConcurrentHashMap<>(INITIAL_CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL); } /** diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/concurrent/CustomEntryConcurrentHashMap.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/concurrent/CustomEntryConcurrentHashMap.java index ca38a50a6..e2e0a7170 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/concurrent/CustomEntryConcurrentHashMap.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/concurrent/CustomEntryConcurrentHashMap.java @@ -2125,16 +2125,17 @@ public abstract class HashIterator { HashEntry lastReturned; - private HashEntry currentEntry; - private final ArrayList> currentList; - - int currentListIndex; + private HashEntry[] currentList; + private int currentListIndex; + private int currentListLen; + @SuppressWarnings("unchecked") HashIterator() { this.currentSegmentIndex = CustomEntryConcurrentHashMap.this .segments.length; this.nextTableIndex = -1; - this.currentList = new ArrayList<>(4); + this.currentList = new HashEntry[4]; + this.currentListLen = 0; this.currentListIndex = 0; advance(); } @@ -2149,18 +2150,8 @@ public final boolean hasMoreElements() { final void advance() { // GemStone changes BEGIN - if (this.currentListIndex == 0) { - if (this.currentEntry != null) { - this.nextEntry = this.currentEntry; - this.currentListIndex = 1; - return; - } else if (this.currentList.size() > 0) { - this.nextEntry = this.currentList.get(0); - this.currentListIndex = 1; - return; - } - } else if (this.currentListIndex < this.currentList.size()) { - this.nextEntry = this.currentList.get(this.currentListIndex++); + if (this.currentListIndex < this.currentListLen) { + this.nextEntry = this.currentList[this.currentListIndex++]; return; } @@ -2224,29 +2215,18 @@ final void advance() { * Read lock on {@link #currentSegmentIndex}'s listUpdateLock should already be * acquired. */ - private final void copyEntriesToList() { + private void copyEntriesToList() { assert segments[currentSegmentIndex] != null: "unexpected null currentSegment"; assert segments[currentSegmentIndex].listUpdateLock.numReaders() > 0; - this.currentEntry = null; - if (this.currentList.size() > 0) { - this.currentList.clear(); - } + this.currentListLen = 0; this.currentListIndex = 0; - boolean useEntry = true; for (HashEntry p = this.nextEntry.getNextEntry(); p != null; p = p .getNextEntry()) { - if (useEntry) { - if (this.currentEntry == null) { - this.currentEntry = p; - } else { - this.currentList.add(this.currentEntry); - this.currentList.add(p); - this.currentEntry = null; - useEntry = false; - } - } else { - this.currentList.add(p); + this.currentList[this.currentListLen++] = p; + if (this.currentListLen >= this.currentList.length) { + this.currentList = Arrays.copyOf(this.currentList, + this.currentList.length << 1); } } } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java index 0eb3bae41..475d91f63 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/CallbackFactoryProvider.java @@ -184,6 +184,10 @@ public void initMemoryStats(MemoryManagerStats stats) { public void clearConnectionPools() { } + @Override + public void clearCodegenCaches() { + } + @Override public URLClassLoader getLeadClassLoader() { return null; } diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java index 492639b2a..6d5a3db44 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/snappy/StoreCallbacks.java @@ -129,6 +129,11 @@ boolean acquireStorageMemory(String objectName, long numBytes, */ void clearConnectionPools(); + /** + * Clear all code-generation caches. + */ + void clearCodegenCaches(); + /** * Get the class loader of the lead */ diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/util/ArrayUtils.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/util/ArrayUtils.java index fbd5ac8f6..47948d0ad 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/util/ArrayUtils.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/util/ArrayUtils.java @@ -18,6 +18,7 @@ package com.gemstone.gemfire.internal.util; import java.util.Arrays; +import java.util.Objects; import com.gemstone.gemfire.internal.lang.StringUtils; import com.gemstone.gemfire.internal.offheap.annotations.Unretained; @@ -240,13 +241,7 @@ public static void objectStringNonRecursive(@Unretained Object obj, StringBuilde * this class... */ public static boolean objectEquals(Object o1, Object o2) { - if (o1 == o2) { - return true; - } - if (o1 == null) { - return false; - } - return o1.equals(o2); + return Objects.equals(o1, o2); } /** diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/util/concurrent/StoppableReentrantReadWriteLock.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/util/concurrent/StoppableReentrantReadWriteLock.java index c15172622..e8a039d3a 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/util/concurrent/StoppableReentrantReadWriteLock.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/util/concurrent/StoppableReentrantReadWriteLock.java @@ -46,7 +46,7 @@ public class StoppableReentrantReadWriteLock implements /* ReadWriteLock, */ jav /** * This is how often waiters will wake up to check for cancellation */ - private static final long RETRY_TIME = 15 * 1000; // milliseconds + private static final long RETRY_TIME = 5 * 1000; // milliseconds /** * Create a new instance @@ -136,6 +136,7 @@ public void lock() { break; } catch (InterruptedException e) { + stopper.checkCancelInProgress(e); interrupted = true; } } // for @@ -155,6 +156,27 @@ public void lockInterruptibly() throws InterruptedException { } } + /** + * Unlike lock() delay cancellation check to be done only if + * lock acquisition is not successful for a while. + */ + public void lockDelayCancel() { + boolean interrupted = Thread.interrupted(); + try { + for (;;) { + try { + if (lock.tryLock(RETRY_TIME, TimeUnit.MILLISECONDS)) break; + stopper.checkCancelInProgress(null); + } catch (InterruptedException e) { + stopper.checkCancelInProgress(e); + interrupted = true; + } + } // for + } finally { + if (interrupted) Thread.currentThread().interrupt(); + } + } + /** * @return true if the lock was acquired */ @@ -223,6 +245,7 @@ public void lock() { break; } catch (InterruptedException e) { + stopper.checkCancelInProgress(e); interrupted = true; } } // for diff --git a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/unsafe/ChannelBufferUnsafeOutputStream.java b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/unsafe/ChannelBufferUnsafeOutputStream.java index 3cfbab4ad..3b562e4b8 100644 --- a/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/unsafe/ChannelBufferUnsafeOutputStream.java +++ b/gemfire-shared/src/main/java/com/gemstone/gemfire/internal/shared/unsafe/ChannelBufferUnsafeOutputStream.java @@ -263,7 +263,7 @@ protected final void releaseBuffer() { final ByteBuffer buffer = this.buffer; if (buffer != null) { this.buffer = null; - DirectBufferAllocator.instance().release(buffer); + DirectBufferAllocator.releaseBuffer(buffer); } } diff --git a/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientConnection.java b/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientConnection.java index e600a4479..e7489acce 100644 --- a/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientConnection.java +++ b/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientConnection.java @@ -786,6 +786,21 @@ public ClientPreparedStatement prepareStatement(String sql, } } + /* + public ClientPreparedStatement getPrepared(String sql, int numParams, + boolean returnGeneratedKeys, int[] columnIndexes, + String[] columnNames) throws SQLException { + super.lock(); + try { + checkClosedConnection(); + return new ClientPreparedStatement(this, sql, numParams, + returnGeneratedKeys, columnIndexes, columnNames); + } finally { + super.unlock(); + } + } + */ + /** * {@inheritDoc} */ diff --git a/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientPreparedStatement.java b/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientPreparedStatement.java index f19de21a1..06a039d70 100644 --- a/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientPreparedStatement.java +++ b/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientPreparedStatement.java @@ -116,6 +116,17 @@ public class ClientPreparedStatement extends ClientStatement implements this.numParams = prepare(); } + /* + ClientPreparedStatement(ClientConnection conn, String sql, int numParams, + boolean getAutoInc, int[] autoIncColumns, String[] autoIncColumnNames) + throws SQLException { + super(conn); + this.preparedSQL = sql; + setAutoIncAttributes(getAutoInc, autoIncColumns, autoIncColumnNames); + this.numParams = numParams; + } + */ + protected Map getOutputParameters() { return Collections.emptyMap(); } @@ -221,6 +232,45 @@ public boolean execute() throws SQLException { } } + /* + public final boolean prepareAndExecute() throws SQLException { + checkClosed(); + reset(); + this.attrs.setPoolable(true); + try { + StatementResult sr = this.service.prepareAndExecute( + // don't throw exception in getLobSource rather return null and + // service will failover to new node and do re-prepare as required + getLobSource(false, "executePrepared"), this.preparedSQL, + Collections.singletonList(this.paramsList), getOutputParameters(), + getAttributes()); + clearPendingTransactionAttrs(); + int numParams = setPrepareResult(sr.getPreparedResult()); + if (numParams != this.numParams) { + throw ThriftExceptionUtil.newSQLException( + SQLState.LANG_INVALID_COLUMN_POSITION, new IllegalStateException( + "mismatch in parameter list size = " + this.numParams + + " and prepared " + numParams), this.numParams, numParams); + } + this.warnings = sr.getWarnings(); + if (this.attrs.isRequireAutoIncCols()) { + this.currentGeneratedKeys = sr.getGeneratedKeys(); + } + initializeProcedureOutParams(sr); + final RowSet rs = sr.getResultSet(); + if (rs != null) { + setCurrentRowSet(rs); + return true; + } else { + this.currentUpdateCount = sr.getUpdateCount(); + return false; + } + } catch (SnappyException se) { + throw informListeners(ThriftExceptionUtil.newSQLException(se)); + } + } + */ + /** * {@inheritDoc} */ diff --git a/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientService.java b/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientService.java index 2554a9cff..a8797f2a1 100644 --- a/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientService.java +++ b/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientService.java @@ -1370,10 +1370,9 @@ public UpdateResult executePreparedBatch(HostConnection source, long stmtId, } } - public StatementResult prepareAndExecute(String sql, List paramsBatch, - Map outputParams, StatementAttrs attrs) - throws SnappyException { - HostConnection source = this.currentHostConnection; + public StatementResult prepareAndExecute(HostConnection source, String sql, + List paramsBatch, Map outputParams, + StatementAttrs attrs) throws SnappyException { if (SanityManager.TraceClientStatement) { final long ns = System.nanoTime(); SanityManager.DEBUG_PRINT_COMPACT("prepareAndExecute_S", sql, diff --git a/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientStatement.java b/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientStatement.java index d6c5ecdab..21c7acd84 100644 --- a/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientStatement.java +++ b/gemfirexd/client/src/main/java/io/snappydata/thrift/internal/ClientStatement.java @@ -666,12 +666,12 @@ protected final void setAutoIncAttributes(boolean getAutoInc, this.attrs.setRequireAutoIncCols(false); this.attrs.setRequireAutoIncColsIsSet(false); } - if (autoIncColumns != null) { + if (autoIncColumns != null && autoIncColumns.length > 0) { this.attrs.setAutoIncColumns(getIntegerList(autoIncColumns)); } else { this.attrs.setAutoIncColumns(null); } - if (autoIncColumnNames != null) { + if (autoIncColumnNames != null && autoIncColumnNames.length > 0) { this.attrs.setAutoIncColumnNames(Arrays.asList(autoIncColumnNames)); } else { this.attrs.setAutoIncColumnNames(null); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/GfxdTXStateProxy.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/GfxdTXStateProxy.java index 109e87aed..f58ca45e5 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/GfxdTXStateProxy.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/GfxdTXStateProxy.java @@ -507,12 +507,12 @@ public final boolean execute(final Object k1, final Object k2, } @Override - protected final void cleanup() { + protected final void cleanup(boolean rollback) { transUpdater.set(this, null); if (!this.dbOps.isEmpty()) { this.dbOps.clear(); } - super.cleanup(); + super.cleanup(rollback); } private final void remoteConnCleanup(final boolean rollback, diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/heap/MemHeapScanController.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/heap/MemHeapScanController.java index d288af262..1a4cc9488 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/heap/MemHeapScanController.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/heap/MemHeapScanController.java @@ -848,7 +848,10 @@ public final boolean next() throws StandardException { final boolean isGlobalScan = (this.openMode & GfxdConstants.SCAN_OPENMODE_FOR_GLOBALINDEX) != 0; - LocalRegion owner = null; + final boolean isRowBuffer = this.gfContainer.isRowBuffer(); + final boolean hasQualifier = this.init_qualifier != null && + this.init_qualifier.length > 0; + LocalRegion owner; RegionEntry entry = null; while (entryIterator.hasNext()) { ExecRow row = null; @@ -906,14 +909,27 @@ public final boolean next() throws StandardException { this.currentDataRegion = owner; } if (this.templateCompactExecRow != null) { - row = RegionEntryUtils.fillRowWithoutFaultIn(this.gfContainer, - owner, this.currentRowLocation.getRegionEntry(), - this.templateCompactExecRow) ? templateCompactExecRow : null; - } - else { - row = RegionEntryUtils.getRowWithoutFaultIn(this.gfContainer, - owner, this.currentRowLocation.getRegionEntry(), - this.currentRowLocation.getTableInfo(this.gfContainer)); + // always fault-in for row buffer + if (isRowBuffer) { + row = RegionEntryUtils.fillRowFaultInOptimized(this.gfContainer, + owner, this.currentRowLocation, this.templateCompactExecRow) + ? templateCompactExecRow : null; + } else { + row = RegionEntryUtils.fillRowWithoutFaultIn(this.gfContainer, + owner, this.currentRowLocation.getRegionEntry(), + this.templateCompactExecRow) ? templateCompactExecRow : null; + } + } else { + // always fault-in for row buffer + if (isRowBuffer) { + row = RegionEntryUtils.getRow(this.gfContainer, + owner, this.currentRowLocation.getRegionEntry(), + this.currentRowLocation.getTableInfo(this.gfContainer)); + } else { + row = RegionEntryUtils.getRowWithoutFaultIn(this.gfContainer, + owner, this.currentRowLocation.getRegionEntry(), + this.currentRowLocation.getTableInfo(this.gfContainer)); + } } if (row != null) { this.currentExecRow = row; @@ -974,10 +990,14 @@ public final boolean next() throws StandardException { boolean rowQualified = false; try { - if (this.init_qualifier == null + if (!hasQualifier || RowFormatter.qualifyRow(this.currentExecRow, this.byteArrayStore, this.init_qualifier)) { rowQualified = true; + // fault-in the entry if filters are present + if (hasQualifier && !isRowBuffer) { + this.currentRowLocation.getValueOrOffHeapEntry(owner); + } this.statNumRowsQualified++; if (this.currentDataRegion != null) { this.localTXState.addReadLockForScan(entry, this.readLockMode, diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/index/AbstractRowLocation.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/index/AbstractRowLocation.java index f1107484b..408070066 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/index/AbstractRowLocation.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/access/index/AbstractRowLocation.java @@ -664,12 +664,15 @@ public void writeNullDVD(DataOutput out) throws IOException{ out.writeByte(DSCODE.NULL); out.writeByte(getTypeId()); } - - + @Override public Object getRawValue() { - - throw new UnsupportedOperationException(" Implement the method in concrete class"); + throw new UnsupportedOperationException("Implement the method in concrete class"); + } + + @Override + public boolean isValueNull() { + throw new UnsupportedOperationException("Implement the method in concrete class"); } @Override diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java index a3c742a14..c8c92201e 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java @@ -73,6 +73,7 @@ import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils; import com.pivotal.gemfirexd.internal.engine.distributed.utils.SecurityUtils; import com.pivotal.gemfirexd.internal.engine.jdbc.GemFireXDRuntimeException; +import com.pivotal.gemfirexd.internal.engine.sql.execute.FunctionUtils; import com.pivotal.gemfirexd.internal.engine.store.CustomRowsResultSet; import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer; import com.pivotal.gemfirexd.internal.engine.store.GemFireStore; @@ -96,6 +97,7 @@ import com.pivotal.gemfirexd.internal.iapi.types.HarmonySerialClob; import com.pivotal.gemfirexd.internal.iapi.types.TypeId; import com.pivotal.gemfirexd.internal.iapi.util.IdUtil; +import com.pivotal.gemfirexd.internal.iapi.util.ReuseFactory; import com.pivotal.gemfirexd.internal.iapi.util.StringUtil; import com.pivotal.gemfirexd.internal.impl.jdbc.EmbedConnection; import com.pivotal.gemfirexd.internal.impl.jdbc.EmbedResultSetMetaData; @@ -112,6 +114,8 @@ import com.pivotal.gemfirexd.internal.shared.common.reference.SQLState; import com.pivotal.gemfirexd.internal.shared.common.sanity.SanityManager; import com.pivotal.gemfirexd.load.Import; +import io.snappydata.execute.AcquireBucketMaintLock; +import io.snappydata.execute.ReleaseBucketMaintLocks; import io.snappydata.thrift.BucketOwners; import io.snappydata.thrift.CatalogMetadataDetails; import io.snappydata.thrift.CatalogMetadataRequest; @@ -2342,9 +2346,13 @@ public static void DUMP_STACKS(Boolean all) * This procedure sets the local execution mode for a particular bucket. */ public static void setBucketsForLocalExecution(String tableName, - Set bucketSet, boolean retain, + Set bucketSet, boolean retain, String lockOwner, @Nonnull LanguageConnectionContext lcc) { - Region region = Misc.getRegionForTable(tableName, true); + Region region = Misc.getRegionForTable(tableName, true); + if ((region instanceof PartitionedRegion) && lockOwner != null) { + PartitionedRegion pr = (PartitionedRegion)region; + lockPrimaryForMaintenance(false, lockOwner, pr, bucketSet); + } lcc.setExecuteLocally(bucketSet, region, false, null); lcc.setBucketRetentionForLocalExecution(retain); } @@ -2357,6 +2365,19 @@ public static void setBucketsForLocalExecution(String tableName, public static void SET_BUCKETS_FOR_LOCAL_EXECUTION(String tableName, String buckets, long catalogSchemaVersion) throws SQLException, StandardException { + SET_BUCKETS_FOR_LOCAL_EXECUTION_EX(tableName, buckets, + catalogSchemaVersion, null); + } + + /** + * This procedure sets the local execution mode for a particular bucket. + * To prevent clearing of lcc in case of thin client connections a flag + * BUCKET_RENTION_FOR_LOCAL_EXECUTION is set. It also acquires bucket + * maintenance locks for an update operation. + */ + public static void SET_BUCKETS_FOR_LOCAL_EXECUTION_EX(String tableName, + String buckets, long catalogSchemaVersion, String lockOwner) + throws SQLException, StandardException { if (tableName == null) { throw Util.generateCsSQLException(SQLState.ENTITY_NAME_MISSING); } @@ -2374,12 +2395,48 @@ public static void SET_BUCKETS_FOR_LOCAL_EXECUTION(String tableName, LanguageConnectionContext lcc = ConnectionUtil.getCurrentLCC(); Set bucketSet = new UnifiedSet<>(); StringTokenizer st = new StringTokenizer(buckets,","); - while(st.hasMoreTokens()){ + while (st.hasMoreTokens()) { bucketSet.add(Integer.parseInt(st.nextToken())); } - setBucketsForLocalExecution(tableName, bucketSet, true, lcc); + setBucketsForLocalExecution(tableName, bucketSet, true, lockOwner, lcc); + } + + public static void releaseBucketMaintenanceLocks(String tableName, + boolean forWrite, String lockOwner, Set bucketIds) { + Set allStores = GfxdMessage.getAllDataStores(); + if (allStores != null && !allStores.isEmpty()) { + FunctionService.onMembers(allStores) + .withArgs(new Object[] { tableName, forWrite, lockOwner, bucketIds }) + .execute(ReleaseBucketMaintLocks.ID); + } } + public static void lockPrimaryForMaintenance(boolean forWrite, + Object owner, PartitionedRegion pr, Set bucketIds) { + PartitionedRegionDataStore ds = pr.getDataStore(); + // lock regions, if required, before acquiring the snapshot + for (Integer bucketId : bucketIds) { + BucketRegion br = ds != null ? ds.getLocalBucketById(bucketId) : null; + // if bucket is present and primary, then lock locally else send to primary + if (br != null && br.getBucketAdvisor().isPrimary()) { + br.lockForMaintenance(forWrite, Long.MAX_VALUE, owner); + // primary check for bucket after acquiring the lock + if (br.getBucketAdvisor().isPrimary()) { + continue; + } else { + br.unlockAfterMaintenance(forWrite, owner); + } + } + try { + FunctionUtils.executeFunctionOnMembers(pr.getSystem(), + new AcquireBucketMaintLock.GetPrimaryMember(pr, bucketId), + new Object[] { pr.getFullPath(), forWrite, owner, bucketId }, + AcquireBucketMaintLock.ID, null, false, false, true, false); + } catch (StandardException se) { + throw new GemFireXDRuntimeException(se); + } + } + } /** * This procedure sets the Nanotimer type. NanoTimer are used extensively while @@ -2409,11 +2466,12 @@ public static void SET_NANOTIMER_TYPE(Boolean useNativeTimer, String nativeTimer // register the call backs with the JDBCSource so that // bucket region can insert into the column table public static void flushLocalBuckets(String resolvedName, boolean forceFlush) { - PartitionedRegion pr = (PartitionedRegion)Misc.getRegionForTable( + LocalRegion region = (LocalRegion)Misc.getRegionForTable( resolvedName, false); PartitionedRegionDataStore ds; - if (pr != null && (ds = pr.getDataStore()) != null) { - TXStateInterface tx = pr.getTXState(); + if ((region instanceof PartitionedRegion) + && (ds = ((PartitionedRegion)region).getDataStore()) != null) { + TXStateInterface tx = region.getTXState(); for (BucketRegion bucketRegion : ds.getAllLocalPrimaryBucketRegions()) { if (forceFlush || bucketRegion.checkForColumnBatchCreation(null)) { bucketRegion.createAndInsertColumnBatch(tx, forceFlush); @@ -2712,6 +2770,20 @@ private static void runCatalogConsistencyChecks(boolean removeInconsistentEntrie } } + public static void PURGE_CODEGEN_CACHES() throws SQLException, StandardException { + if (GemFireXDUtils.TraceExecution) { + SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_EXECUTION, + "in procedure PURGE_CODEGEN_CACHES()"); + } + Object[] emptyParams = ReuseFactory.getZeroLenObjectArray(); + GfxdSystemProcedureMessage.SysProcMethod.purgeCodegenCaches + .processMessage(emptyParams, Misc.getMyId()); + // then publish to other members excluding locators + publishMessage(emptyParams, false, + GfxdSystemProcedureMessage.SysProcMethod.purgeCodegenCaches, false, + false); + } + /** * Cancel a statement asynchronously on all nodes i.e. this will not wait for * the response of cancel message. diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/messages/GfxdSystemProcedureMessage.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/messages/GfxdSystemProcedureMessage.java index f3a8bc6fe..8005b51a9 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/messages/GfxdSystemProcedureMessage.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/messages/GfxdSystemProcedureMessage.java @@ -79,6 +79,7 @@ import com.pivotal.gemfirexd.internal.iapi.sql.dictionary.TableDescriptor; import com.pivotal.gemfirexd.internal.iapi.sql.dictionary.TablePermsDescriptor; import com.pivotal.gemfirexd.internal.iapi.store.access.TransactionController; +import com.pivotal.gemfirexd.internal.iapi.util.ReuseFactory; import com.pivotal.gemfirexd.internal.impl.jdbc.EmbedConnection; import com.pivotal.gemfirexd.internal.impl.sql.catalog.SYSROUTINEPERMSRowFactory; import com.pivotal.gemfirexd.internal.impl.sql.catalog.SYSTABLEPERMSRowFactory; @@ -1930,6 +1931,38 @@ String getSQLStatement(Object[] params) throws StandardException { .append("')").toString(); } }, + + purgeCodegenCaches { + @Override + boolean allowExecution(Object[] params) { + // only on process nodes + return GemFireXDUtils.getMyVMKind().isAccessorOrStore(); + } + + @Override + boolean preprocess() { + return false; + } + + @Override + public void processMessage(Object[] params, DistributedMember sender) { + CallbackFactoryProvider.getStoreCallbacks().clearCodegenCaches(); + } + + @Override + Object[] readParams(DataInput in, short flags) { + return ReuseFactory.getZeroLenObjectArray(); + } + + @Override + void writeParams(Object[] params, DataOutput out) { + } + + @Override + String getSQLStatement(Object[] params) { + return "CALL SYS.PURGE_CODEGEN_CACHES()"; + } + }, ; static final int[][] tableActions = new int[][] { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GfxdFunctionMessage.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GfxdFunctionMessage.java index d18f8f5c9..f8cca0867 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GfxdFunctionMessage.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/distributed/message/GfxdFunctionMessage.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import com.gemstone.gemfire.DataSerializer; import com.gemstone.gemfire.GemFireCheckedException; @@ -75,6 +76,7 @@ import com.pivotal.gemfirexd.internal.iapi.error.StandardException; import com.pivotal.gemfirexd.internal.iapi.services.monitor.Monitor; import com.pivotal.gemfirexd.internal.iapi.sql.conn.LanguageConnectionContext; +import com.pivotal.gemfirexd.internal.iapi.util.ReuseFactory; import com.pivotal.gemfirexd.internal.impl.sql.execute.xplain.XPLAINUtil; import com.pivotal.gemfirexd.internal.shared.common.reference.SQLState; import com.pivotal.gemfirexd.internal.shared.common.sanity.SanityManager; @@ -1567,8 +1569,6 @@ protected static final class ListOfReplies { protected int lastFlushedId; - private static final Object[] zeroLenArray = new Object[0]; - /** * The array buffer into which the elements of the list are stored. */ @@ -1577,7 +1577,7 @@ protected static final class ListOfReplies { private int size; public ListOfReplies(int capacity) { - this.elementData = capacity == 0 ? zeroLenArray + this.elementData = capacity == 0 ? ReuseFactory.getZeroLenObjectArray() : new Object[capacity]; } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/locks/impl/GfxdReentrantReadWriteLock.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/locks/impl/GfxdReentrantReadWriteLock.java index f25eb448f..8485c92d7 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/locks/impl/GfxdReentrantReadWriteLock.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/locks/impl/GfxdReentrantReadWriteLock.java @@ -135,7 +135,7 @@ public final class GfxdReentrantReadWriteLock extends AtomicInteger implements /** * The current owner of the lock. */ - private Object lockOwner; + private volatile Object lockOwner; /** * The value of "ack-wait-threshold" GemFire property that is also used to log @@ -147,12 +147,12 @@ public final class GfxdReentrantReadWriteLock extends AtomicInteger implements * Flags for this lock object including whether this lock is in global map of * {@link GfxdLocalLockService}. */ - byte flags; + private byte flags; /** * If true then indicates that lock tracing is on for this lock. */ - byte traceLock; + private byte traceLock; /** * Class prefix used for toString() output. diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/GemFireInsertResultSet.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/GemFireInsertResultSet.java index 560385d22..f07a8dd51 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/GemFireInsertResultSet.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/sql/execute/GemFireInsertResultSet.java @@ -170,7 +170,8 @@ public void open() throws StandardException { TXStateInterface tx = this.gfContainer.getActiveTXState(this.tran); // TOOD: decide on autocommit or this flag: Discuss - if (tx == null && (this.gfContainer.isRowBuffer() || reg.getCache().snapshotEnabledForTest())) { + if (tx == null && (this.gfContainer.isRowBuffer() || + (reg != null && reg.getCache().snapshotEnabledForTest()))) { this.tran.getTransactionManager().begin(IsolationLevel.SNAPSHOT, null); ((GemFireTransaction)lcc.getTransactionExecute()).setActiveTXState(TXManagerImpl.getCurrentTXState(), false); this.tran.setImplicitSnapshotTxStarted(true); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/AbstractRowLocationAdapter.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/AbstractRowLocationAdapter.java index f399ca3a7..9871abee5 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/AbstractRowLocationAdapter.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/AbstractRowLocationAdapter.java @@ -618,10 +618,14 @@ public Object getValueOrOffHeapEntry(LocalRegion owner) { @Override public Object getRawValue() { - return null; } + @Override + public boolean isValueNull() { + return true; + } + @Override public void markDeleteFromIndexInProgress() { //NOOP diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireContainer.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireContainer.java index 03f0fd6d4..d476f0c69 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireContainer.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireContainer.java @@ -4765,6 +4765,10 @@ public final boolean open(GemFireTransaction tran, int mode) return false; } + public final GfxdLockable getContainerLockingObject() { + return this.locking != null ? this.locking.containerLockObject : null; + } + /** * Close the container at TX end releasing any held locks on the container. */ diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java index 377c38c92..4d398cf41 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/GemFireStore.java @@ -174,6 +174,8 @@ import com.pivotal.gemfirexd.internal.shared.common.SharedUtils; import com.pivotal.gemfirexd.internal.shared.common.sanity.SanityManager; import com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider; +import io.snappydata.execute.AcquireBucketMaintLock; +import io.snappydata.execute.ReleaseBucketMaintLocks; import static com.gemstone.gemfire.distributed.internal.InternalLocator.FORCE_LOCATOR_DM_TYPE; @@ -1192,6 +1194,8 @@ else if (!hostData) { FunctionService.registerFunction(new GfxdCacheLoader.GetRowFunction()); FunctionService.registerFunction(new QueryCancelFunction()); FunctionService.registerFunction(new SnappyRegionStatsCollectorFunction()); + FunctionService.registerFunction(new AcquireBucketMaintLock()); + FunctionService.registerFunction(new ReleaseBucketMaintLocks()); FunctionService.registerFunction(new DiskStoreIDs.DiskStoreIDFunction()); final ConnectionSignaller signaller = ConnectionSignaller.getInstance(); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java index bd3896715..079a3f0f8 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/store/RegionEntryUtils.java @@ -31,7 +31,6 @@ import com.gemstone.gemfire.cache.Region; import com.gemstone.gemfire.distributed.DistributedMember; import com.gemstone.gemfire.distributed.internal.DistributionConfig; -import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember; import com.gemstone.gemfire.internal.Assert; import com.gemstone.gemfire.internal.ByteArrayDataInput; import com.gemstone.gemfire.internal.DSCODE; @@ -589,12 +588,26 @@ else if (Delta.class.isAssignableFrom(valClass)) { return false; } + public static boolean fillRowFaultInOptimized( + final GemFireContainer baseContainer, final LocalRegion region, + final RowLocation entry, final AbstractCompactExecRow row) + throws StandardException { + return fillRowOptimized(entry.getValueOrOffHeapEntry(region), + baseContainer, region, entry, row); + } + public static boolean fillRowWithoutFaultInOptimized( final GemFireContainer baseContainer, final LocalRegion region, final RowLocation entry, final AbstractCompactExecRow row) throws StandardException { + return fillRowOptimized(entry.getValueWithoutFaultInOrOffHeapEntry(region), + baseContainer, region, entry, row); + } - final Object value = entry.getValueWithoutFaultInOrOffHeapEntry(region); + private static boolean fillRowOptimized(final Object value, + final GemFireContainer baseContainer, final LocalRegion region, + final RowLocation entry, final AbstractCompactExecRow row) + throws StandardException { if (value != null) { final Class valClass = value.getClass(); if (valClass == byte[][].class) { diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/iapi/types/RowLocation.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/iapi/types/RowLocation.java index d45e32adf..7024242f3 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/iapi/types/RowLocation.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/iapi/types/RowLocation.java @@ -195,6 +195,8 @@ public ExecRow getRowWithoutFaultIn(GemFireContainer baseContainer) */ public Object getRawValue(); + public boolean isValueNull(); + public void markDeleteFromIndexInProgress(); public void unmarkDeleteFromIndexInProgress(); public boolean useRowLocationForIndexKey(); diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java index b0326bd60..7c18550a8 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java @@ -1639,7 +1639,7 @@ private void createGfxdSystemProcedures(TransactionController tc, argTypes, 0, 0, RoutineAliasInfo.NO_SQL, null, newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, true); } - + { // void SET_BUCKETS_FOR_LOCAL_EXECUTION(TableName, buckets) String[] argNames = new String[] { "TABLE_NAME", "BUCKETS", "CATALOG_VERSION"}; @@ -1653,6 +1653,39 @@ private void createGfxdSystemProcedures(TransactionController tc, tc, GFXD_SYS_PROC_CLASSNAME, false); } + { + // void SET_BUCKETS_FOR_LOCAL_EXECUTION_EX(tableName, buckets, + // relDestroyVersion, lockOwner) + String[] argNames = new String[] { "TABLE_NAME", "BUCKETS", "CATALOG_VERSION", + "LOCK_OWNER" }; + TypeDescriptor[] argTypes = new TypeDescriptor[] { + DataTypeDescriptor.getCatalogType(Types.VARCHAR), + DataTypeDescriptor.getCatalogType(Types.VARCHAR), + DataTypeDescriptor.getCatalogType(Types.BIGINT), + DataTypeDescriptor.getCatalogType(Types.VARCHAR) + }; + super.createSystemProcedureOrFunction("SET_BUCKETS_FOR_LOCAL_EXECUTION_EX", + sysUUID, argNames, argTypes, 0, 0, RoutineAliasInfo.NO_SQL, null, + newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, false); + } + + { + // void RELEASE_BUCKET_MAINTENANCE_LOCKS(tableName, forWrite, lockOwner, buckets) + String[] argNames = new String[] { "TABLE_NAME", "FOR_WRITE", + "LOCK_OWNER", "BUCKETS" }; + TypeDescriptor[] argTypes = new TypeDescriptor[] { + DataTypeDescriptor.getBuiltInDataTypeDescriptor( + Types.VARCHAR, false).getCatalogType(), + DataTypeDescriptor.getBuiltInDataTypeDescriptor( + Types.BOOLEAN, false).getCatalogType(), + DataTypeDescriptor.getCatalogType(Types.VARCHAR), + DataTypeDescriptor.getCatalogType(Types.VARCHAR) + }; + super.createSystemProcedureOrFunction("RELEASE_BUCKET_MAINTENANCE_LOCKS", + sysUUID, argNames, argTypes, 0, 0, RoutineAliasInfo.NO_SQL, null, + newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, false); + } + { // SQLJ.INSTALL_JAR_BYTES(Blob,String) String[] argNames = new String[] { "JAR_BYTES", "JAR_NAME" }; diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/store/access/heap/HeapRowLocation.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/store/access/heap/HeapRowLocation.java index cc00212b9..31e37d79c 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/store/access/heap/HeapRowLocation.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/store/access/heap/HeapRowLocation.java @@ -550,25 +550,30 @@ public TXId getTXId() { return null; } - @Override public Object getValueWithoutFaultInOrOffHeapEntry(LocalRegion owner) { throw new IllegalStateException( - "HeapRowLocation:getValueWithoutFaultIn: method should not get invoked"); + "HeapRowLocation:getValueWithoutFaultInOrOffHeapEntry: method should not get invoked"); } - + @Override public Object getValueOrOffHeapEntry(LocalRegion owner) { throw new IllegalStateException( - "HeapRowLocation:getValueWithoutFaultIn: method should not get invoked"); + "HeapRowLocation:getValueOrOffHeapEntry: method should not get invoked"); } - + @Override public Object getRawValue() { throw new IllegalStateException( - "HeapRowLocation:getValueWithoutFaultIn: method should not get invoked"); + "HeapRowLocation:getRawValue: method should not get invoked"); } - + + @Override + public boolean isValueNull() { + throw new IllegalStateException( + "HeapRowLocation:isValueNull: method should not get invoked"); + } + @Override public void markDeleteFromIndexInProgress() { //NOOP diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ColumnBatchKey.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ColumnBatchKey.java index c1d7758fd..6d8e774df 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ColumnBatchKey.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/snappy/ColumnBatchKey.java @@ -16,10 +16,10 @@ */ package com.pivotal.gemfirexd.internal.snappy; -import com.gemstone.gemfire.internal.cache.AbstractRegionEntry; import com.gemstone.gemfire.internal.cache.BucketRegion; +import com.gemstone.gemfire.internal.cache.RegionEntry; import com.gemstone.gemfire.internal.cache.lru.Sizeable; -import com.gemstone.gemfire.internal.cache.partitioned.PREntriesIterator; +import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer; /** * Interface for a key object in the column store. @@ -30,11 +30,11 @@ public interface ColumnBatchKey extends Sizeable { * Get the number of columns defined for the given * column table (qualified name). */ - int getNumColumnsInTable(String columnTableName); + int getNumColumnsInTable(GemFireContainer columnTable); /** * Get the number of rows in this column batch. */ - int getColumnBatchRowCount(BucketRegion bucketRegion, AbstractRegionEntry re, + int getColumnBatchRowCount(BucketRegion bucketRegion, RegionEntry re, int numColumnsInTable); } diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java index c42926219..b2da05c61 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/tools/sizer/ObjectSizer.java @@ -755,8 +755,7 @@ else if (!isSingleEntrySizeComputed) { if (value != null) { if (!isValueTypeEvaluated) { if (isColumnTable) { - numColumnsInColumnTable = batchKey.getNumColumnsInTable( - c.getQualifiedTableName()); + numColumnsInColumnTable = batchKey.getNumColumnsInTable(c); } else if (value instanceof DataValueDescriptor[]) { dvdArraySize = SIZE_OF_UTIL.sizeof(value); } else if (value instanceof GatewaySenderEventImpl) { @@ -768,8 +767,10 @@ else if (!isSingleEntrySizeComputed) { } if (isColumnTable) { int valueSize = ((Sizeable)value).getSizeInBytes(); - columnRowCount += batchKey.getColumnBatchRowCount(prEntryIter.getHostedBucketRegion(), - re, numColumnsInColumnTable); + if (!re.isDestroyedOrRemoved()) { + columnRowCount += batchKey.getColumnBatchRowCount(prEntryIter + .getHostedBucketRegion(), re, numColumnsInColumnTable); + } valInMemoryCount++; valInMemorySize += valueSize; if (valueSize > maxSize) { diff --git a/gemfirexd/core/src/main/java/io/snappydata/execute/AcquireBucketMaintLock.java b/gemfirexd/core/src/main/java/io/snappydata/execute/AcquireBucketMaintLock.java new file mode 100644 index 000000000..557196f82 --- /dev/null +++ b/gemfirexd/core/src/main/java/io/snappydata/execute/AcquireBucketMaintLock.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2018 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package io.snappydata.execute; + +import java.util.Collections; +import java.util.Properties; +import java.util.Set; + +import com.gemstone.gemfire.cache.Declarable; +import com.gemstone.gemfire.cache.execute.Function; +import com.gemstone.gemfire.cache.execute.FunctionContext; +import com.gemstone.gemfire.distributed.DistributedMember; +import com.gemstone.gemfire.internal.cache.BucketRegion; +import com.gemstone.gemfire.internal.cache.PartitionedRegion; +import com.gemstone.gemfire.internal.cache.PartitionedRegionDataStore; +import com.gemstone.gemfire.internal.cache.execute.BucketMovedException; +import com.pivotal.gemfirexd.internal.engine.Misc; +import com.pivotal.gemfirexd.internal.engine.sql.execute.FunctionUtils; + +/** + * Distributed function to acquire a given bucket lock on primary. + *

+ * Expects three arguments: regionPath, forWrite, lockOwner, bucketId + */ +public class AcquireBucketMaintLock implements Declarable, Function { + + public static final String ID = "AcquireBucketMaintLock"; + + @Override + public void init(Properties props) { + } + + @Override + public void execute(FunctionContext context) { + // three arguments: regionPath, forWrite, lockOwner, bucketId + Object[] args = (Object[])context.getArguments(); + String regionPath = (String)args[0]; + boolean forWrite = (Boolean)args[1]; + String lockOwner = (String)args[2]; + Integer bucketId = (Integer)args[3]; + + PartitionedRegion pr = (PartitionedRegion)Misc.getRegion( + regionPath, true, false); + PartitionedRegionDataStore ds = pr.getDataStore(); + if (ds != null) { + try { + BucketRegion br = ds.getLocalBucketById(bucketId); + // if bucket is present and primary, then lock locally else send to primary + if (br != null && br.getBucketAdvisor().isPrimary()) { + br.lockForMaintenance(forWrite, Long.MAX_VALUE, lockOwner); + // primary check for bucket after acquiring the lock + if (br.getBucketAdvisor().isPrimary()) { + context.getResultSender().lastResult(Boolean.TRUE); + return; + } else { + br.unlockAfterMaintenance(forWrite, lockOwner); + } + } + } catch (Exception e) { + pr.getLogWriterI18n().warning(e); + } + } + // force retry in case of lock failure + throw new BucketMovedException("Primary not found for locking by " + + lockOwner, bucketId, regionPath); + } + + @Override + public String getId() { + return ID; + } + + @Override + public boolean hasResult() { + return true; + } + + @Override + public boolean optimizeForWrite() { + return true; + } + + @Override + public boolean isHA() { + return true; + } + + public static final class GetPrimaryMember + implements FunctionUtils.GetFunctionMembers { + + private final PartitionedRegion region; + private final int bucketId; + + public GetPrimaryMember(PartitionedRegion region, int bucketId) { + this.region = region; + this.bucketId = bucketId; + } + + @Override + public Set getMembers() { + return Collections.singleton(region.getBucketPrimary(bucketId)); + } + + @Override + public Set getServerGroups() { + return null; + } + + @Override + public void postExecutionCallback() { + } + } +} diff --git a/gemfirexd/core/src/main/java/io/snappydata/execute/ReleaseBucketMaintLocks.java b/gemfirexd/core/src/main/java/io/snappydata/execute/ReleaseBucketMaintLocks.java new file mode 100644 index 000000000..6a3956407 --- /dev/null +++ b/gemfirexd/core/src/main/java/io/snappydata/execute/ReleaseBucketMaintLocks.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2018 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package io.snappydata.execute; + +import java.util.Properties; +import java.util.Set; + +import com.gemstone.gemfire.cache.Declarable; +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.execute.Function; +import com.gemstone.gemfire.cache.execute.FunctionContext; +import com.gemstone.gemfire.internal.cache.BucketRegion; +import com.gemstone.gemfire.internal.cache.PartitionedRegion; +import com.gemstone.gemfire.internal.cache.PartitionedRegionDataStore; +import com.pivotal.gemfirexd.internal.engine.Misc; + +/** + * Distributed function to release bucket locks with given IDs (or null for all) + * acquired by some owner across the distributed system. + *

+ * Expects three arguments: tableName, forWrite, lockOwner, bucketIds + */ +public class ReleaseBucketMaintLocks implements Declarable, Function { + + public static final String ID = "ReleaseBucketMaintLocks"; + + @Override + public void init(Properties props) { + } + + @Override + public void execute(FunctionContext context) { + // three arguments: tableName, forWrite, lockOwner, bucketIds + Object[] args = (Object[])context.getArguments(); + String tableName = (String)args[0]; + boolean forWrite = (Boolean)args[1]; + String lockOwner = (String)args[2]; + @SuppressWarnings("unchecked") Set bucketIds = (Set)args[3]; + + Region region = Misc.getRegionForTable(tableName, false); + if (region instanceof PartitionedRegion) { + PartitionedRegion pr = (PartitionedRegion)region; + PartitionedRegionDataStore ds = pr.getDataStore(); + if (ds != null) { + for (BucketRegion br : ds.getAllLocalBucketRegions()) { + try { + if (lockOwner == null) { + // release all locks + br.unlockAllAfterMaintenance(forWrite); + } else if ((bucketIds == null || bucketIds.contains(br.getId())) + && br.hasMaintenanceLock(forWrite, lockOwner)) { + br.unlockAfterMaintenance(forWrite, lockOwner); + } + } catch (Exception e) { + pr.getLogWriterI18n().warning(e); + } + } + } + } + } + + @Override + public String getId() { + return ID; + } + + @Override + public boolean hasResult() { + return false; + } + + @Override + public boolean optimizeForWrite() { + return false; + } + + @Override + public boolean isHA() { + return false; + } +} diff --git a/gemfirexd/core/src/main/java/io/snappydata/thrift/server/SnappyDataServiceImpl.java b/gemfirexd/core/src/main/java/io/snappydata/thrift/server/SnappyDataServiceImpl.java index d6731572d..52f604558 100644 --- a/gemfirexd/core/src/main/java/io/snappydata/thrift/server/SnappyDataServiceImpl.java +++ b/gemfirexd/core/src/main/java/io/snappydata/thrift/server/SnappyDataServiceImpl.java @@ -1625,6 +1625,9 @@ private StatementAttrs applyMergeAttributes(StatementAttrs source, if (source.isSetSnapshotTransactionId() && !target.isSetSnapshotTransactionId()) { target.setSnapshotTransactionId(source.getSnapshotTransactionId()); } + if (source.isSetLockOwner() && !target.isSetLockOwner()) { + target.setLockOwner(source.getLockOwner()); + } } if (target != null) { // apply the attributes to statement and connection @@ -1645,7 +1648,8 @@ private StatementAttrs applyMergeAttributes(StatementAttrs source, if (target.isSetBucketIds()) { GfxdSystemProcedures.setBucketsForLocalExecution( target.getBucketIdsTable(), target.getBucketIds(), - target.isRetainBucketIds(), conn.getLanguageConnectionContext()); + target.isRetainBucketIds(), target.getLockOwner(), + conn.getLanguageConnectionContext()); } if (target.isSetCatalogVersion()) { final ExternalCatalog catalog = Misc.getMemStore().getExistingExternalCatalog(); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/BucketOwners.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/BucketOwners.java index 3b3671e95..a6576524e 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/BucketOwners.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/BucketOwners.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class BucketOwners implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("BucketOwners"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogFunctionObject.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogFunctionObject.java index 2e9e14db9..aadb4911d 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogFunctionObject.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogFunctionObject.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class CatalogFunctionObject implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CatalogFunctionObject"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogMetadataDetails.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogMetadataDetails.java index cec7f002a..23f2034ea 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogMetadataDetails.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogMetadataDetails.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class CatalogMetadataDetails implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CatalogMetadataDetails"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogMetadataRequest.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogMetadataRequest.java index db582c507..6c18f1d20 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogMetadataRequest.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogMetadataRequest.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class CatalogMetadataRequest implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CatalogMetadataRequest"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogPartitionObject.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogPartitionObject.java index 53002fd2c..40c214c6f 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogPartitionObject.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogPartitionObject.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class CatalogPartitionObject implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CatalogPartitionObject"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogSchemaObject.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogSchemaObject.java index 6b28583f7..93e344f6b 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogSchemaObject.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogSchemaObject.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class CatalogSchemaObject implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CatalogSchemaObject"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogStorage.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogStorage.java index b61607238..a248f674b 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogStorage.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogStorage.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class CatalogStorage implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CatalogStorage"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogTableObject.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogTableObject.java index d6844c9f6..60a3ce580 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogTableObject.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/CatalogTableObject.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class CatalogTableObject implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("CatalogTableObject"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/ClobChunk.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/ClobChunk.java index d2f7fa1a7..634f177d3 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/ClobChunk.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/ClobChunk.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class ClobChunk implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ClobChunk"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/ColumnDescriptor.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/ColumnDescriptor.java index 7871fe5a9..09ff8999d 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/ColumnDescriptor.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/ColumnDescriptor.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class ColumnDescriptor implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ColumnDescriptor"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/ConnectionProperties.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/ConnectionProperties.java index bf3724107..658a93f38 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/ConnectionProperties.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/ConnectionProperties.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class ConnectionProperties implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ConnectionProperties"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/Decimal.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/Decimal.java index 650739891..fed003d26 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/Decimal.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/Decimal.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class Decimal implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("Decimal"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/EntityId.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/EntityId.java index 3a9241896..c54815f43 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/EntityId.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/EntityId.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class EntityId implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("EntityId"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/LocatorService.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/LocatorService.java index bb9e2e7c0..d0b27eda8 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/LocatorService.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/LocatorService.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class LocatorService { public interface Iface { diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/OpenConnectionArgs.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/OpenConnectionArgs.java index 3c7b18f39..06b74490b 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/OpenConnectionArgs.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/OpenConnectionArgs.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class OpenConnectionArgs implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("OpenConnectionArgs"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/OutputParameter.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/OutputParameter.java index 949d8919b..5a372465a 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/OutputParameter.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/OutputParameter.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class OutputParameter implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("OutputParameter"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/PrepareResult.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/PrepareResult.java index 3b1d40a3b..0e86a37ee 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/PrepareResult.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/PrepareResult.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class PrepareResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("PrepareResult"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/RowSet.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/RowSet.java index e1e045aa0..074461dd8 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/RowSet.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/RowSet.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class RowSet implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RowSet"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/ServiceMetaData.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/ServiceMetaData.java index b0b36a1e7..18ab000c2 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/ServiceMetaData.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/ServiceMetaData.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class ServiceMetaData implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ServiceMetaData"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/ServiceMetaDataArgs.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/ServiceMetaDataArgs.java index 5739505de..b18163999 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/ServiceMetaDataArgs.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/ServiceMetaDataArgs.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class ServiceMetaDataArgs implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("ServiceMetaDataArgs"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyDataService.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyDataService.java index a48b38162..a726292b9 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyDataService.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyDataService.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class SnappyDataService { public interface Iface { diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyException.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyException.java index 94539c770..03df9b882 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyException.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyException.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class SnappyException extends TException implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("SnappyException"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyExceptionData.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyExceptionData.java index 50139d94e..765139d34 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyExceptionData.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/SnappyExceptionData.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class SnappyExceptionData implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("SnappyExceptionData"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/StatementAttrs.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/StatementAttrs.java index 21c0de4bb..3fea9f823 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/StatementAttrs.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/StatementAttrs.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class StatementAttrs implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("StatementAttrs"); @@ -61,6 +61,7 @@ public class StatementAttrs implements org.apache.thrift.TBase, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -91,6 +92,7 @@ public class StatementAttrs implements org.apache.thrift.TBase byName = new HashMap(); @@ -177,6 +180,8 @@ public static _Fields findByThriftId(int fieldId) { return SNAPSHOT_TRANSACTION_ID; case 23: // CATALOG_VERSION return CATALOG_VERSION; + case 24: // LOCK_OWNER + return LOCK_OWNER; default: return null; } @@ -234,7 +239,7 @@ public String getFieldName() { private static final int __METADATAVERSION_ISSET_ID = 14; private static final int __CATALOGVERSION_ISSET_ID = 15; public short __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.RESULT_SET_TYPE,_Fields.UPDATABLE,_Fields.HOLD_CURSORS_OVER_COMMIT,_Fields.REQUIRE_AUTO_INC_COLS,_Fields.AUTO_INC_COLUMNS,_Fields.AUTO_INC_COLUMN_NAMES,_Fields.BATCH_SIZE,_Fields.FETCH_REVERSE,_Fields.LOB_CHUNK_SIZE,_Fields.MAX_ROWS,_Fields.MAX_FIELD_SIZE,_Fields.TIMEOUT,_Fields.CURSOR_NAME,_Fields.POSSIBLE_DUPLICATE,_Fields.POOLABLE,_Fields.DO_ESCAPE_PROCESSING,_Fields.PENDING_TRANSACTION_ATTRS,_Fields.BUCKET_IDS,_Fields.BUCKET_IDS_TABLE,_Fields.RETAIN_BUCKET_IDS,_Fields.METADATA_VERSION,_Fields.SNAPSHOT_TRANSACTION_ID,_Fields.CATALOG_VERSION}; + private static final _Fields optionals[] = {_Fields.RESULT_SET_TYPE,_Fields.UPDATABLE,_Fields.HOLD_CURSORS_OVER_COMMIT,_Fields.REQUIRE_AUTO_INC_COLS,_Fields.AUTO_INC_COLUMNS,_Fields.AUTO_INC_COLUMN_NAMES,_Fields.BATCH_SIZE,_Fields.FETCH_REVERSE,_Fields.LOB_CHUNK_SIZE,_Fields.MAX_ROWS,_Fields.MAX_FIELD_SIZE,_Fields.TIMEOUT,_Fields.CURSOR_NAME,_Fields.POSSIBLE_DUPLICATE,_Fields.POOLABLE,_Fields.DO_ESCAPE_PROCESSING,_Fields.PENDING_TRANSACTION_ATTRS,_Fields.BUCKET_IDS,_Fields.BUCKET_IDS_TABLE,_Fields.RETAIN_BUCKET_IDS,_Fields.METADATA_VERSION,_Fields.SNAPSHOT_TRANSACTION_ID,_Fields.CATALOG_VERSION,_Fields.LOCK_OWNER}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -289,6 +294,8 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.CATALOG_VERSION, new org.apache.thrift.meta_data.FieldMetaData("catalogVersion", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); + tmpMap.put(_Fields.LOCK_OWNER, new org.apache.thrift.meta_data.FieldMetaData("lockOwner", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(StatementAttrs.class, metaDataMap); } @@ -355,6 +362,9 @@ public StatementAttrs(StatementAttrs other) { this.snapshotTransactionId = other.snapshotTransactionId; } this.catalogVersion = other.catalogVersion; + if (other.isSetLockOwner()) { + this.lockOwner = other.lockOwner; + } } public StatementAttrs deepCopy() { @@ -402,6 +412,7 @@ public void clear() { this.snapshotTransactionId = null; setCatalogVersionIsSet(false); this.catalogVersion = 0; + this.lockOwner = null; } public byte getResultSetType() { @@ -997,6 +1008,30 @@ public void setCatalogVersionIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __CATALOGVERSION_ISSET_ID, value); } + public String getLockOwner() { + return this.lockOwner; + } + + public StatementAttrs setLockOwner(String lockOwner) { + this.lockOwner = lockOwner; + return this; + } + + public void unsetLockOwner() { + this.lockOwner = null; + } + + /** Returns true if field lockOwner is set (has been assigned a value) and false otherwise */ + public boolean isSetLockOwner() { + return this.lockOwner != null; + } + + public void setLockOwnerIsSet(boolean value) { + if (!value) { + this.lockOwner = null; + } + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case RESULT_SET_TYPE: @@ -1183,6 +1218,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case LOCK_OWNER: + if (value == null) { + unsetLockOwner(); + } else { + setLockOwner((String)value); + } + break; + } } @@ -1257,6 +1300,9 @@ public Object getFieldValue(_Fields field) { case CATALOG_VERSION: return getCatalogVersion(); + case LOCK_OWNER: + return getLockOwner(); + } throw new IllegalStateException(); } @@ -1314,6 +1360,8 @@ public boolean isSet(_Fields field) { return isSetSnapshotTransactionId(); case CATALOG_VERSION: return isSetCatalogVersion(); + case LOCK_OWNER: + return isSetLockOwner(); } throw new IllegalStateException(); } @@ -1538,6 +1586,15 @@ public boolean equals(StatementAttrs that) { return false; } + boolean this_present_lockOwner = true && this.isSetLockOwner(); + boolean that_present_lockOwner = true && that.isSetLockOwner(); + if (this_present_lockOwner || that_present_lockOwner) { + if (!(this_present_lockOwner && that_present_lockOwner)) + return false; + if (!this.lockOwner.equals(that.lockOwner)) + return false; + } + return true; } @@ -1660,6 +1717,11 @@ public int hashCode() { if (present_catalogVersion) list.add(catalogVersion); + boolean present_lockOwner = true && (isSetLockOwner()); + list.add(present_lockOwner); + if (present_lockOwner) + list.add(lockOwner); + return list.hashCode(); } @@ -1901,6 +1963,16 @@ public int compareTo(StatementAttrs other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetLockOwner()).compareTo(other.isSetLockOwner()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetLockOwner()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.lockOwner, other.lockOwner); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -2086,6 +2158,16 @@ public String toString() { sb.append(this.catalogVersion); first = false; } + if (isSetLockOwner()) { + if (!first) sb.append(", "); + sb.append("lockOwner:"); + if (this.lockOwner == null) { + sb.append("null"); + } else { + sb.append(this.lockOwner); + } + first = false; + } sb.append(")"); return sb.toString(); } @@ -2357,6 +2439,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, StatementAttrs stru org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 24: // LOCK_OWNER + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.lockOwner = iprot.readString(); + struct.setLockOwnerIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -2530,6 +2620,13 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, StatementAttrs str oprot.writeI64(struct.catalogVersion); oprot.writeFieldEnd(); } + if (struct.lockOwner != null) { + if (struct.isSetLockOwner()) { + oprot.writeFieldBegin(LOCK_OWNER_FIELD_DESC); + oprot.writeString(struct.lockOwner); + oprot.writeFieldEnd(); + } + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -2617,7 +2714,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, StatementAttrs stru if (struct.isSetCatalogVersion()) { optionals.set(22); } - oprot.writeBitSet(optionals, 23); + if (struct.isSetLockOwner()) { + optionals.set(23); + } + oprot.writeBitSet(optionals, 24); if (struct.isSetResultSetType()) { oprot.writeByte(struct.resultSetType); } @@ -2712,12 +2812,15 @@ public void write(org.apache.thrift.protocol.TProtocol prot, StatementAttrs stru if (struct.isSetCatalogVersion()) { oprot.writeI64(struct.catalogVersion); } + if (struct.isSetLockOwner()) { + oprot.writeString(struct.lockOwner); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, StatementAttrs struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(23); + BitSet incoming = iprot.readBitSet(24); if (incoming.get(0)) { struct.resultSetType = iprot.readByte(); struct.setResultSetTypeIsSet(true); @@ -2848,6 +2951,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, StatementAttrs struc struct.catalogVersion = iprot.readI64(); struct.setCatalogVersionIsSet(true); } + if (incoming.get(23)) { + struct.lockOwner = iprot.readString(); + struct.setLockOwnerIsSet(true); + } } } diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/StatementResult.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/StatementResult.java index 3b1f858c8..131150524 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/StatementResult.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/StatementResult.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class StatementResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("StatementResult"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/TransactionXid.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/TransactionXid.java index 107a0e643..f076ae089 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/TransactionXid.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/TransactionXid.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class TransactionXid implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TransactionXid"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/UpdateResult.java b/gemfirexd/shared/src/main/java/io/snappydata/thrift/UpdateResult.java index c46b0aad0..a874ab278 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/UpdateResult.java +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/UpdateResult.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-10") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2018-12-22") public class UpdateResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("UpdateResult"); diff --git a/gemfirexd/shared/src/main/java/io/snappydata/thrift/common/snappydata.thrift b/gemfirexd/shared/src/main/java/io/snappydata/thrift/common/snappydata.thrift index 105c80b43..28466b4c2 100644 --- a/gemfirexd/shared/src/main/java/io/snappydata/thrift/common/snappydata.thrift +++ b/gemfirexd/shared/src/main/java/io/snappydata/thrift/common/snappydata.thrift @@ -522,6 +522,8 @@ struct StatementAttrs { // the last catalog meta-data version recorded by client which will throw exception // on mismatch so that caller can refresh catalog meta-data (if being cached) 23: optional i64 catalogVersion + // column table updates/deletes can use this as the owner for bucket read locks + 24: optional string lockOwner } union ColumnValue { diff --git a/gemfirexd/tools/src/test/resources/codeAnalysis/excludedClasses.txt b/gemfirexd/tools/src/test/resources/codeAnalysis/excludedClasses.txt index 107881219..0fab11ea7 100644 --- a/gemfirexd/tools/src/test/resources/codeAnalysis/excludedClasses.txt +++ b/gemfirexd/tools/src/test/resources/codeAnalysis/excludedClasses.txt @@ -13,7 +13,6 @@ com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapreduce.GFRwHoplogRecordReader com.gemstone.gemfire.cache.hdfs.internal.hoplog.mapreduce.GFStreamRecordReader com.gemstone.gemfire.internal.cache.BucketRegionIndexCleaner com.gemstone.gemfire.internal.cache.ObjectEqualsHashingStrategy -com.gemstone.gemfire.internal.cache.TObjectLongHashMapWithIndex com.gemstone.gemfire.internal.cache.TXRegionState com.gemstone.gemfire.internal.concurrent.ConcurrentTHashSegment com.gemstone.gemfire.internal.concurrent.ConcurrentTHashSet diff --git a/lgpl/gemfire-trove/src/main/java/com/gemstone/gemfire/internal/cache/TObjectLongHashMapWithIndex.java b/lgpl/gemfire-trove/src/main/java/com/gemstone/gemfire/internal/cache/TObjectLongHashMapWithIndex.java deleted file mode 100644 index 220cf13c1..000000000 --- a/lgpl/gemfire-trove/src/main/java/com/gemstone/gemfire/internal/cache/TObjectLongHashMapWithIndex.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2010-2015 Pivotal Software, Inc. All rights reserved. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - */ -/* - * Contains code from GNU Trove having the license below. - * - * Copyright (c) 2001, Eric D. Friedman All Rights Reserved. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - */ -package com.gemstone.gemfire.internal.cache; - -import com.gemstone.gnu.trove.TObjectLongHashMap; - -/** - * A simple extension to {@link TObjectLongHashMap} that allows to get the - * index of value, and value from index directly. This avoids having to do a - * contains() + get() to determine a valid value in the map. - * - * @author swale - * @since 7.0 - */ -public final class TObjectLongHashMapWithIndex extends - TObjectLongHashMap { - - private static final long serialVersionUID = -2831843784355749476L; - - public TObjectLongHashMapWithIndex() { - } - - public TObjectLongHashMapWithIndex(final int initialCapacity) { - super(initialCapacity); - } - - /** - * Get the index of given key or -1 if it does not exist. - */ - public final int getIndex(final Object val) { - return super.index(val); - } - - /** - * Get the long value for given index obtained using - * {@link #getIndex(Object)}. - */ - public final long getValueForIndex(final int index) { - return _values[index]; - } - - /** - * @see #insertionIndex(Object) - */ - public final int getInsertionIndex(final Object key) { - return super.insertionIndex(key); - } - - /** - * Put at an index returned by {@link #getInsertionIndex(Object)}. - */ - public final Object putAtIndex(final Object key, long value, int index) { - long previous = this.defaultValue; - boolean isNewMapping = true; - if (index < 0) { - index = -index -1; - previous = _values[index]; - isNewMapping = false; - } - Object oldKey = _set[index]; - _set[index] = key; - _values[index] = value; - - if (isNewMapping) { - postInsertHook(oldKey == null); - } - return previous; - } -} diff --git a/native/src/snappyclient/cpp/thrift/snappydata_struct_StatementAttrs.cpp b/native/src/snappyclient/cpp/thrift/snappydata_struct_StatementAttrs.cpp index 7813d5f14..ce541001e 100644 --- a/native/src/snappyclient/cpp/thrift/snappydata_struct_StatementAttrs.cpp +++ b/native/src/snappyclient/cpp/thrift/snappydata_struct_StatementAttrs.cpp @@ -122,6 +122,26 @@ void StatementAttrs::__set_bucketIdsTable(const std::string& val) { __isset.bucketIdsTable = true; } +void StatementAttrs::__set_retainBucketIds(const bool val) { + this->retainBucketIds = val; +__isset.retainBucketIds = true; +} + +void StatementAttrs::__set_metadataVersion(const int32_t val) { + this->metadataVersion = val; +__isset.metadataVersion = true; +} + +void StatementAttrs::__set_snapshotTransactionId(const std::string& val) { + this->snapshotTransactionId = val; +__isset.snapshotTransactionId = true; +} + +void StatementAttrs::__set_lockOwner(const std::string& val) { + this->lockOwner = val; +__isset.lockOwner = true; +} + uint32_t StatementAttrs::read(::apache::thrift::protocol::TProtocol* iprot) { uint32_t xfer = 0; @@ -348,6 +368,38 @@ uint32_t StatementAttrs::read(::apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; + case 20: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->retainBucketIds); + this->__isset.retainBucketIds = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 21: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->metadataVersion); + this->__isset.metadataVersion = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 22: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->snapshotTransactionId); + this->__isset.snapshotTransactionId = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 23: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->lockOwner); + this->__isset.lockOwner = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -492,6 +544,26 @@ uint32_t StatementAttrs::write(::apache::thrift::protocol::TProtocol* oprot) con xfer += oprot->writeString(this->bucketIdsTable); xfer += oprot->writeFieldEnd(); } + if (this->__isset.retainBucketIds) { + xfer += oprot->writeFieldBegin("retainBucketIds", ::apache::thrift::protocol::T_BOOL, 20); + xfer += oprot->writeBool(this->retainBucketIds); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.metadataVersion) { + xfer += oprot->writeFieldBegin("metadataVersion", ::apache::thrift::protocol::T_I32, 21); + xfer += oprot->writeI32(this->metadataVersion); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.snapshotTransactionId) { + xfer += oprot->writeFieldBegin("snapshotTransactionId", ::apache::thrift::protocol::T_STRING, 22); + xfer += oprot->writeString(this->snapshotTransactionId); + xfer += oprot->writeFieldEnd(); + } + if (this->__isset.lockOwner) { + xfer += oprot->writeFieldBegin("lockOwner", ::apache::thrift::protocol::T_STRING, 23); + xfer += oprot->writeString(this->lockOwner); + xfer += oprot->writeFieldEnd(); + } xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -519,6 +591,10 @@ void swap(StatementAttrs &a, StatementAttrs &b) noexcept { swap(a.pendingTransactionAttrs, b.pendingTransactionAttrs); swap(a.bucketIds, b.bucketIds); swap(a.bucketIdsTable, b.bucketIdsTable); + swap(a.retainBucketIds, b.retainBucketIds); + swap(a.metadataVersion, b.metadataVersion); + swap(a.snapshotTransactionId, b.snapshotTransactionId); + swap(a.lockOwner, b.lockOwner); swap(a.__isset, b.__isset); } @@ -542,6 +618,10 @@ StatementAttrs::StatementAttrs(const StatementAttrs& other181) { pendingTransactionAttrs = other181.pendingTransactionAttrs; bucketIds = other181.bucketIds; bucketIdsTable = other181.bucketIdsTable; + retainBucketIds = other181.retainBucketIds; + metadataVersion = other181.metadataVersion; + snapshotTransactionId = other181.snapshotTransactionId; + lockOwner = other181.lockOwner; __isset = other181.__isset; } StatementAttrs::StatementAttrs( StatementAttrs&& other182) noexcept { @@ -564,6 +644,10 @@ StatementAttrs::StatementAttrs( StatementAttrs&& other182) noexcept { pendingTransactionAttrs = std::move(other182.pendingTransactionAttrs); bucketIds = std::move(other182.bucketIds); bucketIdsTable = std::move(other182.bucketIdsTable); + retainBucketIds = std::move(other182.retainBucketIds); + metadataVersion = std::move(other182.metadataVersion); + snapshotTransactionId = std::move(other182.snapshotTransactionId); + lockOwner = std::move(other182.lockOwner); __isset = std::move(other182.__isset); } StatementAttrs& StatementAttrs::operator=(const StatementAttrs& other183) { @@ -586,6 +670,10 @@ StatementAttrs& StatementAttrs::operator=(const StatementAttrs& other183) { pendingTransactionAttrs = other183.pendingTransactionAttrs; bucketIds = other183.bucketIds; bucketIdsTable = other183.bucketIdsTable; + retainBucketIds = other183.retainBucketIds; + metadataVersion = other183.metadataVersion; + snapshotTransactionId = other183.snapshotTransactionId; + lockOwner = other183.lockOwner; __isset = other183.__isset; return *this; } @@ -609,6 +697,10 @@ StatementAttrs& StatementAttrs::operator=(StatementAttrs&& other184) noexcept { pendingTransactionAttrs = std::move(other184.pendingTransactionAttrs); bucketIds = std::move(other184.bucketIds); bucketIdsTable = std::move(other184.bucketIdsTable); + retainBucketIds = std::move(other184.retainBucketIds); + metadataVersion = std::move(other184.metadataVersion); + snapshotTransactionId = std::move(other184.snapshotTransactionId); + lockOwner = std::move(other184.lockOwner); __isset = std::move(other184.__isset); return *this; } @@ -634,6 +726,10 @@ void StatementAttrs::printTo(std::ostream& out) const { out << ", " << "pendingTransactionAttrs="; (__isset.pendingTransactionAttrs ? (out << to_string(pendingTransactionAttrs)) : (out << "")); out << ", " << "bucketIds="; (__isset.bucketIds ? (out << to_string(bucketIds)) : (out << "")); out << ", " << "bucketIdsTable="; (__isset.bucketIdsTable ? (out << to_string(bucketIdsTable)) : (out << "")); + out << ", " << "retainBucketIds="; (__isset.retainBucketIds ? (out << to_string(retainBucketIds)) : (out << "")); + out << ", " << "metadataVersion="; (__isset.metadataVersion ? (out << to_string(metadataVersion)) : (out << "")); + out << ", " << "snapshotTransactionId="; (__isset.snapshotTransactionId ? (out << to_string(snapshotTransactionId)) : (out << "")); + out << ", " << "lockOwner="; (__isset.lockOwner ? (out << to_string(lockOwner)) : (out << "")); out << ")"; } diff --git a/native/src/snappyclient/headers/snappydata_struct_StatementAttrs.h b/native/src/snappyclient/headers/snappydata_struct_StatementAttrs.h index cbe574e49..18e48082d 100644 --- a/native/src/snappyclient/headers/snappydata_struct_StatementAttrs.h +++ b/native/src/snappyclient/headers/snappydata_struct_StatementAttrs.h @@ -25,7 +25,7 @@ namespace io { namespace snappydata { namespace thrift { typedef struct _StatementAttrs__isset { - _StatementAttrs__isset() : resultSetType(false), updatable(false), holdCursorsOverCommit(false), requireAutoIncCols(false), autoIncColumns(false), autoIncColumnNames(false), batchSize(true), fetchReverse(false), lobChunkSize(false), maxRows(false), maxFieldSize(false), timeout(false), cursorName(false), possibleDuplicate(false), poolable(false), doEscapeProcessing(false), pendingTransactionAttrs(false), bucketIds(false), bucketIdsTable(false) {} + _StatementAttrs__isset() : resultSetType(false), updatable(false), holdCursorsOverCommit(false), requireAutoIncCols(false), autoIncColumns(false), autoIncColumnNames(false), batchSize(true), fetchReverse(false), lobChunkSize(false), maxRows(false), maxFieldSize(false), timeout(false), cursorName(false), possibleDuplicate(false), poolable(false), doEscapeProcessing(false), pendingTransactionAttrs(false), bucketIds(false), bucketIdsTable(false), retainBucketIds(false), metadataVersion(false), snapshotTransactionId(false), lockOwner(false) {} bool resultSetType :1; bool updatable :1; bool holdCursorsOverCommit :1; @@ -45,6 +45,10 @@ typedef struct _StatementAttrs__isset { bool pendingTransactionAttrs :1; bool bucketIds :1; bool bucketIdsTable :1; + bool retainBucketIds :1; + bool metadataVersion :1; + bool snapshotTransactionId :1; + bool lockOwner :1; } _StatementAttrs__isset; class StatementAttrs { @@ -54,7 +58,7 @@ class StatementAttrs { StatementAttrs(StatementAttrs&&) noexcept; StatementAttrs& operator=(const StatementAttrs&); StatementAttrs& operator=(StatementAttrs&&) noexcept; - StatementAttrs() : resultSetType(0), updatable(0), holdCursorsOverCommit(0), requireAutoIncCols(0), batchSize(8192), fetchReverse(0), lobChunkSize(0), maxRows(0), maxFieldSize(0), timeout(0), cursorName(), possibleDuplicate(0), poolable(0), doEscapeProcessing(0), bucketIdsTable() { + StatementAttrs() : resultSetType(0), updatable(0), holdCursorsOverCommit(0), requireAutoIncCols(0), batchSize(8192), fetchReverse(0), lobChunkSize(0), maxRows(0), maxFieldSize(0), timeout(0), cursorName(), possibleDuplicate(0), poolable(0), doEscapeProcessing(0), bucketIdsTable(), retainBucketIds(0), metadataVersion(0), snapshotTransactionId(), lockOwner() { } virtual ~StatementAttrs() noexcept; @@ -77,6 +81,10 @@ class StatementAttrs { std::map pendingTransactionAttrs; std::set bucketIds; std::string bucketIdsTable; + bool retainBucketIds; + int32_t metadataVersion; + std::string snapshotTransactionId; + std::string lockOwner; _StatementAttrs__isset __isset; @@ -118,6 +126,14 @@ class StatementAttrs { void __set_bucketIdsTable(const std::string& val); + void __set_retainBucketIds(const bool val); + + void __set_metadataVersion(const int32_t val); + + void __set_snapshotTransactionId(const std::string& val); + + void __set_lockOwner(const std::string& val); + bool operator == (const StatementAttrs & rhs) const { if (__isset.resultSetType != rhs.__isset.resultSetType) @@ -196,6 +212,22 @@ class StatementAttrs { return false; else if (__isset.bucketIdsTable && !(bucketIdsTable == rhs.bucketIdsTable)) return false; + if (__isset.retainBucketIds != rhs.__isset.retainBucketIds) + return false; + else if (__isset.retainBucketIds && !(retainBucketIds == rhs.retainBucketIds)) + return false; + if (__isset.metadataVersion != rhs.__isset.metadataVersion) + return false; + else if (__isset.metadataVersion && !(metadataVersion == rhs.metadataVersion)) + return false; + if (__isset.snapshotTransactionId != rhs.__isset.snapshotTransactionId) + return false; + else if (__isset.snapshotTransactionId && !(snapshotTransactionId == rhs.snapshotTransactionId)) + return false; + if (__isset.lockOwner != rhs.__isset.lockOwner) + return false; + else if (__isset.lockOwner && !(lockOwner == rhs.lockOwner)) + return false; return true; } bool operator != (const StatementAttrs &rhs) const {