Skip to content

Commit d6c26f7

Browse files
aaronriekenbergaaronriekenberg
authored
Merge pull request #10 from aaronriekenberg/060722_flatbuffers_demo
Add offheap-flatbuffers demo
2 parents 1cec3f3 + 0733fb9 commit d6c26f7

File tree

11 files changed

+427
-1
lines changed

11 files changed

+427
-1
lines changed

‎examples/map/README.md‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,26 @@ java -jar build/libs/*shadowjar*.jar
2323
* Same as onheap but uses NativeMemoryMap with off-heap storage.
2424
* Uses the ConcurrentHashMap backend for NativeMemoryMap, which does not support eviction.
2525
* Total of 10GB of data in off-heap memory.
26+
* At end of test after full GC uses ~130MB of Java heap space.
2627

2728
## offheap-eviction example
2829

2930
* Same as offheap but uses the Caffeine backend with maximumSize of 10,000 entries.
3031
* Expect 10,000 map entries at end of run with 10,000 total evictions.
32+
* Total of 5GB of data in off-heap memory.
33+
* At end of test after full GC uses ~90MB of Java heap space.
3134

3235
## offheap-eviction-operationcounters example
3336

34-
* Same as offheap-eviction but enables operation counters and logs them at the end of the demo.
37+
* Same as offheap-eviction but enables operation counters and logs them at the end of the demo.
38+
* Total of 5GB of data in off-heap memory.
39+
* At end of test after full GC uses ~90MB of Java heap space.
40+
41+
## offheap-flatbuffers example
42+
43+
* Example using [Google FlatBuffers](https://google.github.io/flatbuffers/) for serialization.
44+
* Put 20,000 DemoCacheObject instances into a NativeMemoryMap. Each DemoCacheObject contains a list of 2,000
45+
DemoCacheObjectListEntry objects.
46+
* Total of ~11GB of data in off-heap memory.
47+
* DemoCacheObjectSerializer does conversions between DemoCacheObject and FlatBuffer objects.
48+
* At end of test after full GC uses ~130MB of Java heap space.

‎examples/map/offheap-flatbuffers/build.gradle.kts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
2+
3+
plugins {
4+
kotlin("jvm")
5+
id("com.github.johnrengelman.shadow") version "7.1.2"
6+
}
7+
8+
repositories {
9+
mavenCentral()
10+
}
11+
12+
dependencies {
13+
implementation(project(":examples:map:utils"))
14+
implementation("com.google.flatbuffers:flatbuffers-java:2.0.3")
15+
}
16+
17+
tasks {
18+
named<ShadowJar>("shadowJar") {
19+
archiveBaseName.set("offheap-flatbuffers-shadowjar")
20+
manifest {
21+
attributes(mapOf("Main-Class" to "com.target.nativememoryallocator.examples.map.offheap.flatbuffers.OffHeapFlatBuffersKt"))
22+
}
23+
}
24+
}
25+
26+
tasks {
27+
build {
28+
dependsOn(shadowJar)
29+
}
30+
}

‎examples/map/offheap-flatbuffers/generate_fbs_code.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
# Script to invoke the FlatBuffers compiler flatc to generate Kotlin FlatBuffer classes.
4+
# You must have flatc installed locally and in your PATH before running this script.
5+
6+
BASE_DIRECTORY=$(dirname $0)
7+
8+
rm -fr $BASE_DIRECTORY/src/main/kotlin/com/target/nativememoryallocator/examples/map/offheap/flatbuffers/generated
9+
10+
flatc --kotlin -o $BASE_DIRECTORY/src/main/kotlin $BASE_DIRECTORY/src/main/flatbuffers/DemoSchema.fbs

‎examples/map/offheap-flatbuffers/src/main/flatbuffers/DemoSchema.fbs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
namespace com.target.nativememoryallocator.examples.map.offheap.flatbuffers.generated;
2+
3+
table FlatBufferDemoCacheObjectListEntry {
4+
id: int;
5+
boolean_field: bool;
6+
string_field: string;
7+
}
8+
9+
table FlatBufferDemoCacheObject {
10+
id: int;
11+
entry_list: [FlatBufferDemoCacheObjectListEntry];
12+
}

‎examples/map/offheap-flatbuffers/src/main/kotlin/com/target/nativememoryallocator/examples/map/offheap/flatbuffers/OffHeapFlatBuffers.kt‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
package com.target.nativememoryallocator.examples.map.offheap.flatbuffers
2+
3+
import com.target.nativememoryallocator.allocator.NativeMemoryAllocatorBuilder
4+
import com.target.nativememoryallocator.examples.map.offheap.flatbuffers.model.DemoCacheObject
5+
import com.target.nativememoryallocator.examples.map.offheap.flatbuffers.model.DemoCacheObjectListEntry
6+
import com.target.nativememoryallocator.examples.map.offheap.flatbuffers.serializer.DemoCacheObjectSerializer
7+
import com.target.nativememoryallocator.examples.map.utils.buildRandomString
8+
import com.target.nativememoryallocator.map.NativeMemoryMapBackend
9+
import com.target.nativememoryallocator.map.NativeMemoryMapBuilder
10+
import kotlinx.coroutines.*
11+
import mu.KotlinLogging
12+
import java.util.concurrent.ThreadLocalRandom
13+
import kotlin.random.Random
14+
15+
private val logger = KotlinLogging.logger {}
16+
17+
18+
/**
19+
* Demo application that puts 20,000 [DemoCacheObject] instances into a [NativeMemoryMap].
20+
*
21+
* This demo uses the ConcurrentHashMap backend for [NativeMemoryMap].
22+
*
23+
* Each [DemoCacheObject] instance contains a list of 2,000 [DemoCacheObjectListEntry]s.
24+
*
25+
* This is a total of ~11 GB of data in off-heap memory.
26+
*/
27+
private class OffHeapFlatBuffers {
28+
29+
private val numMapEntries = 20_000
30+
31+
private val numEntriesPerObject = 2_000
32+
33+
private val randomIndex = Random.nextInt(0, numMapEntries)
34+
35+
private var randomPutValue: DemoCacheObject? = null
36+
37+
private val nativeMemoryAllocator = NativeMemoryAllocatorBuilder(
38+
pageSizeBytes = 4_096, // 4 KB
39+
nativeMemorySizeBytes = (20L * 1_024L * 1_024L * 1_024L), // 20 GB
40+
).build()
41+
42+
private val nativeMemoryMap = NativeMemoryMapBuilder<Int, DemoCacheObject>(
43+
valueSerializer = DemoCacheObjectSerializer(),
44+
nativeMemoryAllocator = nativeMemoryAllocator,
45+
backend = NativeMemoryMapBackend.CONCURRENT_HASH_MAP,
46+
).build()
47+
48+
private fun buildDemoCacheObject(i: Int) =
49+
DemoCacheObject(
50+
id = i,
51+
entryList = (0 until numEntriesPerObject).map { j ->
52+
DemoCacheObjectListEntry(
53+
id = j,
54+
booleanField = ThreadLocalRandom.current().nextBoolean(),
55+
stringField = buildRandomString(length = 250),
56+
)
57+
}
58+
)
59+
60+
private fun putValueIntoMap(i: Int) {
61+
if ((i % 100) == 0) {
62+
logger.info { "put i = $i" }
63+
}
64+
val demoCacheObject = buildDemoCacheObject(i = i)
65+
if (i == randomIndex) {
66+
logger.info { "put randomIndex = $randomIndex demoCacheObject.entryList.size = ${demoCacheObject.entryList.size}" }
67+
logger.info {
68+
"substring = ${demoCacheObject.entryList[0].stringField.substring(0, 20)}"
69+
}
70+
randomPutValue = demoCacheObject
71+
}
72+
nativeMemoryMap.put(
73+
key = i,
74+
value = demoCacheObject,
75+
)
76+
}
77+
78+
suspend fun run() {
79+
logger.info { "begin run randomIndex = $randomIndex" }
80+
81+
coroutineScope {
82+
(0 until numMapEntries).forEach { i ->
83+
launch {
84+
putValueIntoMap(i = i)
85+
}
86+
}
87+
}
88+
89+
logger.info { "nativeMemoryMap.size = ${nativeMemoryMap.size}" }
90+
logger.info { "nativeMemoryAllocator.nativeMemoryAllocatorMetadata = ${nativeMemoryAllocator.nativeMemoryAllocatorMetadata}" }
91+
92+
val randomIndexValue = nativeMemoryMap.get(key = randomIndex)
93+
randomIndexValue?.let {
94+
logger.info { "get randomIndex = $randomIndex" }
95+
logger.info { "randomIndexValue.entryList.size = ${it.entryList.size}" }
96+
logger.info { "substring = ${it.entryList[0].stringField.substring(0, 20)}" }
97+
logger.info { "randomIndexValue == randomPutValue = ${it == randomPutValue}" }
98+
}
99+
100+
while (true) {
101+
delay(1_000)
102+
}
103+
}
104+
}
105+
106+
suspend fun main() {
107+
withContext(Dispatchers.Default) {
108+
OffHeapFlatBuffers().run()
109+
}
110+
}

‎examples/map/offheap-flatbuffers/src/main/kotlin/com/target/nativememoryallocator/examples/map/offheap/flatbuffers/generated/FlatBufferDemoCacheObject.kt‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// automatically generated by the FlatBuffers compiler, do not modify
2+
3+
package com.target.nativememoryallocator.examples.map.offheap.flatbuffers.generated
4+
5+
import java.nio.*
6+
import kotlin.math.sign
7+
import com.google.flatbuffers.*
8+
9+
@Suppress("unused")
10+
class FlatBufferDemoCacheObject : Table() {
11+
12+
fun __init(_i: Int, _bb: ByteBuffer) {
13+
__reset(_i, _bb)
14+
}
15+
fun __assign(_i: Int, _bb: ByteBuffer) : FlatBufferDemoCacheObject {
16+
__init(_i, _bb)
17+
return this
18+
}
19+
val id : Int
20+
get() {
21+
val o = __offset(4)
22+
return if(o != 0) bb.getInt(o + bb_pos) else 0
23+
}
24+
fun entryList(j: Int) : com.target.nativememoryallocator.examples.map.offheap.flatbuffers.generated.FlatBufferDemoCacheObjectListEntry? = entryList(com.target.nativememoryallocator.examples.map.offheap.flatbuffers.generated.FlatBufferDemoCacheObjectListEntry(), j)
25+
fun entryList(obj: com.target.nativememoryallocator.examples.map.offheap.flatbuffers.generated.FlatBufferDemoCacheObjectListEntry, j: Int) : com.target.nativememoryallocator.examples.map.offheap.flatbuffers.generated.FlatBufferDemoCacheObjectListEntry? {
26+
val o = __offset(6)
27+
return if (o != 0) {
28+
obj.__assign(__indirect(__vector(o) + j * 4), bb)
29+
} else {
30+
null
31+
}
32+
}
33+
val entryListLength : Int
34+
get() {
35+
val o = __offset(6); return if (o != 0) __vector_len(o) else 0
36+
}
37+
companion object {
38+
fun validateVersion() = Constants.FLATBUFFERS_2_0_0()
39+
fun getRootAsFlatBufferDemoCacheObject(_bb: ByteBuffer): FlatBufferDemoCacheObject = getRootAsFlatBufferDemoCacheObject(_bb, FlatBufferDemoCacheObject())
40+
fun getRootAsFlatBufferDemoCacheObject(_bb: ByteBuffer, obj: FlatBufferDemoCacheObject): FlatBufferDemoCacheObject {
41+
_bb.order(ByteOrder.LITTLE_ENDIAN)
42+
return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb))
43+
}
44+
fun createFlatBufferDemoCacheObject(builder: FlatBufferBuilder, id: Int, entryListOffset: Int) : Int {
45+
builder.startTable(2)
46+
addEntryList(builder, entryListOffset)
47+
addId(builder, id)
48+
return endFlatBufferDemoCacheObject(builder)
49+
}
50+
fun startFlatBufferDemoCacheObject(builder: FlatBufferBuilder) = builder.startTable(2)
51+
fun addId(builder: FlatBufferBuilder, id: Int) = builder.addInt(0, id, 0)
52+
fun addEntryList(builder: FlatBufferBuilder, entryList: Int) = builder.addOffset(1, entryList, 0)
53+
fun createEntryListVector(builder: FlatBufferBuilder, data: IntArray) : Int {
54+
builder.startVector(4, data.size, 4)
55+
for (i in data.size - 1 downTo 0) {
56+
builder.addOffset(data[i])
57+
}
58+
return builder.endVector()
59+
}
60+
fun startEntryListVector(builder: FlatBufferBuilder, numElems: Int) = builder.startVector(4, numElems, 4)
61+
fun endFlatBufferDemoCacheObject(builder: FlatBufferBuilder) : Int {
62+
val o = builder.endTable()
63+
return o
64+
}
65+
}
66+
}

‎examples/map/offheap-flatbuffers/src/main/kotlin/com/target/nativememoryallocator/examples/map/offheap/flatbuffers/generated/FlatBufferDemoCacheObjectListEntry.kt‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// automatically generated by the FlatBuffers compiler, do not modify
2+
3+
package com.target.nativememoryallocator.examples.map.offheap.flatbuffers.generated
4+
5+
import java.nio.*
6+
import kotlin.math.sign
7+
import com.google.flatbuffers.*
8+
9+
@Suppress("unused")
10+
class FlatBufferDemoCacheObjectListEntry : Table() {
11+
12+
fun __init(_i: Int, _bb: ByteBuffer) {
13+
__reset(_i, _bb)
14+
}
15+
fun __assign(_i: Int, _bb: ByteBuffer) : FlatBufferDemoCacheObjectListEntry {
16+
__init(_i, _bb)
17+
return this
18+
}
19+
val id : Int
20+
get() {
21+
val o = __offset(4)
22+
return if(o != 0) bb.getInt(o + bb_pos) else 0
23+
}
24+
val booleanField : Boolean
25+
get() {
26+
val o = __offset(6)
27+
return if(o != 0) 0.toByte() != bb.get(o + bb_pos) else false
28+
}
29+
val stringField : String?
30+
get() {
31+
val o = __offset(8)
32+
return if (o != 0) __string(o + bb_pos) else null
33+
}
34+
val stringFieldAsByteBuffer : ByteBuffer get() = __vector_as_bytebuffer(8, 1)
35+
fun stringFieldInByteBuffer(_bb: ByteBuffer) : ByteBuffer = __vector_in_bytebuffer(_bb, 8, 1)
36+
companion object {
37+
fun validateVersion() = Constants.FLATBUFFERS_2_0_0()
38+
fun getRootAsFlatBufferDemoCacheObjectListEntry(_bb: ByteBuffer): FlatBufferDemoCacheObjectListEntry = getRootAsFlatBufferDemoCacheObjectListEntry(_bb, FlatBufferDemoCacheObjectListEntry())
39+
fun getRootAsFlatBufferDemoCacheObjectListEntry(_bb: ByteBuffer, obj: FlatBufferDemoCacheObjectListEntry): FlatBufferDemoCacheObjectListEntry {
40+
_bb.order(ByteOrder.LITTLE_ENDIAN)
41+
return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb))
42+
}
43+
fun createFlatBufferDemoCacheObjectListEntry(builder: FlatBufferBuilder, id: Int, booleanField: Boolean, stringFieldOffset: Int) : Int {
44+
builder.startTable(3)
45+
addStringField(builder, stringFieldOffset)
46+
addId(builder, id)
47+
addBooleanField(builder, booleanField)
48+
return endFlatBufferDemoCacheObjectListEntry(builder)
49+
}
50+
fun startFlatBufferDemoCacheObjectListEntry(builder: FlatBufferBuilder) = builder.startTable(3)
51+
fun addId(builder: FlatBufferBuilder, id: Int) = builder.addInt(0, id, 0)
52+
fun addBooleanField(builder: FlatBufferBuilder, booleanField: Boolean) = builder.addBoolean(1, booleanField, false)
53+
fun addStringField(builder: FlatBufferBuilder, stringField: Int) = builder.addOffset(2, stringField, 0)
54+
fun endFlatBufferDemoCacheObjectListEntry(builder: FlatBufferBuilder) : Int {
55+
val o = builder.endTable()
56+
return o
57+
}
58+
}
59+
}

‎examples/map/offheap-flatbuffers/src/main/kotlin/com/target/nativememoryallocator/examples/map/offheap/flatbuffers/mapper/FlatBuffersMapper.kt‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package com.target.nativememoryallocator.examples.map.offheap.flatbuffers.mapper
2+
3+
import com.google.flatbuffers.FlatBufferBuilder
4+
import com.target.nativememoryallocator.examples.map.offheap.flatbuffers.generated.FlatBufferDemoCacheObject
5+
import com.target.nativememoryallocator.examples.map.offheap.flatbuffers.generated.FlatBufferDemoCacheObjectListEntry
6+
import com.target.nativememoryallocator.examples.map.offheap.flatbuffers.model.DemoCacheObject
7+
import com.target.nativememoryallocator.examples.map.offheap.flatbuffers.model.DemoCacheObjectListEntry
8+
import java.nio.ByteBuffer
9+
10+
/**
11+
* Mapper between [DemoCacheObject] and [FlatBufferDemoCacheObject].
12+
*/
13+
object FlatBuffersMapper {
14+
15+
fun demoCacheObjectToFlatBuffer(
16+
demoCacheObject: DemoCacheObject,
17+
): FlatBufferDemoCacheObject {
18+
val fbb = FlatBufferBuilder()
19+
fbb.forceDefaults(false)
20+
21+
val entryListOffset = FlatBufferDemoCacheObject.createEntryListVector(
22+
builder = fbb,
23+
data = demoCacheObject.entryList.map { entry ->
24+
FlatBufferDemoCacheObjectListEntry.createFlatBufferDemoCacheObjectListEntry(
25+
builder = fbb,
26+
id = entry.id,
27+
booleanField = entry.booleanField,
28+
stringFieldOffset = fbb.createString(entry.stringField),
29+
)
30+
}.toIntArray(),
31+
)
32+
33+
val flatBufferDemoCacheObject = FlatBufferDemoCacheObject.createFlatBufferDemoCacheObject(
34+
builder = fbb,
35+
id = demoCacheObject.id,
36+
entryListOffset = entryListOffset,
37+
)
38+
39+
fbb.finish(flatBufferDemoCacheObject)
40+
41+
return FlatBufferDemoCacheObject.getRootAsFlatBufferDemoCacheObject(
42+
ByteBuffer.wrap(
43+
fbb.sizedByteArray(),
44+
)
45+
)
46+
}
47+
48+
fun flatBufferToDemoCacheObject(
49+
flatBufferDemoCacheObject: FlatBufferDemoCacheObject,
50+
): DemoCacheObject {
51+
52+
val entryList = (0 until flatBufferDemoCacheObject.entryListLength).mapNotNull { i ->
53+
val flatBufferEntry = flatBufferDemoCacheObject.entryList(i)
54+
if (flatBufferEntry == null) {
55+
null
56+
} else {
57+
DemoCacheObjectListEntry(
58+
id = flatBufferEntry.id,
59+
booleanField = flatBufferEntry.booleanField,
60+
stringField = flatBufferEntry.stringField.orEmpty(),
61+
)
62+
}
63+
}
64+
65+
return DemoCacheObject(
66+
id = flatBufferDemoCacheObject.id,
67+
entryList = entryList,
68+
)
69+
}
70+
}

0 commit comments

Comments
 (0)