diff --git a/.gitignore b/.gitignore
index c9eafe344..109e451c2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,6 @@ bld*/
# Vim
*.swp
+
+# Python
+python/**/__pycache__/
diff --git a/Doxyfile b/Doxyfile
index 70dfde65f..9550ce554 100644
--- a/Doxyfile
+++ b/Doxyfile
@@ -112,7 +112,7 @@ WARN_LOGFILE =
#---------------------------------------------------------------------------
# Configuration options related to the input files
#---------------------------------------------------------------------------
-INPUT = include lib
+INPUT = include lib python
INPUT_ENCODING = UTF-8
FILE_PATTERNS = *.c \
*.cc \
diff --git a/python/benchmark.py b/python/benchmark.py
new file mode 100644
index 000000000..4ad8faf83
--- /dev/null
+++ b/python/benchmark.py
@@ -0,0 +1,31 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from julea.benchmark import (
+ distributed_object_bench, item_bench, kv_bench, object_bench)
+from julea import juleadir
+import subprocess
+
+setupfile = juleadir.joinpath('scripts', 'setup.sh')
+
+subprocess.run([setupfile, "start"])
+
+kv_bench()
+distributed_object_bench()
+object_bench()
+item_bench()
+
+subprocess.run([setupfile, "stop"])
diff --git a/python/hello_world.py b/python/hello_world.py
new file mode 100644
index 000000000..92ba4a735
--- /dev/null
+++ b/python/hello_world.py
@@ -0,0 +1,36 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from julea.object import JObject, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT
+
+
+def main():
+ object = JObject("hello", "world")
+ hello_world = "Hello World!"
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ object.create(batch)
+ bw = object.write(hello_world.encode('utf-8'), 0, batch)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ out = object.read(bw.value, 0, batch)
+
+ print("Object contains: {buffer} ({length} bytes)".format(
+ buffer=out[0].raw.decode('utf-8'), length=out[1].value))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/julea/__init__.py b/python/julea/__init__.py
new file mode 100644
index 000000000..b6d08320a
--- /dev/null
+++ b/python/julea/__init__.py
@@ -0,0 +1,26 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+import ctypes
+import pathlib
+
+juleadir = pathlib.Path(__file__).parent.parent.parent
+libdir = juleadir.joinpath('bld')
+
+JULEA = ctypes.CDLL(libdir.joinpath('libjulea.so'))
+JULEA_ITEM = ctypes.CDLL(libdir.joinpath('libjulea-item.so'))
+JULEA_KV = ctypes.CDLL(libdir.joinpath('libjulea-kv.so'))
+JULEA_OBJECT = ctypes.CDLL(libdir.joinpath('libjulea-object.so'))
diff --git a/python/julea/benchmark/__init__.py b/python/julea/benchmark/__init__.py
new file mode 100644
index 000000000..606804bd0
--- /dev/null
+++ b/python/julea/benchmark/__init__.py
@@ -0,0 +1,20 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from .bench_distributed_object import distributed_object_bench
+from .bench_item import item_bench
+from .bench_kv import kv_bench
+from .bench_object import object_bench
diff --git a/python/julea/benchmark/bench_distributed_object.py b/python/julea/benchmark/bench_distributed_object.py
new file mode 100644
index 000000000..ed26c45c9
--- /dev/null
+++ b/python/julea/benchmark/bench_distributed_object.py
@@ -0,0 +1,199 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+# Copyright (C) 2017-2020 Michael Kuhn
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from ..object import (JDistributedObject, JBatch, JDistribution,
+ J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN)
+from ..lib.common import format_benchmark
+import time
+
+
+def bench_create(use_batch):
+ if use_batch:
+ n = 100000
+ else:
+ n = 1000
+
+ distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch:
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ o = JDistributedObject("python", name, distribution)
+ o.create(batch)
+ o.delete(delete_batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ if use_batch:
+ return "/object/distributed-object/create-batch", end - start, n
+ else:
+ return "/object/distributed-object/create", end - start, n
+
+
+def bench_delete(use_batch):
+ n = 10000
+ distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ o = JDistributedObject("python", name, distribution)
+ o.create(batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ o = JDistributedObject("python", name, distribution)
+ o.delete(batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ if use_batch:
+ return "/object/distributed-object/delete-batch", end - start, n
+ else:
+ return "/object/distributed-object/delete", end - start, n
+
+
+def bench_status(use_batch):
+ n = 1000
+ distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+ dummy = bytes(1)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o = JDistributedObject("python", "benchmark", distribution)
+ o.delete(batch)
+ o.create(batch)
+ o.write(dummy, 0, batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for _ in range(0, n):
+ o.status(batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o.delete(batch)
+
+ if use_batch:
+ return "/object/distributed-object/status-batch", end - start, n
+ else:
+ return "/object/distributed-object/status", end - start, n
+
+
+def bench_read(use_batch, block_size):
+ n = 25000
+ distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+ dummy = bytes(block_size)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o = JDistributedObject("python", "benchmark", distribution)
+ o.delete(batch)
+ o.create(batch)
+
+ for i in range(0, n):
+ o.write(dummy, i * block_size, batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ o.read(block_size, i * block_size, batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o.delete(batch)
+
+ if use_batch:
+ name = "/object/distributed-object/read-batch"
+ return name, end - start, n, n * block_size
+ else:
+ name = "/object/distributed-object/read"
+ return name, end - start, n, n * block_size
+
+
+def bench_write(use_batch, block_size):
+ n = 25000
+ distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+ dummy = bytes(block_size)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o = JDistributedObject("python", "benchmark", distribution)
+ o.delete(batch)
+ o.create(batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ o.write(dummy, i * block_size, batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o.delete(batch)
+
+ if use_batch:
+ name = "/object/distributed-object/write-batch"
+ return name, end - start, n, n * block_size
+ else:
+ name = "/object/distributed-object/write"
+ return name, end - start, n, n * block_size
+
+
+def distributed_object_bench():
+ create = bench_create(False)
+ print(format_benchmark(create))
+ create_batch = bench_create(True)
+ print(format_benchmark(create_batch))
+
+ delete = bench_delete(False)
+ print(format_benchmark(delete))
+ delete_batch = bench_delete(True)
+ print(format_benchmark(delete_batch))
+
+ status = bench_status(False)
+ print(format_benchmark(status))
+ status_batch = bench_status(True)
+ print(format_benchmark(status_batch))
+
+ read = bench_read(False, 4 * 1024)
+ print(format_benchmark(read))
+ read_batch = bench_read(True, 4 * 1024)
+ print(format_benchmark(read_batch))
+
+ write = bench_write(False, 4 * 1024)
+ print(format_benchmark(write))
+ write_batch = bench_write(True, 4 * 1024)
+ print(format_benchmark(write_batch))
diff --git a/python/julea/benchmark/bench_item.py b/python/julea/benchmark/bench_item.py
new file mode 100644
index 000000000..a36eb4dfb
--- /dev/null
+++ b/python/julea/benchmark/bench_item.py
@@ -0,0 +1,235 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+# Copyright (C) 2010-2020 Michael Kuhn
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from ..item import (JItem, JCollection, JBatch, JDistribution,
+ J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN)
+from ..lib.common import format_benchmark
+import time
+
+
+def bench_create(use_batch):
+ if use_batch:
+ n = 100000
+ else:
+ n = 1000
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ collection = JCollection("benchmark", batch, False)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch:
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ item = JItem(collection, name, batch, False, None)
+ item.delete(delete_batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ collection.delete(batch)
+
+ if use_batch:
+ return "/item/item/create-batch", end - start, n
+ else:
+ return "/item/item/create", end - start, n
+
+
+def bench_delete(use_batch):
+ n = 10000
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ collection = JCollection("benchmark", batch, False)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ JItem(collection, name, batch, False, None)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as get_batch:
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ item = JItem(collection, name, get_batch, True)
+ get_batch.execute()
+
+ item.delete(batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ collection.delete(batch)
+
+ if use_batch:
+ return "/item/item/delete-batch", end - start, n
+ else:
+ return "/item/item/delete", end - start, n
+
+
+def bench_delete_batch_without_get():
+ n = 10000
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ collection = JCollection("benchmark", batch, False)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ item = JItem(collection, name, batch, False, None)
+
+ item.delete(batch)
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ collection.delete(batch)
+
+ return "/item/item/delete-batch-without-get", end - start, n
+
+
+def bench_status(use_batch):
+ n = 1000
+ dummy = bytes(1)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ collection = JCollection("benchmark", batch, False)
+ batch.execute()
+ item = JItem(collection, "benchmark", batch, False, None)
+ item.write(dummy, 0, batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for _ in range(0, n):
+ item.status(batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ item.delete(batch)
+ collection.delete(batch)
+
+ if use_batch:
+ return "/item/item/status-batch", end - start, n
+ else:
+ return "/item/item/status", end - start, n
+
+
+def bench_read(use_batch, block_size):
+ n = 25000
+ dummy = bytes(block_size)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ collection = JCollection("benchmark", batch, False)
+ item = JItem(collection, "benchmark", batch, False, None)
+
+ for i in range(0, n):
+ item.write(dummy, i * block_size, batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ # Unused variable required, otherwise glibc crashes somehow
+ out = item.read(block_size, # pylint: disable=unused-variable
+ i * block_size, batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ item.delete(batch)
+ collection.delete(batch)
+
+ if use_batch:
+ name = "/item/item/read-batch"
+ return name, end - start, n, n * block_size
+ else:
+ name = "/item/item/read"
+ return name, end - start, n, n * block_size
+
+
+def bench_write(use_batch, block_size):
+ n = 25000
+ dummy = bytes(block_size)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ collection = JCollection("benchmark", batch, False)
+ batch.execute()
+ item = JItem(collection, "benchmark", batch, False, None)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ item.write(dummy, i * block_size, batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ item.delete(batch)
+ collection.delete(batch)
+
+ if use_batch:
+ name = "/item/item/write-batch"
+ return name, end - start, n, n * block_size
+ else:
+ name = "/item/item/write"
+ return name, end - start, n, n * block_size
+
+
+def item_bench():
+ create = bench_create(False)
+ print(format_benchmark(create))
+ create_batch = bench_create(True)
+ print(format_benchmark(create_batch))
+
+ delete = bench_delete(False)
+ print(format_benchmark(delete))
+ delete_batch = bench_delete(True)
+ print(format_benchmark(delete_batch))
+ delete_batch_without_get = bench_delete_batch_without_get()
+ print(format_benchmark(delete_batch_without_get))
+
+ status = bench_status(False)
+ print(format_benchmark(status))
+ status_batch = bench_status(True)
+ print(format_benchmark(status_batch))
+
+ read = bench_read(False, 4 * 1024)
+ print(format_benchmark(read))
+ read_batch = bench_read(True, 4 * 1024)
+ print(format_benchmark(read_batch))
+
+ write = bench_write(False, 4 * 1024)
+ print(format_benchmark(write))
+ write_batch = bench_write(True, 4 * 1024)
+ print(format_benchmark(write_batch))
diff --git a/python/julea/benchmark/bench_kv.py b/python/julea/benchmark/bench_kv.py
new file mode 100644
index 000000000..051438c49
--- /dev/null
+++ b/python/julea/benchmark/bench_kv.py
@@ -0,0 +1,116 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+# Copyright (C) 2017-2020 Michael Kuhn
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from ..kv import JKV, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT
+from ..lib.common import format_benchmark
+import time
+
+
+def bench_put(use_batch):
+ n = 200000
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch:
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ k = JKV("python", name)
+ k.put("empty".encode('utf-8'), batch)
+ k.delete(delete_batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ if use_batch:
+ return "/kv/kv/put-batch", end - start, n
+ else:
+ return "/kv/kv/put", end - start, n
+
+
+def bench_get(use_batch):
+ n = 200000
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch:
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ k = JKV("python", name)
+ k.put(name.encode('utf-8'), batch)
+ k.delete(delete_batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ k = JKV("python", name)
+ k.get(batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ if use_batch:
+ return "/kv/kv/get-batch", end - start, n
+ else:
+ return "/kv/kv/get", end - start, n
+
+
+def bench_delete(use_batch):
+ n = 200000
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ k = JKV("python", name)
+ k.put("empty".encode('utf-8'), batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ k = JKV("python", name)
+ k.delete(batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ if use_batch:
+ return "/kv/kv/delete-batch", end - start, n
+ else:
+ return "/kv/kv/delete", end - start, n
+
+
+def kv_bench():
+ create = bench_put(False)
+ print(format_benchmark(create))
+ create_batch = bench_put(True)
+ print(format_benchmark(create_batch))
+
+ read = bench_get(False)
+ print(format_benchmark(read))
+ read_batch = bench_get(True)
+ print(format_benchmark(read_batch))
+
+ delete = bench_delete(False)
+ print(format_benchmark(delete))
+ delete_batch = bench_delete(True)
+ print(format_benchmark(delete_batch))
diff --git a/python/julea/benchmark/bench_object.py b/python/julea/benchmark/bench_object.py
new file mode 100644
index 000000000..a3ef09423
--- /dev/null
+++ b/python/julea/benchmark/bench_object.py
@@ -0,0 +1,185 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+# Copyright (C) 2017-2020 Michael Kuhn
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from ..object import JObject, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT
+from ..lib.common import format_benchmark
+import time
+
+
+def bench_create(use_batch):
+ n = 100000
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch:
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ o = JObject("python", name)
+ o.create(batch)
+ o.delete(delete_batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ if use_batch:
+ return "/object/object/create-batch", end - start, n
+ else:
+ return "/object/object/create", end - start, n
+
+
+def bench_delete(use_batch):
+ n = 100000
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ o = JObject("python", name)
+ o.create(batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ name = "benchmark-" + str(i)
+ o = JObject("python", name)
+ o.delete(batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ if use_batch:
+ return "/object/object/delete-batch", end - start, n
+ else:
+ return "/object/object/delete", end - start, n
+
+
+def bench_status(use_batch):
+ n = 200000
+ dummy = bytes(1)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o = JObject("python", "benchmark")
+ o.delete(batch)
+ o.create(batch)
+ o.write(dummy, 0, batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for _ in range(0, n):
+ o.status(batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o.delete(batch)
+
+ if use_batch:
+ return "/object/object/status-batch", end - start, n
+ else:
+ return "/object/object/status", end - start, n
+
+
+def bench_read(use_batch, block_size):
+ n = 200000
+ dummy = bytes(block_size)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o = JObject("python", "benchmark")
+ o.delete(batch)
+ o.create(batch)
+
+ for i in range(0, n):
+ o.write(dummy, i * block_size, batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ o.read(block_size, i * block_size, batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o.delete(batch)
+
+ if use_batch:
+ return "/object/object/read-batch", end - start, n, n * block_size
+ else:
+ return "/object/object/read", end - start, n, n * block_size
+
+
+def bench_write(use_batch, block_size):
+ n = 200000
+ dummy = bytes(block_size)
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o = JObject("python", "benchmark")
+ o.delete(batch)
+ o.create(batch)
+
+ start = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ for i in range(0, n):
+ o.write(dummy, i * block_size, batch)
+ if not use_batch:
+ batch.execute()
+
+ end = time.time()
+
+ with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+ o.delete(batch)
+
+ if use_batch:
+ return "/object/object/write-batch", end - start, n, n * block_size
+ else:
+ return "/object/object/write", end - start, n, n * block_size
+
+
+def object_bench():
+ create = bench_create(False)
+ print(format_benchmark(create))
+ create_batch = bench_create(True)
+ print(format_benchmark(create_batch))
+
+ delete = bench_delete(False)
+ print(format_benchmark(delete))
+ delete_batch = bench_delete(True)
+ print(format_benchmark(delete_batch))
+
+ status = bench_status(False)
+ print(format_benchmark(status))
+ status_batch = bench_status(True)
+ print(format_benchmark(status_batch))
+
+ read = bench_read(False, 4 * 1024)
+ print(format_benchmark(read))
+ read_batch = bench_read(True, 4 * 1024)
+ print(format_benchmark(read_batch))
+
+ write = bench_write(False, 4 * 1024)
+ print(format_benchmark(write))
+ write_batch = bench_write(True, 4 * 1024)
+ print(format_benchmark(write_batch))
diff --git a/python/julea/item/__init__.py b/python/julea/item/__init__.py
new file mode 100644
index 000000000..7286d3dfd
--- /dev/null
+++ b/python/julea/item/__init__.py
@@ -0,0 +1,20 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from .collection import JCollection
+from .item import JItem
+from ..lib.batch import *
+from ..lib.distribution import *
diff --git a/python/julea/item/collection.py b/python/julea/item/collection.py
new file mode 100644
index 000000000..4c024944d
--- /dev/null
+++ b/python/julea/item/collection.py
@@ -0,0 +1,61 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+import ctypes
+from .. import JULEA_ITEM
+from ..lib.common import my_void_p
+
+JULEA_ITEM.j_collection_create.restype = my_void_p
+
+
+class JCollection:
+ def __init__(self, name, batch, get):
+ """Initializes and creates a collection
+
+ Args:
+ name: The name of the collection
+ batch: A batch
+ get: A boolean, False when the collection should be created,
+ True if get should be called
+ """
+ enc_name = name.encode('utf-8')
+ if get:
+ self.collection = my_void_p
+ JULEA_ITEM.j_collection_get(
+ ctypes.byref(self.collection), enc_name, batch.get_pointer())
+ else:
+ self.collection = JULEA_ITEM.j_collection_create(
+ enc_name, batch.get_pointer())
+
+ def __del__(self):
+ """Dereferences a collection when it is deleted in Python"""
+ JULEA_ITEM.j_collection_unref(self.collection)
+
+ def delete(self, batch):
+ """Deletes a collection
+
+ Args:
+ batch: A batch
+ """
+ JULEA_ITEM.j_collection_delete(self.collection, batch.get_pointer())
+
+ def get_pointer(self):
+ """Gives a pointer to a collection
+
+ Returns:
+ collection: A pointer to the collection
+ """
+ return self.collection
diff --git a/python/julea/item/item.py b/python/julea/item/item.py
new file mode 100644
index 000000000..d72b661e2
--- /dev/null
+++ b/python/julea/item/item.py
@@ -0,0 +1,134 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+import ctypes
+from .. import JULEA_ITEM
+from ..lib.common import my_void_p
+
+JULEA_ITEM.j_item_create.restype = my_void_p
+JULEA_ITEM.j_item_get_size.restype = ctypes.c_ulonglong
+JULEA_ITEM.j_item_get_modification_time.restype = ctypes.c_longlong
+
+JULEA_ITEM.j_item_get.argtypes = [my_void_p, ctypes.POINTER(my_void_p),
+ ctypes.c_char_p, my_void_p]
+
+
+class JItem:
+ def __init__(self, collection, name, batch, get, distribution=None):
+ """Initializes and creates an item
+
+ Args:
+ collection: The collection of the item
+ name: The name of the item
+ batch: A batch
+ get: A boolean, False when the item should be created,
+ True if get should be called
+ distribution: The distribution of the item(only for create)
+ """
+ enc_name = name.encode('utf-8')
+ if get:
+ self.item = my_void_p(0)
+ JULEA_ITEM.j_item_get(collection.get_pointer(),
+ ctypes.byref(self.item), enc_name,
+ batch.get_pointer())
+ else:
+ if distribution is None:
+ self.item = JULEA_ITEM.j_item_create(
+ collection.get_pointer(), enc_name,
+ distribution, batch.get_pointer())
+ else:
+ self.item = JULEA_ITEM.j_item_create(
+ collection.get_pointer(), enc_name,
+ distribution.get_pointer(), batch.get_pointer())
+
+ def __del__(self):
+ """Dereferences an item when it is deleted in Python"""
+ JULEA_ITEM.j_item_unref(self.item)
+
+ def delete(self, batch):
+ """Deletes an item from the server
+
+ Args:
+ batch: A batch
+ """
+ JULEA_ITEM.j_item_delete(self.item, batch.get_pointer())
+
+ def status(self, batch):
+ """Gets the status of an item
+
+ Args:
+ batch: A batch
+ """
+ JULEA_ITEM.j_item_get_status(self.item, batch.get_pointer())
+
+ def get_size(self):
+ """Gets the size of an item
+ Requires status to be called earlier
+
+ Returns:
+ size: The size of the item
+ """
+ size = JULEA_ITEM.j_item_get_size(self.item)
+ return size
+
+ def get_modification_time(self):
+ """Gets the modification time of an item
+ Requires status to be called earlier
+
+ Returns:
+ modtime: The size of the item
+ """
+ modtime = JULEA_ITEM.j_item_get_modification_time(self.item)
+ return modtime
+
+ def write(self, value, offset, batch):
+ """Writes an item
+
+ Args:
+ value: bytes holding the data to write
+ offset: The offset where the data is written
+ batch: A batch
+ Returns:
+ bytes_written: A c_ulonglong with the bytes written
+ (Access with bytes_written.value)
+ """
+ length = ctypes.c_ulonglong(len(value))
+ value2 = ctypes.c_char_p(value)
+ bytes_written = ctypes.c_ulonglong(0)
+ JULEA_ITEM.j_item_write(
+ self.item, value2, length, offset,
+ ctypes.byref(bytes_written), batch.get_pointer())
+ return bytes_written
+
+ def read(self, length, offset, batch):
+ """Reads an item
+
+ Args:
+ length: The length of the data to be read
+ offset: The offset where the data is to be read
+ batch: A batch
+ Returns:
+ value: The ctype with the data that was read
+ (Access data with value.value)
+ bytes_read: A c_ulonglong with the bytes read
+ (Access with bytes_read.value)
+ """
+ value = ctypes.c_char_p(bytes(length))
+ bytes_read = ctypes.c_ulonglong(0)
+ JULEA_ITEM.j_item_read(
+ self.item, value, length, offset,
+ ctypes.byref(bytes_read), batch.get_pointer())
+ return value, bytes_read
diff --git a/python/julea/kv/__init__.py b/python/julea/kv/__init__.py
new file mode 100644
index 000000000..6b266fe1a
--- /dev/null
+++ b/python/julea/kv/__init__.py
@@ -0,0 +1,18 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from .kv import JKV
+from ..lib.batch import *
diff --git a/python/julea/kv/kv.py b/python/julea/kv/kv.py
new file mode 100644
index 000000000..98e311bc4
--- /dev/null
+++ b/python/julea/kv/kv.py
@@ -0,0 +1,77 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+import ctypes
+from .. import JULEA_KV
+from ..lib.common import my_void_p
+
+JULEA_KV.j_kv_new.restype = my_void_p
+
+
+class JKV:
+
+ def __init__(self, namespace, key):
+ """Initializes the Key-Value pair and
+ gets the new KV pair from the C Interface
+
+ Args:
+ namespace: The namespace of the KV pair
+ key: The name of the KV pair
+ """
+ enc_namespace = namespace.encode('utf-8')
+ enc_key = key.encode('utf-8')
+ self.kv = JULEA_KV.j_kv_new(enc_namespace, enc_key)
+
+ def __del__(self):
+ """Dereferences a KV pair when it is deleted in Python"""
+ JULEA_KV.j_kv_unref(self.kv)
+
+ def delete(self, batch):
+ """Deletes a Key-Value pair from the server
+
+ Args:
+ batch: A batch
+ """
+ JULEA_KV.j_kv_delete(self.kv, batch.get_pointer())
+
+ def put(self, value, batch):
+ """Writes a Key-Value pair
+
+ Args:
+ value: bytes of data to be written in the KV pair
+ batch: A batch
+ """
+ length = ctypes.c_ulong(len(value))
+ value2 = ctypes.c_char_p(value)
+ JULEA_KV.j_kv_put(self.kv, value2,
+ length, None, batch.get_pointer())
+
+ def get(self, batch):
+ """Reads a Key-Value pair
+
+ Args:
+ batch: A batch
+ Returns:
+ value: The ctype with the data that was read
+ (Access data with value.value[:length.value])
+ length: The length of the data
+ (Access with length.value)
+ """
+ value = ctypes.c_char_p()
+ length = ctypes.c_ulong(0)
+ JULEA_KV.j_kv_get(self.kv, ctypes.byref(value),
+ ctypes.byref(length), batch.get_pointer())
+ return value, length
diff --git a/python/julea/lib/__init__.py b/python/julea/lib/__init__.py
new file mode 100644
index 000000000..85c7d6777
--- /dev/null
+++ b/python/julea/lib/__init__.py
@@ -0,0 +1,15 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
diff --git a/python/julea/lib/batch.py b/python/julea/lib/batch.py
new file mode 100644
index 000000000..eab95efe9
--- /dev/null
+++ b/python/julea/lib/batch.py
@@ -0,0 +1,62 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+import ctypes
+from .. import JULEA
+from .common import my_void_p
+
+(J_SEMANTICS_TEMPLATE_DEFAULT, J_SEMANTICS_TEMPLATE_POSIX,
+ J_SEMANTICS_TEMPLATE_TEMPORARY_LOCAL) = (0, 1, 2)
+
+JULEA.j_batch_new_for_template.restype = my_void_p
+
+
+class JBatch:
+ def __init__(self, template):
+ """Initializes a batch
+
+ Args:
+ template: The template defined in JSemanticsTemplate (C Interface)
+ """
+ self.batch = JULEA.j_batch_new_for_template(template)
+
+ def __del__(self):
+ """Dereferences a batch when it is deleted in Python"""
+ JULEA.j_batch_unref(self.batch)
+
+ def __enter__(self):
+ """Returns itself when a with-statement is entered
+
+ Returns:
+ self: The batch itself
+ """
+ return self
+
+ def __exit__(self, type, value, traceback):
+ """Executes the Batch when the with-statement is left"""
+ self.execute()
+
+ def execute(self):
+ """Executes the Batch"""
+ ret = JULEA.j_batch_execute(self.batch)
+ if ret == 1:
+ return True
+ else:
+ return False
+
+ def get_pointer(self):
+ """Returns the C Pointer to the batch"""
+ return self.batch
diff --git a/python/julea/lib/common.py b/python/julea/lib/common.py
new file mode 100644
index 000000000..13512937a
--- /dev/null
+++ b/python/julea/lib/common.py
@@ -0,0 +1,54 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+import ctypes
+
+
+# Workaround so Python doesn't convert the pointer to a Python datatype
+class my_void_p(ctypes.c_void_p):
+ pass
+
+
+def format_benchmark(input):
+ name = input[0] + ':'
+ ops = round(input[2] / input[1])
+ if len(input) == 3:
+ output = "{name: <{width}}{time} seconds ({ops}/s)"
+ return output.format(name=name, width=51, time=round(input[1], 3),
+ ops=ops)
+ elif len(input) == 4:
+ rate = input[3] / input[1]
+ unit = "B"
+ if rate >= 1024:
+ rate = rate / 1024
+ unit = "KB"
+ if rate >= 1024:
+ rate = rate / 1024
+ unit = "MB"
+ if rate >= 1024:
+ rate = rate / 1024
+ unit = "GB"
+ if rate >= 1024:
+ rate = rate / 1024
+ unit = "TB"
+ rate = round(rate, 1)
+ output = "{name: <{width}}{time} seconds ({ops}/s) ({rate} {unit}/s)"
+ return output.format(name=name, width=51, time=round(input[1], 3),
+ ops=ops, rate=rate, unit=unit)
+
+
+def print_test(name):
+ print(" {name: <{width}}".format(name=name, width=69), end='', flush=True)
diff --git a/python/julea/lib/distribution.py b/python/julea/lib/distribution.py
new file mode 100644
index 000000000..ca205732d
--- /dev/null
+++ b/python/julea/lib/distribution.py
@@ -0,0 +1,62 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+import ctypes
+from .. import JULEA
+from .common import my_void_p
+
+(J_DISTRIBUTION_ROUND_ROBIN, J_DISTRIBUTION_SINGLE_SERVER,
+ J_DISTRIBUTION_WEIGHTED) = (0, 1, 2)
+
+JULEA.j_distribution_new.restype = my_void_p
+JULEA.j_distribution_new_from_bson.restype = my_void_p
+JULEA.j_distribution_serialize.restype = my_void_p
+
+
+class JDistribution:
+
+ def __init__(self, input):
+ """Creates a new distribution
+
+ Args:
+ input: The type of the distribution or a pointer to a
+ serialized distribution
+ """
+ if isinstance(input, int):
+ self.distribution = JULEA.j_distribution_new(input)
+ else:
+ self.distribution = JULEA.j_distribution_new_from_bson(input)
+
+ def __del__(self):
+ """Dereferences a distribution when it is deleted in Python"""
+ JULEA.j_distribution_unref(self.distribution)
+
+ def serialize(self):
+ """Serializes the distribution into a BSON.
+
+ Returns:
+ b: A pointer to the BSON object in C.
+ """
+ b = JULEA.j_distribution_serialize(self.distribution)
+ return b
+
+ def get_pointer(self):
+ """Gets the pointer to the distribution
+
+ Returns:
+ distribution: A pointer to the distribution
+ """
+ return self.distribution
diff --git a/python/julea/object/__init__.py b/python/julea/object/__init__.py
new file mode 100644
index 000000000..63e7b6da1
--- /dev/null
+++ b/python/julea/object/__init__.py
@@ -0,0 +1,20 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from .distributed_object import JDistributedObject
+from .object import JObject
+from ..lib.batch import *
+from ..lib.distribution import *
diff --git a/python/julea/object/distributed_object.py b/python/julea/object/distributed_object.py
new file mode 100644
index 000000000..381f97f71
--- /dev/null
+++ b/python/julea/object/distributed_object.py
@@ -0,0 +1,115 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+import ctypes
+from .. import JULEA_OBJECT
+from ..lib.common import my_void_p
+
+JULEA_OBJECT.j_distributed_object_new.restype = my_void_p
+
+
+class JDistributedObject:
+
+ def __init__(self, namespace, name, distribution):
+ """Initializes the objects and gets the new object from the C Interface
+
+ Args:
+ namespace: The namespace of the object
+ name: The name of the object
+ """
+ enc_namespace = namespace.encode('utf-8')
+ enc_name = name.encode('utf-8')
+ dist = distribution.get_pointer()
+ self.object = JULEA_OBJECT.j_distributed_object_new(
+ enc_namespace, enc_name, dist)
+
+ def __del__(self):
+ """Dereferences an object when it is deleted in Python"""
+ JULEA_OBJECT.j_distributed_object_unref(self.object)
+
+ def create(self, batch):
+ """Creates an object
+
+ Args:
+ batch: A batch
+ """
+ JULEA_OBJECT.j_distributed_object_create(
+ self.object, batch.get_pointer())
+
+ def delete(self, batch):
+ """Deletes an object from the server
+
+ Args:
+ batch: A batch
+ """
+ JULEA_OBJECT.j_distributed_object_delete(
+ self.object, batch.get_pointer())
+
+ def status(self, batch):
+ """Gets the status of an object
+
+ Args:
+ batch: A batch
+ Returns:
+ modification_time: The modification time of the object
+ (Access data with modification_time.value)
+ size: The size of the object (Access data with size.value)
+ """
+ modification_time = ctypes.c_longlong(0)
+ size = ctypes.c_ulonglong(0)
+ JULEA_OBJECT.j_distributed_object_status(
+ self.object, ctypes.byref(modification_time),
+ ctypes.byref(size), batch.get_pointer())
+ return modification_time, size
+
+ def write(self, value, offset, batch):
+ """Writes an object
+
+ Args:
+ value: bytes holding the data to write
+ offset: The offset where the data is written
+ batch: A batch
+ Returns:
+ bytes_written: A c_ulonglong with the bytes written
+ (Access with bytes_written.value)
+ """
+ length = ctypes.c_ulonglong(len(value))
+ value2 = ctypes.c_char_p(value)
+ bytes_written = ctypes.c_ulonglong(0)
+ JULEA_OBJECT.j_distributed_object_write(
+ self.object, value2, length, offset,
+ ctypes.byref(bytes_written), batch.get_pointer())
+ return bytes_written
+
+ def read(self, length, offset, batch):
+ """Reads an object
+
+ Args:
+ length: The length of the data to be read
+ offset: The offset where the data is to be read
+ batch: A batch
+ Returns:
+ value: The ctype with the data that was read
+ (Access data with value.value)
+ bytes_read: A c_ulonglong with the bytes read
+ (Access with bytes_read.value)
+ """
+ value = ctypes.c_char_p(bytes(length))
+ bytes_read = ctypes.c_ulonglong(0)
+ JULEA_OBJECT.j_distributed_object_read(
+ self.object, value, length, offset,
+ ctypes.byref(bytes_read), batch.get_pointer())
+ return value, bytes_read
diff --git a/python/julea/object/object.py b/python/julea/object/object.py
new file mode 100644
index 000000000..f9ed975e3
--- /dev/null
+++ b/python/julea/object/object.py
@@ -0,0 +1,111 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+import ctypes
+from .. import JULEA_OBJECT
+from ..lib.common import my_void_p
+
+JULEA_OBJECT.j_object_new.restype = my_void_p
+
+
+class JObject:
+
+ def __init__(self, namespace, name):
+ """Initializes the object and gets the new object from the C Interface
+
+ Args:
+ namespace: The namespace of the object
+ name: The name of the object
+ """
+ enc_namespace = namespace.encode('utf-8')
+ enc_name = name.encode('utf-8')
+ self.object = JULEA_OBJECT.j_object_new(enc_namespace, enc_name)
+
+ def __del__(self):
+ """Dereferences an object when it is deleted in Python"""
+ JULEA_OBJECT.j_object_unref(self.object)
+
+ def create(self, batch):
+ """Creates an object
+
+ Args:
+ batch: A batch
+ """
+ JULEA_OBJECT.j_object_create(self.object, batch.get_pointer())
+
+ def delete(self, batch):
+ """Deletes an object from the server
+
+ Args:
+ batch: A batch
+ """
+ JULEA_OBJECT.j_object_delete(self.object, batch.get_pointer())
+
+ def status(self, batch):
+ """Gets the status of an object
+
+ Args:
+ batch: A batch
+ Returns:
+ modification_time: The modification time of the object
+ (Access data with modification_time.value)
+ size: The size of the object (Access data with size.value)
+ """
+ modification_time = ctypes.c_longlong(0)
+ size = ctypes.c_ulonglong(0)
+ JULEA_OBJECT.j_object_status(
+ self.object, ctypes.byref(modification_time),
+ ctypes.byref(size), batch.get_pointer())
+ return modification_time, size
+
+ def write(self, value, offset, batch):
+ """Writes an object
+
+ Args:
+ value: bytes holding the data to write
+ offset: The offset where the data is written
+ batch: A batch
+ Returns:
+ bytes_written: A c_ulonglong with the bytes written
+ (Access with bytes_written.value)
+ """
+ length = ctypes.c_ulonglong(len(value))
+ value2 = ctypes.c_char_p(value)
+ bytes_written = ctypes.c_ulonglong(0)
+ JULEA_OBJECT.j_object_write(
+ self.object, value2, length, offset,
+ ctypes.byref(bytes_written), batch.get_pointer())
+ return bytes_written
+
+ def read(self, length, offset, batch):
+ """Reads an object
+
+ Args:
+ length: The length of the data to be read
+ offset: The offset where the data is to be read
+ batch: A batch
+ Returns:
+ value: The ctype with the data that was read
+ (Access data with value.value)
+ bytes_read: A c_ulonglong with the bytes read
+ (Access with bytes_read.value)
+ """
+ value = ctypes.c_char_p(bytes(length))
+ bytes_read = ctypes.c_ulonglong(0)
+ JULEA_OBJECT.j_object_read(
+ self.object, value, length, offset,
+ ctypes.byref(bytes_read), batch.get_pointer())
+ return value, bytes_read
diff --git a/python/julea/test/__init__.py b/python/julea/test/__init__.py
new file mode 100644
index 000000000..9efa356eb
--- /dev/null
+++ b/python/julea/test/__init__.py
@@ -0,0 +1,20 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from .test_distributed_object import distributed_object_test
+from .test_item import item_test
+from .test_kv import kv_test
+from .test_object import object_test
diff --git a/python/julea/test/test_distributed_object.py b/python/julea/test/test_distributed_object.py
new file mode 100644
index 000000000..0da8b90cf
--- /dev/null
+++ b/python/julea/test/test_distributed_object.py
@@ -0,0 +1,86 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from ..object import (JDistributedObject, JBatch, JDistribution,
+ J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN)
+from ..lib.common import print_test
+
+
+def test_create_delete():
+ print_test("/object/distributed-object/create_delete")
+ batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT)
+ d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+ o = JDistributedObject("python", "test", d)
+ o.create(batch)
+ batch.execute()
+
+ o.delete(batch)
+ batch.execute()
+ del o
+ print("OK")
+
+
+def test_read_write():
+ print_test("/object/distributed-object/read_write")
+ batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT)
+ d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+ o = JDistributedObject("python", "test", d)
+ o.create(batch)
+ batch.execute()
+
+ bw = o.write("dummy".encode('utf-8'), 0, batch)
+ batch.execute()
+ assert bw.value == 5
+
+ out = o.read(5, 0, batch)
+ batch.execute()
+ assert out[0].value.decode('utf-8') == "dummy"
+ assert out[1].value == 5
+
+ o.delete(batch)
+ batch.execute()
+ del o
+ print("OK")
+
+
+def test_status():
+ print_test("/object/distributed-object/status")
+ batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT)
+ d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+ o = JDistributedObject("python", "test", d)
+ o.create(batch)
+ batch.execute()
+
+ dummy = bytes(42)
+ o.write(dummy, 0, batch)
+ batch.execute()
+
+ modtime, size = o.status(batch)
+ batch.execute()
+
+ assert modtime.value != 0
+ assert size.value == 42
+
+ o.delete(batch)
+ batch.execute()
+ del o
+ print("OK")
+
+
+def distributed_object_test():
+ test_create_delete()
+ test_read_write()
+ test_status()
diff --git a/python/julea/test/test_item.py b/python/julea/test/test_item.py
new file mode 100644
index 000000000..2a6463b9d
--- /dev/null
+++ b/python/julea/test/test_item.py
@@ -0,0 +1,88 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from ..item import (JItem, JCollection, JBatch, JDistribution,
+ J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN)
+from ..lib.common import print_test
+
+
+def test_create_delete():
+ print_test("/item/item/create_delete")
+ batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT)
+ d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+ c = JCollection("test-collection", batch, False)
+ i = JItem(c, "test-item", batch, False, d)
+ batch.execute()
+
+ i.delete(batch)
+ batch.execute()
+ del i
+ print("OK")
+
+
+def test_read_write():
+ print_test("/item/item/read_write")
+ batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT)
+ d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+ c = JCollection("test-collection", batch, False)
+ i = JItem(c, "test-item", batch, False, d)
+ batch.execute()
+
+ bw = i.write("dummy".encode('utf-8'), 0, batch)
+ batch.execute()
+ assert bw.value == 5
+
+ out = i.read(5, 0, batch)
+ batch.execute()
+ assert out[0].value.decode('utf-8') == "dummy"
+ assert out[1].value == 5
+
+ i.delete(batch)
+ batch.execute()
+ del i
+ print("OK")
+
+
+def test_status():
+ print_test("/item/item/status")
+ batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT)
+ d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+ c = JCollection("test-collection", batch, False)
+ i = JItem(c, "test-item", batch, False, d)
+ batch.execute()
+
+ dummy = bytes(42)
+ i.write(dummy, 0, batch)
+ batch.execute()
+
+ i.status(batch)
+ batch.execute()
+ size = i.get_size()
+ modtime = i.get_modification_time()
+
+ assert modtime != 0
+ assert size == 42
+
+ i.delete(batch)
+ batch.execute()
+ del i
+ print("OK")
+
+
+def item_test():
+ test_create_delete()
+ test_read_write()
+ test_status()
diff --git a/python/julea/test/test_kv.py b/python/julea/test/test_kv.py
new file mode 100644
index 000000000..2821d8f56
--- /dev/null
+++ b/python/julea/test/test_kv.py
@@ -0,0 +1,54 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from ..kv import JKV, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT
+from ..lib.common import print_test
+
+
+def test_put_delete():
+ print_test("/kv/kv/put_delete")
+ batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT)
+ k = JKV("python", "test")
+ k.put("dummy".encode('utf-8'), batch)
+ batch.execute()
+
+ k.delete(batch)
+ batch.execute()
+ del k
+ print("OK")
+
+
+def test_put_get():
+ print_test("/kv/kv/put_get")
+ batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT)
+ k = JKV("python", "test")
+ k.put("dummy".encode('utf-8'), batch)
+ batch.execute()
+
+ out = k.get(batch)
+ batch.execute()
+ assert out[0].value[:out[1].value].decode('utf-8') == "dummy"
+ assert out[1].value == 5
+
+ k.delete(batch)
+ batch.execute()
+ del k
+ print("OK")
+
+
+def kv_test():
+ test_put_delete()
+ test_put_get()
diff --git a/python/julea/test/test_object.py b/python/julea/test/test_object.py
new file mode 100644
index 000000000..485c3758d
--- /dev/null
+++ b/python/julea/test/test_object.py
@@ -0,0 +1,82 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from ..object import JObject, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT
+from ..lib.common import print_test
+
+
+def test_create_delete():
+ print_test("/object/object/create_delete")
+ batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT)
+ o = JObject("python", "test")
+ o.create(batch)
+ batch.execute()
+
+ o.delete(batch)
+ batch.execute()
+ del o
+ print("OK")
+
+
+def test_read_write():
+ print_test("/object/object/read_write")
+ batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT)
+ o = JObject("python", "test")
+ o.create(batch)
+ batch.execute()
+
+ bw = o.write("dummy".encode('utf-8'), 0, batch)
+ batch.execute()
+ assert bw.value == 5
+
+ out = o.read(5, 0, batch)
+ batch.execute()
+ assert out[0].value.decode('utf-8') == "dummy"
+ assert out[1].value == 5
+
+ o.delete(batch)
+ batch.execute()
+ del o
+ print("OK")
+
+
+def test_status():
+ print_test("/object/object/status")
+ batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT)
+ o = JObject("python", "test")
+ o.create(batch)
+ batch.execute()
+
+ dummy = bytes(42)
+ o.write(dummy, 0, batch)
+ batch.execute()
+
+ modtime, size = o.status(batch)
+ batch.execute()
+
+ assert modtime.value != 0
+ assert size.value == 42
+
+ o.delete(batch)
+ batch.execute()
+ del o
+ print("OK")
+
+
+def object_test():
+ test_create_delete()
+ test_read_write()
+ test_status()
diff --git a/python/test.py b/python/test.py
new file mode 100644
index 000000000..d8e444b43
--- /dev/null
+++ b/python/test.py
@@ -0,0 +1,30 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see .
+
+from julea.test import distributed_object_test, item_test, kv_test, object_test
+from julea import juleadir
+import subprocess
+
+setupfile = juleadir.joinpath('scripts', 'setup.sh')
+
+subprocess.run([setupfile, "start"])
+
+object_test()
+distributed_object_test()
+item_test()
+kv_test()
+
+subprocess.run([setupfile, "stop"])