diff --git a/.gitignore b/.gitignore index c9eafe344..109e451c2 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ bld*/ # Vim *.swp + +# Python +python/**/__pycache__/ diff --git a/Doxyfile b/Doxyfile index 70dfde65f..9550ce554 100644 --- a/Doxyfile +++ b/Doxyfile @@ -112,7 +112,7 @@ WARN_LOGFILE = #--------------------------------------------------------------------------- # Configuration options related to the input files #--------------------------------------------------------------------------- -INPUT = include lib +INPUT = include lib python INPUT_ENCODING = UTF-8 FILE_PATTERNS = *.c \ *.cc \ diff --git a/python/benchmark.py b/python/benchmark.py new file mode 100644 index 000000000..4ad8faf83 --- /dev/null +++ b/python/benchmark.py @@ -0,0 +1,31 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from julea.benchmark import ( + distributed_object_bench, item_bench, kv_bench, object_bench) +from julea import juleadir +import subprocess + +setupfile = juleadir.joinpath('scripts', 'setup.sh') + +subprocess.run([setupfile, "start"]) + +kv_bench() +distributed_object_bench() +object_bench() +item_bench() + +subprocess.run([setupfile, "stop"]) diff --git a/python/hello_world.py b/python/hello_world.py new file mode 100644 index 000000000..92ba4a735 --- /dev/null +++ b/python/hello_world.py @@ -0,0 +1,36 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from julea.object import JObject, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT + + +def main(): + object = JObject("hello", "world") + hello_world = "Hello World!" + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + object.create(batch) + bw = object.write(hello_world.encode('utf-8'), 0, batch) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + out = object.read(bw.value, 0, batch) + + print("Object contains: {buffer} ({length} bytes)".format( + buffer=out[0].raw.decode('utf-8'), length=out[1].value)) + + +if __name__ == "__main__": + main() diff --git a/python/julea/__init__.py b/python/julea/__init__.py new file mode 100644 index 000000000..b6d08320a --- /dev/null +++ b/python/julea/__init__.py @@ -0,0 +1,26 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +import pathlib + +juleadir = pathlib.Path(__file__).parent.parent.parent +libdir = juleadir.joinpath('bld') + +JULEA = ctypes.CDLL(libdir.joinpath('libjulea.so')) +JULEA_ITEM = ctypes.CDLL(libdir.joinpath('libjulea-item.so')) +JULEA_KV = ctypes.CDLL(libdir.joinpath('libjulea-kv.so')) +JULEA_OBJECT = ctypes.CDLL(libdir.joinpath('libjulea-object.so')) diff --git a/python/julea/benchmark/__init__.py b/python/julea/benchmark/__init__.py new file mode 100644 index 000000000..606804bd0 --- /dev/null +++ b/python/julea/benchmark/__init__.py @@ -0,0 +1,20 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from .bench_distributed_object import distributed_object_bench +from .bench_item import item_bench +from .bench_kv import kv_bench +from .bench_object import object_bench diff --git a/python/julea/benchmark/bench_distributed_object.py b/python/julea/benchmark/bench_distributed_object.py new file mode 100644 index 000000000..ed26c45c9 --- /dev/null +++ b/python/julea/benchmark/bench_distributed_object.py @@ -0,0 +1,199 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# Copyright (C) 2017-2020 Michael Kuhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..object import (JDistributedObject, JBatch, JDistribution, + J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN) +from ..lib.common import format_benchmark +import time + + +def bench_create(use_batch): + if use_batch: + n = 100000 + else: + n = 1000 + + distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch: + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JDistributedObject("python", name, distribution) + o.create(batch) + o.delete(delete_batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/object/distributed-object/create-batch", end - start, n + else: + return "/object/distributed-object/create", end - start, n + + +def bench_delete(use_batch): + n = 10000 + distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JDistributedObject("python", name, distribution) + o.create(batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JDistributedObject("python", name, distribution) + o.delete(batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/object/distributed-object/delete-batch", end - start, n + else: + return "/object/distributed-object/delete", end - start, n + + +def bench_status(use_batch): + n = 1000 + distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + dummy = bytes(1) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JDistributedObject("python", "benchmark", distribution) + o.delete(batch) + o.create(batch) + o.write(dummy, 0, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for _ in range(0, n): + o.status(batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + return "/object/distributed-object/status-batch", end - start, n + else: + return "/object/distributed-object/status", end - start, n + + +def bench_read(use_batch, block_size): + n = 25000 + distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + dummy = bytes(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JDistributedObject("python", "benchmark", distribution) + o.delete(batch) + o.create(batch) + + for i in range(0, n): + o.write(dummy, i * block_size, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + o.read(block_size, i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + name = "/object/distributed-object/read-batch" + return name, end - start, n, n * block_size + else: + name = "/object/distributed-object/read" + return name, end - start, n, n * block_size + + +def bench_write(use_batch, block_size): + n = 25000 + distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + dummy = bytes(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JDistributedObject("python", "benchmark", distribution) + o.delete(batch) + o.create(batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + o.write(dummy, i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + name = "/object/distributed-object/write-batch" + return name, end - start, n, n * block_size + else: + name = "/object/distributed-object/write" + return name, end - start, n, n * block_size + + +def distributed_object_bench(): + create = bench_create(False) + print(format_benchmark(create)) + create_batch = bench_create(True) + print(format_benchmark(create_batch)) + + delete = bench_delete(False) + print(format_benchmark(delete)) + delete_batch = bench_delete(True) + print(format_benchmark(delete_batch)) + + status = bench_status(False) + print(format_benchmark(status)) + status_batch = bench_status(True) + print(format_benchmark(status_batch)) + + read = bench_read(False, 4 * 1024) + print(format_benchmark(read)) + read_batch = bench_read(True, 4 * 1024) + print(format_benchmark(read_batch)) + + write = bench_write(False, 4 * 1024) + print(format_benchmark(write)) + write_batch = bench_write(True, 4 * 1024) + print(format_benchmark(write_batch)) diff --git a/python/julea/benchmark/bench_item.py b/python/julea/benchmark/bench_item.py new file mode 100644 index 000000000..a36eb4dfb --- /dev/null +++ b/python/julea/benchmark/bench_item.py @@ -0,0 +1,235 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# Copyright (C) 2010-2020 Michael Kuhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..item import (JItem, JCollection, JBatch, JDistribution, + J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN) +from ..lib.common import format_benchmark +import time + + +def bench_create(use_batch): + if use_batch: + n = 100000 + else: + n = 1000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch: + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + item = JItem(collection, name, batch, False, None) + item.delete(delete_batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection.delete(batch) + + if use_batch: + return "/item/item/create-batch", end - start, n + else: + return "/item/item/create", end - start, n + + +def bench_delete(use_batch): + n = 10000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + JItem(collection, name, batch, False, None) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as get_batch: + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + item = JItem(collection, name, get_batch, True) + get_batch.execute() + + item.delete(batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection.delete(batch) + + if use_batch: + return "/item/item/delete-batch", end - start, n + else: + return "/item/item/delete", end - start, n + + +def bench_delete_batch_without_get(): + n = 10000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + item = JItem(collection, name, batch, False, None) + + item.delete(batch) + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection.delete(batch) + + return "/item/item/delete-batch-without-get", end - start, n + + +def bench_status(use_batch): + n = 1000 + dummy = bytes(1) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + batch.execute() + item = JItem(collection, "benchmark", batch, False, None) + item.write(dummy, 0, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for _ in range(0, n): + item.status(batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + item.delete(batch) + collection.delete(batch) + + if use_batch: + return "/item/item/status-batch", end - start, n + else: + return "/item/item/status", end - start, n + + +def bench_read(use_batch, block_size): + n = 25000 + dummy = bytes(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + item = JItem(collection, "benchmark", batch, False, None) + + for i in range(0, n): + item.write(dummy, i * block_size, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + # Unused variable required, otherwise glibc crashes somehow + out = item.read(block_size, # pylint: disable=unused-variable + i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + item.delete(batch) + collection.delete(batch) + + if use_batch: + name = "/item/item/read-batch" + return name, end - start, n, n * block_size + else: + name = "/item/item/read" + return name, end - start, n, n * block_size + + +def bench_write(use_batch, block_size): + n = 25000 + dummy = bytes(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + collection = JCollection("benchmark", batch, False) + batch.execute() + item = JItem(collection, "benchmark", batch, False, None) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + item.write(dummy, i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + item.delete(batch) + collection.delete(batch) + + if use_batch: + name = "/item/item/write-batch" + return name, end - start, n, n * block_size + else: + name = "/item/item/write" + return name, end - start, n, n * block_size + + +def item_bench(): + create = bench_create(False) + print(format_benchmark(create)) + create_batch = bench_create(True) + print(format_benchmark(create_batch)) + + delete = bench_delete(False) + print(format_benchmark(delete)) + delete_batch = bench_delete(True) + print(format_benchmark(delete_batch)) + delete_batch_without_get = bench_delete_batch_without_get() + print(format_benchmark(delete_batch_without_get)) + + status = bench_status(False) + print(format_benchmark(status)) + status_batch = bench_status(True) + print(format_benchmark(status_batch)) + + read = bench_read(False, 4 * 1024) + print(format_benchmark(read)) + read_batch = bench_read(True, 4 * 1024) + print(format_benchmark(read_batch)) + + write = bench_write(False, 4 * 1024) + print(format_benchmark(write)) + write_batch = bench_write(True, 4 * 1024) + print(format_benchmark(write_batch)) diff --git a/python/julea/benchmark/bench_kv.py b/python/julea/benchmark/bench_kv.py new file mode 100644 index 000000000..051438c49 --- /dev/null +++ b/python/julea/benchmark/bench_kv.py @@ -0,0 +1,116 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# Copyright (C) 2017-2020 Michael Kuhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..kv import JKV, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT +from ..lib.common import format_benchmark +import time + + +def bench_put(use_batch): + n = 200000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch: + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + k = JKV("python", name) + k.put("empty".encode('utf-8'), batch) + k.delete(delete_batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/kv/kv/put-batch", end - start, n + else: + return "/kv/kv/put", end - start, n + + +def bench_get(use_batch): + n = 200000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch: + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + k = JKV("python", name) + k.put(name.encode('utf-8'), batch) + k.delete(delete_batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + k = JKV("python", name) + k.get(batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/kv/kv/get-batch", end - start, n + else: + return "/kv/kv/get", end - start, n + + +def bench_delete(use_batch): + n = 200000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + k = JKV("python", name) + k.put("empty".encode('utf-8'), batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + k = JKV("python", name) + k.delete(batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/kv/kv/delete-batch", end - start, n + else: + return "/kv/kv/delete", end - start, n + + +def kv_bench(): + create = bench_put(False) + print(format_benchmark(create)) + create_batch = bench_put(True) + print(format_benchmark(create_batch)) + + read = bench_get(False) + print(format_benchmark(read)) + read_batch = bench_get(True) + print(format_benchmark(read_batch)) + + delete = bench_delete(False) + print(format_benchmark(delete)) + delete_batch = bench_delete(True) + print(format_benchmark(delete_batch)) diff --git a/python/julea/benchmark/bench_object.py b/python/julea/benchmark/bench_object.py new file mode 100644 index 000000000..a3ef09423 --- /dev/null +++ b/python/julea/benchmark/bench_object.py @@ -0,0 +1,185 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# Copyright (C) 2017-2020 Michael Kuhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..object import JObject, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT +from ..lib.common import format_benchmark +import time + + +def bench_create(use_batch): + n = 100000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch: + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JObject("python", name) + o.create(batch) + o.delete(delete_batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/object/object/create-batch", end - start, n + else: + return "/object/object/create", end - start, n + + +def bench_delete(use_batch): + n = 100000 + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JObject("python", name) + o.create(batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + name = "benchmark-" + str(i) + o = JObject("python", name) + o.delete(batch) + if not use_batch: + batch.execute() + + end = time.time() + + if use_batch: + return "/object/object/delete-batch", end - start, n + else: + return "/object/object/delete", end - start, n + + +def bench_status(use_batch): + n = 200000 + dummy = bytes(1) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JObject("python", "benchmark") + o.delete(batch) + o.create(batch) + o.write(dummy, 0, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for _ in range(0, n): + o.status(batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + return "/object/object/status-batch", end - start, n + else: + return "/object/object/status", end - start, n + + +def bench_read(use_batch, block_size): + n = 200000 + dummy = bytes(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JObject("python", "benchmark") + o.delete(batch) + o.create(batch) + + for i in range(0, n): + o.write(dummy, i * block_size, batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + o.read(block_size, i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + return "/object/object/read-batch", end - start, n, n * block_size + else: + return "/object/object/read", end - start, n, n * block_size + + +def bench_write(use_batch, block_size): + n = 200000 + dummy = bytes(block_size) + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o = JObject("python", "benchmark") + o.delete(batch) + o.create(batch) + + start = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + for i in range(0, n): + o.write(dummy, i * block_size, batch) + if not use_batch: + batch.execute() + + end = time.time() + + with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch: + o.delete(batch) + + if use_batch: + return "/object/object/write-batch", end - start, n, n * block_size + else: + return "/object/object/write", end - start, n, n * block_size + + +def object_bench(): + create = bench_create(False) + print(format_benchmark(create)) + create_batch = bench_create(True) + print(format_benchmark(create_batch)) + + delete = bench_delete(False) + print(format_benchmark(delete)) + delete_batch = bench_delete(True) + print(format_benchmark(delete_batch)) + + status = bench_status(False) + print(format_benchmark(status)) + status_batch = bench_status(True) + print(format_benchmark(status_batch)) + + read = bench_read(False, 4 * 1024) + print(format_benchmark(read)) + read_batch = bench_read(True, 4 * 1024) + print(format_benchmark(read_batch)) + + write = bench_write(False, 4 * 1024) + print(format_benchmark(write)) + write_batch = bench_write(True, 4 * 1024) + print(format_benchmark(write_batch)) diff --git a/python/julea/item/__init__.py b/python/julea/item/__init__.py new file mode 100644 index 000000000..7286d3dfd --- /dev/null +++ b/python/julea/item/__init__.py @@ -0,0 +1,20 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from .collection import JCollection +from .item import JItem +from ..lib.batch import * +from ..lib.distribution import * diff --git a/python/julea/item/collection.py b/python/julea/item/collection.py new file mode 100644 index 000000000..4c024944d --- /dev/null +++ b/python/julea/item/collection.py @@ -0,0 +1,61 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA_ITEM +from ..lib.common import my_void_p + +JULEA_ITEM.j_collection_create.restype = my_void_p + + +class JCollection: + def __init__(self, name, batch, get): + """Initializes and creates a collection + + Args: + name: The name of the collection + batch: A batch + get: A boolean, False when the collection should be created, + True if get should be called + """ + enc_name = name.encode('utf-8') + if get: + self.collection = my_void_p + JULEA_ITEM.j_collection_get( + ctypes.byref(self.collection), enc_name, batch.get_pointer()) + else: + self.collection = JULEA_ITEM.j_collection_create( + enc_name, batch.get_pointer()) + + def __del__(self): + """Dereferences a collection when it is deleted in Python""" + JULEA_ITEM.j_collection_unref(self.collection) + + def delete(self, batch): + """Deletes a collection + + Args: + batch: A batch + """ + JULEA_ITEM.j_collection_delete(self.collection, batch.get_pointer()) + + def get_pointer(self): + """Gives a pointer to a collection + + Returns: + collection: A pointer to the collection + """ + return self.collection diff --git a/python/julea/item/item.py b/python/julea/item/item.py new file mode 100644 index 000000000..d72b661e2 --- /dev/null +++ b/python/julea/item/item.py @@ -0,0 +1,134 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA_ITEM +from ..lib.common import my_void_p + +JULEA_ITEM.j_item_create.restype = my_void_p +JULEA_ITEM.j_item_get_size.restype = ctypes.c_ulonglong +JULEA_ITEM.j_item_get_modification_time.restype = ctypes.c_longlong + +JULEA_ITEM.j_item_get.argtypes = [my_void_p, ctypes.POINTER(my_void_p), + ctypes.c_char_p, my_void_p] + + +class JItem: + def __init__(self, collection, name, batch, get, distribution=None): + """Initializes and creates an item + + Args: + collection: The collection of the item + name: The name of the item + batch: A batch + get: A boolean, False when the item should be created, + True if get should be called + distribution: The distribution of the item(only for create) + """ + enc_name = name.encode('utf-8') + if get: + self.item = my_void_p(0) + JULEA_ITEM.j_item_get(collection.get_pointer(), + ctypes.byref(self.item), enc_name, + batch.get_pointer()) + else: + if distribution is None: + self.item = JULEA_ITEM.j_item_create( + collection.get_pointer(), enc_name, + distribution, batch.get_pointer()) + else: + self.item = JULEA_ITEM.j_item_create( + collection.get_pointer(), enc_name, + distribution.get_pointer(), batch.get_pointer()) + + def __del__(self): + """Dereferences an item when it is deleted in Python""" + JULEA_ITEM.j_item_unref(self.item) + + def delete(self, batch): + """Deletes an item from the server + + Args: + batch: A batch + """ + JULEA_ITEM.j_item_delete(self.item, batch.get_pointer()) + + def status(self, batch): + """Gets the status of an item + + Args: + batch: A batch + """ + JULEA_ITEM.j_item_get_status(self.item, batch.get_pointer()) + + def get_size(self): + """Gets the size of an item + Requires status to be called earlier + + Returns: + size: The size of the item + """ + size = JULEA_ITEM.j_item_get_size(self.item) + return size + + def get_modification_time(self): + """Gets the modification time of an item + Requires status to be called earlier + + Returns: + modtime: The size of the item + """ + modtime = JULEA_ITEM.j_item_get_modification_time(self.item) + return modtime + + def write(self, value, offset, batch): + """Writes an item + + Args: + value: bytes holding the data to write + offset: The offset where the data is written + batch: A batch + Returns: + bytes_written: A c_ulonglong with the bytes written + (Access with bytes_written.value) + """ + length = ctypes.c_ulonglong(len(value)) + value2 = ctypes.c_char_p(value) + bytes_written = ctypes.c_ulonglong(0) + JULEA_ITEM.j_item_write( + self.item, value2, length, offset, + ctypes.byref(bytes_written), batch.get_pointer()) + return bytes_written + + def read(self, length, offset, batch): + """Reads an item + + Args: + length: The length of the data to be read + offset: The offset where the data is to be read + batch: A batch + Returns: + value: The ctype with the data that was read + (Access data with value.value) + bytes_read: A c_ulonglong with the bytes read + (Access with bytes_read.value) + """ + value = ctypes.c_char_p(bytes(length)) + bytes_read = ctypes.c_ulonglong(0) + JULEA_ITEM.j_item_read( + self.item, value, length, offset, + ctypes.byref(bytes_read), batch.get_pointer()) + return value, bytes_read diff --git a/python/julea/kv/__init__.py b/python/julea/kv/__init__.py new file mode 100644 index 000000000..6b266fe1a --- /dev/null +++ b/python/julea/kv/__init__.py @@ -0,0 +1,18 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from .kv import JKV +from ..lib.batch import * diff --git a/python/julea/kv/kv.py b/python/julea/kv/kv.py new file mode 100644 index 000000000..98e311bc4 --- /dev/null +++ b/python/julea/kv/kv.py @@ -0,0 +1,77 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA_KV +from ..lib.common import my_void_p + +JULEA_KV.j_kv_new.restype = my_void_p + + +class JKV: + + def __init__(self, namespace, key): + """Initializes the Key-Value pair and + gets the new KV pair from the C Interface + + Args: + namespace: The namespace of the KV pair + key: The name of the KV pair + """ + enc_namespace = namespace.encode('utf-8') + enc_key = key.encode('utf-8') + self.kv = JULEA_KV.j_kv_new(enc_namespace, enc_key) + + def __del__(self): + """Dereferences a KV pair when it is deleted in Python""" + JULEA_KV.j_kv_unref(self.kv) + + def delete(self, batch): + """Deletes a Key-Value pair from the server + + Args: + batch: A batch + """ + JULEA_KV.j_kv_delete(self.kv, batch.get_pointer()) + + def put(self, value, batch): + """Writes a Key-Value pair + + Args: + value: bytes of data to be written in the KV pair + batch: A batch + """ + length = ctypes.c_ulong(len(value)) + value2 = ctypes.c_char_p(value) + JULEA_KV.j_kv_put(self.kv, value2, + length, None, batch.get_pointer()) + + def get(self, batch): + """Reads a Key-Value pair + + Args: + batch: A batch + Returns: + value: The ctype with the data that was read + (Access data with value.value[:length.value]) + length: The length of the data + (Access with length.value) + """ + value = ctypes.c_char_p() + length = ctypes.c_ulong(0) + JULEA_KV.j_kv_get(self.kv, ctypes.byref(value), + ctypes.byref(length), batch.get_pointer()) + return value, length diff --git a/python/julea/lib/__init__.py b/python/julea/lib/__init__.py new file mode 100644 index 000000000..85c7d6777 --- /dev/null +++ b/python/julea/lib/__init__.py @@ -0,0 +1,15 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . diff --git a/python/julea/lib/batch.py b/python/julea/lib/batch.py new file mode 100644 index 000000000..eab95efe9 --- /dev/null +++ b/python/julea/lib/batch.py @@ -0,0 +1,62 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA +from .common import my_void_p + +(J_SEMANTICS_TEMPLATE_DEFAULT, J_SEMANTICS_TEMPLATE_POSIX, + J_SEMANTICS_TEMPLATE_TEMPORARY_LOCAL) = (0, 1, 2) + +JULEA.j_batch_new_for_template.restype = my_void_p + + +class JBatch: + def __init__(self, template): + """Initializes a batch + + Args: + template: The template defined in JSemanticsTemplate (C Interface) + """ + self.batch = JULEA.j_batch_new_for_template(template) + + def __del__(self): + """Dereferences a batch when it is deleted in Python""" + JULEA.j_batch_unref(self.batch) + + def __enter__(self): + """Returns itself when a with-statement is entered + + Returns: + self: The batch itself + """ + return self + + def __exit__(self, type, value, traceback): + """Executes the Batch when the with-statement is left""" + self.execute() + + def execute(self): + """Executes the Batch""" + ret = JULEA.j_batch_execute(self.batch) + if ret == 1: + return True + else: + return False + + def get_pointer(self): + """Returns the C Pointer to the batch""" + return self.batch diff --git a/python/julea/lib/common.py b/python/julea/lib/common.py new file mode 100644 index 000000000..13512937a --- /dev/null +++ b/python/julea/lib/common.py @@ -0,0 +1,54 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes + + +# Workaround so Python doesn't convert the pointer to a Python datatype +class my_void_p(ctypes.c_void_p): + pass + + +def format_benchmark(input): + name = input[0] + ':' + ops = round(input[2] / input[1]) + if len(input) == 3: + output = "{name: <{width}}{time} seconds ({ops}/s)" + return output.format(name=name, width=51, time=round(input[1], 3), + ops=ops) + elif len(input) == 4: + rate = input[3] / input[1] + unit = "B" + if rate >= 1024: + rate = rate / 1024 + unit = "KB" + if rate >= 1024: + rate = rate / 1024 + unit = "MB" + if rate >= 1024: + rate = rate / 1024 + unit = "GB" + if rate >= 1024: + rate = rate / 1024 + unit = "TB" + rate = round(rate, 1) + output = "{name: <{width}}{time} seconds ({ops}/s) ({rate} {unit}/s)" + return output.format(name=name, width=51, time=round(input[1], 3), + ops=ops, rate=rate, unit=unit) + + +def print_test(name): + print(" {name: <{width}}".format(name=name, width=69), end='', flush=True) diff --git a/python/julea/lib/distribution.py b/python/julea/lib/distribution.py new file mode 100644 index 000000000..ca205732d --- /dev/null +++ b/python/julea/lib/distribution.py @@ -0,0 +1,62 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA +from .common import my_void_p + +(J_DISTRIBUTION_ROUND_ROBIN, J_DISTRIBUTION_SINGLE_SERVER, + J_DISTRIBUTION_WEIGHTED) = (0, 1, 2) + +JULEA.j_distribution_new.restype = my_void_p +JULEA.j_distribution_new_from_bson.restype = my_void_p +JULEA.j_distribution_serialize.restype = my_void_p + + +class JDistribution: + + def __init__(self, input): + """Creates a new distribution + + Args: + input: The type of the distribution or a pointer to a + serialized distribution + """ + if isinstance(input, int): + self.distribution = JULEA.j_distribution_new(input) + else: + self.distribution = JULEA.j_distribution_new_from_bson(input) + + def __del__(self): + """Dereferences a distribution when it is deleted in Python""" + JULEA.j_distribution_unref(self.distribution) + + def serialize(self): + """Serializes the distribution into a BSON. + + Returns: + b: A pointer to the BSON object in C. + """ + b = JULEA.j_distribution_serialize(self.distribution) + return b + + def get_pointer(self): + """Gets the pointer to the distribution + + Returns: + distribution: A pointer to the distribution + """ + return self.distribution diff --git a/python/julea/object/__init__.py b/python/julea/object/__init__.py new file mode 100644 index 000000000..63e7b6da1 --- /dev/null +++ b/python/julea/object/__init__.py @@ -0,0 +1,20 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from .distributed_object import JDistributedObject +from .object import JObject +from ..lib.batch import * +from ..lib.distribution import * diff --git a/python/julea/object/distributed_object.py b/python/julea/object/distributed_object.py new file mode 100644 index 000000000..381f97f71 --- /dev/null +++ b/python/julea/object/distributed_object.py @@ -0,0 +1,115 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA_OBJECT +from ..lib.common import my_void_p + +JULEA_OBJECT.j_distributed_object_new.restype = my_void_p + + +class JDistributedObject: + + def __init__(self, namespace, name, distribution): + """Initializes the objects and gets the new object from the C Interface + + Args: + namespace: The namespace of the object + name: The name of the object + """ + enc_namespace = namespace.encode('utf-8') + enc_name = name.encode('utf-8') + dist = distribution.get_pointer() + self.object = JULEA_OBJECT.j_distributed_object_new( + enc_namespace, enc_name, dist) + + def __del__(self): + """Dereferences an object when it is deleted in Python""" + JULEA_OBJECT.j_distributed_object_unref(self.object) + + def create(self, batch): + """Creates an object + + Args: + batch: A batch + """ + JULEA_OBJECT.j_distributed_object_create( + self.object, batch.get_pointer()) + + def delete(self, batch): + """Deletes an object from the server + + Args: + batch: A batch + """ + JULEA_OBJECT.j_distributed_object_delete( + self.object, batch.get_pointer()) + + def status(self, batch): + """Gets the status of an object + + Args: + batch: A batch + Returns: + modification_time: The modification time of the object + (Access data with modification_time.value) + size: The size of the object (Access data with size.value) + """ + modification_time = ctypes.c_longlong(0) + size = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_distributed_object_status( + self.object, ctypes.byref(modification_time), + ctypes.byref(size), batch.get_pointer()) + return modification_time, size + + def write(self, value, offset, batch): + """Writes an object + + Args: + value: bytes holding the data to write + offset: The offset where the data is written + batch: A batch + Returns: + bytes_written: A c_ulonglong with the bytes written + (Access with bytes_written.value) + """ + length = ctypes.c_ulonglong(len(value)) + value2 = ctypes.c_char_p(value) + bytes_written = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_distributed_object_write( + self.object, value2, length, offset, + ctypes.byref(bytes_written), batch.get_pointer()) + return bytes_written + + def read(self, length, offset, batch): + """Reads an object + + Args: + length: The length of the data to be read + offset: The offset where the data is to be read + batch: A batch + Returns: + value: The ctype with the data that was read + (Access data with value.value) + bytes_read: A c_ulonglong with the bytes read + (Access with bytes_read.value) + """ + value = ctypes.c_char_p(bytes(length)) + bytes_read = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_distributed_object_read( + self.object, value, length, offset, + ctypes.byref(bytes_read), batch.get_pointer()) + return value, bytes_read diff --git a/python/julea/object/object.py b/python/julea/object/object.py new file mode 100644 index 000000000..f9ed975e3 --- /dev/null +++ b/python/julea/object/object.py @@ -0,0 +1,111 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import ctypes +from .. import JULEA_OBJECT +from ..lib.common import my_void_p + +JULEA_OBJECT.j_object_new.restype = my_void_p + + +class JObject: + + def __init__(self, namespace, name): + """Initializes the object and gets the new object from the C Interface + + Args: + namespace: The namespace of the object + name: The name of the object + """ + enc_namespace = namespace.encode('utf-8') + enc_name = name.encode('utf-8') + self.object = JULEA_OBJECT.j_object_new(enc_namespace, enc_name) + + def __del__(self): + """Dereferences an object when it is deleted in Python""" + JULEA_OBJECT.j_object_unref(self.object) + + def create(self, batch): + """Creates an object + + Args: + batch: A batch + """ + JULEA_OBJECT.j_object_create(self.object, batch.get_pointer()) + + def delete(self, batch): + """Deletes an object from the server + + Args: + batch: A batch + """ + JULEA_OBJECT.j_object_delete(self.object, batch.get_pointer()) + + def status(self, batch): + """Gets the status of an object + + Args: + batch: A batch + Returns: + modification_time: The modification time of the object + (Access data with modification_time.value) + size: The size of the object (Access data with size.value) + """ + modification_time = ctypes.c_longlong(0) + size = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_object_status( + self.object, ctypes.byref(modification_time), + ctypes.byref(size), batch.get_pointer()) + return modification_time, size + + def write(self, value, offset, batch): + """Writes an object + + Args: + value: bytes holding the data to write + offset: The offset where the data is written + batch: A batch + Returns: + bytes_written: A c_ulonglong with the bytes written + (Access with bytes_written.value) + """ + length = ctypes.c_ulonglong(len(value)) + value2 = ctypes.c_char_p(value) + bytes_written = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_object_write( + self.object, value2, length, offset, + ctypes.byref(bytes_written), batch.get_pointer()) + return bytes_written + + def read(self, length, offset, batch): + """Reads an object + + Args: + length: The length of the data to be read + offset: The offset where the data is to be read + batch: A batch + Returns: + value: The ctype with the data that was read + (Access data with value.value) + bytes_read: A c_ulonglong with the bytes read + (Access with bytes_read.value) + """ + value = ctypes.c_char_p(bytes(length)) + bytes_read = ctypes.c_ulonglong(0) + JULEA_OBJECT.j_object_read( + self.object, value, length, offset, + ctypes.byref(bytes_read), batch.get_pointer()) + return value, bytes_read diff --git a/python/julea/test/__init__.py b/python/julea/test/__init__.py new file mode 100644 index 000000000..9efa356eb --- /dev/null +++ b/python/julea/test/__init__.py @@ -0,0 +1,20 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from .test_distributed_object import distributed_object_test +from .test_item import item_test +from .test_kv import kv_test +from .test_object import object_test diff --git a/python/julea/test/test_distributed_object.py b/python/julea/test/test_distributed_object.py new file mode 100644 index 000000000..0da8b90cf --- /dev/null +++ b/python/julea/test/test_distributed_object.py @@ -0,0 +1,86 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..object import (JDistributedObject, JBatch, JDistribution, + J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN) +from ..lib.common import print_test + + +def test_create_delete(): + print_test("/object/distributed-object/create_delete") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + o = JDistributedObject("python", "test", d) + o.create(batch) + batch.execute() + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def test_read_write(): + print_test("/object/distributed-object/read_write") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + o = JDistributedObject("python", "test", d) + o.create(batch) + batch.execute() + + bw = o.write("dummy".encode('utf-8'), 0, batch) + batch.execute() + assert bw.value == 5 + + out = o.read(5, 0, batch) + batch.execute() + assert out[0].value.decode('utf-8') == "dummy" + assert out[1].value == 5 + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def test_status(): + print_test("/object/distributed-object/status") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + o = JDistributedObject("python", "test", d) + o.create(batch) + batch.execute() + + dummy = bytes(42) + o.write(dummy, 0, batch) + batch.execute() + + modtime, size = o.status(batch) + batch.execute() + + assert modtime.value != 0 + assert size.value == 42 + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def distributed_object_test(): + test_create_delete() + test_read_write() + test_status() diff --git a/python/julea/test/test_item.py b/python/julea/test/test_item.py new file mode 100644 index 000000000..2a6463b9d --- /dev/null +++ b/python/julea/test/test_item.py @@ -0,0 +1,88 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..item import (JItem, JCollection, JBatch, JDistribution, + J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN) +from ..lib.common import print_test + + +def test_create_delete(): + print_test("/item/item/create_delete") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + c = JCollection("test-collection", batch, False) + i = JItem(c, "test-item", batch, False, d) + batch.execute() + + i.delete(batch) + batch.execute() + del i + print("OK") + + +def test_read_write(): + print_test("/item/item/read_write") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + c = JCollection("test-collection", batch, False) + i = JItem(c, "test-item", batch, False, d) + batch.execute() + + bw = i.write("dummy".encode('utf-8'), 0, batch) + batch.execute() + assert bw.value == 5 + + out = i.read(5, 0, batch) + batch.execute() + assert out[0].value.decode('utf-8') == "dummy" + assert out[1].value == 5 + + i.delete(batch) + batch.execute() + del i + print("OK") + + +def test_status(): + print_test("/item/item/status") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + d = JDistribution(J_DISTRIBUTION_ROUND_ROBIN) + c = JCollection("test-collection", batch, False) + i = JItem(c, "test-item", batch, False, d) + batch.execute() + + dummy = bytes(42) + i.write(dummy, 0, batch) + batch.execute() + + i.status(batch) + batch.execute() + size = i.get_size() + modtime = i.get_modification_time() + + assert modtime != 0 + assert size == 42 + + i.delete(batch) + batch.execute() + del i + print("OK") + + +def item_test(): + test_create_delete() + test_read_write() + test_status() diff --git a/python/julea/test/test_kv.py b/python/julea/test/test_kv.py new file mode 100644 index 000000000..2821d8f56 --- /dev/null +++ b/python/julea/test/test_kv.py @@ -0,0 +1,54 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..kv import JKV, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT +from ..lib.common import print_test + + +def test_put_delete(): + print_test("/kv/kv/put_delete") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + k = JKV("python", "test") + k.put("dummy".encode('utf-8'), batch) + batch.execute() + + k.delete(batch) + batch.execute() + del k + print("OK") + + +def test_put_get(): + print_test("/kv/kv/put_get") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + k = JKV("python", "test") + k.put("dummy".encode('utf-8'), batch) + batch.execute() + + out = k.get(batch) + batch.execute() + assert out[0].value[:out[1].value].decode('utf-8') == "dummy" + assert out[1].value == 5 + + k.delete(batch) + batch.execute() + del k + print("OK") + + +def kv_test(): + test_put_delete() + test_put_get() diff --git a/python/julea/test/test_object.py b/python/julea/test/test_object.py new file mode 100644 index 000000000..485c3758d --- /dev/null +++ b/python/julea/test/test_object.py @@ -0,0 +1,82 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from ..object import JObject, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT +from ..lib.common import print_test + + +def test_create_delete(): + print_test("/object/object/create_delete") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + o = JObject("python", "test") + o.create(batch) + batch.execute() + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def test_read_write(): + print_test("/object/object/read_write") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + o = JObject("python", "test") + o.create(batch) + batch.execute() + + bw = o.write("dummy".encode('utf-8'), 0, batch) + batch.execute() + assert bw.value == 5 + + out = o.read(5, 0, batch) + batch.execute() + assert out[0].value.decode('utf-8') == "dummy" + assert out[1].value == 5 + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def test_status(): + print_test("/object/object/status") + batch = JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) + o = JObject("python", "test") + o.create(batch) + batch.execute() + + dummy = bytes(42) + o.write(dummy, 0, batch) + batch.execute() + + modtime, size = o.status(batch) + batch.execute() + + assert modtime.value != 0 + assert size.value == 42 + + o.delete(batch) + batch.execute() + del o + print("OK") + + +def object_test(): + test_create_delete() + test_read_write() + test_status() diff --git a/python/test.py b/python/test.py new file mode 100644 index 000000000..d8e444b43 --- /dev/null +++ b/python/test.py @@ -0,0 +1,30 @@ +# JULEA - Flexible storage framework +# Copyright (C) 2019-2020 Johannes Coym +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +from julea.test import distributed_object_test, item_test, kv_test, object_test +from julea import juleadir +import subprocess + +setupfile = juleadir.joinpath('scripts', 'setup.sh') + +subprocess.run([setupfile, "start"]) + +object_test() +distributed_object_test() +item_test() +kv_test() + +subprocess.run([setupfile, "stop"])