Skip to content

Commit c3b7bde

Browse files
vladbologavladbologa
authored
Add archive-builder functionality to flatdata-py and CLI writer tool (#206)
Add rudimentary writing capabilities to flatdata-py: * Add support for bit-level writing * Create resources from JSON (slow) * Add CLI app for direct usage Signed-off-by: Vlad Bologa <[email protected]> Signed-off-by: Vishesh Nag <[email protected]>
1 parent 45c1f81 commit c3b7bde

29 files changed

+2944
-95
lines changed

‎flatdata-generator/flatdata/generator/templates/py/python.jinja2

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import flatdata.lib as flatdata
1111
class {{ tree.namespace_path(struct, "_") }}_{{ struct.name }}(flatdata.structure.Structure):
1212
"""{{ struct.doc|safe_py_string_line }}"""
1313
_SCHEMA = """{{ tree.schema(struct) }}"""
14+
_NAME = "{{ tree.namespace_path(struct, "_") }}_{{ struct.name }}"
1415
_SIZE_IN_BITS = {{ struct.size_in_bits }}
1516
_SIZE_IN_BYTES = {{ struct.size_in_bytes }}
1617
_FIELDS = {
@@ -56,10 +57,40 @@ class {{ tree.namespace_path(archive, "_") }}_{{ archive.name }}(flatdata.archiv
5657
flatdata.archive.Archive.__init__(self, resource_storage)
5758
{% endmacro %}
5859

60+
{% macro archive_builder_definition(tree, archive) %}
61+
62+
class {{ tree.namespace_path(archive, "_") }}_{{ archive.name }}Builder(flatdata.archive_builder.ArchiveBuilder):
63+
_SCHEMA = """{{ tree.schema(archive)|safe_py_string_line }}"""
64+
{% for resource in archive.resources if not resource|is_bound_resource %}
65+
_{{ resource.name|upper }}_SCHEMA = """{{ tree.schema(resource)|safe_py_string_line }}"""
66+
_{{ resource.name|upper }}_DOC = """{{ resource.doc|safe_py_string_line }}"""
67+
{% endfor %}
68+
_NAME = "{{ archive.name }}"
69+
_RESOURCES = {
70+
"{{ archive.name }}.archive" : flatdata.archive_builder.ResourceSignature(
71+
container=flatdata.resources.RawData,
72+
initializer=None,
73+
schema=_SCHEMA,
74+
is_optional=False,
75+
doc="Archive signature"),
76+
{% for resource in archive.resources if not resource|is_bound_resource %}
77+
"{{ resource.name }}": flatdata.archive_builder.ResourceSignature(container={{ resource|to_container}},
78+
initializer={{ resource|to_initializer(tree) }},
79+
schema=_{{ resource.name|upper }}_SCHEMA,
80+
is_optional={{ resource.optional }},
81+
doc=_{{ resource.name|upper }}_DOC),
82+
{% endfor %}
83+
}
84+
85+
def __init__(self, resource_storage):
86+
flatdata.archive_builder.ArchiveBuilder.__init__(self, resource_storage)
87+
{% endmacro %}
88+
5989
{% for node in nodes %}
6090
{%- if node|is_structure %}
6191
{{- structure_definition(tree, node) }}
6292
{%- elif node|is_archive %}
6393
{{- archive_definition(tree, node) }}
94+
{{- archive_builder_definition(tree, node) }}
6495
{% endif %}
6596
{% endfor %}

‎flatdata-generator/setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
setup(
88
name="flatdata-generator",
9-
version="0.4.3",
9+
version="0.4.4",
1010
author="Flatdata Developers",
1111
description="Generate source code for C++, Rust, Go or Python from a Flatdata schema file",
1212
long_description=open(os.path.join(SOURCE_FILEPATH, "README.md")).read(),

‎flatdata-generator/tests/generators/py_expectations/archives/comments.py

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,42 @@ class n_Foo(flatdata.archive.Archive):
3434
def __init__(self, resource_storage):
3535
flatdata.archive.Archive.__init__(self, resource_storage)
3636

37+
class n_FooBuilder(flatdata.archive_builder.ArchiveBuilder):
38+
_SCHEMA = """namespace n {
39+
archive Foo
40+
{
41+
bar : raw_data;
42+
}
43+
}
44+
45+
"""
46+
_BAR_SCHEMA = """namespace n {
47+
archive Foo
48+
{
49+
bar : raw_data;
50+
}
51+
}
52+
53+
"""
54+
_BAR_DOC = """// this is a comment about foo.bar"""
55+
_NAME = "Foo"
56+
_RESOURCES = {
57+
"Foo.archive" : flatdata.archive_builder.ResourceSignature(
58+
container=flatdata.resources.RawData,
59+
initializer=None,
60+
schema=_SCHEMA,
61+
is_optional=False,
62+
doc="Archive signature"),
63+
"bar": flatdata.archive_builder.ResourceSignature(container=flatdata.resources.RawData,
64+
initializer=None,
65+
schema=_BAR_SCHEMA,
66+
is_optional=False,
67+
doc=_BAR_DOC),
68+
}
69+
70+
def __init__(self, resource_storage):
71+
flatdata.archive_builder.ArchiveBuilder.__init__(self, resource_storage)
72+
3773
class n_Bar(flatdata.archive.Archive):
3874
_SCHEMA = """namespace n {
3975
archive Bar
@@ -72,3 +108,41 @@ class n_Bar(flatdata.archive.Archive):
72108
def __init__(self, resource_storage):
73109
flatdata.archive.Archive.__init__(self, resource_storage)
74110

111+
class n_BarBuilder(flatdata.archive_builder.ArchiveBuilder):
112+
_SCHEMA = """namespace n {
113+
archive Bar
114+
{
115+
foo : raw_data;
116+
}
117+
}
118+
119+
"""
120+
_FOO_SCHEMA = """namespace n {
121+
archive Bar
122+
{
123+
foo : raw_data;
124+
}
125+
}
126+
127+
"""
128+
_FOO_DOC = """/*
129+
* this is a comment about bar.foo
130+
*/"""
131+
_NAME = "Bar"
132+
_RESOURCES = {
133+
"Bar.archive" : flatdata.archive_builder.ResourceSignature(
134+
container=flatdata.resources.RawData,
135+
initializer=None,
136+
schema=_SCHEMA,
137+
is_optional=False,
138+
doc="Archive signature"),
139+
"foo": flatdata.archive_builder.ResourceSignature(container=flatdata.resources.RawData,
140+
initializer=None,
141+
schema=_FOO_SCHEMA,
142+
is_optional=False,
143+
doc=_FOO_DOC),
144+
}
145+
146+
def __init__(self, resource_storage):
147+
flatdata.archive_builder.ArchiveBuilder.__init__(self, resource_storage)
148+

‎flatdata-generator/tests/generators/py_expectations/archives/multivector.py

Lines changed: 130 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ class n_S(flatdata.structure.Structure):
88
}
99
1010
"""
11+
_NAME = "n_S"
1112
_SIZE_IN_BITS = 64
1213
_SIZE_IN_BYTES = 8
1314
_FIELDS = {
@@ -27,6 +28,7 @@ class n_T(flatdata.structure.Structure):
2728
}
2829
2930
"""
31+
_NAME = "n_T"
3032
_SIZE_IN_BITS = 64
3133
_SIZE_IN_BYTES = 8
3234
_FIELDS = {
@@ -39,6 +41,7 @@ class n_T(flatdata.structure.Structure):
3941
class _builtin_multivector_IndexType8(flatdata.structure.Structure):
4042
"""/** Builtin type to for MultiVector index */"""
4143
_SCHEMA = """"""
44+
_NAME = "_builtin_multivector_IndexType8"
4245
_SIZE_IN_BITS = 8
4346
_SIZE_IN_BYTES = 1
4447
_FIELDS = {
@@ -51,6 +54,7 @@ class _builtin_multivector_IndexType8(flatdata.structure.Structure):
5154
class _builtin_multivector_IndexType16(flatdata.structure.Structure):
5255
"""/** Builtin type to for MultiVector index */"""
5356
_SCHEMA = """"""
57+
_NAME = "_builtin_multivector_IndexType16"
5458
_SIZE_IN_BITS = 16
5559
_SIZE_IN_BYTES = 2
5660
_FIELDS = {
@@ -63,6 +67,7 @@ class _builtin_multivector_IndexType16(flatdata.structure.Structure):
6367
class _builtin_multivector_IndexType64(flatdata.structure.Structure):
6468
"""/** Builtin type to for MultiVector index */"""
6569
_SCHEMA = """"""
70+
_NAME = "_builtin_multivector_IndexType64"
6671
_SIZE_IN_BITS = 64
6772
_SIZE_IN_BYTES = 8
6873
_FIELDS = {
@@ -193,4 +198,128 @@ class n_A(flatdata.archive.Archive):
193198
}
194199

195200
def __init__(self, resource_storage):
196-
flatdata.archive.Archive.__init__(self, resource_storage)
201+
flatdata.archive.Archive.__init__(self, resource_storage)
202+
203+
class n_ABuilder(flatdata.archive_builder.ArchiveBuilder):
204+
_SCHEMA = """namespace n {
205+
struct S
206+
{
207+
x : u64 : 64;
208+
}
209+
}
210+
211+
namespace n {
212+
struct T
213+
{
214+
x : u64 : 64;
215+
}
216+
}
217+
218+
namespace n {
219+
archive A
220+
{
221+
data : multivector< 8, .n.S, .n.T >;
222+
@optional
223+
optional_data : multivector< 16, .n.S, .n.T >;
224+
data_u64_index : multivector< 64, .n.S, .n.T >;
225+
}
226+
}
227+
228+
"""
229+
_DATA_SCHEMA = """namespace n {
230+
struct S
231+
{
232+
x : u64 : 64;
233+
}
234+
}
235+
236+
namespace n {
237+
struct T
238+
{
239+
x : u64 : 64;
240+
}
241+
}
242+
243+
namespace n {
244+
archive A
245+
{
246+
data : multivector< 8, .n.S, .n.T >;
247+
}
248+
}
249+
250+
"""
251+
_DATA_DOC = """"""
252+
_OPTIONAL_DATA_SCHEMA = """namespace n {
253+
struct S
254+
{
255+
x : u64 : 64;
256+
}
257+
}
258+
259+
namespace n {
260+
struct T
261+
{
262+
x : u64 : 64;
263+
}
264+
}
265+
266+
namespace n {
267+
archive A
268+
{
269+
@optional
270+
optional_data : multivector< 16, .n.S, .n.T >;
271+
}
272+
}
273+
274+
"""
275+
_OPTIONAL_DATA_DOC = """"""
276+
_DATA_U64_INDEX_SCHEMA = """namespace n {
277+
struct S
278+
{
279+
x : u64 : 64;
280+
}
281+
}
282+
283+
namespace n {
284+
struct T
285+
{
286+
x : u64 : 64;
287+
}
288+
}
289+
290+
namespace n {
291+
archive A
292+
{
293+
data_u64_index : multivector< 64, .n.S, .n.T >;
294+
}
295+
}
296+
297+
"""
298+
_DATA_U64_INDEX_DOC = """"""
299+
_NAME = "A"
300+
_RESOURCES = {
301+
"A.archive" : flatdata.archive_builder.ResourceSignature(
302+
container=flatdata.resources.RawData,
303+
initializer=None,
304+
schema=_SCHEMA,
305+
is_optional=False,
306+
doc="Archive signature"),
307+
"data": flatdata.archive_builder.ResourceSignature(container=flatdata.resources.Multivector,
308+
initializer=[_builtin_multivector_IndexType8,n_S,n_T],
309+
schema=_DATA_SCHEMA,
310+
is_optional=False,
311+
doc=_DATA_DOC),
312+
"optional_data": flatdata.archive_builder.ResourceSignature(container=flatdata.resources.Multivector,
313+
initializer=[_builtin_multivector_IndexType16,n_S,n_T],
314+
schema=_OPTIONAL_DATA_SCHEMA,
315+
is_optional=True,
316+
doc=_OPTIONAL_DATA_DOC),
317+
"data_u64_index": flatdata.archive_builder.ResourceSignature(container=flatdata.resources.Multivector,
318+
initializer=[_builtin_multivector_IndexType64,n_S,n_T],
319+
schema=_DATA_U64_INDEX_SCHEMA,
320+
is_optional=False,
321+
doc=_DATA_U64_INDEX_DOC),
322+
}
323+
324+
def __init__(self, resource_storage):
325+
flatdata.archive_builder.ArchiveBuilder.__init__(self, resource_storage)

0 commit comments

Comments
 (0)