Skip to content

Commit 4e261ff

Browse files
nielspardonnielspardon
committed
[GLUTEN-12597][CORE] Migrate Substrait extension referencing from URI to URN
Adopt the 0.98 URN-based simple-extension referencing model (substrait-io/substrait#971): - extensions.proto: SimpleExtensionURI -> SimpleExtensionURN; extension_uri_anchor -> extension_urn_anchor; uri -> urn; and in ExtensionType/ExtensionTypeVariation/ExtensionFunction, extension_uri_reference (field 1) is reserved and replaced by extension_urn_reference (field 4). - plan.proto / extended_expression.proto: extension_uris (field 1) is reserved, replaced by extension_urns = 8 (repeated SimpleExtensionURN). Velox producer (SubstraitExtensionCollector, VeloxToSubstraitPlan) now emits extension_urns / extension_urn_reference. Gluten still maps every function to one catch-all anchor and resolves by name, so the emitted URN is a single placeholder in the required extension:<OWNER>:<ID> format (extension:org.apache.gluten:functions) rather than an empty string; consuming the upstream io.substrait function extensions is a follow-up. Gluten's JVM producer never emitted extension_uris, so it needs no changes beyond recompiling against the regenerated classes. Test fixtures updated to URN field names and proper spec URN values (extension:io.substrait:<file>). AdvancedExtension.optimization (also repeated in 0.98) and the additive Plan fields (parameter_bindings/type_aliases/execution_behavior) are handled in follow-up increments. Part of #12597. Validated locally: gluten-substrait JVM build + Velox native build (libgluten/libvelox).
1 parent ff3641f commit 4e261ff

8 files changed

Lines changed: 72 additions & 56 deletions

File tree

‎cpp/velox/substrait/SubstraitExtensionCollector.cc‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,15 +39,18 @@ bool SubstraitExtensionCollector::BiDirectionHashMap<T>::putIfAbsent(const int&
3939
}
4040

4141
void SubstraitExtensionCollector::addExtensionsToPlan(::substrait::Plan* plan) const {
42-
using SimpleExtensionURI = ::substrait::extensions::SimpleExtensionURI;
43-
// Currently we don't introduce any substrait extension YAML files, so always
44-
// only have one URI.
45-
SimpleExtensionURI* extensionUri = plan->add_extension_uris();
46-
extensionUri->set_extension_uri_anchor(1);
42+
using SimpleExtensionURN = ::substrait::extensions::SimpleExtensionURN;
43+
// Currently we don't map functions to their individual Substrait extension
44+
// YAML files, so we emit a single catch-all URN and resolve functions by
45+
// name. The URN follows the required extension:<OWNER>:<ID> format; consuming
46+
// the upstream io.substrait function extensions is left to a follow-up.
47+
SimpleExtensionURN* extensionUrn = plan->add_extension_urns();
48+
extensionUrn->set_extension_urn_anchor(1);
49+
extensionUrn->set_urn("extension:org.apache.gluten:functions");
4750

4851
for (const auto& [referenceNum, functionId] : extensionFunctions_->forwardMap()) {
4952
auto extensionFunction = plan->add_extensions()->mutable_extension_function();
50-
extensionFunction->set_extension_uri_reference(extensionUri->extension_uri_anchor());
53+
extensionFunction->set_extension_urn_reference(extensionUrn->extension_urn_anchor());
5154
extensionFunction->set_function_anchor(referenceNum);
5255
extensionFunction->set_name(functionId.signature);
5356
}

‎cpp/velox/substrait/VeloxToSubstraitPlan.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ ::substrait::Plan& VeloxToSubstraitPlanConvertor::toSubstrait(
103103
// Add unknown type in extension.
104104
auto unknownType = substraitPlan->add_extensions()->mutable_extension_type();
105105

106-
unknownType->set_extension_uri_reference(0);
106+
unknownType->set_extension_urn_reference(0);
107107
unknownType->set_type_anchor(0);
108108
unknownType->set_name("UNKNOWN");
109109

‎cpp/velox/tests/data/q1_first_stage.json‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,70 +1,70 @@
11
{
2-
"extension_uris": [
2+
"extension_urns": [
33
{
4-
"extension_uri_anchor": 1,
5-
"uri": "/functions_datetime.yaml"
4+
"extension_urn_anchor": 1,
5+
"urn": "extension:io.substrait:functions_datetime"
66
}
77
],
88
"extensions": [
99
{
1010
"extension_function": {
11-
"extension_uri_reference": 1,
11+
"extension_urn_reference": 1,
1212
"function_anchor": 1,
1313
"name": "lte:fp64_fp64"
1414
}
1515
},
1616
{
1717
"extension_function": {
18-
"extension_uri_reference": 1,
18+
"extension_urn_reference": 1,
1919
"function_anchor": 6,
2020
"name": "sum:opt_fp64"
2121
}
2222
},
2323
{
2424
"extension_function": {
25-
"extension_uri_reference": 1,
25+
"extension_urn_reference": 1,
2626
"function_anchor": 3,
2727
"name": "subtract:opt_fp64_fp64"
2828
}
2929
},
3030
{
3131
"extension_function": {
32-
"extension_uri_reference": 1,
32+
"extension_urn_reference": 1,
3333
"function_anchor": 9,
3434
"name": "is_not_null:fp64"
3535
}
3636
},
3737
{
3838
"extension_function": {
39-
"extension_uri_reference": 1,
39+
"extension_urn_reference": 1,
4040
"function_anchor": 2,
4141
"name": "and:bool_bool"
4242
}
4343
},
4444
{
4545
"extension_function": {
46-
"extension_uri_reference": 1,
46+
"extension_urn_reference": 1,
4747
"function_anchor": 5,
4848
"name": "add:opt_fp64_fp64"
4949
}
5050
},
5151
{
5252
"extension_function": {
53-
"extension_uri_reference": 1,
53+
"extension_urn_reference": 1,
5454
"function_anchor": 7,
5555
"name": "count:opt_fp64"
5656
}
5757
},
5858
{
5959
"extension_function": {
60-
"extension_uri_reference": 1,
60+
"extension_urn_reference": 1,
6161
"function_anchor": 4,
6262
"name": "multiply:opt_fp64_fp64"
6363
}
6464
},
6565
{
6666
"extension_function": {
67-
"extension_uri_reference": 1,
67+
"extension_urn_reference": 1,
6868
"function_anchor": 8,
6969
"name": "count:opt_i32"
7070
}

‎cpp/velox/tests/data/q6_first_stage.json‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,56 @@
11
{
2-
"extension_uris": [
2+
"extension_urns": [
33
{
4-
"extension_uri_anchor": 1,
5-
"uri": "/functions_boolean.yaml"
4+
"extension_urn_anchor": 1,
5+
"urn": "extension:io.substrait:functions_boolean"
66
}
77
],
88
"extensions": [
99
{
1010
"extension_function": {
11-
"extension_uri_reference": 1,
11+
"extension_urn_reference": 1,
1212
"function_anchor": 4,
1313
"name": "lte:fp64_fp64"
1414
}
1515
},
1616
{
1717
"extension_function": {
18-
"extension_uri_reference": 1,
18+
"extension_urn_reference": 1,
1919
"function_anchor": 6,
2020
"name": "sum:opt_fp64"
2121
}
2222
},
2323
{
2424
"extension_function": {
25-
"extension_uri_reference": 1,
25+
"extension_urn_reference": 1,
2626
"function_anchor": 3,
2727
"name": "lt:fp64_fp64"
2828
}
2929
},
3030
{
3131
"extension_function": {
32-
"extension_uri_reference": 1,
32+
"extension_urn_reference": 1,
3333
"function_anchor": 7,
3434
"name": "is_not_null:fp64"
3535
}
3636
},
3737
{
3838
"extension_function": {
39-
"extension_uri_reference": 1,
39+
"extension_urn_reference": 1,
4040
"function_anchor": 1,
4141
"name": "and:bool_bool"
4242
}
4343
},
4444
{
4545
"extension_function": {
46-
"extension_uri_reference": 1,
46+
"extension_urn_reference": 1,
4747
"function_anchor": 2,
4848
"name": "gte:fp64_fp64"
4949
}
5050
},
5151
{
5252
"extension_function": {
53-
"extension_uri_reference": 1,
53+
"extension_urn_reference": 1,
5454
"function_anchor": 5,
5555
"name": "multiply:opt_fp64_fp64"
5656
}

‎cpp/velox/tests/data/substrait_virtualTable.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"extension_uris": [],
2+
"extension_urns": [],
33
"extensions": [],
44
"relations": [
55
{

‎gluten-substrait/src/main/resources/substrait/proto/substrait/extended_expression.proto‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import "substrait/plan.proto";
99
import "substrait/type.proto";
1010

1111
option csharp_namespace = "Substrait.Protobuf";
12-
option go_package = "github.com/substrait-io/substrait-go/proto";
12+
option go_package = "github.com/substrait-io/substrait-protobuf/go/substraitpb";
1313
option java_multiple_files = true;
1414
option java_package = "io.substrait.proto";
1515

@@ -25,12 +25,15 @@ message ExpressionReference {
2525
// Describe a set of operations to complete.
2626
// For compactness sake, identifiers are normalized at the plan level.
2727
message ExtendedExpression {
28+
reserved 1;
29+
2830
// Substrait version of the expression. Optional up to 0.17.0, required for later
2931
// versions.
3032
Version version = 7;
3133

32-
// a list of yaml specifications this expression may depend on
33-
repeated substrait.extensions.SimpleExtensionURI extension_uris = 1;
34+
// a list of extension specifications this expression may depend on,
35+
// referenced by Extension URN
36+
repeated substrait.extensions.SimpleExtensionURN extension_urns = 8;
3437

3538
// a list of extensions this expression may depend on
3639
repeated substrait.extensions.SimpleExtensionDeclaration extensions = 2;

‎gluten-substrait/src/main/resources/substrait/proto/substrait/extensions/extensions.proto‎

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,22 @@ package substrait.extensions;
66
import "google/protobuf/any.proto";
77

88
option csharp_namespace = "Substrait.Protobuf";
9-
option go_package = "github.com/substrait-io/substrait-go/proto/extensions";
9+
option go_package = "github.com/substrait-io/substrait-protobuf/go/substraitpb/extensions";
1010
option java_multiple_files = true;
1111
option java_package = "io.substrait.proto";
1212

13-
message SimpleExtensionURI {
13+
message SimpleExtensionURN {
1414
// A surrogate key used in the context of a single plan used to reference the
15-
// URI associated with an extension.
16-
uint32 extension_uri_anchor = 1;
15+
// URN associated with an extension.
16+
// 0 is a valid anchor/reference, but prefer non-zero values for ergonomics.
17+
uint32 extension_urn_anchor = 1;
1718

18-
// The URI where this extension YAML can be retrieved. This is the "namespace"
19-
// of this extension.
20-
string uri = 2;
19+
// The extension URN that uniquely identifies this extension. This must follow the
20+
// format extension:<OWNER>:<ID> and serves as the "namespace" of this extension.
21+
string urn = 2;
2122
}
2223

23-
// Describes a mapping between a specific extension entity and the uri where
24+
// Describes a mapping between a specific extension entity and the URN where
2425
// that extension can be found.
2526
message SimpleExtensionDeclaration {
2627
oneof mapping_type {
@@ -31,40 +32,47 @@ message SimpleExtensionDeclaration {
3132

3233
// Describes a Type
3334
message ExtensionType {
34-
// references the extension_uri_anchor defined for a specific extension URI.
35-
uint32 extension_uri_reference = 1;
35+
reserved 1;
36+
37+
// references the extension_urn_anchor defined for a specific extension URN.
38+
uint32 extension_urn_reference = 4;
3639

3740
// A surrogate key used in the context of a single plan to reference a
38-
// specific extension type
41+
// specific extension type.
42+
// 0 is a valid anchor/reference, but prefer non-zero values for ergonomics.
3943
uint32 type_anchor = 2;
4044

4145
// the name of the type in the defined extension YAML.
4246
string name = 3;
4347
}
4448

4549
message ExtensionTypeVariation {
46-
// references the extension_uri_anchor defined for a specific extension URI.
47-
uint32 extension_uri_reference = 1;
50+
reserved 1;
51+
52+
// references the extension_urn_anchor defined for a specific extension URN.
53+
uint32 extension_urn_reference = 4;
4854

4955
// A surrogate key used in the context of a single plan to reference a
50-
// specific type variation
56+
// specific type variation.
57+
// Use non-zero values; 0 is reserved for the system-preferred variation.
5158
uint32 type_variation_anchor = 2;
5259

5360
// the name of the type in the defined extension YAML.
5461
string name = 3;
5562
}
5663

5764
message ExtensionFunction {
58-
// references the extension_uri_anchor defined for a specific extension URI.
59-
uint32 extension_uri_reference = 1;
65+
reserved 1;
66+
67+
// references the extension_urn_anchor defined for a specific extension URN.
68+
uint32 extension_urn_reference = 4;
6069

6170
// A surrogate key used in the context of a single plan to reference a
62-
// specific function
71+
// specific function.
72+
// 0 is a valid anchor/reference, but prefer non-zero values for ergonomics.
6373
uint32 function_anchor = 2;
6474

65-
// A simple name if there is only one impl for the function within the YAML.
66-
// A compound name, referencing that includes type short names if there is
67-
// more than one impl per name in the YAML.
75+
// A function signature
6876
string name = 3;
6977
}
7078
}

‎gluten-substrait/src/main/resources/substrait/proto/substrait/plan.proto‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import "substrait/algebra.proto";
77
import "substrait/extensions/extensions.proto";
88

99
option csharp_namespace = "Substrait.Protobuf";
10-
option go_package = "github.com/substrait-io/substrait-go/proto";
10+
option go_package = "github.com/substrait-io/substrait-protobuf/go/substraitpb";
1111
option java_multiple_files = true;
1212
option java_package = "io.substrait.proto";
1313

@@ -24,12 +24,14 @@ message PlanRel {
2424
// Describe a set of operations to complete.
2525
// For compactness sake, identifiers are normalized at the plan level.
2626
message Plan {
27+
reserved 1;
28+
2729
// Substrait version of the plan. Optional up to 0.17.0, required for later
2830
// versions.
2931
Version version = 6;
3032

31-
// a list of yaml specifications this plan may depend on
32-
repeated substrait.extensions.SimpleExtensionURI extension_uris = 1;
33+
// a list of extension URNs this plan may depend on
34+
repeated substrait.extensions.SimpleExtensionURN extension_urns = 8;
3335

3436
// a list of extensions this plan may depend on
3537
repeated substrait.extensions.SimpleExtensionDeclaration extensions = 2;

0 commit comments

Comments
 (0)