Projection query DSL implementation (#394)

* chore: initial projection query DSL implementation

* feat: support complex queries

* feat: add support to multi arch images

* chore: update deps

* fix: projection repo start and some tests

* fix: add child spec to native projection adapter

* feat: added more tests

* fix: String.chars not implemented

* fix: convert to chars

* try fix

* handle tuple args

* fix: assert tests

* fix: formatting

* translate comments

* feat: add support to group by

* feat: added support to having clause

* refactor: fix some compilation warnings

* trying nimble_parsec to parse queries

* some progress in query dsl parser

* feat: create projection tables from protobuf data

* refactor: minor adjust

* feat: added dynamic table creator

* refactor: rename module

* working projection table with upsert and query

* refact

* refact

* merge

* querying and table creation working on projection actor initialization

* saving state after callback

* pagination to queries

* tests fix

* remove ununsed code

* fix

* fix test

* fix some errors

---------

Co-authored-by: Adriano Santos <[email protected]>
Co-authored-by: Elias <[email protected]>

Author: 3 people

.github/workflows/ci.yaml

@@ -67,6 +67,13 @@ jobs:
           cd ../../
 
       - name: Run tests spawn_statestores
+        run: |
+          cd spawn_statestores/statestores
+          mix deps.get
+          MIX_ENV=test PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name [email protected] -S mix test
+          cd ../../
+
+      - name: Run tests statestores_native
         run: |
           cd spawn_statestores/statestores_native
           mix deps.get

.github/workflows/release.yaml

@@ -59,8 +59,11 @@ jobs:
           file_glob: true
           body: ${{ steps.changelog.outputs.changes }}
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
@@ -73,16 +76,17 @@ jobs:
         run: echo ${{ secrets.GHCR_PAT }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
       - name: Build and push spawn proxy image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
+          platforms: linux/amd64,linux/arm64
           context: .
           file: ./Dockerfile-proxy
           push: true
           tags: ghcr.io/eigr/spawn-proxy:latest,ghcr.io/eigr/spawn-proxy:1.4.3
           github-token: ${{ secrets.GHCR_PAT }}
 
       - name: Build and push spawn operator image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile-operator

.tool-versions

@@ -1,3 +1,2 @@
-elixir 1.17.2
-erlang 26.1.2
-
+elixir 1.18
+erlang 27.2

Makefile

@@ -174,6 +174,9 @@ test-proxy:
 test-statestores_native:
 	cd spawn_statestores/statestores_native && MIX_ENV=test mix deps.get && MIX_ENV=test PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name [email protected] -S mix test
 
+test-statestores:
+	cd spawn_statestores/statestores && MIX_ENV=test mix deps.get && MIX_ENV=test PROXY_CLUSTER_STRATEGY=gossip PROXY_HTTP_PORT=9005 SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= elixir --name [email protected] -S mix test
+
 test-statestores_mariadb:
 	cd spawn_statestores/statestores_mariadb && \
 	MIX_ENV=test mix deps.get && \
@@ -344,8 +347,10 @@ run-sdk-local4:
 	PROXY_CLUSTER_STRATEGY=epmd \
 	SPAWN_USE_INTERNAL_NATS=true \
 	SPAWN_PUBSUB_ADAPTER=nats \
-	PROXY_DATABASE_TYPE=mariadb \
-	PROXY_DATABASE_PORT=3307 \
+	PROXY_DATABASE_TYPE=postgres \
+	PROXY_DATABASE_PORT=5432 \
+	PROXY_DATABASE_USERNAME=postgres \
+	PROXY_DATABASE_SECRET=postgres \
 	SPAWN_STATESTORE_KEY=3Jnb0hZiHIzHTOih7t2cTEPEpY98Tu1wvQkPfq/XwqE= \
 	SPAWN_SUPERVISORS_STATE_HANDOFF_CONTROLLER=nats \
 	iex --name [email protected] -S mix

compile-pb.sh

@@ -32,6 +32,7 @@ PROTOS=("
     priv/protos/spawn/actors/protocol.proto 
     priv/protos/spawn/actors/state.proto 
     priv/protos/spawn/actors/healthcheck.proto
+    priv/protos/spawn/actors/test.proto
 ")
 
 BASE_PATH=`pwd`

lib/_generated/spawn/actors/extensions.pb.ex

@@ -1,13 +1,15 @@
 defmodule Spawn.Actors.PbExtension do
   @moduledoc false
-  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+  use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3
 
   extend(Google.Protobuf.FieldOptions, :actor_id, 9999,
     optional: true,
     type: :bool,
     json_name: "actorId"
   )
 
+  extend(Google.Protobuf.FieldOptions, :searchable, 4_890_129, optional: true, type: :bool)
+
   extend(Google.Protobuf.MethodOptions, :view, 4_890_127,
     optional: true,
     type: Spawn.Actors.ActorViewOption
@@ -19,63 +21,9 @@ defmodule Spawn.Actors.PbExtension do
   )
 end
 
-defmodule Spawn.Actors.ActorViewOption do
-  @moduledoc false
-  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
-
-  def descriptor do
-    # credo:disable-for-next-line
-    %Google.Protobuf.DescriptorProto{
-      name: "ActorViewOption",
-      field: [
-        %Google.Protobuf.FieldDescriptorProto{
-          name: "query",
-          extendee: nil,
-          number: 1,
-          label: :LABEL_OPTIONAL,
-          type: :TYPE_STRING,
-          type_name: nil,
-          default_value: nil,
-          options: nil,
-          oneof_index: nil,
-          json_name: "query",
-          proto3_optional: nil,
-          __unknown_fields__: []
-        },
-        %Google.Protobuf.FieldDescriptorProto{
-          name: "map_to",
-          extendee: nil,
-          number: 2,
-          label: :LABEL_OPTIONAL,
-          type: :TYPE_STRING,
-          type_name: nil,
-          default_value: nil,
-          options: nil,
-          oneof_index: nil,
-          json_name: "mapTo",
-          proto3_optional: nil,
-          __unknown_fields__: []
-        }
-      ],
-      nested_type: [],
-      enum_type: [],
-      extension_range: [],
-      extension: [],
-      options: nil,
-      oneof_decl: [],
-      reserved_range: [],
-      reserved_name: [],
-      __unknown_fields__: []
-    }
-  end
-
-  field(:query, 1, type: :string)
-  field(:map_to, 2, type: :string, json_name: "mapTo")
-end
-
 defmodule Spawn.Actors.ActorOpts do
   @moduledoc false
-  use Protobuf, syntax: :proto3, protoc_gen_elixir_version: "0.13.0"
+  use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3
 
   def descriptor do
     # credo:disable-for-next-line
@@ -236,3 +184,72 @@ defmodule Spawn.Actors.ActorOpts do
   field(:subjects, 8, repeated: true, type: Spawn.Actors.ProjectionSubject)
   field(:kind, 9, type: Spawn.Actors.Kind, enum: true)
 end
+
+defmodule Spawn.Actors.ActorViewOption do
+  @moduledoc false
+  use Protobuf, protoc_gen_elixir_version: "0.13.0", syntax: :proto3
+
+  def descriptor do
+    # credo:disable-for-next-line
+    %Google.Protobuf.DescriptorProto{
+      name: "ActorViewOption",
+      field: [
+        %Google.Protobuf.FieldDescriptorProto{
+          name: "query",
+          extendee: nil,
+          number: 1,
+          label: :LABEL_OPTIONAL,
+          type: :TYPE_STRING,
+          type_name: nil,
+          default_value: nil,
+          options: nil,
+          oneof_index: nil,
+          json_name: "query",
+          proto3_optional: nil,
+          __unknown_fields__: []
+        },
+        %Google.Protobuf.FieldDescriptorProto{
+          name: "map_to",
+          extendee: nil,
+          number: 2,
+          label: :LABEL_OPTIONAL,
+          type: :TYPE_STRING,
+          type_name: nil,
+          default_value: nil,
+          options: nil,
+          oneof_index: nil,
+          json_name: "mapTo",
+          proto3_optional: nil,
+          __unknown_fields__: []
+        },
+        %Google.Protobuf.FieldDescriptorProto{
+          name: "page_size",
+          extendee: nil,
+          number: 3,
+          label: :LABEL_OPTIONAL,
+          type: :TYPE_INT32,
+          type_name: nil,
+          default_value: nil,
+          options: nil,
+          oneof_index: nil,
+          json_name: "pageSize",
+          proto3_optional: nil,
+          __unknown_fields__: []
+        }
+      ],
+      nested_type: [],
+      enum_type: [],
+      extension_range: [],
+      extension: [],
+      options: nil,
+      oneof_decl: [],
+      reserved_range: [],
+      reserved_name: [],
+      __unknown_fields__: []
+    }
+  end
+
+  field(:query, 1, type: :string)
+  field(:map_to, 2, type: :string, json_name: "mapTo")
+  field(:page_size, 3, type: :int32, json_name: "pageSize")
+end