feat: add complete python SDK and example

Problem: we will want to interact from Python
Solution: add a complete python SDK and client example.
I am also adding the gopher logo (branding).

Signed-off-by: vsoch <[email protected]>

.devcontainer/Dockerfile

@@ -27,9 +27,11 @@ RUN flux keygen
 RUN git clone https://github.com/flux-framework/flux-sched.git /opt/flux-sched
 
 # Go dependencies for protobuf
-RUN apt -y update && apt -y upgrade && apt install --no-install-recommends -y protobuf-compiler curl && \
-    go install google.golang.org/protobuf/cmd/[email protected] && \ 
-    go install google.golang.org/grpc/cmd/[email protected]
+RUN apt -y update && apt -y upgrade && apt install --no-install-recommends -y protobuf-compiler python3-pip python3-venv curl && \
+    go install google.golang.org/protobuf/cmd/[email protected] && \
+    go install google.golang.org/grpc/cmd/[email protected] && \
+    ln -s /usr/bin/python3 /usr/bin/python && \
+    python3 -m pip install grpcio-tools
 
 # These need to be on the LD_LIBRARY_PATH for the server to find at runtime
 ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/lib:/usr/lib/flux

.devcontainer/devcontainer.json

@@ -17,4 +17,4 @@
     }
   },
   "postStartCommand": "git config --global --add safe.directory /workspaces/fluxion"
-}
+}

.github/workflows/build-deploy.yaml

@@ -29,7 +29,7 @@ jobs:
       if: (github.event_name == 'release')
       run: |
         tag=${GITHUB_REF#refs/tags/}
-        echo "Tagging and releasing ${{ env.container}}:${tag}"        
+        echo "Tagging and releasing ${{ env.container}}:${tag}"
         docker tag ${{ env.container }}:latest ${{ env.container }}:${tag}
 
     - name: GHCR Login

.github/workflows/main.yaml

@@ -0,0 +1,26 @@
+name: test rainbow
+
+on:
+  pull_request: []
+
+jobs:
+  formatting:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Setup black linter
+      run: conda create --quiet --name black pyflakes
+
+    - name: Check Spelling
+      uses: crate-ci/typos@7ad296c72fa8265059cc03d1eda562fbdfcd6df2 # v1.9.0
+      with:
+        files: ./README.md
+
+    - name: Lint and format Python code
+      run: |
+        export PATH="/usr/share/miniconda/bin:$PATH"
+        source activate black
+        pip install -r .github/dev-requirements.txt
+        cd python/v1
+        pre-commit run --all-files

.gitignore

@@ -1,2 +1,5 @@
 vendor
 bin
+.eggs
+__pycache__
+env

Dockerfile

@@ -18,7 +18,7 @@ RUN git clone https://github.com/flux-framework/flux-sched.git /opt/flux-sched
 
 # Go dependencies for protobuf
 RUN apt -y update && apt -y upgrade && apt install --no-install-recommends -y protobuf-compiler curl && \
-    go install google.golang.org/protobuf/cmd/[email protected] && \ 
+    go install google.golang.org/protobuf/cmd/[email protected] && \
     go install google.golang.org/grpc/cmd/[email protected]
 
 # These need to be on the LD_LIBRARY_PATH for the server to find at runtime

LICENSE

@@ -199,4 +199,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

Makefile

@@ -18,19 +18,28 @@ $(LOCALBIN):
 	mkdir -p $(LOCALBIN)
 
 .PHONY: build
-build: 
+build:
 	docker build --build-arg ARCH="amd64" --build-arg RELEASE_VERSION="$(RELEASE_VERSION)" -t $(REGISTRY)/$(IMAGE) .
 
 .PHONY: server
-server: 
-	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o bin/server cmd/main.go
+server:
+	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o bin/server cmd/server/server.go
 
 .PHONY: protoc
 protoc: $(LOCALBIN)
 	GOBIN=$(LOCALBIN) go install google.golang.org/protobuf/cmd/[email protected]
 	GOBIN=$(LOCALBIN) go install google.golang.org/grpc/cmd/[email protected]
-	
+
 # You can use make protoc to download proto
 .PHONY: proto
 proto: protoc
 	PATH=$(LOCALBIN):${PATH} protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative pkg/fluxion-grpc/fluxion.proto
+
+.PHONY: python
+python: python ## Generate python proto files in python
+	# pip install grpcio-tools
+	# pip freeze | grep grpcio-tools
+	mkdir -p python/v1/fluxion/protos
+	cd python/v1/fluxion/protos
+	python -m grpc_tools.protoc -I./pkg/fluxion-grpc --python_out=./python/v1/fluxion/protos --pyi_out=./python/v1/fluxion/protos --grpc_python_out=./python/v1/fluxion/protos ./pkg/fluxion-grpc/fluxion.proto
+	sed -i 's/import fluxion_pb2 as fluxion__pb2/from . import fluxion_pb2 as fluxion__pb2/' ./python/v1/fluxion/protos/fluxion_pb2_grpc.py

README.md

@@ -1,16 +1,17 @@
 # Fluxion
 
-> Hierarchical Memory Graph Database
+> Hierarchical Memory Graph Database 🕶️
+
+![img/fluxion-banner.jpg](img/fluxion-banner.jpg)
 
 Fluxion is the [flux-sched](https://github.com/flux-framework/flux-sched) project packaged and provided as a containerized graph database. You can interact with it using any language that can interact with the gRPC endpoints. Example and client libraries are provided here.
 
-🚧️ **under development** 🚧️
 
 ## Usage
 
 ### Build
 
-If you have flux-sched locally you can build on your local machine, however it's recommended to use a VS Code developer environment with the included [.devcontainer](.devcontainer) directory. 
+If you have flux-sched locally you can build on your local machine, however it's recommended to use a VS Code developer environment with the included [.devcontainer](.devcontainer) directory.
 
 To build the container:
 
@@ -42,6 +43,8 @@ This is the fluxion graph server
 [GRPCServer] gRPC Listening on [::]:4242
 ```
 
+### Go Example
+
 In another terminal you can try one of the client examples in [examples](examples). For example:
 
 ```bash
@@ -83,28 +86,53 @@ GOOS=linux CGO_CFLAGS="-I/opt/flux-sched -I/opt/flux-sched/resource/reapi/bindin
 
 And that's it!
 
-### Run the Container
+### Container Example
 
 As an alternative, you can run the container service instead.
 
 ```bash
-docker run -p 51003 ghcr.io/converged-computing/fluxion --port 51003
+docker run -p 4242:4242 ghcr.io/converged-computing/fluxion  --host 0.0.0.0
+```
+
+And from a different terminal (make sure you've closed VSCode which sometimes can claim ports).
+
+```bash
+go run examples/go/example.go  --jobspec examples/go/jobspec.yaml
+```
+
+And you will get the same response as above.
+
+### Python Example
+
+Finally, we have the same gRPC endpoints exposed via Python. With the server running:
+
+```bash
+make server && ./bin/server
+```
+
+Run the Python example:
+
+```bash
+python ./examples/python/example.py
+```
+
+Note that the protocol buffers are built (or updated) for Python like:
+
+```bash
+make python
 ```
 
-More coming soon for how to use the container and Python examples!
+You shouldn't need to run this command unless you change something.
 
 ## TODO
 
-- create clients for Python
 - get grow "unpack" bindings into fluxion-go and add update here
-- cute gopher logo!
-- assess use cases for service grpc
 - when flux-sched supports JGF v2, upgrade here.
 
 ## Thank you
 
 This code is based off of the work done for [fluence](https://github.com/flux-framework/flux-k8s) by the same authors.
-The code base will change significantly with development.
+The code base has changed significantly with development.
 
 ## License
 

cmd/main.go → cmd/server/server.go

@@ -49,7 +49,7 @@ func main() {
 			MaxConnectionIdle: 5 * time.Minute,
 		}),
 	)
-	pb.RegisterFluxcliServiceServer(s, &flux)
+	pb.RegisterFluxionServiceServer(s, &flux)
 	fmt.Printf("[GRPCServer] gRPC Listening on %s\n", lis.Addr().String())
 	if err := s.Serve(lis); err != nil {
 		fmt.Printf("[GRPCServer] failed to serve: %v\n", err)

examples/python/example.py

@@ -0,0 +1,74 @@
+import argparse
+import os
+import sys
+
+from fluxion.protos import fluxion_pb2
+from fluxion.client import FluxionClient
+import fluxion.utils as utils
+import time
+
+# Config file from a few directories up
+here = os.path.abspath(os.path.dirname(__file__))
+root = here
+
+# examples directory
+root = os.path.dirname(root)
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(description="🦩️ Fluxion Python client example")
+    parser.add_argument(
+        "--host", help="host of fluxion graph database", default="localhost:4242"
+    )
+    parser.add_argument(
+        "--jobspec",
+        help="Path to example jobspec to use",
+        default=os.path.join(root, "go", "jobspec.yaml"),
+    )
+    parser.add_argument(
+        "--cluster-nodes",
+        help="Nodes to provide to initialize graph",
+        default=os.path.join(root, "go", "cluster-nodes.json"),
+    )
+    return parser
+
+
+def main():
+    parser = get_parser()
+    args, _ = parser.parse_known_args()
+    cli = FluxionClient(host=args.host)
+
+    # Read in the jobspec and nodes to raw text
+    jobspec = utils.read_file(args.jobspec)
+    nodes_jgf = utils.read_file(args.cluster_nodes)
+
+    # Step 1: Init the fluxion graph
+    response = cli.init(nodes_jgf)
+    if response.status == fluxion_pb2.InitResponse.ResultType.INIT_SUCCESS:
+        print("✅️ Init of Fluxion resource graph success!")
+    else:
+        sys.exit(f"Issue with init, return code {response.status}")
+
+    # Step 2: Do a match
+    response = cli.match(jobspec)
+    if response.status == fluxion_pb2.MatchResponse.ResultType.MATCH_SUCCESS:
+        print("✅️ Match of jobspec to Fluxion graph success!")
+    else:
+        sys.exit(f"Issue with match, return code {response.status}")
+
+    print("😴️ Sleeping for 3 seconds before cancel...")
+    time.sleep(3)
+
+    # Step 3: cancel the job
+    jobid = response.jobid
+    response = cli.cancel(jobid=jobid)
+    if response.status == fluxion_pb2.CancelResponse.ResultType.CANCEL_SUCCESS:
+        print(f"✅️ Cancel of jobid {jobid} success!")
+    else:
+        sys.exit(f"Issue with cancel, return code {response.status}")
+
+    print("👋️ That's all folks!")
+
+
+if __name__ == "__main__":
+    main()

go.mod

@@ -3,28 +3,20 @@ module github.com/converged-computing/fluxion
 go 1.21
 
 require (
-	github.com/compspec/jobspec-go v0.0.0-20240502225416-90dd9994d697
 	github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe
 	github.com/flux-framework/fluxion-go v0.32.1-0.20240420052153-909523c84ca2
 	github.com/pkg/errors v0.9.1
 	google.golang.org/grpc v1.38.0
 	google.golang.org/protobuf v1.26.0
-	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/klog/v2 v2.9.0
 )
 
 require (
-	github.com/go-logr/logr v0.4.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/kr/text v0.2.0 // indirect
-	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
-	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
 	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
-	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
 )
 
 replace (

go.sum

@@ -3,20 +3,15 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/compspec/jobspec-go v0.0.0-20240502225416-90dd9994d697 h1:06bMa7DR6/yN0KWYudwUHO4Vn60feZtRtngj2dMZmU8=
-github.com/compspec/jobspec-go v0.0.0-20240502225416-90dd9994d697/go.mod h1:BaJyxaOhESe2DD4lqBdwTEWOw0TaTZVJGPrFh6KyXQM=
 github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe h1:Tk//RW3uKn4A7N8gpHRXs+ZGlR7Fxkwh+4/Iml0GBV4=
 github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe/go.mod h1:+DhVyLXGVfBsfta4185jd33jqa94inshCcdvsXK2Irk=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/flux-framework/fluxion-go v0.32.1-0.20240420052153-909523c84ca2 h1:Yz/vVX0XfB2q51ZLh2p8YI5vphvv0rZF4PqtKPscvsY=
 github.com/flux-framework/fluxion-go v0.32.1-0.20240420052153-909523c84ca2/go.mod h1:jA5+kOSLxchFzixzYEvMAGjkXB5yszO/HxUwdhX/5/U=
-github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
-github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -40,18 +35,10 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
-github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
@@ -130,14 +117,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
-k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
-sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

pkg/client/client.go

@@ -15,7 +15,7 @@ import (
 type FluxionClient struct {
 	host       string
 	connection *grpc.ClientConn
-	service    pb.FluxcliServiceClient
+	service    pb.FluxionServiceClient
 }
 
 var _ Client = (*FluxionClient)(nil)
@@ -44,7 +44,7 @@ func NewClient(host string) (Client, error) {
 	}
 
 	c.connection = conn
-	c.service = pb.NewFluxcliServiceClient(conn)
+	c.service = pb.NewFluxionServiceClient(conn)
 	return c, nil
 }