diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 0896ec1..310fcec 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -27,9 +27,11 @@ RUN flux keygen RUN git clone https://github.com/flux-framework/flux-sched.git /opt/flux-sched # Go dependencies for protobuf -RUN apt -y update && apt -y upgrade && apt install --no-install-recommends -y protobuf-compiler curl && \ - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 && \ - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 +RUN apt -y update && apt -y upgrade && apt install --no-install-recommends -y protobuf-compiler python3-pip python3-venv curl && \ + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 && \ + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 && \ + ln -s /usr/bin/python3 /usr/bin/python && \ + python3 -m pip install grpcio-tools # These need to be on the LD_LIBRARY_PATH for the server to find at runtime ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/lib:/usr/lib/flux diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 134f001..f18aa23 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,4 +17,4 @@ } }, "postStartCommand": "git config --global --add safe.directory /workspaces/fluxion" -} \ No newline at end of file +} diff --git a/.github/workflows/build-deploy.yaml b/.github/workflows/build-deploy.yaml index 8fbc8ff..7f15c61 100644 --- a/.github/workflows/build-deploy.yaml +++ b/.github/workflows/build-deploy.yaml @@ -29,7 +29,7 @@ jobs: if: (github.event_name == 'release') run: | tag=${GITHUB_REF#refs/tags/} - echo "Tagging and releasing ${{ env.container}}:${tag}" + echo "Tagging and releasing ${{ env.container}}:${tag}" docker tag ${{ env.container }}:latest ${{ env.container }}:${tag} - name: GHCR Login diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..9f65304 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,26 @@ +name: test rainbow + +on: + pull_request: [] + +jobs: + formatting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup black linter + run: conda create --quiet --name black pyflakes + + - name: Check Spelling + uses: crate-ci/typos@7ad296c72fa8265059cc03d1eda562fbdfcd6df2 # v1.9.0 + with: + files: ./README.md + + - name: Lint and format Python code + run: | + export PATH="/usr/share/miniconda/bin:$PATH" + source activate black + pip install -r .github/dev-requirements.txt + cd python/v1 + pre-commit run --all-files diff --git a/.gitignore b/.gitignore index b873546..a79d091 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ vendor bin +.eggs +__pycache__ +env diff --git a/Dockerfile b/Dockerfile index 8991dff..d77f5de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN git clone https://github.com/flux-framework/flux-sched.git /opt/flux-sched # Go dependencies for protobuf RUN apt -y update && apt -y upgrade && apt install --no-install-recommends -y protobuf-compiler curl && \ - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 && \ + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 && \ go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 # These need to be on the LD_LIBRARY_PATH for the server to find at runtime diff --git a/LICENSE b/LICENSE index 7a4a3ea..d645695 100644 --- a/LICENSE +++ b/LICENSE @@ -199,4 +199,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. diff --git a/Makefile b/Makefile index 6b8c478..8ec86f3 100644 --- a/Makefile +++ b/Makefile @@ -18,19 +18,28 @@ $(LOCALBIN): mkdir -p $(LOCALBIN) .PHONY: build -build: +build: docker build --build-arg ARCH="amd64" --build-arg RELEASE_VERSION="$(RELEASE_VERSION)" -t $(REGISTRY)/$(IMAGE) . .PHONY: server -server: - $(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o bin/server cmd/main.go +server: + $(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o bin/server cmd/server/server.go .PHONY: protoc protoc: $(LOCALBIN) GOBIN=$(LOCALBIN) go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 GOBIN=$(LOCALBIN) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - + # You can use make protoc to download proto .PHONY: proto proto: protoc PATH=$(LOCALBIN):${PATH} protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative pkg/fluxion-grpc/fluxion.proto + +.PHONY: python +python: python ## Generate python proto files in python + # pip install grpcio-tools + # pip freeze | grep grpcio-tools + mkdir -p python/v1/fluxion/protos + cd python/v1/fluxion/protos + python -m grpc_tools.protoc -I./pkg/fluxion-grpc --python_out=./python/v1/fluxion/protos --pyi_out=./python/v1/fluxion/protos --grpc_python_out=./python/v1/fluxion/protos ./pkg/fluxion-grpc/fluxion.proto + sed -i 's/import fluxion_pb2 as fluxion__pb2/from . import fluxion_pb2 as fluxion__pb2/' ./python/v1/fluxion/protos/fluxion_pb2_grpc.py diff --git a/README.md b/README.md index 736796a..aebb60b 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,17 @@ # Fluxion -> Hierarchical Memory Graph Database +> Hierarchical Memory Graph Database đŸ•ļī¸ + +![img/fluxion-banner.jpg](img/fluxion-banner.jpg) Fluxion is the [flux-sched](https://github.com/flux-framework/flux-sched) project packaged and provided as a containerized graph database. You can interact with it using any language that can interact with the gRPC endpoints. Example and client libraries are provided here. -🚧ī¸ **under development** 🚧ī¸ ## Usage ### Build -If you have flux-sched locally you can build on your local machine, however it's recommended to use a VS Code developer environment with the included [.devcontainer](.devcontainer) directory. +If you have flux-sched locally you can build on your local machine, however it's recommended to use a VS Code developer environment with the included [.devcontainer](.devcontainer) directory. To build the container: @@ -42,6 +43,8 @@ This is the fluxion graph server [GRPCServer] gRPC Listening on [::]:4242 ``` +### Go Example + In another terminal you can try one of the client examples in [examples](examples). For example: ```bash @@ -83,28 +86,53 @@ GOOS=linux CGO_CFLAGS="-I/opt/flux-sched -I/opt/flux-sched/resource/reapi/bindin And that's it! -### Run the Container +### Container Example As an alternative, you can run the container service instead. ```bash -docker run -p 51003 ghcr.io/converged-computing/fluxion --port 51003 +docker run -p 4242:4242 ghcr.io/converged-computing/fluxion --host 0.0.0.0 +``` + +And from a different terminal (make sure you've closed VSCode which sometimes can claim ports). + +```bash +go run examples/go/example.go --jobspec examples/go/jobspec.yaml +``` + +And you will get the same response as above. + +### Python Example + +Finally, we have the same gRPC endpoints exposed via Python. With the server running: + +```bash +make server && ./bin/server +``` + +Run the Python example: + +```bash +python ./examples/python/example.py +``` + +Note that the protocol buffers are built (or updated) for Python like: + +```bash +make python ``` -More coming soon for how to use the container and Python examples! +You shouldn't need to run this command unless you change something. ## TODO -- create clients for Python - get grow "unpack" bindings into fluxion-go and add update here -- cute gopher logo! -- assess use cases for service grpc - when flux-sched supports JGF v2, upgrade here. ## Thank you This code is based off of the work done for [fluence](https://github.com/flux-framework/flux-k8s) by the same authors. -The code base will change significantly with development. +The code base has changed significantly with development. ## License diff --git a/cmd/main.go b/cmd/server/server.go similarity index 96% rename from cmd/main.go rename to cmd/server/server.go index 781d74f..60cea6e 100644 --- a/cmd/main.go +++ b/cmd/server/server.go @@ -49,7 +49,7 @@ func main() { MaxConnectionIdle: 5 * time.Minute, }), ) - pb.RegisterFluxcliServiceServer(s, &flux) + pb.RegisterFluxionServiceServer(s, &flux) fmt.Printf("[GRPCServer] gRPC Listening on %s\n", lis.Addr().String()) if err := s.Serve(lis); err != nil { fmt.Printf("[GRPCServer] failed to serve: %v\n", err) diff --git a/examples/python/example.py b/examples/python/example.py new file mode 100644 index 0000000..df0cb51 --- /dev/null +++ b/examples/python/example.py @@ -0,0 +1,74 @@ +import argparse +import os +import sys + +from fluxion.protos import fluxion_pb2 +from fluxion.client import FluxionClient +import fluxion.utils as utils +import time + +# Config file from a few directories up +here = os.path.abspath(os.path.dirname(__file__)) +root = here + +# examples directory +root = os.path.dirname(root) + + +def get_parser(): + parser = argparse.ArgumentParser(description="đŸĻŠī¸ Fluxion Python client example") + parser.add_argument( + "--host", help="host of fluxion graph database", default="localhost:4242" + ) + parser.add_argument( + "--jobspec", + help="Path to example jobspec to use", + default=os.path.join(root, "go", "jobspec.yaml"), + ) + parser.add_argument( + "--cluster-nodes", + help="Nodes to provide to initialize graph", + default=os.path.join(root, "go", "cluster-nodes.json"), + ) + return parser + + +def main(): + parser = get_parser() + args, _ = parser.parse_known_args() + cli = FluxionClient(host=args.host) + + # Read in the jobspec and nodes to raw text + jobspec = utils.read_file(args.jobspec) + nodes_jgf = utils.read_file(args.cluster_nodes) + + # Step 1: Init the fluxion graph + response = cli.init(nodes_jgf) + if response.status == fluxion_pb2.InitResponse.ResultType.INIT_SUCCESS: + print("✅ī¸ Init of Fluxion resource graph success!") + else: + sys.exit(f"Issue with init, return code {response.status}") + + # Step 2: Do a match + response = cli.match(jobspec) + if response.status == fluxion_pb2.MatchResponse.ResultType.MATCH_SUCCESS: + print("✅ī¸ Match of jobspec to Fluxion graph success!") + else: + sys.exit(f"Issue with match, return code {response.status}") + + print("😴ī¸ Sleeping for 3 seconds before cancel...") + time.sleep(3) + + # Step 3: cancel the job + jobid = response.jobid + response = cli.cancel(jobid=jobid) + if response.status == fluxion_pb2.CancelResponse.ResultType.CANCEL_SUCCESS: + print(f"✅ī¸ Cancel of jobid {jobid} success!") + else: + sys.exit(f"Issue with cancel, return code {response.status}") + + print("👋ī¸ That's all folks!") + + +if __name__ == "__main__": + main() diff --git a/go.mod b/go.mod index 86e2bc6..a6146b4 100644 --- a/go.mod +++ b/go.mod @@ -3,28 +3,20 @@ module github.com/converged-computing/fluxion go 1.21 require ( - github.com/compspec/jobspec-go v0.0.0-20240502225416-90dd9994d697 github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe github.com/flux-framework/fluxion-go v0.32.1-0.20240420052153-909523c84ca2 github.com/pkg/errors v0.9.1 google.golang.org/grpc v1.38.0 google.golang.org/protobuf v1.26.0 - gopkg.in/yaml.v2 v2.4.0 - k8s.io/klog/v2 v2.9.0 ) require ( - github.com/go-logr/logr v0.4.0 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect + github.com/google/go-cmp v0.5.9 // indirect golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect golang.org/x/text v0.3.6 // indirect google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect - gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect - sigs.k8s.io/yaml v1.4.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index 4d1c85a..13e385c 100644 --- a/go.sum +++ b/go.sum @@ -3,11 +3,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/compspec/jobspec-go v0.0.0-20240502225416-90dd9994d697 h1:06bMa7DR6/yN0KWYudwUHO4Vn60feZtRtngj2dMZmU8= -github.com/compspec/jobspec-go v0.0.0-20240502225416-90dd9994d697/go.mod h1:BaJyxaOhESe2DD4lqBdwTEWOw0TaTZVJGPrFh6KyXQM= github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe h1:Tk//RW3uKn4A7N8gpHRXs+ZGlR7Fxkwh+4/Iml0GBV4= github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe/go.mod h1:+DhVyLXGVfBsfta4185jd33jqa94inshCcdvsXK2Irk= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -15,8 +12,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/flux-framework/fluxion-go v0.32.1-0.20240420052153-909523c84ca2 h1:Yz/vVX0XfB2q51ZLh2p8YI5vphvv0rZF4PqtKPscvsY= github.com/flux-framework/fluxion-go v0.32.1-0.20240420052153-909523c84ca2/go.mod h1:jA5+kOSLxchFzixzYEvMAGjkXB5yszO/HxUwdhX/5/U= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -40,18 +35,10 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -130,14 +117,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/img/fluxion-banner.jpg b/img/fluxion-banner.jpg new file mode 100644 index 0000000..fd27b44 Binary files /dev/null and b/img/fluxion-banner.jpg differ diff --git a/img/fluxion-banner.pdf b/img/fluxion-banner.pdf new file mode 100644 index 0000000..8a465a3 Binary files /dev/null and b/img/fluxion-banner.pdf differ diff --git a/img/fluxion-banner.png b/img/fluxion-banner.png new file mode 100644 index 0000000..63124fa Binary files /dev/null and b/img/fluxion-banner.png differ diff --git a/img/fluxion-banner.webp b/img/fluxion-banner.webp new file mode 100644 index 0000000..2b7672a Binary files /dev/null and b/img/fluxion-banner.webp differ diff --git a/img/fluxion-small.png b/img/fluxion-small.png new file mode 100644 index 0000000..f54600c Binary files /dev/null and b/img/fluxion-small.png differ diff --git a/img/fluxion.png b/img/fluxion.png new file mode 100644 index 0000000..a16e2ab Binary files /dev/null and b/img/fluxion.png differ diff --git a/img/fluxion.xcf b/img/fluxion.xcf new file mode 100644 index 0000000..4db52e0 Binary files /dev/null and b/img/fluxion.xcf differ diff --git a/pkg/client/client.go b/pkg/client/client.go index 1ecc7dd..ea3646b 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -15,7 +15,7 @@ import ( type FluxionClient struct { host string connection *grpc.ClientConn - service pb.FluxcliServiceClient + service pb.FluxionServiceClient } var _ Client = (*FluxionClient)(nil) @@ -44,7 +44,7 @@ func NewClient(host string) (Client, error) { } c.connection = conn - c.service = pb.NewFluxcliServiceClient(conn) + c.service = pb.NewFluxionServiceClient(conn) return c, nil } diff --git a/pkg/fluxion-grpc/fluxion.pb.go b/pkg/fluxion-grpc/fluxion.pb.go index f94894a..0877f56 100644 --- a/pkg/fluxion-grpc/fluxion.pb.go +++ b/pkg/fluxion-grpc/fluxion.pb.go @@ -592,7 +592,7 @@ var file_pkg_fluxion_grpc_fluxion_proto_rawDesc = []byte{ 0x52, 0x4f, 0x52, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x5f, 0x44, 0x45, 0x4e, 0x49, 0x45, 0x44, 0x10, 0x03, 0x32, 0xbe, 0x01, 0x0a, 0x0e, 0x46, - 0x6c, 0x75, 0x78, 0x63, 0x6c, 0x69, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x38, 0x0a, + 0x6c, 0x75, 0x78, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x38, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x15, 0x2e, 0x66, 0x6c, 0x75, 0x78, 0x69, 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x66, 0x6c, 0x75, 0x78, 0x69, 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, @@ -639,12 +639,12 @@ var file_pkg_fluxion_grpc_fluxion_proto_depIdxs = []int32{ 0, // 0: fluxion.InitResponse.status:type_name -> fluxion.InitResponse.ResultType 1, // 1: fluxion.MatchResponse.status:type_name -> fluxion.MatchResponse.ResultType 2, // 2: fluxion.CancelResponse.status:type_name -> fluxion.CancelResponse.ResultType - 5, // 3: fluxion.FluxcliService.Match:input_type -> fluxion.MatchRequest - 7, // 4: fluxion.FluxcliService.Cancel:input_type -> fluxion.CancelRequest - 3, // 5: fluxion.FluxcliService.Init:input_type -> fluxion.InitRequest - 6, // 6: fluxion.FluxcliService.Match:output_type -> fluxion.MatchResponse - 8, // 7: fluxion.FluxcliService.Cancel:output_type -> fluxion.CancelResponse - 4, // 8: fluxion.FluxcliService.Init:output_type -> fluxion.InitResponse + 5, // 3: fluxion.FluxionService.Match:input_type -> fluxion.MatchRequest + 7, // 4: fluxion.FluxionService.Cancel:input_type -> fluxion.CancelRequest + 3, // 5: fluxion.FluxionService.Init:input_type -> fluxion.InitRequest + 6, // 6: fluxion.FluxionService.Match:output_type -> fluxion.MatchResponse + 8, // 7: fluxion.FluxionService.Cancel:output_type -> fluxion.CancelResponse + 4, // 8: fluxion.FluxionService.Init:output_type -> fluxion.InitResponse 6, // [6:9] is the sub-list for method output_type 3, // [3:6] is the sub-list for method input_type 3, // [3:3] is the sub-list for extension type_name diff --git a/pkg/fluxion-grpc/fluxion.proto b/pkg/fluxion-grpc/fluxion.proto index 3627495..b9f260e 100644 --- a/pkg/fluxion-grpc/fluxion.proto +++ b/pkg/fluxion-grpc/fluxion.proto @@ -5,7 +5,7 @@ option go_package = "github.com/converged-computing/fluxion/pkg/fluxion-grpc"; package fluxion; // Service definition for Fluxclient -service FluxcliService { +service FluxionService { // Sends a Match command rpc Match(MatchRequest) returns (MatchResponse) {} rpc Cancel(CancelRequest) returns (CancelResponse) {} @@ -45,7 +45,7 @@ message MatchResponse { MATCH_ERROR = 1; MATCH_DENIED = 2; } - ResultType status = 6; + ResultType status = 6; } message CancelRequest { @@ -63,7 +63,5 @@ message CancelResponse { CANCEL_ERROR = 2; CANCEL_DENIED = 3; } - ResultType status = 3; + ResultType status = 3; } - - diff --git a/pkg/fluxion-grpc/fluxion_grpc.pb.go b/pkg/fluxion-grpc/fluxion_grpc.pb.go index 8aab63e..ad80727 100644 --- a/pkg/fluxion-grpc/fluxion_grpc.pb.go +++ b/pkg/fluxion-grpc/fluxion_grpc.pb.go @@ -18,160 +18,160 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -// FluxcliServiceClient is the client API for FluxcliService service. +// FluxionServiceClient is the client API for FluxionService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type FluxcliServiceClient interface { +type FluxionServiceClient interface { // Sends a Match command Match(ctx context.Context, in *MatchRequest, opts ...grpc.CallOption) (*MatchResponse, error) Cancel(ctx context.Context, in *CancelRequest, opts ...grpc.CallOption) (*CancelResponse, error) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*InitResponse, error) } -type fluxcliServiceClient struct { +type fluxionServiceClient struct { cc grpc.ClientConnInterface } -func NewFluxcliServiceClient(cc grpc.ClientConnInterface) FluxcliServiceClient { - return &fluxcliServiceClient{cc} +func NewFluxionServiceClient(cc grpc.ClientConnInterface) FluxionServiceClient { + return &fluxionServiceClient{cc} } -func (c *fluxcliServiceClient) Match(ctx context.Context, in *MatchRequest, opts ...grpc.CallOption) (*MatchResponse, error) { +func (c *fluxionServiceClient) Match(ctx context.Context, in *MatchRequest, opts ...grpc.CallOption) (*MatchResponse, error) { out := new(MatchResponse) - err := c.cc.Invoke(ctx, "/fluxion.FluxcliService/Match", in, out, opts...) + err := c.cc.Invoke(ctx, "/fluxion.FluxionService/Match", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *fluxcliServiceClient) Cancel(ctx context.Context, in *CancelRequest, opts ...grpc.CallOption) (*CancelResponse, error) { +func (c *fluxionServiceClient) Cancel(ctx context.Context, in *CancelRequest, opts ...grpc.CallOption) (*CancelResponse, error) { out := new(CancelResponse) - err := c.cc.Invoke(ctx, "/fluxion.FluxcliService/Cancel", in, out, opts...) + err := c.cc.Invoke(ctx, "/fluxion.FluxionService/Cancel", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *fluxcliServiceClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*InitResponse, error) { +func (c *fluxionServiceClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*InitResponse, error) { out := new(InitResponse) - err := c.cc.Invoke(ctx, "/fluxion.FluxcliService/Init", in, out, opts...) + err := c.cc.Invoke(ctx, "/fluxion.FluxionService/Init", in, out, opts...) if err != nil { return nil, err } return out, nil } -// FluxcliServiceServer is the server API for FluxcliService service. -// All implementations must embed UnimplementedFluxcliServiceServer +// FluxionServiceServer is the server API for FluxionService service. +// All implementations must embed UnimplementedFluxionServiceServer // for forward compatibility -type FluxcliServiceServer interface { +type FluxionServiceServer interface { // Sends a Match command Match(context.Context, *MatchRequest) (*MatchResponse, error) Cancel(context.Context, *CancelRequest) (*CancelResponse, error) Init(context.Context, *InitRequest) (*InitResponse, error) - mustEmbedUnimplementedFluxcliServiceServer() + mustEmbedUnimplementedFluxionServiceServer() } -// UnimplementedFluxcliServiceServer must be embedded to have forward compatible implementations. -type UnimplementedFluxcliServiceServer struct { +// UnimplementedFluxionServiceServer must be embedded to have forward compatible implementations. +type UnimplementedFluxionServiceServer struct { } -func (UnimplementedFluxcliServiceServer) Match(context.Context, *MatchRequest) (*MatchResponse, error) { +func (UnimplementedFluxionServiceServer) Match(context.Context, *MatchRequest) (*MatchResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Match not implemented") } -func (UnimplementedFluxcliServiceServer) Cancel(context.Context, *CancelRequest) (*CancelResponse, error) { +func (UnimplementedFluxionServiceServer) Cancel(context.Context, *CancelRequest) (*CancelResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Cancel not implemented") } -func (UnimplementedFluxcliServiceServer) Init(context.Context, *InitRequest) (*InitResponse, error) { +func (UnimplementedFluxionServiceServer) Init(context.Context, *InitRequest) (*InitResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Init not implemented") } -func (UnimplementedFluxcliServiceServer) mustEmbedUnimplementedFluxcliServiceServer() {} +func (UnimplementedFluxionServiceServer) mustEmbedUnimplementedFluxionServiceServer() {} -// UnsafeFluxcliServiceServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to FluxcliServiceServer will +// UnsafeFluxionServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to FluxionServiceServer will // result in compilation errors. -type UnsafeFluxcliServiceServer interface { - mustEmbedUnimplementedFluxcliServiceServer() +type UnsafeFluxionServiceServer interface { + mustEmbedUnimplementedFluxionServiceServer() } -func RegisterFluxcliServiceServer(s grpc.ServiceRegistrar, srv FluxcliServiceServer) { - s.RegisterService(&FluxcliService_ServiceDesc, srv) +func RegisterFluxionServiceServer(s grpc.ServiceRegistrar, srv FluxionServiceServer) { + s.RegisterService(&FluxionService_ServiceDesc, srv) } -func _FluxcliService_Match_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _FluxionService_Match_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(MatchRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(FluxcliServiceServer).Match(ctx, in) + return srv.(FluxionServiceServer).Match(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/fluxion.FluxcliService/Match", + FullMethod: "/fluxion.FluxionService/Match", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(FluxcliServiceServer).Match(ctx, req.(*MatchRequest)) + return srv.(FluxionServiceServer).Match(ctx, req.(*MatchRequest)) } return interceptor(ctx, in, info, handler) } -func _FluxcliService_Cancel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _FluxionService_Cancel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CancelRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(FluxcliServiceServer).Cancel(ctx, in) + return srv.(FluxionServiceServer).Cancel(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/fluxion.FluxcliService/Cancel", + FullMethod: "/fluxion.FluxionService/Cancel", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(FluxcliServiceServer).Cancel(ctx, req.(*CancelRequest)) + return srv.(FluxionServiceServer).Cancel(ctx, req.(*CancelRequest)) } return interceptor(ctx, in, info, handler) } -func _FluxcliService_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _FluxionService_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(InitRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(FluxcliServiceServer).Init(ctx, in) + return srv.(FluxionServiceServer).Init(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/fluxion.FluxcliService/Init", + FullMethod: "/fluxion.FluxionService/Init", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(FluxcliServiceServer).Init(ctx, req.(*InitRequest)) + return srv.(FluxionServiceServer).Init(ctx, req.(*InitRequest)) } return interceptor(ctx, in, info, handler) } -// FluxcliService_ServiceDesc is the grpc.ServiceDesc for FluxcliService service. +// FluxionService_ServiceDesc is the grpc.ServiceDesc for FluxionService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) -var FluxcliService_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "fluxion.FluxcliService", - HandlerType: (*FluxcliServiceServer)(nil), +var FluxionService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "fluxion.FluxionService", + HandlerType: (*FluxionServiceServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "Match", - Handler: _FluxcliService_Match_Handler, + Handler: _FluxionService_Match_Handler, }, { MethodName: "Cancel", - Handler: _FluxcliService_Cancel_Handler, + Handler: _FluxionService_Cancel_Handler, }, { MethodName: "Init", - Handler: _FluxcliService_Init_Handler, + Handler: _FluxionService_Init_Handler, }, }, Streams: []grpc.StreamDesc{}, diff --git a/pkg/fluxion/fluxion.go b/pkg/fluxion/fluxion.go index c1ba2e7..8fd260d 100644 --- a/pkg/fluxion/fluxion.go +++ b/pkg/fluxion/fluxion.go @@ -13,7 +13,7 @@ import ( type Fluxion struct { cli *fluxcli.ReapiClient - pb.UnimplementedFluxcliServiceServer + pb.UnimplementedFluxionServiceServer } // Init creates a new client to interaction with the fluxion API diff --git a/python/v1/.pre-commit-config.yaml b/python/v1/.pre-commit-config.yaml new file mode 100644 index 0000000..142c17d --- /dev/null +++ b/python/v1/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +exclude: "examples" +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-added-large-files + args: ["--maxkb=2000"] + - id: check-case-conflict + - id: check-docstring-first + - id: end-of-file-fixer + - id: trailing-whitespace + - id: mixed-line-ending + + - repo: local + hooks: + - id: black + name: black + language: python + types: [python] + entry: black + + - id: isort + name: isort + args: [--filter-files] + language: python + types: [python] + entry: isort diff --git a/python/v1/MANIFEST.in b/python/v1/MANIFEST.in new file mode 100644 index 0000000..7dcbf99 --- /dev/null +++ b/python/v1/MANIFEST.in @@ -0,0 +1,6 @@ +recursive-include fluxion * +prune env* +global-exclude .env +global-exclude *.py[co] +recursive-exclude .git * +global-exclude __pycache__ diff --git a/python/v1/fluxion/__init__.py b/python/v1/fluxion/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/v1/fluxion/client.py b/python/v1/fluxion/client.py new file mode 100644 index 0000000..5c5e982 --- /dev/null +++ b/python/v1/fluxion/client.py @@ -0,0 +1,64 @@ +import json +import os + +import grpc + +import fluxion.utils as utils +from fluxion.protos import fluxion_pb2, fluxion_pb2_grpc + + +class FluxionClient: + """ + A FluxionClient is able to interact with a Fluxion cluster + """ + + def __init__(self, host="localhost:4242"): + """ + Create a new rainbow client to interact with a rainbow cluster. + """ + self.host = host + + def match(self, jobspec, request="allocate"): + """ + Run a match for a jobspec + """ + if os.path.exists(jobspec): + jobspec = utils.read_file(jobspec) + + match_request = fluxion_pb2.MatchRequest(jobspec=jobspec, request=request, count=1) + with grpc.insecure_channel(self.host) as channel: + stub = fluxion_pb2_grpc.FluxionServiceStub(channel) + response = stub.Match(match_request) + return response + + def init(self, cluster_nodes): + """ + Init Fluxion with a cluster nodes graph. + """ + # Be forgiving if a filename is provided - try to read in + if os.path.exists(cluster_nodes): + cluster_nodes = utils.read_file(cluster_nodes) + + if not cluster_nodes: + raise ValueError("Loaded cluster nodes are required") + + # These are the variables for our cluster - name for now + request = fluxion_pb2.InitRequest(jgf=cluster_nodes) + with grpc.insecure_channel(self.host) as channel: + stub = fluxion_pb2_grpc.FluxionServiceStub(channel) + response = stub.Init(request) + return response + + def cancel(self, jobid): + """ + Cancel a job + """ + if not jobid: + raise ValueError("A jobid is required") + + # These are the variables for our cluster - name for now + request = fluxion_pb2.CancelRequest(jobID=jobid) + with grpc.insecure_channel(self.host) as channel: + stub = fluxion_pb2_grpc.FluxionServiceStub(channel) + response = stub.Cancel(request) + return response diff --git a/python/v1/fluxion/protos/fluxion_pb2.py b/python/v1/fluxion/protos/fluxion_pb2.py new file mode 100644 index 0000000..a89ffc0 --- /dev/null +++ b/python/v1/fluxion/protos/fluxion_pb2.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: fluxion.proto +# Protobuf Python Version: 5.26.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\rfluxion.proto\x12\x07\x66luxion"*\n\x0bInitRequest\x12\x0e\n\x06policy\x18\x01 \x01(\t\x12\x0b\n\x03jgf\x18\x02 \x01(\t"\x81\x01\n\x0cInitResponse\x12\x30\n\x06status\x18\x01 \x01(\x0e\x32 .fluxion.InitResponse.ResultType"?\n\nResultType\x12\x10\n\x0cINIT_SUCCESS\x10\x00\x12\x0e\n\nINIT_ERROR\x10\x01\x12\x0f\n\x0bINIT_DENIED\x10\x02"?\n\x0cMatchRequest\x12\x0f\n\x07jobspec\x18\x01 \x01(\t\x12\x0f\n\x07request\x18\x02 \x01(\t\x12\r\n\x05\x63ount\x18\x03 \x01(\x05"\xd9\x01\n\rMatchResponse\x12\x12\n\nallocation\x18\x01 \x01(\t\x12\r\n\x05jobid\x18\x02 \x01(\x03\x12\x10\n\x08reserved\x18\x03 \x01(\x08\x12\n\n\x02\x61t\x18\x04 \x01(\x03\x12\x10\n\x08overhead\x18\x05 \x01(\x02\x12\x31\n\x06status\x18\x06 \x01(\x0e\x32!.fluxion.MatchResponse.ResultType"B\n\nResultType\x12\x11\n\rMATCH_SUCCESS\x10\x00\x12\x0f\n\x0bMATCH_ERROR\x10\x01\x12\x10\n\x0cMATCH_DENIED\x10\x02"\x1e\n\rCancelRequest\x12\r\n\x05jobID\x18\x02 \x01(\x03"\xc3\x01\n\x0e\x43\x61ncelResponse\x12\r\n\x05jobID\x18\x01 \x01(\x03\x12\r\n\x05\x65rror\x18\x02 \x01(\x05\x12\x32\n\x06status\x18\x03 \x01(\x0e\x32".fluxion.CancelResponse.ResultType"_\n\nResultType\x12\x12\n\x0e\x43\x41NCEL_SUCCESS\x10\x00\x12\x18\n\x14\x43\x41NCEL_REQUEST_ERROR\x10\x01\x12\x10\n\x0c\x43\x41NCEL_ERROR\x10\x02\x12\x11\n\rCANCEL_DENIED\x10\x03\x32\xbe\x01\n\x0e\x46luxionService\x12\x38\n\x05Match\x12\x15.fluxion.MatchRequest\x1a\x16.fluxion.MatchResponse"\x00\x12;\n\x06\x43\x61ncel\x12\x16.fluxion.CancelRequest\x1a\x17.fluxion.CancelResponse"\x00\x12\x35\n\x04Init\x12\x14.fluxion.InitRequest\x1a\x15.fluxion.InitResponse"\x00\x42\x39Z7github.com/converged-computing/fluxion/pkg/fluxion-grpcb\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "fluxion_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals[ + "DESCRIPTOR" + ]._serialized_options = b"Z7github.com/converged-computing/fluxion/pkg/fluxion-grpc" + _globals["_INITREQUEST"]._serialized_start = 26 + _globals["_INITREQUEST"]._serialized_end = 68 + _globals["_INITRESPONSE"]._serialized_start = 71 + _globals["_INITRESPONSE"]._serialized_end = 200 + _globals["_INITRESPONSE_RESULTTYPE"]._serialized_start = 137 + _globals["_INITRESPONSE_RESULTTYPE"]._serialized_end = 200 + _globals["_MATCHREQUEST"]._serialized_start = 202 + _globals["_MATCHREQUEST"]._serialized_end = 265 + _globals["_MATCHRESPONSE"]._serialized_start = 268 + _globals["_MATCHRESPONSE"]._serialized_end = 485 + _globals["_MATCHRESPONSE_RESULTTYPE"]._serialized_start = 419 + _globals["_MATCHRESPONSE_RESULTTYPE"]._serialized_end = 485 + _globals["_CANCELREQUEST"]._serialized_start = 487 + _globals["_CANCELREQUEST"]._serialized_end = 517 + _globals["_CANCELRESPONSE"]._serialized_start = 520 + _globals["_CANCELRESPONSE"]._serialized_end = 715 + _globals["_CANCELRESPONSE_RESULTTYPE"]._serialized_start = 620 + _globals["_CANCELRESPONSE_RESULTTYPE"]._serialized_end = 715 + _globals["_FLUXIONSERVICE"]._serialized_start = 718 + _globals["_FLUXIONSERVICE"]._serialized_end = 908 +# @@protoc_insertion_point(module_scope) diff --git a/python/v1/fluxion/protos/fluxion_pb2.pyi b/python/v1/fluxion/protos/fluxion_pb2.pyi new file mode 100644 index 0000000..67a0fd2 --- /dev/null +++ b/python/v1/fluxion/protos/fluxion_pb2.pyi @@ -0,0 +1,88 @@ +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class InitRequest(_message.Message): + __slots__ = ("policy", "jgf") + POLICY_FIELD_NUMBER: _ClassVar[int] + JGF_FIELD_NUMBER: _ClassVar[int] + policy: str + jgf: str + def __init__(self, policy: _Optional[str] = ..., jgf: _Optional[str] = ...) -> None: ... + +class InitResponse(_message.Message): + __slots__ = ("status",) + class ResultType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + INIT_SUCCESS: _ClassVar[InitResponse.ResultType] + INIT_ERROR: _ClassVar[InitResponse.ResultType] + INIT_DENIED: _ClassVar[InitResponse.ResultType] + INIT_SUCCESS: InitResponse.ResultType + INIT_ERROR: InitResponse.ResultType + INIT_DENIED: InitResponse.ResultType + STATUS_FIELD_NUMBER: _ClassVar[int] + status: InitResponse.ResultType + def __init__(self, status: _Optional[_Union[InitResponse.ResultType, str]] = ...) -> None: ... + +class MatchRequest(_message.Message): + __slots__ = ("jobspec", "request", "count") + JOBSPEC_FIELD_NUMBER: _ClassVar[int] + REQUEST_FIELD_NUMBER: _ClassVar[int] + COUNT_FIELD_NUMBER: _ClassVar[int] + jobspec: str + request: str + count: int + def __init__(self, jobspec: _Optional[str] = ..., request: _Optional[str] = ..., count: _Optional[int] = ...) -> None: ... + +class MatchResponse(_message.Message): + __slots__ = ("allocation", "jobid", "reserved", "at", "overhead", "status") + class ResultType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + MATCH_SUCCESS: _ClassVar[MatchResponse.ResultType] + MATCH_ERROR: _ClassVar[MatchResponse.ResultType] + MATCH_DENIED: _ClassVar[MatchResponse.ResultType] + MATCH_SUCCESS: MatchResponse.ResultType + MATCH_ERROR: MatchResponse.ResultType + MATCH_DENIED: MatchResponse.ResultType + ALLOCATION_FIELD_NUMBER: _ClassVar[int] + JOBID_FIELD_NUMBER: _ClassVar[int] + RESERVED_FIELD_NUMBER: _ClassVar[int] + AT_FIELD_NUMBER: _ClassVar[int] + OVERHEAD_FIELD_NUMBER: _ClassVar[int] + STATUS_FIELD_NUMBER: _ClassVar[int] + allocation: str + jobid: int + reserved: bool + at: int + overhead: float + status: MatchResponse.ResultType + def __init__(self, allocation: _Optional[str] = ..., jobid: _Optional[int] = ..., reserved: bool = ..., at: _Optional[int] = ..., overhead: _Optional[float] = ..., status: _Optional[_Union[MatchResponse.ResultType, str]] = ...) -> None: ... + +class CancelRequest(_message.Message): + __slots__ = ("jobID",) + JOBID_FIELD_NUMBER: _ClassVar[int] + jobID: int + def __init__(self, jobID: _Optional[int] = ...) -> None: ... + +class CancelResponse(_message.Message): + __slots__ = ("jobID", "error", "status") + class ResultType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + CANCEL_SUCCESS: _ClassVar[CancelResponse.ResultType] + CANCEL_REQUEST_ERROR: _ClassVar[CancelResponse.ResultType] + CANCEL_ERROR: _ClassVar[CancelResponse.ResultType] + CANCEL_DENIED: _ClassVar[CancelResponse.ResultType] + CANCEL_SUCCESS: CancelResponse.ResultType + CANCEL_REQUEST_ERROR: CancelResponse.ResultType + CANCEL_ERROR: CancelResponse.ResultType + CANCEL_DENIED: CancelResponse.ResultType + JOBID_FIELD_NUMBER: _ClassVar[int] + ERROR_FIELD_NUMBER: _ClassVar[int] + STATUS_FIELD_NUMBER: _ClassVar[int] + jobID: int + error: int + status: CancelResponse.ResultType + def __init__(self, jobID: _Optional[int] = ..., error: _Optional[int] = ..., status: _Optional[_Union[CancelResponse.ResultType, str]] = ...) -> None: ... diff --git a/python/v1/fluxion/protos/fluxion_pb2_grpc.py b/python/v1/fluxion/protos/fluxion_pb2_grpc.py new file mode 100644 index 0000000..bd696de --- /dev/null +++ b/python/v1/fluxion/protos/fluxion_pb2_grpc.py @@ -0,0 +1,202 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import warnings + +import grpc + +from . import fluxion_pb2 as fluxion__pb2 + +GRPC_GENERATED_VERSION = "1.63.0" +GRPC_VERSION = grpc.__version__ +EXPECTED_ERROR_RELEASE = "1.65.0" +SCHEDULED_RELEASE_DATE = "June 25, 2024" +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + warnings.warn( + f"The grpc package installed is at version {GRPC_VERSION}," + + f" but the generated code in fluxion_pb2_grpc.py depends on" + + f" grpcio>={GRPC_GENERATED_VERSION}." + + f" Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}" + + f" or downgrade your generated code using grpcio-tools<={GRPC_VERSION}." + + f" This warning will become an error in {EXPECTED_ERROR_RELEASE}," + + f" scheduled for release on {SCHEDULED_RELEASE_DATE}.", + RuntimeWarning, + ) + + +class FluxionServiceStub(object): + """Service definition for Fluxclient""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Match = channel.unary_unary( + "/fluxion.FluxionService/Match", + request_serializer=fluxion__pb2.MatchRequest.SerializeToString, + response_deserializer=fluxion__pb2.MatchResponse.FromString, + _registered_method=True, + ) + self.Cancel = channel.unary_unary( + "/fluxion.FluxionService/Cancel", + request_serializer=fluxion__pb2.CancelRequest.SerializeToString, + response_deserializer=fluxion__pb2.CancelResponse.FromString, + _registered_method=True, + ) + self.Init = channel.unary_unary( + "/fluxion.FluxionService/Init", + request_serializer=fluxion__pb2.InitRequest.SerializeToString, + response_deserializer=fluxion__pb2.InitResponse.FromString, + _registered_method=True, + ) + + +class FluxionServiceServicer(object): + """Service definition for Fluxclient""" + + def Match(self, request, context): + """Sends a Match command""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def Cancel(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def Init(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + +def add_FluxionServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + "Match": grpc.unary_unary_rpc_method_handler( + servicer.Match, + request_deserializer=fluxion__pb2.MatchRequest.FromString, + response_serializer=fluxion__pb2.MatchResponse.SerializeToString, + ), + "Cancel": grpc.unary_unary_rpc_method_handler( + servicer.Cancel, + request_deserializer=fluxion__pb2.CancelRequest.FromString, + response_serializer=fluxion__pb2.CancelResponse.SerializeToString, + ), + "Init": grpc.unary_unary_rpc_method_handler( + servicer.Init, + request_deserializer=fluxion__pb2.InitRequest.FromString, + response_serializer=fluxion__pb2.InitResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + "fluxion.FluxionService", rpc_method_handlers + ) + server.add_generic_rpc_handlers((generic_handler,)) + + +# This class is part of an EXPERIMENTAL API. +class FluxionService(object): + """Service definition for Fluxclient""" + + @staticmethod + def Match( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/fluxion.FluxionService/Match", + fluxion__pb2.MatchRequest.SerializeToString, + fluxion__pb2.MatchResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True, + ) + + @staticmethod + def Cancel( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/fluxion.FluxionService/Cancel", + fluxion__pb2.CancelRequest.SerializeToString, + fluxion__pb2.CancelResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True, + ) + + @staticmethod + def Init( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/fluxion.FluxionService/Init", + fluxion__pb2.InitRequest.SerializeToString, + fluxion__pb2.InitResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True, + ) diff --git a/python/v1/fluxion/server.py b/python/v1/fluxion/server.py new file mode 100644 index 0000000..8e1b29e --- /dev/null +++ b/python/v1/fluxion/server.py @@ -0,0 +1,28 @@ +import logging +from concurrent import futures + +import grpc + +from fluxion.protos import fluxion_pb2_grpc + + +class FluxionServicer(fluxion_pb2_grpc.FluxionServicer): + """ + Unimplemented server - let us know if you need this, but largely + you should use Go for the serve and python for a client only. + """ + + pass + + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + api_pb2_grpc.add_RainbowSchedulerServicer_to_server(FluxionServicer(), server) + server.add_insecure_port("[::]:4242") + server.start() + server.wait_for_termination() + + +if __name__ == "__main__": + logging.basicConfig() + serve() diff --git a/python/v1/fluxion/utils.py b/python/v1/fluxion/utils.py new file mode 100644 index 0000000..4844c91 --- /dev/null +++ b/python/v1/fluxion/utils.py @@ -0,0 +1,36 @@ +import json + +import yaml + + +def read_json(filename): + """ + Read json from file + """ + return json.loads(read_file(filename)) + + +def read_file(filename): + """ + Read in a file content + """ + with open(filename, "r") as fd: + content = fd.read() + return content + + +def read_yaml(filename): + """ + Read yaml from file + """ + with open(filename, "r") as fd: + content = yaml.safe_load(fd) + return content + + +def write_yaml(obj, filename): + """ + Read yaml to file + """ + with open(filename, "w") as fd: + yaml.dump(obj, fd) diff --git a/python/v1/pyproject.toml b/python/v1/pyproject.toml new file mode 100644 index 0000000..58bcf18 --- /dev/null +++ b/python/v1/pyproject.toml @@ -0,0 +1,9 @@ +[tool.black] +profile = "black" +exclude = ["^env/"] +line-length = 100 + +[tool.isort] +profile = "black" # needed for black/isort compatibility +line_length = 100 +skip = [] diff --git a/python/v1/setup.cfg b/python/v1/setup.cfg new file mode 100644 index 0000000..1dde4c8 --- /dev/null +++ b/python/v1/setup.cfg @@ -0,0 +1,6 @@ +[flake8] +exclude = docs +max-line-length = 100 +ignore = E1 E2 E5 W5 +per-file-ignores = + fluxion/__init__.py:F401 diff --git a/python/v1/setup.py b/python/v1/setup.py new file mode 100644 index 0000000..dbe6114 --- /dev/null +++ b/python/v1/setup.py @@ -0,0 +1,48 @@ +import os + +from setuptools import find_packages, setup # noqa: H301 + +DESCRIPTION = "Python gRPC functions for Fluxion, the Flux Framework flux-sched project" + +# Try to read description, otherwise fallback to short description +try: + with open(os.path.abspath("README.md")) as filey: + LONG_DESCRIPTION = filey.read() +except Exception: + LONG_DESCRIPTION = DESCRIPTION + +################################################################################ +# MAIN ######################################################################### +################################################################################ + +if __name__ == "__main__": + setup( + name="fluxion", + version="0.0.0", + author="Vanessasaurus", + author_email="vsoch@users.noreply.github.com", + maintainer="Vanessasaurus", + packages=find_packages(), + include_package_data=True, + zip_safe=False, + url="https://github.com/converged-computing/fluxion/tree/main/python/v1", + license="MIT", + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + long_description_content_type="text/markdown", + keywords="flux-framework, scheduler", + setup_requires=["pytest-runner"], + install_requires=["grpcio", "grpcio-tools", "jsonschema", "pyyaml", "jobspec"], + tests_require=["pytest", "pytest-cov"], + classifiers=[ + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: C", + "Programming Language :: Python", + "Topic :: Software Development", + "Topic :: Scientific/Engineering", + "Operating System :: Unix", + "Programming Language :: Python :: 3.7", + ], + )