diff --git a/.github/workflows/testLinuxWindowsMacOS.yml b/.github/workflows/testLinuxWindowsMacOS.yml index 4d98dc27..6906d0c1 100644 --- a/.github/workflows/testLinuxWindowsMacOS.yml +++ b/.github/workflows/testLinuxWindowsMacOS.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Test run: go test -v ./... @@ -29,7 +29,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Test run: go test -v ./... @@ -42,7 +42,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.16 + go-version: 1.17 - name: Test run: go test -v ./... diff --git a/.gitignore b/.gitignore index 8957e4ab..acd641a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,12 @@ *~ -cmd/epmd/epmd coverage.txt coverage.html +tests/coverage.txt +tests/coverage.html *.swp tags .session cover.out +tests/cover.out +examples/sandbox diff --git a/ChangeLog.md b/ChangeLog.md index a59c17c0..0c4e4773 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -4,6 +4,31 @@ All notable changes to this project will be documented in this file. This format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +#### [v2.0.0](https://github.com/ergo-services/ergo/releases/tag/v1.999.200) 2021-10-12 [tag version v1.999.200] #### + +* Added support of Erlang/OTP 24 (including [Alias](https://blog.erlang.org/My-OTP-24-Highlights/#eep-53-process-aliases) feature and [Remote Spawn](https://blog.erlang.org/OTP-23-Highlights/#distributed-spawn-and-the-new-erpc-module) introduced in Erlang/OTP 23) +* **Important**: This release includes refined API (without backward compatibility) for a more convenient way to create OTP-designed microservices. Make sure to update your code. +* **Important**: Project repository has been moved to [https://github.com/ergo-services/ergo](https://github.com/ergo-services/ergo). It is still available on the old URL [https://github.com/halturin/ergo](https://github.com/halturin/ergo) and GitHub will redirect all requests to the new one (thanks to GitHub for this feature). +* Introduced new behavior `gen.Saga`. It implements Saga design pattern - a sequence of transactions that updates each service state and publishes the result (or cancels the transaction or triggers the next transaction step). `gen.Saga` also provides a feature of interim results (can be used as transaction progress or as a part of pipeline processing), time deadline (to limit transaction lifespan), two-phase commit (to make distributed transaction atomic). Here is example [examples/gensaga](examples/gensaga). +* Introduced new methods `Process.Direct` and `Process.DirectWithTimeout` to make direct request to the actor (`gen.Server` or inherited object). If an actor has no implementation of `HandleDirect` callback it returns `ErrUnsupportedRequest` as a error. +* Introduced new callback `HandleDirect` in the `gen.Server` interface as a handler for requests made by `Process.Direct` or `Process.DirectWithTimeout`. It should be easy to interact with actors from outside. +* Introduced new types intended to be used to interact with Erlang/Elixir + * `etf.ListImproper` to support improper lists like `[a|b]` (a cons cell). + * `etf.String` (an alias for the Golang string) encodes as a binary in order to support Elixir string type (which is `binary()` type) + * `etf.Charlist` (an alias for the Golang string) encodes as a list of chars `[]rune` in order to support Erlang string type (which is `charlist()` type) +* Introduced new methods `Node.ProvideRemoteSpawn`, `Node.RevokeRemoteSpawn`, `Process.RemoteSpawn`. +* Introduced new interfaces `Marshaler` (method `MarshalETF`) and `Unmarshaler` (method `UnmarshalETF`) for the custom encoding/decoding data. +* Improved performance for the local messaging (up to 3 times for some cases) +* Added example [examples/http](examples/http) to demonsrate how HTTP server can be integrated into the Ergo node. +* Added example [examples/gendemo](examples/gendemo) - how to create a custom behavior (design pattern) on top of the `gen.Server`. Take inspiration from the [gen/stage.go](gen/stage.go) or [gen/saga.go](gen/saga.go) design patterns. +* Added support FreeBSD, OpenBSD, NetBSD, DragonFly. +* Fixed RPC issue #45 +* Fixed internal timer issue #48 +* Fixed memory leaks #53 +* Fixed double panic issue #52 +* Fixed Atom Cache race conditioned issue #54 +* Fixed ETF encoder issues #64 #66 + #### [1.2.0](https://github.com/ergo-services/ergo/releases/tag/v1.2.0) - 2021-04-07 #### * Added TLS support. Introduced new option `TLSmode` in `ergo.NodeOptions` with the following values: diff --git a/README.md b/README.md index f3976d4b..57aa296a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@

Ergo Framework

-[![GitHub release](https://img.shields.io/github/release/ergo-services/ergo.svg)](https://github.com/ergo-services/ergo/releases/latest) -[![Go Report Card](https://goreportcard.com/badge/github.com/ergo-services/ergo)](https://goreportcard.com/report/github.com/ergo-services/ergo) + [![GoDoc](https://pkg.go.dev/badge/ergo-services/ergo)](https://pkg.go.dev/github.com/ergo-services/ergo) -[![MIT license](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://opensource.org/licenses/MIT) [![Build Status](https://img.shields.io/github/workflow/status/ergo-services/ergo/TestLinuxWindowsMacOS)](https://github.com/ergo-services/ergo/actions/) +[![Telegram Community](https://img.shields.io/badge/Telegram-Community-blue?style=flat&logo=telegram)](https://t.me/ErgoServices) +[![Discord Community](https://img.shields.io/badge/Discord-Community-5865F2?style=flat&logo=discord&logoColor=white)](https://discord.gg/sdscxKGV62) +[![MIT license](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://opensource.org/licenses/MIT) Technologies and design patterns of Erlang/OTP have been proven over the years. Now in Golang. Up to x5 times faster than original Erlang/OTP in terms of network messaging. @@ -35,106 +36,136 @@ The goal of this project is to leverage Erlang/OTP experience with Golang perfor * Transient * `gen.Stage` behavior support (originated from Elixir's [GenStage](https://hexdocs.pm/gen_stage/GenStage.html)). This is abstraction built on top of `gen.Server` to provide a simple way to create a distributed Producer/Consumer architecture, while automatically managing the concept of backpressure. This implementation is fully compatible with Elixir's GenStage. Example is here [examples/genstage](examples/genstage) or just run `go run ./examples/genstage` to see it in action * `gen.Saga` behavior support. It implements Saga design pattern - a sequence of transactions that updates each service state and publishes the result (or cancels the transaction or triggers the next transaction step). `gen.Saga` also provides a feature of interim results (can be used as transaction progress or as a part of pipeline processing), time deadline (to limit transaction lifespan), two-phase commit (to make distributed transaction atomic). Here is example [examples/gensaga](examples/gensaga). -* Connect to (accept connection from) any Erlang node within a cluster +* `gen.Raft` behavior support. It's improved implementation of [Raft consensus algorithm](https://raft.github.io). The key improvement is using quorum under the hood to manage the leader election process and make the Raft cluster more reliable. This implementation supports quorums of 3, 5, 7, 9, or 11 quorum members. Here is an example of this feature [examples/raft](examples/raft). +* Connect to (accept connection from) any Erlang/Elixir node within a cluster * Making sync request `ServerProcess.Call`, async - `ServerProcess.Cast` or `Process.Send` in fashion of `gen_server:call`, `gen_server:cast`, `erlang:send` accordingly -* Monitor processes/nodes - * local -> local - * local -> remote - * remote -> local -* Link processes - * local <-> local - * local <-> remote - * remote <-> local +* Monitor processes/nodes, local/remote +* Link processes local/remote * RPC callbacks support * [embedded EPMD](#epmd) (in order to get rid of erlang' dependencies) -* Experimental [observer support](#observer) * Unmarshalling terms into the struct using `etf.TermIntoStruct`, `etf.TermProplistIntoStruct` or to the string using `etf.TermToString` * Custom marshaling/unmarshaling via `Marshal` and `Unmarshal` interfaces * Encryption (TLS 1.3) support (including autogenerating self-signed certificates) +* Compression support (with customization of compression level and threshold). It can be configured for the node or a particular process. +* Proxy support with end-to-end encryption, includeing compression/fragmentation/linking/monitoring features. * Tested and confirmed support Windows, Darwin (MacOS), Linux, FreeBSD. +* Zero dependencies. All features are implemented using the standard Golang library. ### Requirements ### -* Go 1.15.x and above +* Go 1.17.x and above ### Versioning ### -Golang introduced [v2 rule](https://go.dev/blog/v2-go-modules) a while ago to solve complicated dependency issues. We found this solution very controversial and there is still a lot of discussion around it. So, we decided to keep the old way for the versioning, but have to use the git tag versioning with v1 as a major version (due to "v2 rule" restrictions) . As a starting point for the v2.0.0 we use git tag v1.999.200. Since now, the only "patch version" will be increased for the next releases (e.g. v2.0.1 will be tagged in git as v.1.999.201 and so on, but never be above git tag v1.999 until the moment when Golang developers change the versioning approach) +Golang introduced [v2 rule](https://go.dev/blog/v2-go-modules) a while ago to solve complicated dependency issues. We found this solution very controversial and there is still a lot of discussion around it. So, we decided to keep the old way for the versioning, but have to use the git tag with v1 as a major version (due to "v2 rule" restrictions). Since now we use git tag pattern 1.999.XYZ where X - major number, Y - minor, Z - patch version. ### Changelog ### Here are the changes of latest release. For more details see the [ChangeLog](ChangeLog.md) -#### [v2.0.0](https://github.com/ergo-services/ergo/releases/tag/v1.999.200) 2021-10-12 [tag version v1.999.200] #### - -* Added support of Erlang/OTP 24 (including [Alias](https://blog.erlang.org/My-OTP-24-Highlights/#eep-53-process-aliases) feature and [Remote Spawn](https://blog.erlang.org/OTP-23-Highlights/#distributed-spawn-and-the-new-erpc-module) introduced in Erlang/OTP 23) -* **Important**: This release includes refined API (without backward compatibility) for a more convenient way to create OTP-designed microservices. Make sure to update your code. -* **Important**: Project repository has been moved to [https://github.com/ergo-services/ergo](https://github.com/ergo-services/ergo). It is still available on the old URL [https://github.com/halturin/ergo](https://github.com/halturin/ergo) and GitHub will redirect all requests to the new one (thanks to GitHub for this feature). -* Introduced new behavior `gen.Saga`. It implements Saga design pattern - a sequence of transactions that updates each service state and publishes the result (or cancels the transaction or triggers the next transaction step). `gen.Saga` also provides a feature of interim results (can be used as transaction progress or as a part of pipeline processing), time deadline (to limit transaction lifespan), two-phase commit (to make distributed transaction atomic). Here is example [examples/gensaga](examples/gensaga). -* Introduced new methods `Process.Direct` and `Process.DirectWithTimeout` to make direct request to the actor (`gen.Server` or inherited object). If an actor has no implementation of `HandleDirect` callback it returns `ErrUnsupportedRequest` as a error. -* Introduced new callback `HandleDirect` in the `gen.Server` interface as a handler for requests made by `Process.Direct` or `Process.DirectWithTimeout`. It should be easy to interact with actors from outside. -* Introduced new types intended to be used to interact with Erlang/Elixir - * `etf.ListImproper` to support improper lists like `[a|b]` (a cons cell). - * `etf.String` (an alias for the Golang string) encodes as a binary in order to support Elixir string type (which is `binary()` type) - * `etf.Charlist` (an alias for the Golang string) encodes as a list of chars `[]rune` in order to support Erlang string type (which is `charlist()` type) -* Introduced new methods `Node.ProvideRemoteSpawn`, `Node.RevokeRemoteSpawn`, `Process.RemoteSpawn`. -* Introduced new interfaces `Marshaler` (method `MarshalETF`) and `Unmarshaler` (method `UnmarshalETF`) for the custom encoding/decoding data. -* Improved performance for the local messaging (up to 3 times for some cases) -* Added example [examples/http](examples/http) to demonsrate how HTTP server can be integrated into the Ergo node. -* Added example [examples/gendemo](examples/gendemo) - how to create a custom behavior (design pattern) on top of the `gen.Server`. Take inspiration from the [gen/stage.go](gen/stage.go) or [gen/saga.go](gen/saga.go) design patterns. -* Added support FreeBSD, OpenBSD, NetBSD, DragonFly. -* Fixed RPC issue #45 -* Fixed internal timer issue #48 -* Fixed memory leaks #53 -* Fixed double panic issue #52 -* Fixed Atom Cache race conditioned issue #54 -* Fixed ETF encoder issues #64 #66 +#### [v2.1.0](https://github.com/ergo-services/ergo/releases/tag/v1.999.210) 2022-04-19 [tag version v1.999.210] #### + +* Introduced **compression feature** support. Here are new methods and options to manage this feature: + - `gen.Process`: + - `SetCompression(enable bool)`, `Compression() bool` + - `SetCompressionLevel(level int) bool`, `CompressionLevel() int` + - `SetCompressionThreshold(threshold int) bool`, `CompressionThreshold() int` messages smaller than the threshold will be sent with no compression. The default compression threshold is 1024 bytes. + - `node.Options`: + - `Compression` these settings are used as defaults for the spawning processes + - this feature will be ignored if the receiver is running on either the Erlang or Elixir node +* Introduced **proxy feature** support **with end-to-end encryption**. + - `node.Node` new methods: + - `AddProxyRoute(...)`, `RemoveProxyRoute(...)` + - `ProxyRoute(...)`, `ProxyRoutes()` + - `NodesIndirect()` returns list of connected nodes via proxy connection + - `node.Options`: + - `Proxy` for configuring proxy settings + - includes support (over the proxy connection): compression, fragmentation, link/monitor process, monitor node + - example [examples/proxy](examples/proxy). + - this feature is not available for the Erlang/Elixir nodes +* Introduced **behavior `gen.Raft`**. It's improved implementation of [Raft consensus algorithm](https://raft.github.io). The key improvement is using quorum under the hood to manage the leader election process and make the Raft cluster more reliable. This implementation supports quorums of 3, 5, 7, 9, or 11 quorum members. Here is an example of this feature [examples/raft](examples/raft). +* Introduced **interfaces to customize network layer** + - `Resolver` to replace EPMD routines with your solution (e.g., ZooKeeper or any other service registrar) + - `Handshake` allows customizing authorization/authentication process + - `Proto` provides the way to implement proprietary protocols (e.g., IoT area) +* Other new features: + - `gen.Process` new methods: + - `NodeUptime()`, `NodeName()`, `NodeStop()` + - `gen.ServerProcess` new method: + - `MessageCounter()` shows how many messages have been handled by the `gen.Server` callbacks + - `gen.ProcessOptions` new option: + - `ProcessFallback` allows forward messages to the fallback process if the process mailbox is full. Forwarded messages are wrapped into `gen.MessageFallback` struct. Related to issue #96. + - `gen.SupervisorChildSpec` and `gen.ApplicationChildSpec` got option `gen.ProcessOptions` to customize options for the spawning child processes. +* Improved sending messages by etf.Pid or etf.Alias: methods `gen.Process.Send`, `gen.ServerProcess.Cast`, `gen.ServerProcess.Call` now return `node.ErrProcessIncarnation` if a message is sending to the remote process of the previous incarnation (remote node has been restarted). Making monitor on a remote process of the previous incarnation triggers sending `gen.MessageDown` with reason `incarnation`. +* Introduced type `gen.EnvKey` for the environment variables +* All spawned processes now have the `node.EnvKeyNode` variable to get access to the `node.Node` value. +* **Improved performance** of local messaging (**up to 8 times** for some cases) +* **Important** `node.Options` has changed. Make sure to adjust your code. +* Fixed issue #89 (incorrect handling of Call requests) +* Fixed issues #87, #88 and #93 (closing network socket) +* Fixed issue #96 (silently drops message if process mailbox is full) +* Updated minimal requirement of Golang version to 1.17 (go.mod) +* We still keep the rule **Zero Dependencies** ### Benchmarks ### Here is simple EndToEnd test demonstrates performance of messaging subsystem -#### Sequential Process.Call using two processes running on a single and two nodes +Hardware: workstation with AMD Ryzen Threadripper 3970X (64) @ 3.700GHz -Hardware: laptop with Intel(R) Core(TM) i5-8265U (4 cores. 8 with HT) +``` +❯❯❯❯ go test -bench=NodeParallel -run=XXX -benchtime=10s +goos: linux +goarch: amd64 +pkg: github.com/ergo-services/ergo/tests +cpu: AMD Ryzen Threadripper 3970X 32-Core Processor +BenchmarkNodeParallel-64 4738918 2532 ns/op +BenchmarkNodeParallelSingleNode-64 100000000 429.8 ns/op +PASS +ok github.com/ergo-services/ergo/tests 29.596s ``` -❯❯❯❯ go test -bench=NodeSequential -run=XXX -benchtime=10s + +these numbers show almost **500.000 sync requests per second** for the network messaging via localhost and **10.000.000 sync requests per second** for the local messaging (within a node). + +#### Compression + +This benchmark shows the performance of compression for sending 1MB message between two nodes (via a network). + +``` +❯❯❯❯ go test -bench=NodeCompression -run=XXX -benchtime=10s goos: linux goarch: amd64 -pkg: github.com/ergo-services/ergo -BenchmarkNodeSequential/number-8 256108 48578 ns/op -BenchmarkNodeSequential/string-8 266906 51531 ns/op -BenchmarkNodeSequential/tuple_(PID)-8 233700 58192 ns/op -BenchmarkNodeSequential/binary_1MB-8 5617 2092495 ns/op -BenchmarkNodeSequentialSingleNode/number-8 2527580 4857 ns/op -BenchmarkNodeSequentialSingleNode/string-8 2519410 4760 ns/op -BenchmarkNodeSequentialSingleNode/tuple_(PID)-8 2524701 4757 ns/op -BenchmarkNodeSequentialSingleNode/binary_1MB-8 2521370 4758 ns/op +pkg: github.com/ergo-services/ergo/tests +cpu: AMD Ryzen Threadripper 3970X 32-Core Processor +BenchmarkNodeCompressionDisabled1MBempty-64 2400 4957483 ns/op +BenchmarkNodeCompressionEnabled1MBempty-64 5769 2088051 ns/op +BenchmarkNodeCompressionEnabled1MBstring-64 5202 2077099 ns/op PASS -ok github.com/ergo-services/ergo 120.720s +ok github.com/ergo-services/ergo/tests 56.708s ``` -it means Ergo Framework provides around **25.000 sync requests per second** via localhost for simple data and around 4Gbit/sec for 1MB messages +It demonstrates **more than 2 times** improvement. -#### Parallel Process.Call using 120 pairs of processes running on a single and two nodes +#### Proxy -Hardware: workstation with AMD Ryzen Threadripper 3970X (64) @ 3.700GHz +This benchmark demonstrates how proxy feature and e2e encryption impact a messaging performance. ``` -❯❯❯❯ go test -bench=NodeParallel -run=XXX -benchtime=10s +❯❯❯❯ go test -bench=NodeProxy -run=XXX -benchtime=10s goos: linux goarch: amd64 pkg: github.com/ergo-services/ergo/tests cpu: AMD Ryzen Threadripper 3970X 32-Core Processor -BenchmarkNodeParallel-64 4922430 2440 ns/op -BenchmarkNodeParallelSingleNode-64 16293586 810.0 ns/op +BenchmarkNodeProxy_NodeA_to_NodeC_direct_Message_1KB-64 1908477 6337 ns/op +BenchmarkNodeProxy_NodeA_to_NodeC_via_NodeB_Message_1KB-64 1700984 7062 ns/op +BenchmarkNodeProxy_NodeA_to_NodeC_via_NodeB_Message_1KB_Encrypted-64 1271125 9410 ns/op PASS -ok github.com/ergo-services/ergo/tests 29.596s +ok github.com/ergo-services/ergo/tests 45.649s + ``` -these numbers show almost **500.000 sync requests per second** for the network messaging via localhost and **1.600.000 sync requests per second** for the local messaging (within a node). #### Ergo Framework vs original Erlang/OTP @@ -263,6 +294,7 @@ There are options already defined that you might want to use * `-ergo.trace` - enable extended debug info * `-ergo.norecover` - disable panic catching +* `-ergo.warning` - enable/disable warnings (default: enable) To enable Golang profiler just add `--tags debug` in your `go run` or `go build` like this: diff --git a/debug.go b/debug.go index 8fd6347a..89b4e8a6 100644 --- a/debug.go +++ b/debug.go @@ -1,4 +1,5 @@ -//+build debug +//go:build debug +// +build debug package ergo diff --git a/ergo.go b/ergo.go index a45555f0..5d9655aa 100644 --- a/ergo.go +++ b/ergo.go @@ -6,6 +6,7 @@ import ( "github.com/ergo-services/ergo/erlang" "github.com/ergo-services/ergo/gen" "github.com/ergo-services/ergo/node" + "github.com/ergo-services/ergo/proto/dist" ) // StartNode create new node with name and cookie string @@ -13,16 +14,36 @@ func StartNode(name string, cookie string, opts node.Options) (node.Node, error) return StartNodeWithContext(context.Background(), name, cookie, opts) } -// CreateNodeWithContext create new node with specified context, name and cookie string +// StartNodeWithContext create new node with specified context, name and cookie string func StartNodeWithContext(ctx context.Context, name string, cookie string, opts node.Options) (node.Node, error) { version := node.Version{ Release: Version, Prefix: VersionPrefix, OTP: VersionOTP, } + if opts.Env == nil { + opts.Env = make(map[gen.EnvKey]interface{}) + } + opts.Env[node.EnvKeyVersion] = version // add erlang support application opts.Applications = append([]gen.ApplicationBehavior{&erlang.KernelApp{}}, opts.Applications...) - return node.StartWithContext(context.WithValue(ctx, "version", version), name, cookie, opts) + if opts.Handshake == nil { + // create default handshake for the node (Erlang Dist Handshake) + opts.Handshake = dist.CreateHandshake(dist.HandshakeOptions{}) + } + + if opts.Proto == nil { + // create default proto handler (Erlang Dist Proto) + protoOptions := node.DefaultProtoOptions() + opts.Proto = dist.CreateProto(protoOptions) + } + + if opts.StaticRoutesOnly == false && opts.Resolver == nil { + // create default resolver (with enabled Erlang EPMD server) + opts.Resolver = dist.CreateResolverWithLocalEPMD("", dist.DefaultEPMDPort) + } + + return node.StartWithContext(ctx, name, cookie, opts) } diff --git a/erlang/appmon.go b/erlang/appmon.go index 81925d14..4d912500 100644 --- a/erlang/appmon.go +++ b/erlang/appmon.go @@ -26,7 +26,6 @@ type jobDetails struct { } // Init initializes process state using arbitrary arguments -// Init -> state func (am *appMon) Init(process *gen.ServerProcess, args ...etf.Term) error { lib.Log("APP_MON: Init %#v", args) from := args[0] @@ -37,10 +36,11 @@ func (am *appMon) Init(process *gen.ServerProcess, args ...etf.Term) error { return nil } +// HandleCast func (am *appMon) HandleCast(process *gen.ServerProcess, message etf.Term) gen.ServerStatus { var appState *appMonState = process.State.(*appMonState) lib.Log("APP_MON: HandleCast: %#v", message) - node := process.Env("ergo:Node").(node.Node) + node := process.Env(node.EnvKeyNode).(node.Node) switch message { case "sendStat": @@ -137,7 +137,7 @@ func (am *appMon) HandleCast(process *gen.ServerProcess, message etf.Term) gen.S } func (am *appMon) makeAppTree(process gen.Process, app etf.Atom) etf.Tuple { - node := process.Env("ergo:Node").(node.Node) + node := process.Env(node.EnvKeyNode).(node.Node) appInfo, err := node.ApplicationInfo(string(app)) if err != nil { return nil diff --git a/erlang/erlang.go b/erlang/erlang.go index f2e924df..d445bb2b 100644 --- a/erlang/erlang.go +++ b/erlang/erlang.go @@ -12,11 +12,13 @@ type erlang struct { gen.Server } +// Init func (e *erlang) Init(process *gen.ServerProcess, args ...etf.Term) error { lib.Log("ERLANG: Init: %#v", args) return nil } +// HandleCall func (e *erlang) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, message etf.Term) (etf.Term, gen.ServerStatus) { lib.Log("ERLANG: HandleCall: %#v, From: %#v", message, from) @@ -99,7 +101,7 @@ func processInfo(p gen.Process, pid etf.Pid, property etf.Term) etf.Term { case etf.Atom("priority"): // values = append(values, etf.Tuple{p[i], 0}) case etf.Atom("reductions"): - values = append(values, etf.Tuple{p[i], info.Reductions}) + values = append(values, etf.Tuple{p[i], 0}) case etf.Atom("registered_name"): values = append(values, etf.Tuple{p[i], process.Name()}) case etf.Atom("sequential_trace_token"): diff --git a/erlang/global_name_server.go b/erlang/global_name_server.go index 5acb21e8..e1099654 100644 --- a/erlang/global_name_server.go +++ b/erlang/global_name_server.go @@ -11,6 +11,7 @@ type globalNameServer struct { gen.Server } +// HandleCast func (gns *globalNameServer) HandleCast(process *gen.ServerProcess, message etf.Term) gen.ServerStatus { return gen.ServerStatusOK } diff --git a/erlang/net_kernel.go b/erlang/net_kernel.go index d11816b8..a36400ec 100644 --- a/erlang/net_kernel.go +++ b/erlang/net_kernel.go @@ -13,17 +13,19 @@ import ( "github.com/ergo-services/ergo/lib/osdep" ) +// KernelApp type KernelApp struct { gen.Application } +// Load func (nka *KernelApp) Load(args ...etf.Term) (gen.ApplicationSpec, error) { return gen.ApplicationSpec{ Name: "erlang", Description: "Erlang support app", Version: "v.1.0", Children: []gen.ApplicationChildSpec{ - gen.ApplicationChildSpec{ + { Child: &netKernelSup{}, Name: "net_kernel_sup", }, @@ -31,32 +33,34 @@ func (nka *KernelApp) Load(args ...etf.Term) (gen.ApplicationSpec, error) { }, nil } +// Start func (nka *KernelApp) Start(p gen.Process, args ...etf.Term) {} type netKernelSup struct { gen.Supervisor } +// Init func (nks *netKernelSup) Init(args ...etf.Term) (gen.SupervisorSpec, error) { return gen.SupervisorSpec{ Children: []gen.SupervisorChildSpec{ - gen.SupervisorChildSpec{ + { Name: "net_kernel", Child: &netKernel{}, }, - gen.SupervisorChildSpec{ + { Name: "global_name_server", Child: &globalNameServer{}, }, - gen.SupervisorChildSpec{ + { Name: "rex", Child: &rex{}, }, - gen.SupervisorChildSpec{ + { Name: "observer_backend", Child: &observerBackend{}, }, - gen.SupervisorChildSpec{ + { Name: "erlang", Child: &erlang{}, }, @@ -75,12 +79,14 @@ type netKernel struct { routinesCtx map[etf.Pid]context.CancelFunc } +// Init func (nk *netKernel) Init(process *gen.ServerProcess, args ...etf.Term) error { lib.Log("NET_KERNEL: Init: %#v", args) nk.routinesCtx = make(map[etf.Pid]context.CancelFunc) return nil } +// HandleCall func (nk *netKernel) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, message etf.Term) (reply etf.Term, status gen.ServerStatus) { lib.Log("NET_KERNEL: HandleCall: %#v, From: %#v", message, from) status = gen.ServerStatusOK @@ -124,6 +130,7 @@ func (nk *netKernel) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, return } +// HandleInfo func (nk *netKernel) HandleInfo(process *gen.ServerProcess, message etf.Term) gen.ServerStatus { lib.Log("NET_KERNEL: HandleInfo: %#v", message) switch m := message.(type) { @@ -147,7 +154,7 @@ func sendProcInfo(p gen.Process, to etf.Pid) { etf.Atom("etop_proc_info"), // record name #etop_proc_info list[i].Self(), // pid 0, // mem - info.Reductions, // reds + 0, // reds etf.Atom(list[i].Name()), // etf.Tuple{etf.Atom("ergo"), etf.Atom(list[i].Name()), 0}, // name 0, // runtime info.CurrentFunction, // etf.Tuple{etf.Atom("ergo"), etf.Atom(info.CurrentFunction), 0}, // cf diff --git a/erlang/observer_backend.go b/erlang/observer_backend.go index 0598ee3c..bb72644a 100644 --- a/erlang/observer_backend.go +++ b/erlang/observer_backend.go @@ -19,14 +19,13 @@ type observerBackend struct { } // Init initializes process state using arbitrary arguments -// Init(...) -> state func (o *observerBackend) Init(process *gen.ServerProcess, args ...etf.Term) error { lib.Log("OBSERVER: Init: %#v", args) funProcLibInitialCall := func(a ...etf.Term) etf.Term { return etf.Tuple{etf.Atom("proc_lib"), etf.Atom("init_p"), 5} } - node := process.Env("ergo:Node").(node.Node) + node := process.Env(node.EnvKeyNode).(node.Node) node.ProvideRPC("proc_lib", "translate_initial_call", funProcLibInitialCall) funAppmonInfo := func(a ...etf.Term) etf.Term { @@ -42,6 +41,7 @@ func (o *observerBackend) Init(process *gen.ServerProcess, args ...etf.Term) err return nil } +// HandleCall func (o *observerBackend) HandleCall(state *gen.ServerProcess, from gen.ServerFrom, message etf.Term) (etf.Term, gen.ServerStatus) { lib.Log("OBSERVER: HandleCall: %v, From: %#v", message, from) function := message.(etf.Tuple).Element(1).(etf.Atom) @@ -68,7 +68,7 @@ func (o *observerBackend) HandleCall(state *gen.ServerProcess, from gen.ServerFr func (o *observerBackend) sysInfo(p gen.Process) etf.List { // observer_backend:sys_info() - node := p.Env("ergo:Node").(node.Node) + node := p.Env(node.EnvKeyNode).(node.Node) processCount := etf.Tuple{etf.Atom("process_count"), len(p.ProcessList())} processLimit := etf.Tuple{etf.Atom("process_limit"), 262144} atomCount := etf.Tuple{etf.Atom("atom_count"), 0} diff --git a/erlang/rex.go b/erlang/rex.go index 80fa975b..27db92b0 100644 --- a/erlang/rex.go +++ b/erlang/rex.go @@ -28,6 +28,7 @@ type rex struct { methods map[modFun]gen.RPC } +// Init func (r *rex) Init(process *gen.ServerProcess, args ...etf.Term) error { lib.Log("REX: Init: %#v", args) // Do not overwrite existing methods if this process restarted @@ -42,11 +43,12 @@ func (r *rex) Init(process *gen.ServerProcess, args ...etf.Term) error { } r.methods[mf] = nil } - node := process.Env("ergo:Node").(node.Node) + node := process.Env(node.EnvKeyNode).(node.Node) node.ProvideRemoteSpawn("erpc", &erpc{}) return nil } +// HandleCall func (r *rex) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, message etf.Term) (etf.Term, gen.ServerStatus) { lib.Log("REX: HandleCall: %#v, From: %#v", message, from) switch m := message.(type) { @@ -63,7 +65,7 @@ func (r *rex) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, messag return reply, gen.ServerStatusOK } - to := gen.ProcessID{string(module), process.NodeName()} + to := gen.ProcessID{Name: string(module), Node: process.NodeName()} m := etf.Tuple{m.Element(3), m.Element(4)} reply, err := process.Call(to, m) if err != nil { @@ -78,11 +80,13 @@ func (r *rex) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, messag return reply, gen.ServerStatusOK } +// HandleInfo func (r *rex) HandleInfo(process *gen.ServerProcess, message etf.Term) gen.ServerStatus { // add this handler to suppres any messages from erlang return gen.ServerStatusOK } +// HandleDirect func (r *rex) HandleDirect(process *gen.ServerProcess, message interface{}) (interface{}, error) { switch m := message.(type) { case gen.MessageManageRPC: @@ -179,6 +183,7 @@ type erpc struct { gen.Server } +// Init func (e *erpc) Init(process *gen.ServerProcess, args ...etf.Term) error { lib.Log("ERPC [%v]: Init: %#v", process.Self(), args) mfa := erpcMFA{ @@ -192,6 +197,7 @@ func (e *erpc) Init(process *gen.ServerProcess, args ...etf.Term) error { } +// HandleCast func (e *erpc) HandleCast(process *gen.ServerProcess, message etf.Term) gen.ServerStatus { lib.Log("ERPC [%v]: HandleCast: %#v", process.Self(), message) mfa := message.(erpcMFA) diff --git a/etf/cache.go b/etf/cache.go index 2dc2c27f..2cfb35e5 100644 --- a/etf/cache.go +++ b/etf/cache.go @@ -1,7 +1,6 @@ package etf import ( - "context" "sync" ) @@ -10,30 +9,35 @@ const ( ) type AtomCache struct { + In *AtomCacheIn + Out *AtomCacheOut +} + +type AtomCacheIn struct { + Atoms [maxCacheItems]*Atom +} + +// AtomCache +type AtomCacheOut struct { + sync.RWMutex cacheMap map[Atom]int16 - update chan Atom - lastID int16 + id int16 cacheList [maxCacheItems]Atom - sync.Mutex } +// CacheItem type CacheItem struct { ID int16 Encoded bool Name Atom } -type ListAtomCache struct { - L []CacheItem - original []CacheItem - HasLongAtom bool -} - var ( - listAtomCachePool = &sync.Pool{ + encodingAtomCachePool = &sync.Pool{ New: func() interface{} { - l := &ListAtomCache{ - L: make([]CacheItem, 0, 256), + l := &EncodingAtomCache{ + L: make([]CacheItem, 0, 255), + added: make(map[Atom]uint8), } l.original = l.L return l @@ -41,88 +45,117 @@ var ( } ) -func (a *AtomCache) Append(atom Atom) { - a.Lock() - id := a.lastID - a.Unlock() - if id < maxCacheItems { - a.update <- atom +// NewAtomCache +func NewAtomCache() AtomCache { + return AtomCache{ + In: &AtomCacheIn{}, + Out: &AtomCacheOut{ + cacheMap: make(map[Atom]int16), + id: -1, + }, } - // otherwise ignore } -func (a *AtomCache) GetLastID() int16 { +// Append +func (a *AtomCacheOut) Append(atom Atom) (int16, bool) { a.Lock() - id := a.lastID - a.Unlock() - return id -} + defer a.Unlock() -func NewAtomCache(ctx context.Context) *AtomCache { - var id int16 + if a.id > maxCacheItems-2 { + return 0, false + } - a := &AtomCache{ - cacheMap: make(map[Atom]int16), - update: make(chan Atom, 100), - lastID: -1, + if id, exist := a.cacheMap[atom]; exist { + return id, false } - go func() { - for { - select { - case atom := <-a.update: - if _, ok := a.cacheMap[atom]; ok { - // already exist - continue - } - - id = a.lastID - id++ - a.cacheMap[atom] = id - a.Lock() - a.cacheList[id] = atom - a.lastID = id - a.Unlock() - - case <-ctx.Done(): - return - } - } - }() + a.id++ + a.cacheList[a.id] = atom + a.cacheMap[atom] = a.id - return a + return a.id, true } -func (a *AtomCache) List() [maxCacheItems]Atom { - a.Lock() - l := a.cacheList - a.Unlock() - return l +// LastID +func (a *AtomCacheOut) LastAdded() (Atom, int16) { + a.RLock() + defer a.RUnlock() + l := len(a.cacheList) + if l == 0 { + return "", -1 + } + return a.cacheList[l-1], int16(l - 1) } -func (a *AtomCache) ListSince(id int16) []Atom { +// ListSince +func (a *AtomCacheOut) ListSince(id int16) []Atom { + if id < 0 { + id = 0 + } + if int(id) > len(a.cacheList)-1 { + return nil + } return a.cacheList[id:] } -func TakeListAtomCache() *ListAtomCache { - return listAtomCachePool.Get().(*ListAtomCache) +// EncodingAtomCache +type EncodingAtomCache struct { + L []CacheItem + original []CacheItem + added map[Atom]uint8 + HasLongAtom bool } -func ReleaseListAtomCache(l *ListAtomCache) { +// TakeEncodingAtomCache +func TakeEncodingAtomCache() *EncodingAtomCache { + return encodingAtomCachePool.Get().(*EncodingAtomCache) +} + +// ReleaseEncodingAtomCache +func ReleaseEncodingAtomCache(l *EncodingAtomCache) { l.L = l.original[:0] - listAtomCachePool.Put(l) + if len(l.added) > 0 { + for k, _ := range l.added { + delete(l.added, k) + } + } + encodingAtomCachePool.Put(l) } -func (l *ListAtomCache) Reset() { + +// Reset +func (l *EncodingAtomCache) Reset() { l.L = l.original[:0] l.HasLongAtom = false + if len(l.added) > 0 { + for k, _ := range l.added { + delete(l.added, k) + } + } } -func (l *ListAtomCache) Append(a CacheItem) { + +// Append +func (l *EncodingAtomCache) Append(a CacheItem) uint8 { + id, added := l.added[a.Name] + if added { + return id + } + l.L = append(l.L, a) if !a.Encoded && len(a.Name) > 255 { l.HasLongAtom = true } + id = uint8(len(l.L) - 1) + l.added[a.Name] = id + return id +} + +// Delete +func (l *EncodingAtomCache) Delete(atom Atom) { + // clean up in order to get rid of map reallocation which is pretty expensive + delete(l.added, atom) } -func (l *ListAtomCache) Len() int { +// Len +func (l *EncodingAtomCache) Len() int { return len(l.L) } diff --git a/etf/cache_test.go b/etf/cache_test.go deleted file mode 100644 index e7c04152..00000000 --- a/etf/cache_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package etf - -import ( - "context" - "reflect" - "testing" - "time" -) - -func TestAtomCache(t *testing.T) { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - a := NewAtomCache(ctx) - - a.Append(Atom("test1")) - time.Sleep(100 * time.Millisecond) - - if a.GetLastID() != 0 { - t.Fatal("LastID != 0", a.GetLastID()) - } - - a.Append(Atom("test1")) - time.Sleep(100 * time.Millisecond) - - if a.GetLastID() != 0 { - t.Fatalf("LastID != 0") - } - - a.Append(Atom("test2")) - time.Sleep(100 * time.Millisecond) - - expected := []Atom{"test1", "test2"} - result := a.ListSince(0) - if reflect.DeepEqual(result, expected) { - t.Fatal("got incorrect result", result) - } - - expectedArray := make([]Atom, 2048) - expectedArray[0] = "test1" - expectedArray[1] = "test2" - - resultArray := a.List() - if reflect.DeepEqual(resultArray, expectedArray) { - t.Fatal("got incorrect resultArray", result) - } -} diff --git a/etf/decode.go b/etf/decode.go index b7a36802..f2a5186d 100644 --- a/etf/decode.go +++ b/etf/decode.go @@ -5,6 +5,8 @@ import ( "fmt" "math" "math/big" + + "github.com/ergo-services/ergo/lib" ) // linked list for decoding complex types like list/map/tuple @@ -55,12 +57,12 @@ var ( errInternal = fmt.Errorf("Internal error") ) +// DecodeOptions type DecodeOptions struct { - FlagV4NC bool - FlagBigCreation bool + FlagBigPidRef bool } -// stackless implementation is speeding up it up to x25 times +// stackless implementation is speeding up decoding function up to x25 times // it might looks hard to understand the logic, but // there are only two stages @@ -69,27 +71,30 @@ type DecodeOptions struct { // // see comments within this function +// Decode func Decode(packet []byte, cache []Atom, options DecodeOptions) (retTerm Term, retByte []byte, retErr error) { var term Term var stack *stackElement var child *stackElement var t byte - defer func() { - // We should catch any panic happened during decoding the raw data. - // Some of the Erlang' types can not be supported in Golang. - // As an example: Erlang map with tuple as a key cause a panic - // in Golang runtime with message: - // 'panic: runtime error: hash of unhashable type etf.Tuple' - // The problem is in etf.Tuple type - it is interface type. At the same - // time Golang does support hashable key in map (like struct as a key), - // but it should be known implicitly. It means we can encode such kind - // of data, but can not to decode it back. - if r := recover(); r != nil { - retTerm = nil - retByte = nil - retErr = fmt.Errorf("%v", r) - } - }() + if lib.CatchPanic() { + defer func() { + // We should catch any panic happened during decoding the raw data. + // Some of the Erlang' types can not be supported in Golang. + // As an example: Erlang map with tuple as a key cause a panic + // in Golang runtime with message: + // 'panic: runtime error: hash of unhashable type etf.Tuple' + // The problem is in etf.Tuple type - it is interface type. At the same + // time Golang does support hashable key in map (like struct as a key), + // but it should be known implicitly. It means we can encode such kind + // of data, but can not to decode it back. + if r := recover(); r != nil { + retTerm = nil + retByte = nil + retErr = fmt.Errorf("%v", r) + } + }() + } for { child = nil @@ -205,16 +210,22 @@ func Decode(packet []byte, cache []Atom, options DecodeOptions) (retTerm Term, r n := packet[0] negative := packet[1] == 1 // sign - ///// this block improve the performance at least 4 times - if n < 8 { // treat as an int64 + ///// this block improves performance at least 4 times + if n < 9 { // treat as an int64/uint64 le8 := make([]byte, 8) copy(le8, packet[2:n+2]) smallBig := binary.LittleEndian.Uint64(le8) if negative { smallBig = -smallBig + term = int64(smallBig) + } else { + if smallBig > math.MaxInt64 { + term = uint64(smallBig) + } else { + term = int64(smallBig) + } } - term = int64(smallBig) packet = packet[n+2:] break } @@ -525,7 +536,7 @@ func Decode(packet []byte, cache []Atom, options DecodeOptions) (retTerm Term, r id := uint64(binary.BigEndian.Uint32(packet[:4])) serial := uint64(binary.BigEndian.Uint32(packet[4:8])) - if options.FlagV4NC { + if options.FlagBigPidRef { id = id | (serial << 32) } else { // id 15 bits only 2**15 - 1 = 32767 @@ -554,7 +565,7 @@ func Decode(packet []byte, cache []Atom, options DecodeOptions) (retTerm Term, r Node: name, Creation: binary.BigEndian.Uint32(packet[8:12]), } - if options.FlagV4NC { + if options.FlagBigPidRef { id = id | (serial << 32) } else { // id 15 bits only 2**15 - 1 = 32767 @@ -578,7 +589,7 @@ func Decode(packet []byte, cache []Atom, options DecodeOptions) (retTerm Term, r if l > 5 { return nil, nil, errMalformedRef } - if l > 3 && !options.FlagV4NC { + if l > 3 && !options.FlagBigPidRef { return nil, nil, errMalformedRef } stack.tmp = nil @@ -621,7 +632,7 @@ func Decode(packet []byte, cache []Atom, options DecodeOptions) (retTerm Term, r if l > 5 { return nil, nil, errMalformedRef } - if l > 3 && !options.FlagV4NC { + if l > 3 && !options.FlagBigPidRef { return nil, nil, errMalformedRef } stack.tmp = nil diff --git a/etf/decode_test.go b/etf/decode_test.go index 2408e021..25aaacc1 100644 --- a/etf/decode_test.go +++ b/etf/decode_test.go @@ -92,17 +92,17 @@ func TestDecodeInteger(t *testing.T) { t.Fatal(err) } - //-1234567890987654321 - bigInt := new(big.Int) - bigInt.SetString("-1234567890987654321", 10) - packet = []byte{ettSmallBig, 8, 1, 177, 28, 108, 177, 244, 16, 34, 17} + //-9223372036854775808 + expectedBigInt64 := int64(-9223372036854775808) + packet = []byte{ettSmallBig, 8, 1, 0, 0, 0, 0, 0, 0, 0, 128} term, _, err = Decode(packet, []Atom{}, DecodeOptions{}) - if err != nil || bigInt.Cmp(term.(*big.Int)) != 0 { - t.Fatal(err, term, bigInt) + if err != nil || term != expectedBigInt64 { + t.Fatal(err, term, expectedBigInt64) } largeBigString := "-12345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890987654321012345678909876543210123456789098765432101234567890" + bigInt := new(big.Int) bigInt.SetString(largeBigString, 10) packet = []byte{ettLargeBig, 0, 0, 1, 139, 1, 210, 106, 44, 197, 54, 176, 151, 83, 243, 228, 193, 133, 194, 193, 21, 90, 196, 4, 252, 150, 226, 188, 79, 11, 8, @@ -144,6 +144,14 @@ func TestDecodeInteger(t *testing.T) { if err != nil || term != expectedInt64 { t.Fatal(err, term, expectedInt64) } + + // 18446744073709551615 + expectedUint64 := uint64(18446744073709551615) + packet = []byte{ettSmallBig, 8, 0, 255, 255, 255, 255, 255, 255, 255, 255} + term, _, err = Decode(packet, []Atom{}, DecodeOptions{}) + if err != nil || term != expectedUint64 { + t.Fatal(err, term, expectedUint64) + } } func TestDecodeList(t *testing.T) { @@ -655,12 +663,3 @@ func BenchmarkDecodeTupleRefPidWithAtomCache(b *testing.B) { } } } -func BenchmarkDecodeFunction(b *testing.B) { - packet := packetFunction - for i := 0; i < b.N; i++ { - _, _, err := Decode(packet, []Atom{}, DecodeOptions{}) - if err != nil { - b.Fatal(err) - } - } -} diff --git a/etf/encode.go b/etf/encode.go index e0c5f965..a5915662 100644 --- a/etf/encode.go +++ b/etf/encode.go @@ -21,47 +21,53 @@ var ( marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() ) +// EncodeOptions type EncodeOptions struct { - LinkAtomCache *AtomCache - WriterAtomCache map[Atom]CacheItem - EncodingAtomCache *ListAtomCache + AtomCache *AtomCacheOut + SenderAtomCache map[Atom]CacheItem + EncodingAtomCache *EncodingAtomCache - // FlagV4NC The node accepts a larger amount of data in pids + // FlagBigPidRef The node accepts a larger amount of data in pids // and references (node container types version 4). // In the pid case full 32-bit ID and Serial fields in NEW_PID_EXT // and in the reference case up to 5 32-bit ID words are now // accepted in NEWER_REFERENCE_EXT. Introduced in OTP 24. - FlagV4NC bool + FlagBigPidRef bool // FlagBigCreation The node understands big node creation tags NEW_PID_EXT, // NEWER_REFERENCE_EXT. FlagBigCreation bool + + NodeName string + PeerName string } +// Encode func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { - defer func() { - // We should catch any panic happend during encoding Golang types. - if r := recover(); r != nil { - retErr = fmt.Errorf("%v", r) - } - }() - + if lib.CatchPanic() { + defer func() { + // We should catch any panic happened during encoding Golang types. + if r := recover(); r != nil { + retErr = fmt.Errorf("%v", r) + } + }() + } var stack, child *stackElement - cacheEnabled := options.LinkAtomCache != nil - - cacheIndex := uint16(0) - if options.EncodingAtomCache != nil { - cacheIndex = uint16(len(options.EncodingAtomCache.L)) + cacheEnabled := options.AtomCache != nil + cacheIndex := int16(0) + if cacheEnabled { + cacheIndex = int16(len(options.EncodingAtomCache.L)) } - // Atom cache: (if its enabled: options.LinkAtomCache != nil) + // Atom cache: (if its enabled: options.AtomCache != nil) // 1. check for an atom in options.WriterAtomCache (map) - // 2. if not found in WriterAtomCache call LinkAtomCache.Append(atom), + // 2. if not found in WriterAtomCache call AtomCache.Append(atom), // encode it as a regular atom (ettAtom*) // 3. if found // add options.EncodingAtomCache[i] = CacheItem, where i is just a counter // within this encoding process. + // encode atom as ettCacheRef with value = i for { @@ -101,20 +107,20 @@ func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { buf := b.Extend(9) - // ID a 32-bit big endian unsigned integer. If distribution - // flag DFLAG_V4_NC is not set, only 15 bits may be used + // ID a 32-bit big endian unsigned integer. + // If FlagBigPidRef is not set, only 15 bits may be used // and the rest must be 0. - if options.FlagV4NC { + if options.FlagBigPidRef { binary.BigEndian.PutUint32(buf[:4], uint32(p.ID)) } else { // 15 bits only 2**15 - 1 = 32767 binary.BigEndian.PutUint32(buf[:4], uint32(p.ID)&32767) } - // Serial a 32-bit big endian unsigned integer. If distribution - // flag DFLAG_V4_NC is not set, only 13 bits may be used + // Serial a 32-bit big endian unsigned integer. + // If distribution FlagBigPidRef is not set, only 13 bits may be used // and the rest must be 0. - if options.FlagV4NC { + if options.FlagBigPidRef { binary.BigEndian.PutUint32(buf[4:8], uint32(p.ID>>32)) } else { // 13 bits only 2**13 - 1 = 8191 @@ -138,14 +144,14 @@ func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { buf := b.Extend(12) // ID - if options.FlagV4NC { + if options.FlagBigPidRef { binary.BigEndian.PutUint32(buf[:4], uint32(p.ID)) } else { // 15 bits only 2**15 - 1 = 32767 binary.BigEndian.PutUint32(buf[:4], uint32(p.ID)&32767) } // Serial - if options.FlagV4NC { + if options.FlagBigPidRef { binary.BigEndian.PutUint32(buf[4:8], uint32(p.ID>>32)) } else { // 13 bits only 2**13 - 1 = 8191 @@ -165,9 +171,6 @@ func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { } lenID := 3 - if options.FlagV4NC { - lenID = 5 - } buf := b.Extend(1 + lenID*4) // Only one byte long and only two bits are significant, the rest must be 0. buf[0] = byte(r.Creation & 3) @@ -194,10 +197,10 @@ func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { break } - lenID := 3 - // FIXME Erlang 24 has a bug https://github.com/erlang/otp/issues/5097 + // // FIXME Erlang 24 has a bug https://github.com/erlang/otp/issues/5097 // uncomment once they fix it - //if options.FlagV4NC { + lenID := 3 + //if options.FlagBigPidRef { // lenID = 5 //} buf := b.Extend(4 + lenID*4) @@ -263,23 +266,23 @@ func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { switch t := term.(type) { case bool: - if cacheEnabled && cacheIndex < 256 { + if cacheEnabled && cacheIndex < 255 { value := Atom("false") if t { value = Atom("true") } // looking for CacheItem - ci, found := options.WriterAtomCache[value] + ci, found := options.SenderAtomCache[value] if found { - options.EncodingAtomCache.Append(ci) - b.Append([]byte{ettCacheRef, byte(cacheIndex)}) - cacheIndex++ + i := options.EncodingAtomCache.Append(ci) + cacheIndex = int16(i + 1) + b.Append([]byte{ettCacheRef, byte(i)}) break + } else { + // add it to the cache and encode as usual Atom + options.AtomCache.Append(value) } - // add it to the cache and encode as usual Atom - options.LinkAtomCache.Append(value) - } if t { @@ -420,7 +423,6 @@ func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { buf := []byte{ettSmallBig, 0, negative, 0, 0, 0, 0, 0, 0, 0, 0} binary.LittleEndian.PutUint64(buf[3:], uint64(t)) - switch { case t < 4294967296: buf[1] = 4 @@ -508,19 +510,6 @@ func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { // characters and are always encoded using the UTF-8 external // formats ATOM_UTF8_EXT or SMALL_ATOM_UTF8_EXT. - if cacheEnabled && cacheIndex < 256 { - // looking for CacheItem - ci, found := options.WriterAtomCache[t] - if found { - options.EncodingAtomCache.Append(ci) - b.Append([]byte{ettCacheRef, byte(cacheIndex)}) - cacheIndex++ - break - } - - options.LinkAtomCache.Append(t) - } - // https://erlang.org/doc/apps/erts/erl_ext_dist.html#utf8_atoms // The maximum number of allowed characters in an atom is 255. // In the UTF-8 case, each character can need 4 bytes to be encoded. @@ -528,6 +517,20 @@ func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { return ErrAtomTooLong } + if cacheEnabled && cacheIndex < 255 { + // looking for CacheItem + ci, found := options.SenderAtomCache[t] + if found { + i := options.EncodingAtomCache.Append(ci) + cacheIndex = int16(i + 1) + b.Append([]byte{ettCacheRef, byte(i)}) + break + } else { + // add it to the cache and encode as usual Atom + options.AtomCache.Append(t) + } + } + lenAtom := len(t) if lenAtom < 256 { buf := b.Extend(1 + 1 + lenAtom) @@ -541,7 +544,7 @@ func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { buf := b.Extend(1 + 2 + lenAtom) buf[0] = ettAtomUTF8 binary.BigEndian.PutUint16(buf[1:3], uint16(lenAtom)) - copy(b.B[3:], t) + copy(buf[3:], t) case float32: term = float64(t) @@ -609,11 +612,11 @@ func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { } // LEN a 16-bit big endian unsigned integer not larger - // than 5 when the DFLAG_V4_NC has been set; otherwise not larger than 3. + // than 5 when the FlagBigPidRef has been set; otherwise not larger than 3. // FIXME Erlang 24 has a bug https://github.com/erlang/otp/issues/5097 // uncomment once they fix it - //if options.FlagV4NC { + //if options.FlagBigPidRef { // binary.BigEndian.PutUint16(buf[1:3], 5) //} else { binary.BigEndian.PutUint16(buf[1:3], 3) @@ -709,6 +712,10 @@ func Encode(term Term, b *lib.Buffer, options EncodeOptions) (retErr error) { case reflect.Array, reflect.Slice: lenList := v.Len() + if lenList == 0 { + b.AppendByte(ettNil) + continue + } buf := b.Extend(5) buf[0] = ettList binary.BigEndian.PutUint32(buf[1:], uint32(lenList)) diff --git a/etf/encode_test.go b/etf/encode_test.go index 553f5807..8a59627d 100644 --- a/etf/encode_test.go +++ b/etf/encode_test.go @@ -1,7 +1,6 @@ package etf import ( - "context" "fmt" "math/big" "reflect" @@ -27,20 +26,16 @@ func TestEncodeBoolWithAtomCache(t *testing.T) { b := lib.TakeBuffer() defer lib.ReleaseBuffer(b) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - writerAtomCache := make(map[Atom]CacheItem) - encodingAtomCache := TakeListAtomCache() - defer ReleaseListAtomCache(encodingAtomCache) - linkAtomCache := NewAtomCache(ctx) + senderAtomCache := make(map[Atom]CacheItem) + encodingAtomCache := TakeEncodingAtomCache() + atomCache := NewAtomCache() ci := CacheItem{ID: 499, Encoded: true, Name: "false"} - writerAtomCache["false"] = ci + senderAtomCache["false"] = ci encodeOptions := EncodeOptions{ - LinkAtomCache: linkAtomCache, - WriterAtomCache: writerAtomCache, + AtomCache: atomCache.Out, + SenderAtomCache: senderAtomCache, EncodingAtomCache: encodingAtomCache, } @@ -75,71 +70,71 @@ func integerCases() []integerCase { // // unsigned integers // - integerCase{"uint8::255", uint8(255), []byte{ettSmallInteger, 255}}, - integerCase{"uint16::255", uint16(255), []byte{ettSmallInteger, 255}}, - integerCase{"uint32::255", uint32(255), []byte{ettSmallInteger, 255}}, - integerCase{"uint64::255", uint64(255), []byte{ettSmallInteger, 255}}, - integerCase{"uint::255", uint(255), []byte{ettSmallInteger, 255}}, + {"uint8::255", uint8(255), []byte{ettSmallInteger, 255}}, + {"uint16::255", uint16(255), []byte{ettSmallInteger, 255}}, + {"uint32::255", uint32(255), []byte{ettSmallInteger, 255}}, + {"uint64::255", uint64(255), []byte{ettSmallInteger, 255}}, + {"uint::255", uint(255), []byte{ettSmallInteger, 255}}, - integerCase{"uint16::256", uint16(256), []byte{ettInteger, 0, 0, 1, 0}}, + {"uint16::256", uint16(256), []byte{ettInteger, 0, 0, 1, 0}}, - integerCase{"uint16::65535", uint16(65535), []byte{ettInteger, 0, 0, 255, 255}}, - integerCase{"uint32::65535", uint32(65535), []byte{ettInteger, 0, 0, 255, 255}}, - integerCase{"uint64::65535", uint64(65535), []byte{ettInteger, 0, 0, 255, 255}}, + {"uint16::65535", uint16(65535), []byte{ettInteger, 0, 0, 255, 255}}, + {"uint32::65535", uint32(65535), []byte{ettInteger, 0, 0, 255, 255}}, + {"uint64::65535", uint64(65535), []byte{ettInteger, 0, 0, 255, 255}}, - integerCase{"uint64::65536", uint64(65536), []byte{ettInteger, 0, 1, 0, 0}}, + {"uint64::65536", uint64(65536), []byte{ettInteger, 0, 1, 0, 0}}, // treat as an int32 - integerCase{"uint32::2147483647", uint32(2147483647), []byte{ettInteger, 127, 255, 255, 255}}, - integerCase{"uint64::2147483647", uint64(2147483647), []byte{ettInteger, 127, 255, 255, 255}}, - integerCase{"uint64::2147483648", uint64(2147483648), []byte{ettSmallBig, 4, 0, 0, 0, 0, 128}}, + {"uint32::2147483647", uint32(2147483647), []byte{ettInteger, 127, 255, 255, 255}}, + {"uint64::2147483647", uint64(2147483647), []byte{ettInteger, 127, 255, 255, 255}}, + {"uint64::2147483648", uint64(2147483648), []byte{ettSmallBig, 4, 0, 0, 0, 0, 128}}, - integerCase{"uint32::4294967295", uint32(4294967295), []byte{ettSmallBig, 4, 0, 255, 255, 255, 255}}, - integerCase{"uint64::4294967295", uint64(4294967295), []byte{ettSmallBig, 4, 0, 255, 255, 255, 255}}, - integerCase{"uint64::4294967296", uint64(4294967296), []byte{ettSmallBig, 5, 0, 0, 0, 0, 0, 1}}, + {"uint32::4294967295", uint32(4294967295), []byte{ettSmallBig, 4, 0, 255, 255, 255, 255}}, + {"uint64::4294967295", uint64(4294967295), []byte{ettSmallBig, 4, 0, 255, 255, 255, 255}}, + {"uint64::4294967296", uint64(4294967296), []byte{ettSmallBig, 5, 0, 0, 0, 0, 0, 1}}, - integerCase{"uint64::18446744073709551615", uint64(18446744073709551615), []byte{ettSmallBig, 8, 0, 255, 255, 255, 255, 255, 255, 255, 255}}, + {"uint64::18446744073709551615", uint64(18446744073709551615), []byte{ettSmallBig, 8, 0, 255, 255, 255, 255, 255, 255, 255, 255}}, // // signed integers // // negative is always ettInteger for the numbers within the range of int32 - integerCase{"int8::-127", int8(-127), []byte{ettInteger, 255, 255, 255, 129}}, - integerCase{"int16::-127", int16(-127), []byte{ettInteger, 255, 255, 255, 129}}, - integerCase{"int32::-127", int32(-127), []byte{ettInteger, 255, 255, 255, 129}}, - integerCase{"int64::-127", int64(-127), []byte{ettInteger, 255, 255, 255, 129}}, - integerCase{"int::-127", int(-127), []byte{ettInteger, 255, 255, 255, 129}}, - - // positive within range of int8 treats as ettSmallInteger - integerCase{"int8::127", int8(127), []byte{ettSmallInteger, 127}}, - integerCase{"int16::127", int16(127), []byte{ettSmallInteger, 127}}, - integerCase{"int32::127", int32(127), []byte{ettSmallInteger, 127}}, - integerCase{"int64::127", int64(127), []byte{ettSmallInteger, 127}}, + {"int8::-127", int8(-127), []byte{ettInteger, 255, 255, 255, 129}}, + {"int16::-127", int16(-127), []byte{ettInteger, 255, 255, 255, 129}}, + {"int32::-127", int32(-127), []byte{ettInteger, 255, 255, 255, 129}}, + {"int64::-127", int64(-127), []byte{ettInteger, 255, 255, 255, 129}}, + {"int::-127", int(-127), []byte{ettInteger, 255, 255, 255, 129}}, + + // positive within a range of int8 treats as ettSmallInteger + {"int8::127", int8(127), []byte{ettSmallInteger, 127}}, + {"int16::127", int16(127), []byte{ettSmallInteger, 127}}, + {"int32::127", int32(127), []byte{ettSmallInteger, 127}}, + {"int64::127", int64(127), []byte{ettSmallInteger, 127}}, // a positive int[16,32,64] value within the range of uint8 treats as an uint8 - integerCase{"int16::128", int16(128), []byte{ettSmallInteger, 128}}, - integerCase{"int32::128", int32(128), []byte{ettSmallInteger, 128}}, - integerCase{"int64::128", int64(128), []byte{ettSmallInteger, 128}}, - integerCase{"int::128", int(128), []byte{ettSmallInteger, 128}}, + {"int16::128", int16(128), []byte{ettSmallInteger, 128}}, + {"int32::128", int32(128), []byte{ettSmallInteger, 128}}, + {"int64::128", int64(128), []byte{ettSmallInteger, 128}}, + {"int::128", int(128), []byte{ettSmallInteger, 128}}, // whether its positive or negative value within the range of int16 its treating as an int32 - integerCase{"int16::-32767", int16(-32767), []byte{ettInteger, 255, 255, 128, 1}}, - integerCase{"int16::32767", int16(32767), []byte{ettInteger, 0, 0, 127, 255}}, + {"int16::-32767", int16(-32767), []byte{ettInteger, 255, 255, 128, 1}}, + {"int16::32767", int16(32767), []byte{ettInteger, 0, 0, 127, 255}}, // treat as an int32 - integerCase{"int32::2147483647", int32(2147483647), []byte{ettInteger, 127, 255, 255, 255}}, - integerCase{"int32::-2147483648", int32(-2147483648), []byte{ettInteger, 128, 0, 0, 0}}, - integerCase{"int64::2147483647", int64(2147483647), []byte{ettInteger, 127, 255, 255, 255}}, - integerCase{"int64::-2147483648", int64(-2147483648), []byte{ettInteger, 128, 0, 0, 0}}, + {"int32::2147483647", int32(2147483647), []byte{ettInteger, 127, 255, 255, 255}}, + {"int32::-2147483648", int32(-2147483648), []byte{ettInteger, 128, 0, 0, 0}}, + {"int64::2147483647", int64(2147483647), []byte{ettInteger, 127, 255, 255, 255}}, + {"int64::-2147483648", int64(-2147483648), []byte{ettInteger, 128, 0, 0, 0}}, - integerCase{"int64::2147483648", int64(2147483648), []byte{ettSmallBig, 4, 0, 0, 0, 0, 128}}, + {"int64::2147483648", int64(2147483648), []byte{ettSmallBig, 4, 0, 0, 0, 0, 128}}, // int64 treats as ettSmallBig whether its positive or negative - integerCase{"int64::9223372036854775807", int64(9223372036854775807), []byte{ettSmallBig, 8, 0, 255, 255, 255, 255, 255, 255, 255, 127}}, - integerCase{"int64::-9223372036854775808", int64(-9223372036854775808), []byte{ettSmallBig, 8, 1, 0, 0, 0, 0, 0, 0, 0, 128}}, + {"int64::9223372036854775807", int64(9223372036854775807), []byte{ettSmallBig, 8, 0, 255, 255, 255, 255, 255, 255, 255, 127}}, + {"int64::-9223372036854775808", int64(-9223372036854775808), []byte{ettSmallBig, 8, 1, 0, 0, 0, 0, 0, 0, 0, 128}}, - integerCase{"big.int::-9223372036854775807123456789", bigIntNegative, []byte{ettSmallBig, 12, 1, 21, 3, 193, 203, 255, 255, 255, 255, 255, 100, 205, 29}}, + {"big.int::-9223372036854775807123456789", bigIntNegative, []byte{ettSmallBig, 12, 1, 21, 3, 193, 203, 255, 255, 255, 255, 255, 100, 205, 29}}, } } @@ -260,20 +255,17 @@ func TestEncodeAtomWithCache(t *testing.T) { b := lib.TakeBuffer() defer lib.ReleaseBuffer(b) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + senderAtomCache := make(map[Atom]CacheItem) + encodingAtomCache := TakeEncodingAtomCache() - writerAtomCache := make(map[Atom]CacheItem) - encodingAtomCache := TakeListAtomCache() - defer ReleaseListAtomCache(encodingAtomCache) - linkAtomCache := NewAtomCache(ctx) + atomCache := NewAtomCache() ci := CacheItem{ID: 2020, Encoded: true, Name: "cached atom"} - writerAtomCache["cached atom"] = ci + senderAtomCache["cached atom"] = ci encodeOptions := EncodeOptions{ - LinkAtomCache: linkAtomCache, - WriterAtomCache: writerAtomCache, + AtomCache: atomCache.Out, + SenderAtomCache: senderAtomCache, EncodingAtomCache: encodingAtomCache, } @@ -374,7 +366,23 @@ func TestEncodeSlice(t *testing.T) { fmt.Println("got", b.B) t.Fatal("incorrect value") } + + b.Reset() + + expected = []byte{108, 0, 0, 0, 3, 119, 1, 97, 119, 1, 98, 119, 1, 99, 106} + termAtoms := []Atom{Atom("a"), Atom("b"), Atom("c")} + err = Encode(termAtoms, b, EncodeOptions{}) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(b.B, expected) { + fmt.Println("exp", expected) + fmt.Println("got", b.B) + t.Fatal("incorrect value") + } } + func TestEncodeListNested(t *testing.T) { b := lib.TakeBuffer() defer lib.ReleaseBuffer(b) @@ -641,7 +649,7 @@ func TestEncodeStructWithTags(t *testing.T) { Key1: "Hello World!", Key2: []*Charlist{&value2, &value3, &value4}, Key3: &nested, - Key4: [][]*Charlist{[]*Charlist{&value2, &value3, &value4}, []*Charlist{&value2, &value3, &value4}}, + Key4: [][]*Charlist{{&value2, &value3, &value4}, {&value2, &value3, &value4}}, } err := Encode(term, b, EncodeOptions{}) if err != nil { @@ -658,7 +666,7 @@ func TestEncodePid(t *testing.T) { b := lib.TakeBuffer() defer lib.ReleaseBuffer(b) - // V4NC disabled. max value for ID (15 bits), serial 0 + // FlagBigPidRef disabled. max value for ID (15 bits), serial 0 expected := []byte{ettPid, 119, 18, 101, 114, 108, 45, 100, 101, 109, 111, 64, 49, 50, 55, 46, 48, 46, 48, 46, 49, 0, 0, 127, 255, 0, 0, 0, 0, 2} term := Pid{Node: "erl-demo@127.0.0.1", ID: 32767, Creation: 2} @@ -674,7 +682,7 @@ func TestEncodePid(t *testing.T) { t.Fatal("incorrect value") } - // V4NC disabled. overflowed 15 bit. ID 0, serial 1 + // FlagBigPidRef disabled. overflowed 15 bit. ID 0, serial 1 b.Reset() expected = []byte{ettPid, 119, 18, 101, 114, 108, 45, 100, 101, 109, 111, 64, 49, 50, 55, 46, 48, 46, 48, 46, 49, 0, 0, 0, 0, 0, 0, 0, 1, 2} @@ -691,7 +699,7 @@ func TestEncodePid(t *testing.T) { t.Fatal("incorrect value") } - // BigCreation, V4NC enabled. max value for ID (32 bits), serial 0 + // BigCreation, FlagBigPidRef enabled. max value for ID (32 bits), serial 0 b.Reset() expected = []byte{ettNewPid, 119, 18, 101, 114, 108, 45, 100, 101, 109, 111, 64, 49, 50, 55, 46, 48, 46, 48, 46, 49, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 2} @@ -699,7 +707,7 @@ func TestEncodePid(t *testing.T) { options := EncodeOptions{ FlagBigCreation: true, - FlagV4NC: true, + FlagBigPidRef: true, } err = Encode(term, b, options) if err != nil { @@ -712,7 +720,7 @@ func TestEncodePid(t *testing.T) { t.Fatal("incorrect value") } - // BigCreation, V4NC enabled. max value for ID (32 bits), max value for Serial (32 bits) + // BigCreation, FlagBigPidRef enabled. max value for ID (32 bits), max value for Serial (32 bits) b.Reset() expected = []byte{ettNewPid, 119, 18, 101, 114, 108, 45, 100, 101, 109, 111, 64, 49, 50, 55, 46, 48, 46, 48, 46, 49, 255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 2} @@ -720,7 +728,7 @@ func TestEncodePid(t *testing.T) { options = EncodeOptions{ FlagBigCreation: true, - FlagV4NC: true, + FlagBigPidRef: true, } err = Encode(term, b, options) if err != nil { @@ -741,19 +749,15 @@ func TestEncodePidWithAtomCache(t *testing.T) { expected := []byte{103, 82, 0, 0, 0, 1, 56, 0, 0, 0, 0, 2} term := Pid{Node: "erl-demo@127.0.0.1", ID: 312, Creation: 2} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - writerAtomCache := make(map[Atom]CacheItem) - encodingAtomCache := TakeListAtomCache() - defer ReleaseListAtomCache(encodingAtomCache) - linkAtomCache := NewAtomCache(ctx) + senderAtomCache := make(map[Atom]CacheItem) + encodingAtomCache := TakeEncodingAtomCache() + atomCache := NewAtomCache() ci := CacheItem{ID: 2020, Encoded: true, Name: "erl-demo@127.0.0.1"} - writerAtomCache["erl-demo@127.0.0.1"] = ci + senderAtomCache["erl-demo@127.0.0.1"] = ci encodeOptions := EncodeOptions{ - LinkAtomCache: linkAtomCache, - WriterAtomCache: writerAtomCache, + AtomCache: atomCache.Out, + SenderAtomCache: senderAtomCache, EncodingAtomCache: encodingAtomCache, } err := Encode(term, b, encodeOptions) @@ -777,7 +781,7 @@ func TestEncodeRef(t *testing.T) { b := lib.TakeBuffer() defer lib.ReleaseBuffer(b) - // FlagBigCreation = false, FlagV4NC = false + // FlagBigCreation = false, FlagBigPidRef = false expected := []byte{ettNewRef, 0, 3, 119, 18, 101, 114, 108, 45, 100, 101, 109, 111, 64, 49, 50, 55, 46, 48, 46, 48, 46, 49, 3, 0, 1, 30, 228, 183, 192, 0, 1, 141, 122, 203, 35} @@ -801,7 +805,7 @@ func TestEncodeRef(t *testing.T) { t.Fatal("incorrect value") } - // FlagBigCreation = true, FlagV4NC = false + // FlagBigCreation = true, FlagBigPidRef = false b.Reset() expected = []byte{ettNewerRef, 0, 3, 119, 18, 101, 114, 108, 45, 100, 101, 109, 111, 64, 49, 50, 55, 46, 48, 46, 48, 46, 49, 0, 0, 0, 8, 0, 1, 30, 228, 183, 192, 0, 1, 141, @@ -831,7 +835,7 @@ func TestEncodeRef(t *testing.T) { // FIXME Erlang 24 has a bug https://github.com/erlang/otp/issues/5097 // uncomment once they fix it // - // FlagBigCreation = true, FlagV4NC = true + // FlagBigCreation = true, FlagBigPidRef = true //b.Reset() //expected = []byte{ettNewerRef, 0, 5, 119, 18, 101, 114, 108, 45, 100, 101, 109, 111, 64, // 49, 50, 55, 46, 48, 46, 48, 46, 49, 0, 0, 0, 8, 0, 1, 30, 228, 183, 192, 0, 1, 141, @@ -845,7 +849,7 @@ func TestEncodeRef(t *testing.T) { //options = EncodeOptions{ // FlagBigCreation: true, - // FlagV4NC: true, + // FlagBigPidRef: true, //} //err = Encode(term, b, options) //if err != nil { @@ -952,25 +956,20 @@ func BenchmarkEncodeBoolWithAtomCache(b *testing.B) { buf := lib.TakeBuffer() defer lib.ReleaseBuffer(buf) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + senderAtomCache := make(map[Atom]CacheItem) + encodingAtomCache := TakeEncodingAtomCache() + atomCache := NewAtomCache() - writerAtomCache := make(map[Atom]CacheItem) - encodingAtomCache := TakeListAtomCache() - defer ReleaseListAtomCache(encodingAtomCache) - linkAtomCache := NewAtomCache(ctx) - - writerAtomCache["false"] = CacheItem{ID: 499, Encoded: true, Name: "false"} + senderAtomCache["false"] = CacheItem{ID: 499, Encoded: true, Name: "false"} encodeOptions := EncodeOptions{ - LinkAtomCache: linkAtomCache, - WriterAtomCache: writerAtomCache, + AtomCache: atomCache.Out, + SenderAtomCache: senderAtomCache, EncodingAtomCache: encodingAtomCache, } b.ResetTimer() for i := 0; i < b.N; i++ { err := Encode(false, buf, encodeOptions) - encodingAtomCache.Reset() buf.Reset() if err != nil { b.Fatal(err) @@ -1050,20 +1049,16 @@ func BenchmarkEncodeAtomWithCache(b *testing.B) { buf := lib.TakeBuffer() defer lib.ReleaseBuffer(buf) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - writerAtomCache := make(map[Atom]CacheItem) - encodingAtomCache := TakeListAtomCache() - defer ReleaseListAtomCache(encodingAtomCache) - linkAtomCache := NewAtomCache(ctx) + senderAtomCache := make(map[Atom]CacheItem) + encodingAtomCache := TakeEncodingAtomCache() + atomCache := NewAtomCache() ci := CacheItem{ID: 2020, Encoded: true, Name: "cached atom"} - writerAtomCache["cached atom"] = ci + senderAtomCache["cached atom"] = ci encodeOptions := EncodeOptions{ - LinkAtomCache: linkAtomCache, - WriterAtomCache: writerAtomCache, + AtomCache: atomCache.Out, + SenderAtomCache: senderAtomCache, EncodingAtomCache: encodingAtomCache, } @@ -1071,7 +1066,6 @@ func BenchmarkEncodeAtomWithCache(b *testing.B) { for i := 0; i < b.N; i++ { err := Encode(Atom("cached atom"), buf, encodeOptions) buf.Reset() - encodingAtomCache.Reset() if err != nil { b.Fatal(err) } @@ -1275,20 +1269,16 @@ func BenchmarkEncodePidWithAtomCache(b *testing.B) { term := Pid{Node: "erl-demo@127.0.0.1", ID: 312, Creation: 2} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - writerAtomCache := make(map[Atom]CacheItem) - encodingAtomCache := TakeListAtomCache() - defer ReleaseListAtomCache(encodingAtomCache) - linkAtomCache := NewAtomCache(ctx) + senderAtomCache := make(map[Atom]CacheItem) + encodingAtomCache := TakeEncodingAtomCache() + atomCache := NewAtomCache() ci := CacheItem{ID: 2020, Encoded: true, Name: "erl-demo@127.0.0.1"} - writerAtomCache["erl-demo@127.0.0.1"] = ci + senderAtomCache["erl-demo@127.0.0.1"] = ci encodeOptions := EncodeOptions{ - LinkAtomCache: linkAtomCache, - WriterAtomCache: writerAtomCache, + AtomCache: atomCache.Out, + SenderAtomCache: senderAtomCache, EncodingAtomCache: encodingAtomCache, } @@ -1296,7 +1286,6 @@ func BenchmarkEncodePidWithAtomCache(b *testing.B) { for i := 0; i < b.N; i++ { err := Encode(term, buf, encodeOptions) buf.Reset() - encodingAtomCache.Reset() if err != nil { b.Fatal(err) } @@ -1332,20 +1321,16 @@ func BenchmarkEncodeRefWithAtomCache(b *testing.B) { ID: [5]uint32{73444, 3082813441, 2373634851}, } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - writerAtomCache := make(map[Atom]CacheItem) - encodingAtomCache := TakeListAtomCache() - defer ReleaseListAtomCache(encodingAtomCache) - linkAtomCache := NewAtomCache(ctx) + senderAtomCache := make(map[Atom]CacheItem) + encodingAtomCache := TakeEncodingAtomCache() + atomCache := NewAtomCache() ci := CacheItem{ID: 2020, Encoded: true, Name: "erl-demo@127.0.0.1"} - writerAtomCache["erl-demo@127.0.0.1"] = ci + senderAtomCache["erl-demo@127.0.0.1"] = ci encodeOptions := EncodeOptions{ - LinkAtomCache: linkAtomCache, - WriterAtomCache: writerAtomCache, + AtomCache: atomCache.Out, + SenderAtomCache: senderAtomCache, EncodingAtomCache: encodingAtomCache, } @@ -1353,7 +1338,6 @@ func BenchmarkEncodeRefWithAtomCache(b *testing.B) { for i := 0; i < b.N; i++ { err := Encode(term, buf, encodeOptions) buf.Reset() - encodingAtomCache.Reset() if err != nil { b.Fatal(err) } @@ -1398,20 +1382,17 @@ func BenchmarkEncodeTupleRefPidWithAtomCache(b *testing.B) { ID: 312, Creation: 2}} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - writerAtomCache := make(map[Atom]CacheItem) - encodingAtomCache := TakeListAtomCache() - defer ReleaseListAtomCache(encodingAtomCache) - linkAtomCache := NewAtomCache(ctx) + senderAtomCache := make(map[Atom]CacheItem) + encodingAtomCache := TakeEncodingAtomCache() + defer ReleaseEncodingAtomCache(encodingAtomCache) + atomCache := NewAtomCache() ci := CacheItem{ID: 2020, Encoded: true, Name: "erl-demo@127.0.0.1"} - writerAtomCache["erl-demo@127.0.0.1"] = ci + senderAtomCache["erl-demo@127.0.0.1"] = ci encodeOptions := EncodeOptions{ - LinkAtomCache: linkAtomCache, - WriterAtomCache: writerAtomCache, + AtomCache: atomCache.Out, + SenderAtomCache: senderAtomCache, EncodingAtomCache: encodingAtomCache, } diff --git a/etf/etf.go b/etf/etf.go index b94501a5..aa29c93d 100644 --- a/etf/etf.go +++ b/etf/etf.go @@ -2,20 +2,30 @@ package etf import ( "fmt" - "hash/fnv" + "hash/crc32" "reflect" "strings" ) +// Term type Term interface{} + +// Tuple type Tuple []Term + +// List type List []Term + +// Alias type Alias Ref // ListImproper as a workaround for the Erlang's improper list [a|b]. Intended to be used to interact with Erlang. type ListImproper []Term +// Atom type Atom string + +// Map type Map map[Term]Term // String this type is intended to be used to interact with Erlang. String value encodes as a binary (Erlang type: <<...>>) @@ -24,18 +34,21 @@ type String string // Charlist this type is intended to be used to interact with Erlang. Charlist value encodes as a list of int32 numbers in order to support Erlang string with UTF-8 symbols on an Erlang side (Erlang type: [...]) type Charlist string +// Pid type Pid struct { Node Atom ID uint64 Creation uint32 } +// Port type Port struct { Node Atom ID uint32 Creation uint32 } +// Ref type Ref struct { Node Atom Creation uint32 @@ -72,6 +85,7 @@ type Unmarshaler interface { UnmarshalETF([]byte) error } +// Function type Function struct { Arity byte Unique [16]byte @@ -85,9 +99,10 @@ type Function struct { } var ( - hasher32 = fnv.New32a() + crc32q = crc32.MakeTable(0xD5828281) ) +// Export type Export struct { Module Atom Function Atom @@ -140,18 +155,22 @@ const ( ettFloat = byte(99) // legacy ) +// Element func (m Map) Element(k Term) Term { return m[k] } +// Element func (l List) Element(i int) Term { return l[i-1] } +// Element func (t Tuple) Element(i int) Term { return t[i-1] } +// String func (p Pid) String() string { empty := Pid{} if p == empty { @@ -160,33 +179,30 @@ func (p Pid) String() string { n := uint32(0) if p.Node != "" { - hasher32.Write([]byte(p.Node)) - defer hasher32.Reset() - n = hasher32.Sum32() + n = crc32.Checksum([]byte(p.Node), crc32q) } - return fmt.Sprintf("<%X.%d.%d>", n, int32(p.ID>>32), int32(p.ID)) + return fmt.Sprintf("<%08X.%d.%d>", n, int32(p.ID>>32), int32(p.ID)) } +// String func (r Ref) String() string { n := uint32(0) if r.Node != "" { - hasher32.Write([]byte(r.Node)) - defer hasher32.Reset() - n = hasher32.Sum32() + n = crc32.Checksum([]byte(r.Node), crc32q) } - return fmt.Sprintf("Ref#<%X.%d.%d.%d>", n, r.ID[0], r.ID[1], r.ID[2]) + return fmt.Sprintf("Ref#<%08X.%d.%d.%d>", n, r.ID[0], r.ID[1], r.ID[2]) } +// String func (a Alias) String() string { n := uint32(0) if a.Node != "" { - hasher32.Write([]byte(a.Node)) - defer hasher32.Reset() - n = hasher32.Sum32() + n = crc32.Checksum([]byte(a.Node), crc32q) } - return fmt.Sprintf("Ref#<%X.%d.%d.%d>", n, a.ID[0], a.ID[1], a.ID[2]) + return fmt.Sprintf("Ref#<%08X.%d.%d.%d>", n, a.ID[0], a.ID[1], a.ID[2]) } +// ProplistElement type ProplistElement struct { Name Atom Value Term @@ -215,7 +231,7 @@ func TermToString(t Term) (s string, ok bool) { return } -// ProplistIntoStruct transorms given term into the provided struct 'dest'. +// TermProplistIntoStruct transorms given term into the provided struct 'dest'. // Proplist is the list of Tuple values with two items { Name , Value }, // where Name can be string or Atom and Value must be the same type as // it has the field of 'dest' struct with the equivalent name. Its also @@ -463,7 +479,7 @@ func setProplistField(list List, dest reflect.Value) error { t := dest.Type() numField := t.NumField() fields := make([]reflect.StructField, numField) - for i, _ := range fields { + for i := range fields { fields[i] = t.Field(i) } @@ -496,7 +512,7 @@ func setProplistElementField(proplist []ProplistElement, dest reflect.Value) err t := dest.Type() numField := t.NumField() fields := make([]reflect.StructField, numField) - for i, _ := range fields { + for i := range fields { fields[i] = t.Field(i) } @@ -554,7 +570,7 @@ func setMapStructField(term Map, dest reflect.Value) error { t := dest.Type() numField := t.NumField() fields := make([]reflect.StructField, numField) - for i, _ := range fields { + for i := range fields { fields[i] = t.Field(i) } diff --git a/etf/etf_test.go b/etf/etf_test.go index e78948c5..90ad888b 100644 --- a/etf/etf_test.go +++ b/etf/etf_test.go @@ -446,9 +446,7 @@ func TestEncodeDecodePid(t *testing.T) { t.Fatal(err) } - decodeOptions := DecodeOptions{ - FlagBigCreation: true, - } + decodeOptions := DecodeOptions{} term, _, err = Decode(b.B, []Atom{}, decodeOptions) pidOut, ok = term.(Pid) if !ok { @@ -461,10 +459,10 @@ func TestEncodeDecodePid(t *testing.T) { t.Fatal("incorrect result") } - // enable V4NC + // enable FlagBigPidRef b.Reset() encodeOptions = EncodeOptions{ - FlagV4NC: true, + FlagBigPidRef: true, } err = Encode(pidIn, b, encodeOptions) if err != nil { @@ -472,7 +470,7 @@ func TestEncodeDecodePid(t *testing.T) { } decodeOptions = DecodeOptions{ - FlagV4NC: true, + FlagBigPidRef: true, } term, _, err = Decode(b.B, []Atom{}, decodeOptions) pidOut, ok = term.(Pid) @@ -486,10 +484,10 @@ func TestEncodeDecodePid(t *testing.T) { t.Fatal("incorrect result") } - // enable BigCreation and V4NC + // enable BigCreation and FlagBigPidRef b.Reset() encodeOptions = EncodeOptions{ - FlagV4NC: true, + FlagBigPidRef: true, FlagBigCreation: true, } err = Encode(pidIn, b, encodeOptions) @@ -498,8 +496,7 @@ func TestEncodeDecodePid(t *testing.T) { } decodeOptions = DecodeOptions{ - FlagV4NC: true, - FlagBigCreation: true, + FlagBigPidRef: true, } term, _, err = Decode(b.B, []Atom{}, decodeOptions) pidOut, ok = term.(Pid) diff --git a/examples/application/demoApplication.go b/examples/application/demoApplication.go index dcf70ff6..8bcc1d08 100644 --- a/examples/application/demoApplication.go +++ b/examples/application/demoApplication.go @@ -31,7 +31,7 @@ func (da *demoApp) Load(args ...etf.Term) (gen.ApplicationSpec, error) { Name: "demoApp", Description: "Demo Applicatoin", Version: "v.1.0", - Environment: map[string]interface{}{ + Environment: map[gen.EnvKey]interface{}{ "envName1": 123, "envName2": "Hello world", }, @@ -116,21 +116,14 @@ func (dgs *demoGenServ) HandleCall(process *gen.ServerProcess, from gen.ServerFr } func init() { - flag.IntVar(&ListenRangeBegin, "listen_begin", 15151, "listen port range") - flag.IntVar(&ListenRangeEnd, "listen_end", 25151, "listen port range") flag.StringVar(&NodeName, "name", "demo@127.0.0.1", "node name") - flag.IntVar(&ListenEPMD, "epmd", 4369, "EPMD port") flag.StringVar(&Cookie, "cookie", "123", "cookie for interaction with erlang cluster") } func main() { flag.Parse() - opts := node.Options{ - ListenRangeBegin: uint16(ListenRangeBegin), - ListenRangeEnd: uint16(ListenRangeEnd), - EPMDPort: uint16(ListenEPMD), - } + opts := node.Options{} // Initialize new node with given name, cookie, listening port range and epmd port demoNode, _ := ergo.StartNode(NodeName, Cookie, opts) diff --git a/examples/gensaga/main.go b/examples/gensaga/main.go index e406b358..a2ff0f53 100644 --- a/examples/gensaga/main.go +++ b/examples/gensaga/main.go @@ -67,11 +67,7 @@ func main() { time.Sleep(300 * time.Millisecond) node1.Stop() - node1.Wait() node2.Stop() - node2.Wait() node3.Stop() - node3.Wait() node4.Stop() - node4.Wait() } diff --git a/examples/genserver/server.go b/examples/genserver/server.go index 1b61ba9b..77af00e7 100644 --- a/examples/genserver/server.go +++ b/examples/genserver/server.go @@ -16,14 +16,12 @@ type demo struct { } var ( - ServerName string - NodeName string - Cookie string - err error - ListenRangeBegin int - ListenRangeEnd int = 35000 - Listen string - ListenEPMD int + ServerName string + NodeName string + Cookie string + err error + ListenBegin int + ListenEnd int = 35000 EnableRPC bool ) @@ -61,11 +59,10 @@ func (dgs *demo) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, mes } func init() { - flag.IntVar(&ListenRangeBegin, "listen_begin", 15151, "listen port range") - flag.IntVar(&ListenRangeEnd, "listen_end", 25151, "listen port range") + flag.IntVar(&ListenBegin, "listen_begin", 15151, "listen port range") + flag.IntVar(&ListenEnd, "listen_end", 25151, "listen port range") flag.StringVar(&ServerName, "gen_server_name", "example", "gen_server name") flag.StringVar(&NodeName, "name", "demo@127.0.0.1", "node name") - flag.IntVar(&ListenEPMD, "epmd", 4369, "EPMD port") flag.StringVar(&Cookie, "cookie", "123", "cookie for interaction with erlang cluster") } @@ -73,9 +70,8 @@ func main() { flag.Parse() opts := node.Options{ - ListenRangeBegin: uint16(ListenRangeBegin), - ListenRangeEnd: uint16(ListenRangeEnd), - EPMDPort: uint16(ListenEPMD), + ListenBegin: uint16(ListenBegin), + ListenEnd: uint16(ListenEnd), } // Initialize new node with given name, cookie, listening port range and epmd port diff --git a/examples/genstage/consumer.go b/examples/genstage/consumer.go index 8310b9ee..58c4e805 100644 --- a/examples/genstage/consumer.go +++ b/examples/genstage/consumer.go @@ -30,7 +30,7 @@ func (c *Consumer) InitStage(process *gen.StageProcess, args ...etf.Term) (gen.S fmt.Println("Subscribe consumer", process.Name(), "[", process.Self(), "]", "with min events =", opts.MinDemand, "and max events", opts.MaxDemand) - process.Subscribe(gen.ProcessID{"producer", "node_abc@localhost"}, opts) + process.Subscribe(gen.ProcessID{Name: "producer", Node: "node_abc@localhost"}, opts) return gen.StageOptions{}, nil } func (c *Consumer) HandleEvents(process *gen.StageProcess, subscription gen.StageSubscription, events etf.List) gen.StageStatus { diff --git a/examples/http/app.go b/examples/http/app.go index 21f85401..94728017 100644 --- a/examples/http/app.go +++ b/examples/http/app.go @@ -20,7 +20,6 @@ func (a *App) Load(args ...etf.Term) (gen.ApplicationSpec, error) { Name: "WebApp", Description: "Demo Web Application", Version: "v.1.0", - Environment: map[string]interface{}{}, Children: []gen.ApplicationChildSpec{ gen.ApplicationChildSpec{ Child: handler_sup, diff --git a/examples/http/main.go b/examples/http/main.go index 83b40978..dba1cca5 100644 --- a/examples/http/main.go +++ b/examples/http/main.go @@ -19,21 +19,14 @@ var ( ) func init() { - flag.IntVar(&ListenRangeBegin, "listen_begin", 15151, "listen port range") - flag.IntVar(&ListenRangeEnd, "listen_end", 25151, "listen port range") flag.StringVar(&NodeName, "name", "web@127.0.0.1", "node name") - flag.IntVar(&ListenEPMD, "epmd", 4369, "EPMD port") flag.StringVar(&Cookie, "cookie", "123", "cookie for interaction with erlang cluster") } func main() { flag.Parse() - opts := node.Options{ - ListenRangeBegin: uint16(ListenRangeBegin), - ListenRangeEnd: uint16(ListenRangeEnd), - EPMDPort: uint16(ListenEPMD), - } + opts := node.Options{} // Initialize new node with given name, cookie, listening port range and epmd port nodeHTTP, _ := ergo.StartNode(NodeName, Cookie, opts) diff --git a/examples/nodetls/tlsGenServer.go b/examples/nodetls/tlsGenServer.go index 20efba1c..b5dd8d27 100644 --- a/examples/nodetls/tlsGenServer.go +++ b/examples/nodetls/tlsGenServer.go @@ -16,14 +16,10 @@ type demoGenServ struct { } var ( - GenServerName string - NodeName string - Cookie string - err error - ListenRangeBegin int - ListenRangeEnd int = 35000 - Listen string - ListenEPMD int + GenServerName string + NodeName string + Cookie string + err error EnableRPC bool ) @@ -33,15 +29,26 @@ func (dgs *demoGenServ) HandleCast(process *gen.ServerProcess, message etf.Term) switch message { case etf.Atom("stop"): return gen.ServerStatusStopWithReason("stop they said") + case "test": + node := process.Env(node.EnvKeyNode).(node.Node) + n := node.Nodes() + fmt.Println("nodes: ", n) + if err := node.Disconnect(n[0]); err != nil { + fmt.Println("Cant disconnect", err) + } + if err := node.Connect(n[0]); err != nil { + fmt.Println("Cant connect", err) + } } return gen.ServerStatusOK } -func (dgs *demoGenServ) HandleCall(state *gen.ServerProcess, from gen.ServerFrom, message etf.Term) (etf.Term, gen.ServerStatus) { +func (dgs *demoGenServ) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, message etf.Term) (etf.Term, gen.ServerStatus) { fmt.Printf("HandleCall: %#v, From: %#v\n", message, from) switch message { case etf.Atom("hello"): + process.Cast(process.Self(), "test") return etf.Term("hi"), gen.ServerStatusOK } reply := etf.Tuple{etf.Atom("error"), etf.Atom("unknown_request")} @@ -49,11 +56,8 @@ func (dgs *demoGenServ) HandleCall(state *gen.ServerProcess, from gen.ServerFrom } func init() { - flag.IntVar(&ListenRangeBegin, "listen_begin", 15151, "listen port range") - flag.IntVar(&ListenRangeEnd, "listen_end", 25151, "listen port range") flag.StringVar(&GenServerName, "gen_server_name", "example", "gen_server name") flag.StringVar(&NodeName, "name", "demo@127.0.0.1", "node name") - flag.IntVar(&ListenEPMD, "epmd", 4369, "EPMD port") flag.StringVar(&Cookie, "cookie", "123", "cookie for interaction with erlang cluster") } @@ -61,19 +65,8 @@ func main() { flag.Parse() opts := node.Options{ - ListenRangeBegin: uint16(ListenRangeBegin), - ListenRangeEnd: uint16(ListenRangeEnd), - EPMDPort: uint16(ListenEPMD), - // enables TLS encryption with self-signed certificate - TLSMode: node.TLSModeAuto, - - // set TLSmode to TLSmodeStrict to use custom certificate - // TLSmode: ergo.TLSmodeStrict, - // TLScrtServer: "example.crt", - // TLSkeyServer: "example.key", - // TLScrtClient: "example.crt", - // TLSkeyClient: "example.key", + TLS: node.TLS{Enable: true}, } // Initialize new node with given name, cookie, listening port range and epmd port diff --git a/examples/proxy/README.md b/examples/proxy/README.md new file mode 100644 index 00000000..9a406058 --- /dev/null +++ b/examples/proxy/README.md @@ -0,0 +1,24 @@ +## Demo proxy connection + +This example demonstrates how to connect two nodes belonging to two different clusters by using the proxy feature and making this connection end-to-end encrypted. + +![demo](demo.png) + +Here is output of this example + +``` +❯❯❯❯ go run . +Starting node: node1 (cluster 1) ...OK +Starting node: node2 (cluster 1) with Proxy.Transit = true ...OK +Starting node: node3 (cluster 2) with Proxy.Transit = true ...OK +Starting node: node4 (cluster 2) with Proxy.Cookie = "abc" ...OK +Add static route on node2 to node3 with custom cookie to get access to the cluster 2 ...OK +Add proxy route to node4 via node2 on node1 with proxy cookie = "abc" and enabled encryption ...OK +Add proxy route to node4 via node3 on node2 ...OK +Connect node1 to node4 ...OK +Peers on node1 [node2@localhost node4@localhost] +Peers on node2 [node1@localhost node3@localhost] +Peers on node3 [node2@localhost node4@localhost] +Peers on node4 [node3@localhost node1@localhost] +``` + diff --git a/examples/proxy/demo.png b/examples/proxy/demo.png new file mode 100644 index 00000000..8c0a9694 Binary files /dev/null and b/examples/proxy/demo.png differ diff --git a/examples/proxy/demo.svg b/examples/proxy/demo.svg new file mode 100644 index 00000000..d35469ca --- /dev/null +++ b/examples/proxy/demo.svg @@ -0,0 +1,420 @@ + + + + + + + + + + + + + + + + + + + + + + AddProxyRoute("node4", ...)via "node2" and cookie="abc" + node1 + node2 + node3 + node4 + + AddProxyRoute("node4", ...)via "node3" + + cluster 1 (cookie: "secret1") + cluster 2 (cookie: "secret2") + StartNode("node4", ...)with Proxy.Cookie="abc" + StartNode("node2",...)with Proxy.Transit = true + + AddStaticRoute("node3", ...)with cookie "secret2" + + StartNode("node3",...)with Proxy.Transit = true + + + + + + + diff --git a/examples/proxy/main.go b/examples/proxy/main.go new file mode 100644 index 00000000..f0c51b42 --- /dev/null +++ b/examples/proxy/main.go @@ -0,0 +1,93 @@ +package main + +import ( + "flag" + "fmt" + + "github.com/ergo-services/ergo" + "github.com/ergo-services/ergo/node" +) + +func main() { + flag.Parse() + + fmt.Printf("Starting node: node1 (cluster 1) ...") + node1, err := ergo.StartNode("node1@localhost", "secret1", node.Options{}) + if err != nil { + panic(err) + } + fmt.Println("OK") + + fmt.Printf("Starting node: node2 (cluster 1) with Proxy.Transit = true ...") + opts2 := node.Options{} + opts2.Proxy.Transit = true + node2, err := ergo.StartNode("node2@localhost", "secret1", opts2) + if err != nil { + panic(err) + } + fmt.Println("OK") + + fmt.Printf("Starting node: node3 (cluster 2) with Proxy.Transit = true ...") + opts3 := node.Options{} + opts3.Proxy.Transit = true + node3, err := ergo.StartNode("node3@localhost", "secret2", opts3) + if err != nil { + panic(err) + } + fmt.Println("OK") + + proxyCookie := "abc" + fmt.Printf("Starting node: node4 (cluster 2) with Proxy.Cookie = %q ...", proxyCookie) + opts4 := node.Options{} + opts4.Proxy.Cookie = proxyCookie + node4, err := ergo.StartNode("node4@localhost", "secret2", opts4) + if err != nil { + panic(err) + } + fmt.Println("OK") + + fmt.Printf("Add static route on node2 to node3 with custom cookie to get access to the cluster 2 ...") + routeOptions := node.RouteOptions{ + Cookie: "secret2", + } + if err := node2.AddStaticRouteOptions(node3.Name(), routeOptions); err != nil { + panic(err) + } + fmt.Println("OK") + + fmt.Printf("Add proxy route to node4 via node2 on node1 with proxy cookie = %q and enabled encryption ...", proxyCookie) + proxyRoute1 := node.ProxyRoute{ + Proxy: node2.Name(), + Cookie: proxyCookie, + Flags: node.DefaultProxyFlags(), + } + proxyRoute1.Flags.EnableEncryption = true + if err := node1.AddProxyRoute(node4.Name(), proxyRoute1); err != nil { + panic(err) + } + fmt.Println("OK") + + fmt.Printf("Add proxy route to node4 via node3 on node2 ...") + proxyRoute2 := node.ProxyRoute{ + Proxy: node3.Name(), + } + if err := node2.AddProxyRoute(node4.Name(), proxyRoute2); err != nil { + panic(err) + } + fmt.Println("OK") + + fmt.Printf("Connect node1 to node4 ...") + if err := node1.Connect(node4.Name()); err != nil { + panic(err) + } + fmt.Println("OK") + fmt.Println("Peers on node1", node1.Nodes()) + fmt.Println("Peers on node2", node2.Nodes()) + fmt.Println("Peers on node3", node3.Nodes()) + fmt.Println("Peers on node4", node4.Nodes()) + + node1.Stop() + node2.Stop() + node3.Stop() + node4.Stop() +} diff --git a/examples/raft/README.md b/examples/raft/README.md new file mode 100644 index 00000000..6a831e7f --- /dev/null +++ b/examples/raft/README.md @@ -0,0 +1,57 @@ +## Raft demo scenario ## + +1. Starting 4 nodes with raft processes. They are connecting to each other to build a quorum. Since we use 4 raft processes, only 3 of them will be chosen for the quorum (gen.RaftQuorumState3). One becomes a follower (it won't receive the leader election result). +2. Quorum is built. All cluster members receive infomartion about the new quorum (invoking HandleQuorum callback on each raft process). Quorum members start leader election. +3. Leader is elected (chose the raft process with the latest 'serial'). Every quorum member receives information about the leader and its 'serial' (invoking HandleLeader callback on them). +4. Raft process checks the missing serials on its storage and makes a Get request to the cluster to receive them. +5. Follower makes an Append request to add a new key/value to the cluster (it sends to the quorum member, and the quorum member forwards it further to the leader) +6. All cluster members receive the new key/value with the increased 'serial' by 1. +7. Raft process checks the missing serials on its storage and makes a Get request to the cluster to receive them (follower had no chance to make this check since it doesn't receive information with the election leader result) + +Here is output of this example + +``` +❯❯❯❯ go run . + +to stop press Ctrl-C or wait 10 seconds... + +Starting node: node1@localhost and raft1 process - Serial: 1 PID: <1B30E9C5.0.1008> ...OK +Starting node: node2@localhost and raft2 process - Serial: 4 PID: <08151198.0.1008> ...OK +Starting node: node3@localhost and raft3 process - Serial: 4 PID: <9FF54552.0.1008> ...OK +Starting node: node4@localhost and raft4 process - Serial: 0 PID: <2E5EE122.0.1008> ...OK +<08151198.0.1008> Quorum built - State: 3 Quorum member: true +<9FF54552.0.1008> Quorum built - State: 3 Quorum member: true +<2E5EE122.0.1008> Quorum built - State: 3 Quorum member: false +<2E5EE122.0.1008> since I'm not a quorum member, I won't receive any information about elected leader +<1B30E9C5.0.1008> Quorum built - State: 3 Quorum member: true +<08151198.0.1008> I'm a leader of this quorum +<9FF54552.0.1008> Leader elected: <08151198.0.1008> with serial 4 +<1B30E9C5.0.1008> Leader elected: <08151198.0.1008> with serial 4 +<1B30E9C5.0.1008> Missing serials: 2 .. 4 +<1B30E9C5.0.1008> requested missing serial to the cluster: 2 id: Ref#<1B30E9C5.138519.23452.0> +<1B30E9C5.0.1008> requested missing serial to the cluster: 3 id: Ref#<1B30E9C5.138520.23452.0> +<1B30E9C5.0.1008> requested missing serial to the cluster: 4 id: Ref#<1B30E9C5.138521.23452.0> +<08151198.0.1008> Received request for serial 2 +<08151198.0.1008> Received request for serial 3 +<08151198.0.1008> Received request for serial 4 +<1B30E9C5.0.1008> Received requested serial 2 with key key2 and value value2 +<1B30E9C5.0.1008> Received requested serial 3 with key key3 and value value3 +<1B30E9C5.0.1008> Received requested serial 4 with key key4 and value value4 +<08151198.0.1008> Received append request with serial 5, key "key100" and value "value100" +<2E5EE122.0.1008> Received append request with serial 5, key "key100" and value "value100" +<1B30E9C5.0.1008> Received append request with serial 5, key "key100" and value "value100" +<2E5EE122.0.1008> Missing serial: 1 requested missing serial to the cluster. id Ref#<2E5EE122.40367.23452.0> +<2E5EE122.0.1008> Missing serial: 2 requested missing serial to the cluster. id Ref#<2E5EE122.40368.23452.0> +<2E5EE122.0.1008> Missing serial: 3 requested missing serial to the cluster. id Ref#<2E5EE122.40369.23452.0> +<2E5EE122.0.1008> Missing serial: 4 requested missing serial to the cluster. id Ref#<2E5EE122.40370.23452.0> +<9FF54552.0.1008> Received append request with serial 5, key "key100" and value "value100" +<1B30E9C5.0.1008> Received request for serial 1 +<08151198.0.1008> Received request for serial 2 +<08151198.0.1008> Received request for serial 3 +<08151198.0.1008> Received request for serial 4 +<2E5EE122.0.1008> Received requested serial 1 with key key1 and value value1 +<2E5EE122.0.1008> Received requested serial 2 with key key2 and value value2 +<2E5EE122.0.1008> Received requested serial 3 with key key3 and value value3 +<2E5EE122.0.1008> Received requested serial 4 with key key4 and value value4 + +``` diff --git a/examples/raft/main.go b/examples/raft/main.go new file mode 100644 index 00000000..8088643a --- /dev/null +++ b/examples/raft/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "fmt" + "time" + + "github.com/ergo-services/ergo" + "github.com/ergo-services/ergo/gen" + "github.com/ergo-services/ergo/node" +) + +func main() { + + fmt.Println("") + fmt.Println("to stop press Ctrl-C or wait 10 seconds...") + fmt.Println("") + + fmt.Printf("Starting node: node1@localhost and raft1 process - ") + node1, err := ergo.StartNode("node1@localhost", "cookies", node.Options{}) + if err != nil { + panic(err) + } + defer node1.Stop() + raft1 := &Raft{startSerial: 1} + _, err = node1.Spawn("raft1", gen.ProcessOptions{}, raft1) + if err != nil { + panic(err) + } + fmt.Println("OK") + + fmt.Printf("Starting node: node2@localhost and raft2 process - ") + node2, err := ergo.StartNode("node2@localhost", "cookies", node.Options{}) + if err != nil { + panic(err) + } + defer node2.Stop() + raft2 := &Raft{ + startSerial: 4, + startPeers: []gen.ProcessID{gen.ProcessID{Name: "raft1", Node: "node1@localhost"}}, + } + _, err = node2.Spawn("raft2", gen.ProcessOptions{}, raft2) + if err != nil { + panic(err) + } + fmt.Println("OK") + + fmt.Printf("Starting node: node3@localhost and raft3 process - ") + node3, err := ergo.StartNode("node3@localhost", "cookies", node.Options{}) + if err != nil { + panic(err) + } + defer node3.Stop() + raft3 := &Raft{ + startSerial: 4, + startPeers: []gen.ProcessID{gen.ProcessID{Name: "raft1", Node: "node1@localhost"}}, + } + _, err = node3.Spawn("raft3", gen.ProcessOptions{}, raft3) + if err != nil { + panic(err) + } + fmt.Println("OK") + + fmt.Printf("Starting node: node4@localhost and raft4 process - ") + node4, err := ergo.StartNode("node4@localhost", "cookies", node.Options{}) + if err != nil { + panic(err) + } + defer node4.Stop() + raft4 := &Raft{ + startSerial: 0, + startPeers: []gen.ProcessID{gen.ProcessID{Name: "raft1", Node: "node1@localhost"}}, + } + _, err = node4.Spawn("raft4", gen.ProcessOptions{}, raft4) + if err != nil { + panic(err) + } + fmt.Println("OK") + time.Sleep(10 * time.Second) +} diff --git a/examples/raft/raft.go b/examples/raft/raft.go new file mode 100644 index 00000000..2fdc7633 --- /dev/null +++ b/examples/raft/raft.go @@ -0,0 +1,113 @@ +package main + +import ( + "fmt" + "time" + + "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/gen" +) + +// +// Raft +// +type Raft struct { + gen.Raft + storage *storage + + startSerial uint64 + startPeers []gen.ProcessID +} + +type messageAppend struct { + key string + value string +} + +func (r *Raft) InitRaft(process *gen.RaftProcess, args ...etf.Term) (gen.RaftOptions, error) { + opts := gen.RaftOptions{ + Serial: r.startSerial, + Peers: r.startPeers, + } + + r.storage = &storage{} + r.storage.Init(int(opts.Serial)) + fmt.Printf("Serial: %d PID: %s ...", opts.Serial, process.Self()) + + return opts, nil +} +func (r *Raft) HandleQuorum(process *gen.RaftProcess, quorum *gen.RaftQuorum) gen.RaftStatus { + fmt.Println(process.Self(), "Quorum built - State:", quorum.State, "Quorum member:", quorum.Member) + if quorum.Member == false { + fmt.Println(process.Self(), " since I'm not a quorum member, I won't receive any information about elected leader") + message := messageAppend{key: "key100", value: "value100"} + process.SendAfter(process.Self(), message, 500*time.Millisecond) + } + return gen.RaftStatusOK +} + +func (r *Raft) HandleLeader(process *gen.RaftProcess, leader *gen.RaftLeader) gen.RaftStatus { + if leader != nil && leader.Leader == process.Self() { + fmt.Println(process.Self(), "I'm a leader of this quorum") + return gen.RaftStatusOK + } + + if leader != nil { + fmt.Println(process.Self(), "Leader elected:", leader.Leader, "with serial", leader.Serial) + } + s := process.Serial() + if s < leader.Serial { + fmt.Println(process.Self(), "Missing serials:", s+1, "..", leader.Serial) + for i := s + 1; i <= leader.Serial; i++ { + req, e := process.Get(i) + if e != nil { + panic(e) + } + fmt.Println(process.Self(), " requested missing serial to the cluster:", i, " id:", req) + } + } + return gen.RaftStatusOK +} + +func (r *Raft) HandleAppend(process *gen.RaftProcess, ref etf.Ref, serial uint64, key string, value etf.Term) gen.RaftStatus { + fmt.Printf("%s Received append request with serial %d, key %q and value %q\n", process.Self(), serial, key, value) + r.storage.Append(serial, key, value) + // check if storage has missing items + for i := uint64(1); i < serial; i++ { + _, v := r.storage.Read(i) + if v != nil { + continue + } + req, e := process.Get(i) + if e != nil { + panic(e) + } + fmt.Println(process.Self(), "Missing serial:", i, " requested missing serial to the cluster. id", req) + } + return gen.RaftStatusOK +} +func (r *Raft) HandleGet(process *gen.RaftProcess, serial uint64) (string, etf.Term, gen.RaftStatus) { + var key string + var value etf.Term + fmt.Println(process.Self(), "Received request for serial", serial) + key, value = r.storage.Read(serial) + return key, value, gen.RaftStatusOK +} + +func (r *Raft) HandleRaftInfo(process *gen.RaftProcess, message etf.Term) gen.ServerStatus { + messageAppend, ok := message.(messageAppend) + if ok == false { + return gen.ServerStatusOK + } + if _, err := process.Append(messageAppend.key, messageAppend.value); err != nil { + fmt.Println("can't make append request", err) + return gen.ServerStatusOK + } + return gen.ServerStatusOK +} + +func (r *Raft) HandleSerial(process *gen.RaftProcess, ref etf.Ref, serial uint64, key string, value etf.Term) gen.RaftStatus { + fmt.Println(process.Self(), "Received requested serial", serial, "with key", key, "and value", value) + r.storage.Append(serial, key, value) + return gen.RaftStatusOK +} diff --git a/examples/raft/storage.go b/examples/raft/storage.go new file mode 100644 index 00000000..a4be9125 --- /dev/null +++ b/examples/raft/storage.go @@ -0,0 +1,71 @@ +package main + +import ( + "github.com/ergo-services/ergo/etf" +) + +var ( + data = map[string]dataValueSerial{ + "key0": dataValueSerial{"value0", 0}, + "key1": dataValueSerial{"value1", 1}, + "key2": dataValueSerial{"value2", 2}, + "key3": dataValueSerial{"value3", 3}, + "key4": dataValueSerial{"value4", 4}, + "key5": dataValueSerial{"value5", 5}, + "key6": dataValueSerial{"value6", 6}, + "key7": dataValueSerial{"value7", 7}, + "key8": dataValueSerial{"value8", 8}, + "key9": dataValueSerial{"value9", 9}, + } + keySerials = []string{ + "key0", + "key1", + "key2", + "key3", + "key4", + "key5", + "key6", + "key7", + "key8", + "key9", + } +) + +type storage struct { + data map[string]dataValueSerial // key/value + idx [10]string // keys +} + +type dataValueSerial struct { + value string + serial uint64 +} + +func (s *storage) Init(n int) { + s.data = make(map[string]dataValueSerial) + for i := 0; i <= int(n); i++ { + key := keySerials[i] + s.data[key] = data[key] + s.idx[i] = key + } +} + +func (s *storage) Read(serial uint64) (string, etf.Term) { + var key string + + key = s.idx[int(serial)] + if key == "" { + // no data found + return key, nil + } + data := s.data[key] + return key, data.value +} + +func (s *storage) Append(serial uint64, key string, value etf.Term) { + s.data[key] = dataValueSerial{ + value: value.(string), + serial: serial, + } + s.idx[serial] = key +} diff --git a/examples/supervisor/demoSupervisor.go b/examples/supervisor/demoSupervisor.go index 2de2e9ad..4a192116 100644 --- a/examples/supervisor/demoSupervisor.go +++ b/examples/supervisor/demoSupervisor.go @@ -11,13 +11,9 @@ import ( ) var ( - NodeName string - Cookie string - err error - ListenRangeBegin int - ListenRangeEnd int = 35000 - Listen string - ListenEPMD int + NodeName string + Cookie string + err error EnableRPC bool ) @@ -90,24 +86,15 @@ func (dgs *demoGenServ) Terminate(process *gen.ServerProcess, reason string) { } func init() { - flag.IntVar(&ListenRangeBegin, "listen_begin", 15151, "listen port range") - flag.IntVar(&ListenRangeEnd, "listen_end", 25151, "listen port range") flag.StringVar(&NodeName, "name", "demo@127.0.0.1", "node name") - flag.IntVar(&ListenEPMD, "epmd", 4369, "EPMD port") flag.StringVar(&Cookie, "cookie", "123", "cookie for interaction with erlang cluster") } func main() { flag.Parse() - opts := node.Options{ - ListenRangeBegin: uint16(ListenRangeBegin), - ListenRangeEnd: uint16(ListenRangeEnd), - EPMDPort: uint16(ListenEPMD), - } - // Initialize new node with given name, cookie, listening port range and epmd port - node, _ := ergo.StartNode(NodeName, Cookie, opts) + node, _ := ergo.StartNode(NodeName, Cookie, node.Options{}) // Spawn supervisor process process, _ := node.Spawn("demo_sup", gen.ProcessOptions{}, &demoSup{}) diff --git a/gen/README.md b/gen/README.md index 0fc767d6..58df1761 100644 --- a/gen/README.md +++ b/gen/README.md @@ -7,12 +7,23 @@ ### Supervisor Generic supervisor behavior. +A supervisor is responsible for starting, stopping, and monitoring its child processes. The basic idea of a supervisor is that it is to keep its child processes alive by restarting them when necessary. + ### Application Generic application behavior. ### Stage - Generic stage behavior. + Generic stage behavior (originated from Elixir's [GenStage](https://hexdocs.pm/gen_stage/GenStage.html)). + +This is abstraction built on top of `gen.Server` to provide a simple way to create a distributed Producer/Consumer architecture, while automatically managing the concept of backpressure. This implementation is fully compatible with Elixir's GenStage. Example is here [examples/genstage](examples/genstage) or just run `go run ./examples/genstage` to see it in action ### Saga Generic saga behavior. +It implements Saga design pattern - a sequence of transactions that updates each service state and publishes the result (or cancels the transaction or triggers the next transaction step). `gen.Saga` also provides a feature of interim results (can be used as transaction progress or as a part of pipeline processing), time deadline (to limit transaction lifespan), two-phase commit (to make distributed transaction atomic). Here is example [examples/gensaga](examples/gensaga). + +### Raft + Generic raft behavior. + +It's improved implementation of [Raft consensus algorithm](https://raft.github.io). The key improvement is using quorum under the hood to manage the leader election process and make the Raft cluster more reliable. This implementation supports quorums of 3, 5, 7, 9, or 11 quorum members. Here is an example of this feature [examples/raft](examples/raft). + diff --git a/gen/application.go b/gen/application.go index 81375540..7dffdfcc 100644 --- a/gen/application.go +++ b/gen/application.go @@ -8,6 +8,7 @@ import ( "time" "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/lib" ) type ApplicationStartType = string @@ -29,6 +30,9 @@ const ( // is with any other reason than normal, all other applications and // the runtime system (node) are also terminated. ApplicationStartTransient = "transient" + + // EnvKeySpec + EnvKeySpec EnvKey = "ergo:AppSpec" ) // ApplicationBehavior interface @@ -38,6 +42,7 @@ type ApplicationBehavior interface { Start(process Process, args ...etf.Term) } +// ApplicationSpec type ApplicationSpec struct { sync.Mutex Name string @@ -45,14 +50,16 @@ type ApplicationSpec struct { Version string Lifespan time.Duration Applications []string - Environment map[string]interface{} + Environment map[EnvKey]interface{} Children []ApplicationChildSpec Process Process StartType ApplicationStartType } +// ApplicationChildSpec type ApplicationChildSpec struct { Child ProcessBehavior + Options ProcessOptions Name string Args []etf.Term process Process @@ -61,6 +68,7 @@ type ApplicationChildSpec struct { // Application is implementation of ProcessBehavior interface type Application struct{} +// ApplicationInfo type ApplicationInfo struct { Name string Description string @@ -68,13 +76,14 @@ type ApplicationInfo struct { PID etf.Pid } +// ProcessInit func (a *Application) ProcessInit(p Process, args ...etf.Term) (ProcessState, error) { - spec, ok := p.Env("spec").(*ApplicationSpec) + spec, ok := p.Env(EnvKeySpec).(*ApplicationSpec) if !ok { return ProcessState{}, fmt.Errorf("ProcessInit: not an ApplicationBehavior") } // remove variable from the env - p.SetEnv("spec", nil) + p.SetEnv(EnvKeySpec, nil) p.SetTrapExit(true) @@ -102,6 +111,7 @@ func (a *Application) ProcessInit(p Process, args ...etf.Term) (ProcessState, er }, nil } +// ProcessLoop func (a *Application) ProcessLoop(ps ProcessState, started chan<- bool) string { spec := ps.State.(*ApplicationSpec) defer func() { spec.Process = nil }() @@ -126,7 +136,7 @@ func (a *Application) ProcessLoop(ps ProcessState, started chan<- bool) string { if ex.From == ps.Self() { childrenStopped := a.stopChildren(terminated, spec.Children, reason) if !childrenStopped { - fmt.Printf("Warining: application can't be stopped. Some of the children are still running") + lib.Warning("application %q can't be stopped. Some of the children are still running", spec.Name) continue } return ex.Reason @@ -152,19 +162,19 @@ func (a *Application) ProcessLoop(ps ProcessState, started chan<- bool) string { switch spec.StartType { case ApplicationStartPermanent: a.stopChildren(terminated, spec.Children, string(reason)) - fmt.Printf("Application child %s (at %s) stopped with reason %s (permanent: node is shutting down)\n", + lib.Warning("Application child %s (at %s) stopped with reason %s (permanent: node is shutting down)", terminated, ps.NodeName(), reason) ps.NodeStop() return "shutdown" case ApplicationStartTransient: if reason == "normal" || reason == "shutdown" { - fmt.Printf("Application child %s (at %s) stopped with reason %s (transient)\n", + lib.Warning("Application child %s (at %s) stopped with reason %s (transient)", terminated, ps.NodeName(), reason) continue } a.stopChildren(terminated, spec.Children, reason) - fmt.Printf("Application child %s (at %s) stopped with reason %s. (transient: node is shutting down)\n", + lib.Warning("Application child %s (at %s) stopped with reason %s. (transient: node is shutting down)", terminated, ps.NodeName(), reason) ps.NodeStop() return string(reason) @@ -247,7 +257,7 @@ func (a *Application) startChildren(parent Process, children []ApplicationChildS for i := range children { // i know, it looks weird to use the funcion from supervisor file. // will move it to somewhere else, but let it be there for a while. - p := startChild(parent, children[i].Name, children[i].Child, children[i].Args...) + p := startChild(parent, children[i].Name, children[i].Child, children[i].Options, children[i].Args...) if p == nil { return false } diff --git a/gen/raft.go b/gen/raft.go new file mode 100644 index 00000000..9601f6e6 --- /dev/null +++ b/gen/raft.go @@ -0,0 +1,2616 @@ +package gen + +import ( + "context" + "fmt" + "math/rand" + "sort" + "time" + + "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/lib" +) + +const ( + DefaultRaftGetTimeout = 5 // in seconds + DefaultRaftAppendTimeout = 5 // in seconds + DefaultRaftHeartbeat = 3 // in seconds +) + +var ( + ErrRaftState = fmt.Errorf("incorrect raft state") + ErrRaftNoQuorum = fmt.Errorf("no quorum") + ErrRaftNoLeader = fmt.Errorf("no leader") + ErrRaftNoSerial = fmt.Errorf("no peers with requested serial") + ErrRaftBusy = fmt.Errorf("another append request is in progress") + ErrRaftWrongTimeout = fmt.Errorf("wrong timeout value") +) + +type RaftBehavior interface { + // + // Mandatory callbacks + // + + InitRaft(process *RaftProcess, arr ...etf.Term) (RaftOptions, error) + + // HandleAppend. Invokes on append request. To cancel this request by a leader, it must return RaftStatusDiscard. + HandleAppend(process *RaftProcess, ref etf.Ref, serial uint64, key string, value etf.Term) RaftStatus + + // HandleGet + HandleGet(process *RaftProcess, serial uint64) (string, etf.Term, RaftStatus) + + // + // Optional callbacks + // + + // HandlePeer + HandlePeer(process *RaftProcess, peer etf.Pid, serial uint64) RaftStatus + + // HandleQuorum + HandleQuorum(process *RaftProcess, quorum *RaftQuorum) RaftStatus + + // HandleLeader + HandleLeader(process *RaftProcess, leader *RaftLeader) RaftStatus + + // HandleCancel + HandleCancel(process *RaftProcess, ref etf.Ref, reason string) RaftStatus + + // HandleSerial + HandleSerial(process *RaftProcess, ref etf.Ref, serial uint64, key string, value etf.Term) RaftStatus + + // + // Server's callbacks + // + + // HandleRaftCall this callback is invoked on ServerProcess.Call. This method is optional + // for the implementation + HandleRaftCall(process *RaftProcess, from ServerFrom, message etf.Term) (etf.Term, ServerStatus) + // HandleStageCast this callback is invoked on ServerProcess.Cast. This method is optional + // for the implementation + HandleRaftCast(process *RaftProcess, message etf.Term) ServerStatus + // HandleStageInfo this callback is invoked on Process.Send. This method is optional + // for the implementation + HandleRaftInfo(process *RaftProcess, message etf.Term) ServerStatus + // HandleRaftDirect this callback is invoked on Process.Direct. This method is optional + // for the implementation + HandleRaftDirect(process *RaftProcess, message interface{}) (interface{}, error) +} + +type RaftStatus error +type RaftQuorumState int + +var ( + RaftStatusOK RaftStatus // nil + RaftStatusStop RaftStatus = fmt.Errorf("stop") + RaftStatusDiscard RaftStatus = fmt.Errorf("discard") + + RaftQuorumState3 RaftQuorumState = 3 // minimum quorum that could make leader election + RaftQuorumState5 RaftQuorumState = 5 + RaftQuorumState7 RaftQuorumState = 7 + RaftQuorumState9 RaftQuorumState = 9 + RaftQuorumState11 RaftQuorumState = 11 // maximal quorum + + cleanVoteTimeout = 1 * time.Second + cleanLeaderVoteTimeout = 1 * time.Second + quorumChangeDeferMaxTime = 450 // in millisecond. uses as max value in range of 50.. +) + +type Raft struct { + Server +} + +type RaftProcess struct { + ServerProcess + options RaftOptions + behavior RaftBehavior + + quorum *RaftQuorum + quorumCandidates *quorumCandidates + quorumVotes map[RaftQuorumState]*quorum + quorumChangeDefer bool + quorumChangeAttempt int + + leader etf.Pid + election *leaderElection + round int // "log term" in terms of Raft spec + + // get requests + requests map[etf.Ref]context.CancelFunc + + // append requests + requestsAppend map[string]*requestAppend + requestsAppendQueue []requestAppendQueued + + // leader sends heartbeat messages and keep the last sending timestamp + heartbeatLeader int64 + heartbeatCancel context.CancelFunc +} + +type leaderElection struct { + votes map[etf.Pid]etf.Pid + results map[etf.Pid]bool + round int + leader etf.Pid // leader elected + voted int // number of peers voted for the leader + cancel context.CancelFunc +} + +type requestAppend struct { + ref etf.Ref + from etf.Pid + origin etf.Pid + value etf.Term + peers map[etf.Pid]bool + cancel context.CancelFunc +} + +type requestAppendQueued struct { + from etf.Pid + request *messageRaftRequestAppend +} + +type quorumCandidates struct { + candidates map[etf.Pid]*candidate +} + +type candidate struct { + monitor etf.Ref + serial uint64 + joined bool + heartbeat int64 + failures int +} + +type RaftLeader struct { + Leader etf.Pid + Serial uint64 + State RaftQuorumState +} + +type RaftQuorum struct { + Member bool + State RaftQuorumState + Peers []etf.Pid // the number of participants in quorum could be 3,5,7,9,11 +} +type quorum struct { + RaftQuorum + votes map[etf.Pid]int // 1 - sent, 2 - recv, 3 - sent and recv + origin etf.Pid // where the voting has come from. it must receive our voice in the last order + lastVote int64 // time.Now().UnixMilli() +} + +type RaftOptions struct { + ID string // raft cluster id + Peers []ProcessID + Serial uint64 // serial number ("log id" in terms of Raft spec) +} + +type messageRaft struct { + Request etf.Atom + Pid etf.Pid + Command interface{} +} + +type messageRaftClusterInit struct{} +type messageRaftClusterJoin struct { + ID string // cluster id + Serial uint64 +} +type messageRaftClusterJoinReply struct { + ID string // cluster id + Serial uint64 + Peers []etf.Pid + QuorumState int + QuorumPeers []etf.Pid +} +type messageRaftQuorumVote struct { + ID string // cluster id + Serial uint64 + State int + Candidates []etf.Pid +} +type messageRaftQuorumChange struct{} +type messageRaftQuorumBuilt struct { + ID string // cluster id + State int + Round int // last round + Peers []etf.Pid +} +type messageRaftQuorumLeave struct { + ID string + DueToPid etf.Pid +} + +type messageRaftQuorumCleanVote struct { + state RaftQuorumState +} + +type messageRaftLeaderHeartbeat struct { + ID string + Serial uint64 +} + +type messageRaftLeaderVote struct { + ID string // cluster id + State int //quorum state + Leader etf.Pid // offered leader + Round int +} +type messageRaftLeaderElected struct { + ID string // cluster id + Leader etf.Pid // elected leader + Voted int // number of votes for this leader + Round int +} + +type messageRaftRequestGet struct { + ID string // cluster id + Ref etf.Ref + Origin etf.Pid + Serial uint64 +} +type messageRaftRequestReply struct { + ID string // cluster id + Ref etf.Ref + Serial uint64 + Key string + Value etf.Term +} +type messageRaftRequestAppend struct { + ID string // cluster id + Ref etf.Ref + Origin etf.Pid + Key string + Value etf.Term + Deadline int64 // timestamp in milliseconds +} + +type messageRaftAppendReady struct { + ID string // cluster id + Ref etf.Ref + Key string +} + +type messageRaftAppendCommit struct { + ID string // cluster id + Ref etf.Ref + Key string + Serial uint64 + Broadcast etf.Pid // quorum member who is in charge of broadcasting +} + +type messageRaftAppendBroadcast struct { + ID string + Ref etf.Ref + Serial uint64 + Key string + Value etf.Term +} + +type messageRaftRequestClean struct { + ref etf.Ref +} +type messageRaftAppendClean struct { + key string + ref etf.Ref +} +type messageRaftElectionClean struct { + round int +} +type messageRaftHeartbeat struct{} + +// +// RaftProcess quorum routines and APIs +// + +// Join makes a join requst to the given peer, which is supposed to be in a raft cluster +func (rp *RaftProcess) Join(peer interface{}) error { + // QUODBG fmt.Println(rp.Name(), "CLU send join to", peer) + join := etf.Tuple{ + etf.Atom("$cluster_join"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + }, + } + return rp.Cast(peer, join) +} + +// Peers returns list of the processes in the raft cluster. Note, this list is sorted by the Serial value on them in the descending order +func (rp *RaftProcess) Peers() []etf.Pid { + return rp.quorumCandidates.List() +} + +// Quorum returns current quorum. It returns nil if quorum hasn't built yet. +func (rp *RaftProcess) Quorum() *RaftQuorum { + var q RaftQuorum + if rp.quorum == nil { + return nil + } + q.Member = rp.quorum.Member + q.State = rp.quorum.State + q.Peers = make([]etf.Pid, len(rp.quorum.Peers)) + for i := range rp.quorum.Peers { + q.Peers[i] = rp.quorum.Peers[i] + } + return &q +} + +// Leader returns current leader in the quorum. It returns nil If this process is not a quorum or if leader election is still in progress +func (rp *RaftProcess) Leader() *RaftLeader { + var leader RaftLeader + + if rp.quorum == nil || rp.quorum.Member == false { + return nil + } + + noLeader := etf.Pid{} + if rp.leader == noLeader { + return nil + } + leader.Leader = rp.leader + leader.State = rp.quorum.State + leader.Serial = rp.options.Serial + if rp.leader != rp.Self() { + // must be present among the peers + c := rp.quorumCandidates.GetOnline(rp.leader) + if c == nil { + panic("internal error. elected leader has been lost") + } + leader.Serial = c.serial + } + + return &leader +} + +// Get makes a request to the quorum member to get the data with the given serial number and +// sets the timeout to the DefaultRaftGetTimeout = 5 sec. It returns ErrRaftNoQuorum if quorum +// forming is still in progress. +func (rp *RaftProcess) Get(serial uint64) (etf.Ref, error) { + return rp.GetWithTimeout(serial, DefaultRaftGetTimeout) +} + +// Get makes a request to the quorum member to get the data with the given serial number and +// timeout in seconds. Returns a reference of this request. Once requested data has arrived +// the callback HandleSerial will be invoked. +// If a timeout occurred the callback HandleCancel will be invoked with reason "timeout" +func (rp *RaftProcess) GetWithTimeout(serial uint64, timeout int) (etf.Ref, error) { + var ref etf.Ref + if rp.quorum == nil { + return ref, ErrRaftNoQuorum + } + + peers := []etf.Pid{} + for _, pid := range rp.quorum.Peers { + if pid == rp.Self() { + continue + } + if c := rp.quorumCandidates.GetOnline(pid); c != nil { + if serial > c.serial { + continue + } + peers = append(peers, pid) + } + } + if len(peers) == 0 { + return ref, ErrRaftNoSerial + } + + // get random member of quorum and send the request + n := 0 + if len(peers) > 1 { + rand.Intn(len(peers) - 1) + } + peer := peers[n] + ref = rp.MakeRef() + requestGet := etf.Tuple{ + etf.Atom("$request_get"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + ref, + rp.Self(), // origin + serial, + }, + } + + if err := rp.Cast(peer, requestGet); err != nil { + return ref, err + } + cancel := rp.CastAfter(rp.Self, messageRaftRequestClean{ref: ref}, time.Duration(timeout)*time.Second) + rp.requests[ref] = cancel + return ref, nil +} + +// Append +func (rp *RaftProcess) Append(key string, value etf.Term) (etf.Ref, error) { + return rp.AppendWithTimeout(key, value, DefaultRaftAppendTimeout) +} + +// AppendWithTimeout +func (rp *RaftProcess) AppendWithTimeout(key string, value etf.Term, timeout int) (etf.Ref, error) { + var ref etf.Ref + if timeout < 1 { + return ref, ErrRaftWrongTimeout + } + + if _, exist := rp.requestsAppend[key]; exist { + return ref, ErrRaftBusy + } + if rp.quorum == nil { + return ref, ErrRaftNoQuorum + } + noLeader := etf.Pid{} + if rp.quorum.Member == true && rp.leader == noLeader { + return ref, ErrRaftNoLeader + } + t := int(time.Duration(timeout) * time.Second) + deadline := time.Now().Add(time.Duration(t - t/int(rp.quorum.State))).UnixMilli() + ref = rp.MakeRef() + + // if Append request has made on a leader + if rp.leader == rp.Self() { + // DBGAPN fmt.Println(rp.Self(), "DBGAPN append request", ref, "made on a leader") + dataAppend := &messageRaftRequestAppend{ + Ref: ref, + Origin: rp.Self(), + Key: key, + Value: value, + Deadline: deadline, + } + rp.handleAppendLeader(rp.Self(), dataAppend) + return ref, nil + } + + peer := rp.leader + // if Member == false => rp.leader == noLeader + if rp.quorum.Member == false { + // this raft process runs as a Client. send this request to the quorum member + n := rand.Intn(len(rp.quorum.Peers) - 1) + peer = rp.quorum.Peers[n] + deadline = time.Now().Add(time.Duration(t - t/(int(rp.quorum.State)+1))).UnixMilli() + } + dataAppend := etf.Tuple{ + etf.Atom("$request_append"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + ref, + rp.Self(), + key, + value, + deadline, + }, + } + // DBGAPN fmt.Println(rp.Self(), "DPGAPN sent $request_append", ref, "to the peer", peer) + if err := rp.Cast(peer, dataAppend); err != nil { + return ref, err + } + + peers := make(map[etf.Pid]bool) + if rp.quorum.Member == true { + // this process will be in charge of broadcasting + // so we should keep the set of peers in this quorum in order + // to exlude them on the broadcasting + for _, pid := range rp.quorum.Peers { + if pid == rp.Self() { + continue + } + peers[pid] = true + } + } + + clean := messageRaftAppendClean{key: key, ref: ref} + after := time.Duration(timeout) * time.Second + cancel := rp.CastAfter(rp.Self, clean, after) + requestAppend := &requestAppend{ + ref: ref, + origin: rp.Self(), + value: value, + peers: peers, + cancel: cancel, + } + rp.requestsAppend[key] = requestAppend + return ref, nil +} + +// Serial returns current value of serial for this raft process +func (rp *RaftProcess) Serial() uint64 { + return rp.options.Serial +} + +// private routines + +func (rp *RaftProcess) handleRaftRequest(m messageRaft) error { + switch m.Request { + case etf.Atom("$cluster_join"): + join := &messageRaftClusterJoin{} + if err := etf.TermIntoStruct(m.Command, &join); err != nil { + return ErrUnsupportedRequest + } + + if join.ID != rp.options.ID { + // this peer belongs to another quorum id + return RaftStatusOK + } + + if rp.quorum != nil && rp.quorum.Member { + // if we got $cluster_join from a quorum member, it means + // the quorum we had belonging is not existed anymore + if rp.isQuorumMember(m.Pid) == true { + rp.quorum = nil + rp.handleQuorum() + rp.quorumChangeStart(false) + } + } + + rp.quorumCandidates.Set(rp, m.Pid) + rp.quorumCandidates.SetOnline(rp, m.Pid, join.Serial) + + if status := rp.behavior.HandlePeer(rp, m.Pid, join.Serial); status != RaftStatusOK { + return status + } + + // send peer list even if this peer is already present in our candidates list + // just to exchange updated data + peers := rp.quorumCandidates.List() + quorumState := 0 + quorumPeers := []etf.Pid{} + if rp.quorum != nil { + quorumState = int(rp.quorum.State) + quorumPeers = rp.quorum.Peers + } + reply := etf.Tuple{ + etf.Atom("$cluster_join_reply"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + rp.options.Serial, + peers, + quorumState, + quorumPeers, + }, + } + // QUODBG fmt.Println(rp.Name(), "GOT CLU JOIN from", m.Pid, "send peers", peers) + rp.Cast(m.Pid, reply) + return RaftStatusOK + + case etf.Atom("$cluster_join_reply"): + + reply := &messageRaftClusterJoinReply{} + if err := etf.TermIntoStruct(m.Command, &reply); err != nil { + return ErrUnsupportedRequest + } + + if reply.ID != rp.options.ID { + // this peer belongs to another quorum id. ignore it. + return RaftStatusOK + } + + // QUODBG fmt.Println(rp.Name(), "GOT CLU JOIN REPL from", m.Pid, "got peers", reply.Peers) + canAcceptQuorum := true + + // check if there is another quorum in this cluster + if rp.quorum != nil { + // doesnt matter we compare the number of peers or quorum state + // reply.QuorumState <= rp.quorum.State + if len(reply.QuorumPeers) <= len(rp.quorum.Peers) { + canAcceptQuorum = false + } + } + + // check peers + for _, peer := range reply.Peers { + if peer == rp.Self() { + continue + } + // check if we dont have some of them among the online peers + if c := rp.quorumCandidates.GetOnline(peer); c != nil { + continue + } + rp.quorumCandidates.Set(rp, peer) + canAcceptQuorum = false + } + + rp.quorumCandidates.Set(rp, m.Pid) + rp.quorumCandidates.SetOnline(rp, m.Pid, reply.Serial) + + if status := rp.behavior.HandlePeer(rp, m.Pid, reply.Serial); status != RaftStatusOK { + return status + } + + // try to rebuild quorum since the number of peers has changed + rp.quorumChangeStart(false) + + // accept quorum if this peer is belongs to the existing quorum + // and set membership to false + switch RaftQuorumState(reply.QuorumState) { + case RaftQuorumState3, RaftQuorumState5: + break + case RaftQuorumState7, RaftQuorumState9, RaftQuorumState11: + break + default: + canAcceptQuorum = false + } + if canAcceptQuorum == true { + rp.election = nil + rp.quorum = &RaftQuorum{ + State: RaftQuorumState(reply.QuorumState), + Peers: reply.QuorumPeers, + Member: false, + } + return rp.handleQuorum() + } + return RaftStatusOK + + case etf.Atom("$quorum_vote"): + vote := &messageRaftQuorumVote{} + if err := etf.TermIntoStruct(m.Command, &vote); err != nil { + return ErrUnsupportedRequest + } + if vote.ID != rp.options.ID { + // ignore this request + return RaftStatusOK + } + return rp.quorumVote(m.Pid, vote) + + case etf.Atom("$quorum_built"): + built := &messageRaftQuorumBuilt{} + if err := etf.TermIntoStruct(m.Command, &built); err != nil { + return ErrUnsupportedRequest + } + // QUODBG fmt.Println(rp.Name(), "GOT QUO BUILT from", m.Pid) + if built.ID != rp.options.ID { + // this process is not belong this quorum + return RaftStatusOK + } + duplicates := make(map[etf.Pid]bool) + matchCandidates := true + for _, pid := range built.Peers { + if _, exist := duplicates[pid]; exist { + // duplicate found + return RaftStatusOK + } + if pid == rp.Self() { + panic("raft internal error. got quorum built message") + } + if c := rp.quorumCandidates.GetOnline(pid); c != nil { + c.failures = 0 + c.heartbeat = time.Now().Unix() + continue + } + rp.quorumCandidates.Set(rp, pid) + matchCandidates = false + } + if len(built.Peers) != built.State { + // ignore wrong peer list + lib.Warning("[%s] got quorum state doesn't match with the peer list", rp.Self()) + return RaftStatusOK + } + candidateQuorumState := RaftQuorumState3 + switch built.State { + case 11: + candidateQuorumState = RaftQuorumState11 + case 9: + candidateQuorumState = RaftQuorumState9 + case 7: + candidateQuorumState = RaftQuorumState7 + case 5: + candidateQuorumState = RaftQuorumState5 + case 3: + candidateQuorumState = RaftQuorumState3 + default: + // ignore wrong state + return RaftStatusOK + } + + rp.quorumChangeStart(false) + + if built.Round > rp.round { + // update rp.round + rp.round = built.Round + } + + // we do accept quorum if it was built using + // the peers we got registered as candidates + if matchCandidates == true { + rp.election = nil + if rp.quorum == nil { + rp.quorum = &RaftQuorum{} + rp.quorum.State = candidateQuorumState + rp.quorum.Member = false + rp.quorum.Peers = built.Peers + // QUODBG fmt.Println(rp.Name(), "QUO BUILT. NOT A MEMBER", rp.quorum.State, rp.quorum.Peers) + return rp.handleQuorum() + } + // QUODBG fmt.Println(rp.Name(), "QUO BUILT. NOT A MEMBER", rp.quorum.State, rp.quorum.Peers) + + changed := false + if rp.quorum.State != candidateQuorumState { + changed = true + } + rp.quorum.State = candidateQuorumState + + if rp.quorum.Member != false { + changed = true + } + rp.quorum.Member = false + + rp.quorum.Peers = built.Peers + if changed == true { + return rp.handleQuorum() + } + return RaftStatusOK + } + + if rp.quorum != nil { + rp.quorum = nil + rp.election = nil + return rp.handleQuorum() + } + return RaftStatusOK + + case etf.Atom("$leader_heartbeat"): + heartbeat := &messageRaftLeaderHeartbeat{} + if err := etf.TermIntoStruct(m.Command, &heartbeat); err != nil { + return ErrUnsupportedRequest + } + + if rp.options.ID != heartbeat.ID { + return RaftStatusOK + } + + c := rp.quorumCandidates.GetOnline(m.Pid) + if c == nil { + // HRTDBG fmt.Println(rp.Self(), "HRT from unknown/offline peer", m.Pid) + rp.quorumCandidates.Set(rp, m.Pid) + return RaftStatusOK + } + // HRTDBG fmt.Println(rp.Self(), "HRT from", m.Pid, "serial", c.serial) + c.heartbeat = time.Now().Unix() + c.serial = heartbeat.Serial + c.failures = 0 + return RaftStatusOK + + case etf.Atom("$quorum_leave"): + leave := &messageRaftQuorumLeave{} + if err := etf.TermIntoStruct(m.Command, &leave); err != nil { + return ErrUnsupportedRequest + } + if rp.quorum == nil { + return RaftStatusOK + } + + if rp.options.ID != leave.ID { + return RaftStatusOK + } + + // check if it came from the quorum member + if rp.isQuorumMember(m.Pid) == false { + return RaftStatusOK + } + + // QUODBG fmt.Println(rp.Self(), "QUO got leave from", m.Pid, "due to", leave.DueToPid) + rp.quorumCandidates.SetOffline(rp, leave.DueToPid) + + member := rp.quorum.Member + rp.quorum = nil + rp.handleQuorum() + // only quorum member can restart quorum building if some of the member has left + if member == true { + rp.quorumChangeStart(false) + } + return RaftStatusOK + + case etf.Atom("$leader_vote"): + vote := &messageRaftLeaderVote{} + if err := etf.TermIntoStruct(m.Command, &vote); err != nil { + return ErrUnsupportedRequest + } + + if rp.options.ID != vote.ID { + lib.Warning("[%s] ignore 'leader vote' message being not a member of the given raft cluster (from %s)", rp.Self(), m.Pid) + return RaftStatusOK + } + + if rp.quorum == nil { + rp.election = nil + // no quorum + // LDRDBG fmt.Println(rp.Self(), "LDR NO QUO ignore vote from", m.Pid, "round", vote.Round, "for", vote.Leader) + // Seems we have received leader_vote before the quorum_built message. + // Ignore this vote but update its round value to start a new leader election. + // Otherwise, the new election will be started with the same round value but without + // votes, which have been ignored before the quorum was built. + if vote.Round > rp.round { + rp.round = vote.Round + } + return RaftStatusOK + } + + if rp.quorum.State != RaftQuorumState(vote.State) { + // vote within another quorum. seems the quorum has been changed during this election. + // ignore it + // LDRDBG fmt.Println(rp.Self(), "LDR ignore vote from", m.Pid, "with another quorum", vote.State, "current quorum", rp.quorum.State) + if vote.Round > rp.round { + rp.round = vote.Round + } + return RaftStatusOK + } + if rp.election != nil && rp.election.round > vote.Round { + // ignore it. current election round is greater + // LDRDBG fmt.Println(rp.Self(), "LDR ignore vote from", m.Pid, "with round", vote.Round, "current election round", rp.election.round) + return RaftStatusOK + } + if rp.round > vote.Round { + // newbie is trying to start a new election :) + // LDRDBG fmt.Println(rp.Self(), "LDR ignore vote from newbie", m.Pid, "with round", vote.Round, "current round", rp.round) + return RaftStatusOK + } + + // check if m.Pid is belongs to the quorum + belongs := false + for _, pid := range rp.quorum.Peers { + if pid == m.Pid { + belongs = true + break + } + } + + if belongs == false { + // there might be a case if we got vote message before the quorum_built + lib.Warning("[%s] got ignore from the peer, which doesn't belong to the quorum %s", rp.Self(), m.Pid) + if vote.Round > rp.round { + rp.round = vote.Round + } + return RaftStatusOK + } + + // start new election + new_election := false + switch { + case rp.election == nil: + new_election = true + case rp.election != nil: + // TODO case with existing leader whithin this quorum. if some of the quorum member + // got leader heartbeat timeout it starts new election but this process has no problem + // with the leader. + if vote.Round > rp.election.round { + // overwrite election if it has greater round number + rp.election.cancel() + new_election = true + } + } + if new_election { + // LDRDBG fmt.Println(rp.Self(), "LDR accept election from", m.Pid, "round", vote.Round, " with vote for:", vote.Leader) + rp.election = &leaderElection{ + votes: make(map[etf.Pid]etf.Pid), + results: make(map[etf.Pid]bool), + round: vote.Round, + } + rp.election.cancel = rp.CastAfter(rp.Self, messageRaftElectionClean{round: vote.Round}, cleanLeaderVoteTimeout) + rp.handleElectionVote() + } + + if _, exist := rp.election.votes[m.Pid]; exist { + lib.Warning("[%s] ignore duplicate vote for %s from %s during %d round", rp.Self(), + vote.Leader, m.Pid, vote.Round) + return RaftStatusOK + } + + rp.election.votes[m.Pid] = vote.Leader + // LDRDBG fmt.Println(rp.Self(), "LDR got vote from", m.Pid, "for", vote.Leader, "round", vote.Round, "quorum", vote.State) + if len(rp.quorum.Peers) != len(rp.election.votes) { + // make sure if we got all votes + return RaftStatusOK + } + if len(rp.election.votes) != len(rp.quorum.Peers) { + // waiting for all votes from the quorum members) + return RaftStatusOK + } + + // got all votes. count them to get the quorum leader + countVotes := make(map[etf.Pid]int) + for _, vote_for := range rp.election.votes { + c, _ := countVotes[vote_for] + countVotes[vote_for] = c + 1 + } + leaderPid := etf.Pid{} + leaderVoted := 0 + leaderSplit := false + for leader, voted := range countVotes { + if leaderVoted == voted { + leaderSplit = true + continue + } + if leaderVoted < voted { + leaderVoted = voted + leaderPid = leader + leaderSplit = false + } + } + // LDRDBG fmt.Println(rp.Self(), "LDR got all votes. round", vote.Round, "quorum", vote.State) + if leaderSplit { + // LDRDBG fmt.Println(rp.Self(), "LDR got split voices. round", vote.Round, "quorum", vote.State) + // got more than one leader + // start new leader election with round++ + rp.handleElectionStart(vote.Round + 1) + return RaftStatusOK + } + + noLeader := etf.Pid{} + if rp.election.leader == noLeader { + rp.election.leader = leaderPid + rp.election.voted = leaderVoted + } else { + if rp.election.leader != leaderPid || rp.election.voted != leaderVoted { + // our result defers from the others which we already received + // start new leader election with round++ + lib.Warning("[%s] got different result from %s. cheating detected", rp.Self(), m.Pid) + rp.handleElectionStart(vote.Round + 1) + return RaftStatusOK + } + } + + // LDRDBG fmt.Println(rp.Self(), "LDR election done. round", rp.election.round, "Leader", leaderPid, "with", leaderVoted, "voices", "quorum", vote.State) + rp.election.results[rp.Self()] = true + + // send to all quorum members our choice + elected := etf.Tuple{ + etf.Atom("$leader_elected"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + leaderPid, + leaderVoted, + rp.election.round, + }, + } + for _, pid := range rp.quorum.Peers { + if pid == rp.Self() { + continue + } + rp.Cast(pid, elected) + // LDRDBG fmt.Println(rp.Self(), "LDR elected", leaderPid, "sent result to", pid, "wait the others") + } + + if len(rp.election.votes) != len(rp.election.results) { + // we should wait for result from all the election members + return RaftStatusOK + } + + // leader has been elected + // LDRDBG fmt.Println(rp.Self(), "LDR finished. leader", rp.election.leader, "round", rp.election.round, "quorum", rp.quorum.State) + rp.round = rp.election.round + rp.election.cancel() + if rp.leader != rp.election.leader { + rp.leader = rp.election.leader + l := rp.Leader() + rp.election = nil + return rp.behavior.HandleLeader(rp, l) + } + rp.election = nil + return RaftStatusOK + + case etf.Atom("$leader_elected"): + elected := &messageRaftLeaderElected{} + if err := etf.TermIntoStruct(m.Command, &elected); err != nil { + return ErrUnsupportedRequest + } + + if rp.options.ID != elected.ID { + lib.Warning("[%s] ignore 'leader elected' message being not a member of the given raft cluster (from %s)", rp.Self(), m.Pid) + return RaftStatusOK + } + + if rp.quorum == nil { + rp.election = nil + // no quorum + // LDRDBG fmt.Println(rp.Self, "LDR NO QUO ignore election result", elected, "from", m.Pid) + return RaftStatusOK + } + + if rp.election == nil { + lib.Warning("[%s] ignore election result from %s. no election on this peer", rp.Self(), m.Pid) + return RaftStatusOK + } + + if elected.Round != rp.election.round { + // round value must be the same. seemd another election is started + lib.Warning("[%s] ignore election result from %s with another round value %d (current election round %d)", rp.Self(), m.Pid, elected.Round, rp.election.round) + if elected.Round > rp.round { + // update round value to the greatest one + rp.round = elected.Round + } + return RaftStatusOK + } + + noLeader := etf.Pid{} + if rp.election.leader == noLeader { + rp.election.leader = elected.Leader + rp.election.voted = elected.Voted + } else { + if rp.election.leader != elected.Leader || rp.election.voted != elected.Voted { + // elected leader must be the same in all election results + lib.Warning("[%s] ignore election result from %s with different leader which must be the same", rp.Self(), m.Pid) + return RaftStatusOK + } + } + + if _, exist := rp.election.results[m.Pid]; exist { + // duplicate + lib.Warning("[%s] ignore duplicate election result from %s during %d round", rp.Self(), + m.Pid, elected.Round) + return RaftStatusOK + } + + if _, exist := rp.election.votes[m.Pid]; exist == false { + // Got election result before the vote from m.Pid + // Check if m.Pid belongs to the quorum + if rp.election.round > rp.round { + rp.round = rp.election.round + } + belongs := false + for _, pid := range rp.quorum.Peers { + if pid == m.Pid { + belongs = true + break + } + } + if belongs == false { + // got from unknown peer + lib.Warning("[%s] ignore election result from %s which doesn't belong this quorum", rp.Self(), m.Pid) + return RaftStatusOK + } + + // keep it and wait for the vote from this peer + rp.election.results[m.Pid] = true + return RaftStatusOK + } + rp.election.results[m.Pid] = true + + if len(rp.quorum.Peers) != len(rp.election.votes) { + // make sure if we got all votes + return RaftStatusOK + } + + if len(rp.election.votes) != len(rp.election.results) { + // we should wait for result from all the election members + return RaftStatusOK + } + + // leader has been elected + // LDRDBG fmt.Println(rp.Self(), "LDR finished. leader", rp.election.leader, "round", rp.election.round, "quorum", rp.quorum.State) + rp.election.cancel() // cancel timer + rp.round = rp.election.round + if rp.leader != rp.election.leader { + rp.leader = rp.election.leader + rp.election = nil + l := rp.Leader() + return rp.behavior.HandleLeader(rp, l) + } + rp.election = nil + return RaftStatusOK + + case etf.Atom("$request_get"): + requestGet := &messageRaftRequestGet{} + if err := etf.TermIntoStruct(m.Command, &requestGet); err != nil { + return ErrUnsupportedRequest + } + + if rp.options.ID != requestGet.ID { + lib.Warning("[%s] got 'get' request being not a member of the given raft cluster (from %s)", rp.Self(), m.Pid) + return RaftStatusOK + } + + if rp.quorum == nil { + // no quorum + return RaftStatusOK + } + + if rp.quorum.Member == false { + // not a quorum member. couldn't handle this request + lib.Warning("[%s] got 'get' request being not a member of the quorum (from %s)", rp.Self(), m.Pid) + return RaftStatusOK + } + //fmt.Println(rp.Self(), "GET request", requestGet.Ref, "from", m.Pid, "serial", requestGet.Serial) + + key, value, status := rp.behavior.HandleGet(rp, requestGet.Serial) + if status != RaftStatusOK { + // do nothing + return status + } + if value == nil { + // not found. + if m.Pid != requestGet.Origin { + // its already forwarded request. just ignore it + return RaftStatusOK + } + + // forward this request to another qourum member + forwardGet := etf.Tuple{ + etf.Atom("$request_get"), + rp.Self(), + etf.Tuple{ + requestGet.ID, + requestGet.Ref, + requestGet.Origin, + requestGet.Serial, + }, + } + + // get random quorum member excluding m.Pid and requestGet.Origin + peers := []etf.Pid{} + for _, pid := range rp.quorum.Peers { + if pid == m.Pid { + continue + } + if pid == requestGet.Origin { + continue + } + if pid == rp.Self() { + continue + } + peers = append(peers, pid) + } + + if len(peers) == 0 { + return RaftStatusOK + } + + n := 0 + if len(peers) > 1 { + n = rand.Intn(len(peers) - 1) + } + peer := peers[n] + //fmt.Println(rp.Self(), "GET forward", requestGet.Ref, "to", peer, "serial", requestGet.Serial) + rp.Cast(peer, forwardGet) + return RaftStatusOK + } + + requestReply := etf.Tuple{ + etf.Atom("$request_reply"), + rp.Self(), + etf.Tuple{ + requestGet.ID, + requestGet.Ref, + requestGet.Serial, + key, + value, + }, + } + rp.Cast(requestGet.Origin, requestReply) + + // update serial of this peer + if c := rp.quorumCandidates.GetOnline(requestGet.Origin); c != nil { + if c.serial < requestGet.Serial { + c.serial = requestGet.Serial + } + } else { + rp.quorumCandidates.Set(rp, requestGet.Origin) + } + return RaftStatusOK + + case etf.Atom("$request_reply"): + requestReply := &messageRaftRequestReply{} + if err := etf.TermIntoStruct(m.Command, &requestReply); err != nil { + return ErrUnsupportedRequest + } + + if rp.options.ID != requestReply.ID { + lib.Warning("[%s] got 'reply' being not a member of the given raft cluster (from %s)", rp.Self(), m.Pid) + return RaftStatusOK + } + cancel, exist := rp.requests[requestReply.Ref] + if exist == false { + // might be timed out already. do nothing + return RaftStatusOK + } + // cancel timer + cancel() + if rp.options.Serial < requestReply.Serial { + rp.options.Serial = requestReply.Serial + } + // call HandleSerial + return rp.behavior.HandleSerial(rp, requestReply.Ref, requestReply.Serial, + requestReply.Key, requestReply.Value) + + case etf.Atom("$request_append"): + requestAppend := &messageRaftRequestAppend{} + if err := etf.TermIntoStruct(m.Command, &requestAppend); err != nil { + return ErrUnsupportedRequest + } + + if rp.options.ID != requestAppend.ID { + lib.Warning("[%s] got 'append' request being not a member of the given raft cluster (from %s)", rp.Self(), m.Pid) + return RaftStatusOK + } + + if rp.quorum == nil { + // no quorum. ignore it + return RaftStatusOK + } + + // + // There are 3 options: + // + + // 1) This process is a leader -> handleAppendLeader() + // a) increment serial. send this request to all quorum members (except the origin peer) + // b) wait for the request_append_ready from the quorum peers + // c) call the callback HandleAppend + // d) send request_append_commit(serial) to all quorum members (including the origin peer) + if rp.leader == rp.Self() { + return rp.handleAppendLeader(m.Pid, requestAppend) + } + + // 2) This process is not a leader, is a quorum member, and request has + // received from the leader -> handleAppendQuorum() + // a) accept this request and reply with request_append_ready + // b) wait for the request_append_commit + // c) call the callback HandleAppend + // d) send request_append to the peers that are not in the quorum + if rp.quorum.Member == true && m.Pid == rp.leader { + return rp.handleAppendQuorum(requestAppend) + } + + // 3) This process neither a leader or a quorum member. + // Or this process is a quorum member but request has received not from + // the leader of this quorum. + // It also could happened if quorum has changed during the delivering this request. + + // Forward this request to the quorum member (if this process not a quorum member) + // or to the leader (if this process is a quorum member) + + forwardAppend := etf.Tuple{ + etf.Atom("$request_append"), + rp.Self(), + etf.Tuple{ + requestAppend.ID, + requestAppend.Ref, + requestAppend.Origin, + requestAppend.Key, + requestAppend.Value, + requestAppend.Deadline, + }, + } + + if rp.quorum.Member == true { + // DBGAPN fmt.Println(rp.Self(), "DPGAPN forward $request_append", requestAppend.Ref, "to the leader", rp.leader) + noLeader := etf.Pid{} + if rp.leader == noLeader { + // no leader in this quorum yet. ignore this request + return RaftStatusOK + } + // This request has received not from the quorum leader. + // Forward this request to the leader + rp.Cast(rp.leader, forwardAppend) + return RaftStatusOK + } + + // exclude requestAppend.Origin and m.Pid + peers := []etf.Pid{} + for _, pid := range rp.quorum.Peers { + if pid == m.Pid { + continue + } + if pid == requestAppend.Origin { + continue + } + peers = append(peers, pid) + } + n := rand.Intn(len(peers) - 1) + peer := peers[n] + // DBGAPN fmt.Println(rp.Self(), "DPGAPN forward $request_append", requestAppend.Ref, "to the quorum member", peer) + rp.Cast(peer, forwardAppend) + return RaftStatusOK + + case etf.Atom("$request_append_ready"): + appendReady := &messageRaftAppendReady{} + if err := etf.TermIntoStruct(m.Command, &appendReady); err != nil { + return ErrUnsupportedRequest + } + + if rp.options.ID != appendReady.ID { + lib.Warning("[%s] got 'append_ready' message being not a member of the given raft cluster (from %s)", rp.Self(), m.Pid) + return RaftStatusOK + } + + if rp.quorum == nil { + // no quorum. ignore it + return RaftStatusOK + } + + requestAppend, exist := rp.requestsAppend[appendReady.Key] + if exist == false { + // there might be timeout happened. ignore this message + return RaftStatusOK + } + + if requestAppend.ref != appendReady.Ref { + // there might be timeout happened for the previous append request for this key + // and another append request arrived during previous append request handling + return RaftStatusOK + } + + if rp.leader != rp.Self() { + // i'm not a leader. seems leader election happened during this request handling + requestAppend.cancel() + delete(rp.requestsAppend, appendReady.Key) + return RaftStatusOK + } + requestAppend.peers[m.Pid] = true + commit := true + for _, confirmed := range requestAppend.peers { + if confirmed { + continue + } + commit = false + break + } + + if commit == false { + return RaftStatusOK + } + + // received confirmations from all the peers are involved to this append handling. + // call HandleAppend + status := rp.behavior.HandleAppend(rp, requestAppend.ref, rp.options.Serial+1, + appendReady.Key, requestAppend.value) + switch status { + case RaftStatusOK: + rp.options.Serial++ + // sent them $request_append_commit including the origin + request := etf.Tuple{ + etf.Atom("$request_append_commit"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + requestAppend.ref, + appendReady.Key, + rp.options.Serial, + requestAppend.from, + }, + } + for pid, _ := range requestAppend.peers { + if pid == rp.Self() { + continue + } + rp.Cast(pid, request) + // DBGAPN fmt.Println(rp.Self(), "DBGAPN sent append_commit to", pid, "with serial", rp.options.Serial) + if c := rp.quorumCandidates.GetOnline(pid); c != nil { + if c.serial < rp.options.Serial { + c.serial = rp.options.Serial + } + } + } + requestAppend.cancel() + delete(rp.requestsAppend, appendReady.Key) + if requestAppend.from == rp.Self() { + rp.handleBroadcastCommit(appendReady.Key, requestAppend, rp.options.Serial) + } + if len(rp.requestsAppendQueue) == 0 { + return RaftStatusOK + } + + // handle queued append request + handled := 0 + for i := range rp.requestsAppendQueue { + handled = i + queued := rp.requestsAppendQueue[i] + if queued.request.Deadline < time.Now().UnixMilli() { + // expired request + lib.Warning("[%s] append request %s is expired", rp.Self(), queued.request.Ref) + continue + } + rp.handleAppendLeader(queued.from, queued.request) + break + } + rp.requestsAppendQueue = rp.requestsAppendQueue[handled+1:] + if len(rp.requestsAppendQueue) == 0 { + rp.requestsAppendQueue = nil + } + return RaftStatusOK + + case RaftStatusDiscard: + requestAppend.cancel() + delete(rp.requestsAppend, appendReady.Key) + return RaftStatusOK + } + + return status + + case etf.Atom("$request_append_commit"): + appendCommit := &messageRaftAppendCommit{} + if err := etf.TermIntoStruct(m.Command, &appendCommit); err != nil { + return ErrUnsupportedRequest + } + + if rp.options.ID != appendCommit.ID { + lib.Warning("[%s] got 'append_commit' message being not a member of the given raft cluster (from %s)", rp.Self(), m.Pid) + return RaftStatusOK + } + + requestAppend, exist := rp.requestsAppend[appendCommit.Key] + if exist == false { + // seems timeout happened and this request was cleaned up + return RaftStatusOK + } + requestAppend.cancel() + delete(rp.requestsAppend, appendCommit.Key) + + if rp.options.Serial >= appendCommit.Serial { + lib.Warning("[%s] got append commit with serial (%d) greater or equal we have (%d). fork happened. stopping this process", rp.Self(), appendCommit.Serial, rp.options.Serial) + return fmt.Errorf("raft fork happened") + } + + rp.options.Serial = appendCommit.Serial + status := rp.behavior.HandleAppend(rp, requestAppend.ref, appendCommit.Serial, + appendCommit.Key, requestAppend.value) + if status == RaftStatusDiscard { + lib.Warning("[%s] RaftStatusDiscard can be used by a leader only", rp.Self()) + status = RaftStatusOK + } + if appendCommit.Broadcast != rp.Self() { + return status + } + + rp.handleBroadcastCommit(appendCommit.Key, requestAppend, appendCommit.Serial) + return status + + case etf.Atom("$request_append_broadcast"): + broadcast := &messageRaftAppendBroadcast{} + if err := etf.TermIntoStruct(m.Command, &broadcast); err != nil { + return ErrUnsupportedRequest + } + + if rp.options.ID != broadcast.ID { + lib.Warning("[%s] got 'append_broadcast' message being not a member of the given raft cluster (from %s)", rp.Self(), m.Pid) + return RaftStatusOK + } + + rp.options.Serial = broadcast.Serial + return rp.behavior.HandleAppend(rp, broadcast.Ref, broadcast.Serial, + broadcast.Key, broadcast.Value) + + } + + return ErrUnsupportedRequest +} + +func (rp *RaftProcess) handleElectionStart(round int) { + if rp.quorum == nil { + // no quorum. can't start election + return + } + if rp.quorum.Member == false { + // not a quorum member + return + } + if rp.election != nil { + if rp.election.round >= round { + // already in progress + return + } + rp.election.cancel() + } + if rp.round > round { + round = rp.round + } + // LDRDBG fmt.Println(rp.Self(), "LDR start. round", round, "Q", rp.quorum.State) + rp.election = &leaderElection{ + votes: make(map[etf.Pid]etf.Pid), + results: make(map[etf.Pid]bool), + round: round, + } + rp.handleElectionVote() + cancel := rp.CastAfter(rp.Self, messageRaftElectionClean{round: round}, cleanLeaderVoteTimeout) + rp.election.cancel = cancel +} + +func (rp *RaftProcess) handleElectionVote() { + if rp.quorum == nil || rp.election == nil { + return + } + + mapPeers := make(map[etf.Pid]bool) + for _, p := range rp.quorum.Peers { + mapPeers[p] = true + } + + voted_for := etf.Pid{} + c := rp.quorumCandidates.List() // ordered by serial in desk order + for _, pid := range c { + // check if this candidate is a member of quorum + if _, exist := mapPeers[pid]; exist == false { + continue + } + // get the first member since it has biggest serial + voted_for = pid + break + } + + // LDRDBG fmt.Println(rp.Self(), "LDR voted for:", voted_for, "quorum", rp.quorum.State) + leaderVote := etf.Tuple{ + etf.Atom("$leader_vote"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + int(rp.quorum.State), + voted_for, + rp.election.round, + }, + } + for _, pid := range rp.quorum.Peers { + if pid == rp.Self() { + continue + } + // LDRDBG fmt.Println(rp.Self(), "LDR sent vote for", voted_for, "to", pid, "round", rp.election.round, "quorum", rp.quorum.State) + rp.Cast(pid, leaderVote) + } + rp.election.votes[rp.Self()] = voted_for +} + +func (rp *RaftProcess) handleBroadcastCommit(key string, request *requestAppend, serial uint64) { + // DBGAPN fmt.Println(rp.Self(), "broadcasting", request.ref) + // the origin process is in charge of broadcasting this result among + // the peers who aren't quorum members. + commit := etf.Tuple{ + etf.Atom("$request_append_broadcast"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + request.ref, + serial, + key, + request.value, + }, + } + allPeers := rp.quorumCandidates.List() + for _, pid := range allPeers { + if _, exist := request.peers[pid]; exist { + continue + } + if pid == rp.Self() { + continue + } + rp.Cast(pid, commit) + // DBGAPN fmt.Println(rp.Self(), "DBGAPN sent $request_append_broadcast", request.ref, "to", pid) + c := rp.quorumCandidates.GetOnline(pid) + if c != nil && c.serial < serial { + c.serial = serial + } + } +} + +func (rp *RaftProcess) handleAppendLeader(from etf.Pid, request *messageRaftRequestAppend) RaftStatus { + // DBGAPN fmt.Println(rp.Self(), "DBGAPN handle append", request.Ref, "on leader.", request.Key, request.Value) + if _, exist := rp.requestsAppend[request.Key]; exist { + // another append request with this key is still in progress. append to the queue + queued := requestAppendQueued{ + from: from, + request: request, + } + rp.requestsAppendQueue = append(rp.requestsAppendQueue, queued) + lq := len(rp.requestsAppendQueue) + if lq > 10 { + lib.Warning("[%s] append request queue is getting long. queued request %d", rp.Self(), lq) + } + return RaftStatusOK + } + now := time.Now().UnixMilli() + if now >= request.Deadline { + // deadline has been passed. ignore this request + return RaftStatusOK + } + + sendRequestAppend := etf.Tuple{ + etf.Atom("$request_append"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + request.Ref, + request.Origin, + request.Key, + request.Value, + request.Deadline, + }, + } + + peers := make(map[etf.Pid]bool) + for _, pid := range rp.quorum.Peers { + if pid == rp.Self() { + continue + } + peers[pid] = false + if pid == request.Origin { + peers[pid] = true // do not wait append_ready for the Origin + continue + } + rp.Cast(pid, sendRequestAppend) + } + + // if 'from' is not a quorum member the leader is in charge of broadcasting + if _, exist := peers[from]; exist == false { + from = rp.Self() + } + + after := time.Duration(request.Deadline-now) * time.Millisecond + clean := messageRaftAppendClean{key: request.Key, ref: request.Ref} + cancel := rp.CastAfter(rp.Self(), clean, after) + requestAppend := &requestAppend{ + ref: request.Ref, + from: from, + origin: request.Origin, + value: request.Value, + peers: peers, + cancel: cancel, + } + rp.requestsAppend[request.Key] = requestAppend + + return RaftStatusOK +} + +func (rp *RaftProcess) handleAppendQuorum(request *messageRaftRequestAppend) RaftStatus { + // DBGAPN fmt.Println(rp.Self(), "DBGAPN handle append", request.Ref, "on a quorum member.", request.Key, request.Value) + if r, exist := rp.requestsAppend[request.Key]; exist { + r.cancel() + delete(rp.requestsAppend, request.Key) + } + + ready := etf.Tuple{ + etf.Atom("$request_append_ready"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + request.Ref, + request.Key, + }, + } + rp.Cast(rp.leader, ready) + clean := messageRaftAppendClean{key: request.Key, ref: request.Ref} + after := time.Duration(DefaultRaftAppendTimeout) * time.Second + if d := time.Duration(request.Deadline-time.Now().UnixMilli()) * time.Millisecond; d > after { + after = d + } + cancel := rp.CastAfter(rp.Self, clean, after) + + peers := make(map[etf.Pid]bool) + for _, pid := range rp.quorum.Peers { + peers[pid] = true + } + + requestAppend := &requestAppend{ + ref: request.Ref, + origin: request.Origin, + value: request.Value, + peers: peers, + cancel: cancel, + } + rp.requestsAppend[request.Key] = requestAppend + return RaftStatusOK +} + +func (rp *RaftProcess) quorumChangeStart(nextAttempt bool) { + if rp.quorumChangeDefer == false { + if nextAttempt { + // increase timeout for the next attempt to build a new quorum + rp.quorumChangeAttempt++ + } else { + rp.quorumChangeAttempt = 1 + } + maxTime := rp.quorumChangeAttempt * quorumChangeDeferMaxTime + after := time.Duration(50+rand.Intn(maxTime)) * time.Millisecond + rp.CastAfter(rp.Self(), messageRaftQuorumChange{}, after) + rp.quorumChangeDefer = true + } +} + +func (rp *RaftProcess) quorumChange() RaftStatus { + l := len(rp.quorumCandidates.List()) + + candidateRaftQuorumState := RaftQuorumState3 + switch { + case l > 9: + if rp.quorum != nil && rp.quorum.State == RaftQuorumState11 { + // do nothing + return RaftStatusOK + } + candidateRaftQuorumState = RaftQuorumState11 + l = 10 // to create quorum of 11 we need 10 candidates + itself. + + case l > 7: + if rp.quorum != nil && rp.quorum.State == RaftQuorumState9 { + // do nothing + return RaftStatusOK + } + candidateRaftQuorumState = RaftQuorumState9 + l = 8 // quorum of 9 => 8 candidates + itself + case l > 5: + if rp.quorum != nil && rp.quorum.State == RaftQuorumState7 { + // do nothing + return RaftStatusOK + } + candidateRaftQuorumState = RaftQuorumState7 + l = 6 // quorum of 7 => 6 candidates + itself + case l > 3: + if rp.quorum != nil && rp.quorum.State == RaftQuorumState5 { + // do nothing + return RaftStatusOK + } + candidateRaftQuorumState = RaftQuorumState5 + l = 4 // quorum of 5 => 4 candidates + itself + case l > 1: + if rp.quorum != nil && rp.quorum.State == RaftQuorumState3 { + // do nothing + return RaftStatusOK + } + candidateRaftQuorumState = RaftQuorumState3 + l = 2 // quorum of 3 => 2 candidates + itself + default: + // not enougth candidates to create a quorum + if rp.quorum != nil { + rp.quorum = nil + return rp.handleQuorum() + } + // QUODBG fmt.Println(rp.Name(), "QUO VOTE. NOT ENO CAND", rp.quorumCandidates.List()) + + // try send cluster_join again to receive an updated peer list + rp.CastAfter(rp.Self(), messageRaftClusterInit{}, 5*time.Second) + return RaftStatusOK + } + + if _, exist := rp.quorumVotes[candidateRaftQuorumState]; exist { + // voting for this state is already in progress + return RaftStatusOK + } + + quorumCandidates := make([]etf.Pid, 0, l+1) + quorumCandidates = append(quorumCandidates, rp.Self()) + candidates := rp.quorumCandidates.List() + quorumCandidates = append(quorumCandidates, candidates[:l]...) + // QUODBG fmt.Println(rp.Name(), "QUO VOTE INIT", candidateRaftQuorumState, quorumCandidates) + + // send quorumVote to all candidates (except itself) + quorum := &quorum{ + votes: make(map[etf.Pid]int), + origin: rp.Self(), + } + quorum.State = candidateRaftQuorumState + quorum.Peers = quorumCandidates + rp.quorumVotes[candidateRaftQuorumState] = quorum + rp.quorumSendVote(quorum) + rp.CastAfter(rp.Self(), messageRaftQuorumCleanVote{state: quorum.State}, cleanVoteTimeout) + return RaftStatusOK +} + +func (rp *RaftProcess) quorumSendVote(q *quorum) bool { + empty := etf.Pid{} + if q.origin == empty { + // do not send its vote until the origin vote will be received + return false + } + + allVoted := true + quorumVote := etf.Tuple{ + etf.Atom("$quorum_vote"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + rp.options.Serial, + int(q.State), + q.Peers, + }, + } + + for _, pid := range q.Peers { + if pid == rp.Self() { + continue // do not send to itself + } + + if pid == q.origin { + continue + } + v, _ := q.votes[pid] + + // check if already sent vote to this peer + if v&1 == 0 { + // QUODBG fmt.Println(rp.Name(), "SEND VOTE to", pid, q.Peers) + rp.Cast(pid, quorumVote) + // mark as sent + v |= 1 + q.votes[pid] = v + } + + if v != 3 { // 2(010) - recv, 1(001) - sent, 3(011) - recv & sent + allVoted = false + } + } + + if allVoted == true && q.origin != rp.Self() { + // send vote to origin + // QUODBG fmt.Println(rp.Name(), "SEND VOTE to origin", q.origin, q.Peers) + rp.Cast(q.origin, quorumVote) + } + + return allVoted +} + +func (rp *RaftProcess) quorumVote(from etf.Pid, vote *messageRaftQuorumVote) RaftStatus { + if vote.State != len(vote.Candidates) { + lib.Warning("[%s] quorum state and number of candidates are mismatch", rp.Self()) + rp.quorumCandidates.SetOffline(rp, from) + return RaftStatusOK + } + + if c := rp.quorumCandidates.GetOnline(from); c == nil { + // there is a race conditioned case when we received a vote before + // the quorum_join_reply message. just ignore it. they will start + // another round of quorum forming + return RaftStatusOK + } else { + c.heartbeat = time.Now().Unix() + c.failures = 0 + } + candidatesRaftQuorumState := RaftQuorumState3 + switch vote.State { + case 3: + candidatesRaftQuorumState = RaftQuorumState3 + case 5: + candidatesRaftQuorumState = RaftQuorumState5 + case 7: + candidatesRaftQuorumState = RaftQuorumState7 + case 9: + candidatesRaftQuorumState = RaftQuorumState9 + case 11: + candidatesRaftQuorumState = RaftQuorumState11 + default: + lib.Warning("[%s] wrong number of candidates in the request. removing %s from quorum candidates list", rp.Self(), from) + rp.quorumCandidates.SetOffline(rp, from) + return RaftStatusOK + } + + // do not vote if requested quorum is less than existing one + if rp.quorum != nil && candidatesRaftQuorumState <= rp.quorum.State { + // There is a case when a peer is involved in more than one voting, + // and this peer just sent a vote for another voting process which + // is still in progress. + // Do not send $quorum_voted message if this peer is already a member + // of accepted quorum + member := false + for _, pid := range rp.quorum.Peers { + if pid == from { + member = true + break + } + } + if member == true { + return RaftStatusOK + } + + // QUODBG fmt.Println(rp.Name(), "SKIP VOTE from", from, candidatesRaftQuorumState, rp.quorum.State) + built := etf.Tuple{ + etf.Atom("$quorum_built"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + int(rp.quorum.State), + rp.round, + rp.quorum.Peers, + }, + } + rp.Cast(from, built) + return RaftStatusOK + } + + q, exist := rp.quorumVotes[candidatesRaftQuorumState] + if exist == false { + // + // Received the first vote + // + if len(rp.quorumVotes) > 5 { + // can't be more than 5 (there could be only votes for 3,5,7,9,11) + lib.Warning("[%s] too many votes %#v", rp.quorumVotes) + return RaftStatusOK + } + + q = &quorum{} + q.State = candidatesRaftQuorumState + q.Peers = vote.Candidates + + if from == vote.Candidates[0] { + // Origin vote (received from the peer initiated this voting process). + // Otherwise keep this field empty, which means this quorum + // will be overwritten if we get another voting from the peer + // initiated that voting (with a different set/order of peers) + q.origin = from + } + + if rp.quorumValidateVote(from, q, vote) == false { + // do not create this voting if those peers aren't valid (haven't registered yet) + return RaftStatusOK + } + q.lastVote = time.Now().UnixMilli() + // QUODBG fmt.Println(rp.Name(), "QUO VOTE (NEW)", from, vote) + rp.quorumVotes[candidatesRaftQuorumState] = q + rp.CastAfter(rp.Self(), messageRaftQuorumCleanVote{state: q.State}, cleanVoteTimeout) + + } else { + empty := etf.Pid{} + if q.origin == empty && from == vote.Candidates[0] { + // got origin vote. + q.origin = from + + // check if this vote has the same set of peers + same := true + for i := range q.Peers { + if vote.Candidates[i] != q.Peers[i] { + same = false + break + } + } + // if it differs overwrite quorum by the new voting + if same == false { + q.Peers = vote.Candidates + q.votes = nil + } + } + + if rp.quorumValidateVote(from, q, vote) == false { + return RaftStatusOK + } + q.lastVote = time.Now().UnixMilli() + // QUODBG fmt.Println(rp.Name(), "QUO VOTE", from, vote) + } + + // returns true if we got votes from all the peers whithin this quorum + if rp.quorumSendVote(q) == true { + // + // Quorum built + // + // QUODBG fmt.Println(rp.Name(), "QUO BUILT", q.State, q.Peers) + if rp.quorum == nil { + rp.quorum = &RaftQuorum{} + } + rp.quorum.Member = true + rp.quorum.State = q.State + rp.quorum.Peers = q.Peers + delete(rp.quorumVotes, q.State) + + // all candidates who don't belong to this quorum should be known that quorum is built. + mapPeers := make(map[etf.Pid]bool) + for _, peer := range rp.quorum.Peers { + mapPeers[peer] = true + } + allCandidates := rp.quorumCandidates.List() + for _, peer := range allCandidates { + if _, exist := mapPeers[peer]; exist { + // this peer belongs to the quorum. skip it + continue + } + built := etf.Tuple{ + etf.Atom("$quorum_built"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + int(rp.quorum.State), + rp.round, + rp.quorum.Peers, + }, + } + rp.Cast(peer, built) + + } + + rp.handleElectionStart(rp.round + 1) + return rp.handleQuorum() + } + + return RaftStatusOK +} + +func (rp *RaftProcess) clusterHeal() { + for _, pid := range rp.quorumCandidates.ListOffline() { + // c can't be nil here + c := rp.quorumCandidates.Get(pid) + if c.heartbeat == 0 { + continue + } + diff := time.Now().Unix() - c.heartbeat + switch { + case diff < 0: + // heartbeat was set in the future + continue + case diff > 300: // > 5 min + rp.Join(pid) + // the next attempt will be in an hour + c.heartbeat = time.Now().Unix() + 3600 + } + } +} + +func (rp *RaftProcess) handleQuorum() RaftStatus { + q := rp.Quorum() + if status := rp.behavior.HandleQuorum(rp, q); status != RaftStatusOK { + return status + } + + noLeader := etf.Pid{} + if rp.leader != noLeader { + rp.leader = noLeader + if status := rp.behavior.HandleLeader(rp, nil); status != RaftStatusOK { + return status + } + } + + if q == nil || q.Member == false { + return RaftStatusOK + } + + if rp.election == nil { + rp.handleElectionStart(rp.round + 1) + } + + return RaftStatusOK +} + +func (rp *RaftProcess) handleHeartbeat() { + if rp.heartbeatCancel != nil { + rp.heartbeatCancel() + rp.heartbeatCancel = nil + } + + defer func() { + after := DefaultRaftHeartbeat * time.Second + cancel := rp.CastAfter(rp.Self(), messageRaftHeartbeat{}, after) + rp.heartbeatCancel = cancel + rp.clusterHeal() + }() + + if rp.quorum == nil || rp.quorum.Member == false { + return + } + + noLeader := etf.Pid{} + if rp.leader == noLeader { + // leader election is still in progress. do nothing atm. + return + } + + if rp.leader == rp.Self() { + // send a heartbeat to all quorum members if this process is a leader of this quorum + heartbeat := etf.Tuple{ + etf.Atom("$leader_heartbeat"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + rp.options.Serial, + }, + } + for _, pid := range rp.quorum.Peers { + if pid == rp.Self() { + continue + } + rp.Cast(pid, heartbeat) + } + return + } + + // check leader's heartbeat + c := rp.quorumCandidates.GetOnline(rp.leader) + if c != nil { + diff := time.Now().Unix() - c.heartbeat + if c.heartbeat == 0 { + diff = 0 + } + + if diff < DefaultRaftHeartbeat*3 { + return + } + + // long time no see heartbeats from the leader + c.joined = false + rp.quorumCandidates.SetOffline(rp, rp.leader) + } + + // HRTDBG fmt.Println(rp.Self(), "HRT lost leader", rp.leader) + leave := etf.Tuple{ + etf.Atom("$quorum_leave"), + rp.Self(), + etf.Tuple{ + rp.options.ID, + rp.leader, + }, + } + + // tell everyone in the raft cluster + for _, peer := range rp.quorumCandidates.List() { + rp.Cast(peer, leave) + } + rp.quorum = nil + rp.handleQuorum() + rp.quorumChangeStart(false) +} + +func (rp *RaftProcess) isQuorumMember(pid etf.Pid) bool { + if rp.quorum == nil { + return false + } + for _, peer := range rp.quorum.Peers { + if pid == peer { + return true + } + } + return false +} + +func (rp *RaftProcess) quorumValidateVote(from etf.Pid, q *quorum, vote *messageRaftQuorumVote) bool { + duplicates := make(map[etf.Pid]bool) + validFrom := false + validTo := false + validSerial := false + candidatesMatch := true + newVote := false + if q.votes == nil { + q.votes = make(map[etf.Pid]int) + newVote = true + } + + empty := etf.Pid{} + if q.origin != empty && newVote == true && vote.Candidates[0] != from { + return false + } + + for i, pid := range vote.Candidates { + if pid == rp.Self() { + validTo = true + continue + } + + // quorum peers must be matched with the vote's cadidates + if q.Peers[i] != vote.Candidates[i] { + candidatesMatch = false + } + + // check if received vote has the same set of peers. + // if this is the first vote for the given q.State the pid + // will be added to the vote map + _, exist := q.votes[pid] + if exist == false { + if newVote { + q.votes[pid] = 0 + } else { + candidatesMatch = false + } + } + + if _, exist := duplicates[pid]; exist { + lib.Warning("[%s] got vote with duplicates from %s", rp.Name(), from) + rp.quorumCandidates.SetOffline(rp, from) + return false + } + duplicates[pid] = false + + c := rp.quorumCandidates.GetOnline(pid) + if c == nil { + candidatesMatch = false + rp.quorumCandidates.Set(rp, pid) + continue + } + if pid == from { + if c.serial > vote.Serial { + // invalid serial + continue + } + c.serial = vote.Serial + validFrom = true + validSerial = true + } + } + + if candidatesMatch == false { + // can't accept this vote + // QUODBG fmt.Println(rp.Name(), "QUO CAND MISMATCH", from, vote.Candidates) + return false + } + + if validSerial == false { + lib.Warning("[%s] got vote from %s with invalid serial", rp.Name(), from) + rp.quorumCandidates.SetOffline(rp, from) + return false + } + + if validFrom == false || validTo == false { + lib.Warning("[%s] got vote from %s with invalid data", rp.Name(), from) + rp.quorumCandidates.SetOffline(rp, from) + return false + } + + // mark as recv + v, _ := q.votes[from] + q.votes[from] = v | 2 + + return true +} + +// +// Server callbacks +// + +func (r *Raft) Init(process *ServerProcess, args ...etf.Term) error { + var options RaftOptions + + behavior, ok := process.Behavior().(RaftBehavior) + if !ok { + return fmt.Errorf("Raft: not a RaftBehavior") + } + + raftProcess := &RaftProcess{ + ServerProcess: *process, + behavior: behavior, + quorumCandidates: createQuorumCandidates(), + quorumVotes: make(map[RaftQuorumState]*quorum), + requests: make(map[etf.Ref]context.CancelFunc), + requestsAppend: make(map[string]*requestAppend), + } + + // do not inherit parent State + raftProcess.State = nil + options, err := behavior.InitRaft(raftProcess, args...) + if err != nil { + return err + } + + raftProcess.options = options + process.State = raftProcess + + process.Cast(process.Self(), messageRaftClusterInit{}) + //process.SetTrapExit(true) + raftProcess.handleHeartbeat() + return nil +} + +// HandleCall +func (r *Raft) HandleCall(process *ServerProcess, from ServerFrom, message etf.Term) (etf.Term, ServerStatus) { + rp := process.State.(*RaftProcess) + return rp.behavior.HandleRaftCall(rp, from, message) +} + +// HandleCast +func (r *Raft) HandleCast(process *ServerProcess, message etf.Term) ServerStatus { + var mRaft messageRaft + var status RaftStatus + + rp := process.State.(*RaftProcess) + switch m := message.(type) { + case messageRaftClusterInit: + if rp.quorum != nil { + return ServerStatusOK + } + if len(rp.quorumVotes) > 0 { + return ServerStatusOK + } + for _, peer := range rp.options.Peers { + rp.Join(peer) + } + return ServerStatusOK + + case messageRaftQuorumCleanVote: + q, exist := rp.quorumVotes[m.state] + if exist == true && q.lastVote > 0 { + diff := time.Duration(time.Now().UnixMilli()-q.lastVote) * time.Millisecond + // if voting is still in progress cast itself again with shifted timeout + // according to cleanVoteTimeout + if cleanVoteTimeout > diff { + nextCleanVoteTimeout := cleanVoteTimeout - diff + rp.CastAfter(rp.Self(), messageRaftQuorumCleanVote{state: q.State}, nextCleanVoteTimeout) + return ServerStatusOK + } + } + + if q != nil { + // QUODBG fmt.Println(rp.Name(), "CLN VOTE", m.state, q.Peers) + delete(rp.quorumVotes, m.state) + for _, peer := range q.Peers { + v, _ := q.votes[peer] + if v&2 > 0 { // vote received + continue + } + // no vote from this peer. there are two options + // 1. this peer has switched to the other quorum building + // 2. something wrong with this peer (raft process could be stuck). + c := rp.quorumCandidates.GetOnline(peer) + if c == nil { + // already offline + continue + } + c.failures++ + if c.failures > 10 { + // QUODBG fmt.Println(rp.Self(), "too many failures with", peer) + rp.quorumCandidates.SetOffline(rp, peer) + } + } + } + if len(rp.quorumVotes) == 0 { + // make another attempt to build new quorum + rp.quorumChangeStart(true) + } + case messageRaftQuorumChange: + rp.quorumChangeDefer = false + status = rp.quorumChange() + + case messageRaftRequestClean: + delete(rp.requests, m.ref) + status = rp.behavior.HandleCancel(rp, m.ref, "timeout") + + case messageRaftAppendClean: + request, exist := rp.requestsAppend[m.key] + if exist == false { + // do nothing + return ServerStatusOK + } + if request.ref != m.ref { + return ServerStatusOK + } + if request.origin == rp.Self() { + status = rp.behavior.HandleCancel(rp, request.ref, "timeout") + break + } + delete(rp.requestsAppend, m.key) + return ServerStatusOK + case messageRaftElectionClean: + if rp.quorum == nil { + return ServerStatusOK + } + if rp.election == nil && rp.quorum.Member { + // restart election + rp.handleElectionStart(rp.round + 1) + return ServerStatusOK + } + if m.round != rp.election.round { + // new election round happened + // LDRDBG fmt.Println(rp.Self(), "LDR clean election. skip. new election round", rp.election.round) + return ServerStatusOK + } + // LDRDBG fmt.Println(rp.Self(), "LDR clean election. round", rp.election.round) + rp.election = nil + return ServerStatusOK + + case messageRaftHeartbeat: + rp.handleHeartbeat() + return ServerStatusOK + + default: + if err := etf.TermIntoStruct(message, &mRaft); err != nil { + status = rp.behavior.HandleRaftInfo(rp, message) + break + } + if mRaft.Pid == process.Self() { + lib.Warning("[%s] got raft command from itself %#v", process.Self(), mRaft) + return ServerStatusOK + } + status = rp.handleRaftRequest(mRaft) + if status == ErrUnsupportedRequest { + status = rp.behavior.HandleRaftCast(rp, message) + } + } + + switch status { + case nil, RaftStatusOK: + return ServerStatusOK + case RaftStatusStop: + return ServerStatusStop + case ErrUnsupportedRequest: + return rp.behavior.HandleRaftInfo(rp, message) + default: + return ServerStatus(status) + } + +} + +// HandleInfo +func (r *Raft) HandleInfo(process *ServerProcess, message etf.Term) ServerStatus { + var status RaftStatus + + rp := process.State.(*RaftProcess) + switch m := message.(type) { + case MessageDown: + can := rp.quorumCandidates.GetOnline(m.Pid) + if can == nil { + break + } + if can.monitor != m.Ref { + status = rp.behavior.HandleRaftInfo(rp, message) + break + } + rp.quorumCandidates.SetOffline(rp, m.Pid) + if rp.quorum == nil { + return ServerStatusOK + } + for _, peer := range rp.quorum.Peers { + // check if this pid belongs to the quorum + if peer != m.Pid { + continue + } + + // start to build new quorum + // QUODBG fmt.Println(rp.Name(), "QUO PEER DOWN", m.Pid) + rp.handleQuorum() + rp.quorumChangeStart(false) + break + } + return ServerStatusOK + + default: + status = rp.behavior.HandleRaftInfo(rp, message) + } + + switch status { + case nil, RaftStatusOK: + return ServerStatusOK + case RaftStatusStop: + return ServerStatusStop + default: + return ServerStatus(status) + } +} + +// +// default Raft callbacks +// + +// HandleQuorum +func (r *Raft) HandleQuorum(process *RaftProcess, quorum *RaftQuorum) RaftStatus { + return RaftStatusOK +} + +// HandleLeader +func (r *Raft) HandleLeader(process *RaftProcess, leader *RaftLeader) RaftStatus { + return RaftStatusOK +} + +// HandlePeer +func (r *Raft) HandlePeer(process *RaftProcess, peer etf.Pid, serial uint64) RaftStatus { + return RaftStatusOK +} + +// HandleSerial +func (r *Raft) HandleSerial(process *RaftProcess, ref etf.Ref, serial uint64, key string, value etf.Term) RaftStatus { + lib.Warning("HandleSerial: unhandled key-value message with ref %s and serial %d", ref, serial) + return RaftStatusOK +} + +// HandleCancel +func (r *Raft) HandleCancel(process *RaftProcess, ref etf.Ref, reason string) RaftStatus { + lib.Warning("HandleCancel: unhandled cancel with ref %s and reason %q", ref, reason) + return RaftStatusOK +} + +// HandleRaftCall +func (r *Raft) HandleRaftCall(process *RaftProcess, from ServerFrom, message etf.Term) (etf.Term, ServerStatus) { + lib.Warning("HandleRaftCall: unhandled message (from %#v) %#v", from, message) + return etf.Atom("ok"), ServerStatusOK +} + +// HandleRaftCast +func (r *Raft) HandleRaftCast(process *RaftProcess, message etf.Term) ServerStatus { + lib.Warning("HandleRaftCast: unhandled message %#v", message) + return ServerStatusOK +} + +// HandleRaftInfo +func (r *Raft) HandleRaftInfo(process *RaftProcess, message etf.Term) ServerStatus { + lib.Warning("HandleRaftInfo: unhandled message %#v", message) + return ServerStatusOK +} + +// HandleRaftDirect +func (r *Raft) HandleRaftDirect(process *RaftProcess, message interface{}) (interface{}, error) { + return nil, ErrUnsupportedRequest +} + +// +// internals +// + +func createQuorumCandidates() *quorumCandidates { + qc := &quorumCandidates{ + candidates: make(map[etf.Pid]*candidate), + } + return qc +} + +func (qc *quorumCandidates) Set(rp *RaftProcess, peer etf.Pid) { + c, exist := qc.candidates[peer] + if exist == true { + diff := time.Now().Unix() - c.heartbeat + if diff > DefaultRaftHeartbeat { + rp.Join(peer) + } + return + } + c = &candidate{ + heartbeat: time.Now().Unix(), + } + qc.candidates[peer] = c + rp.Join(peer) +} + +func (qc *quorumCandidates) SetOnline(rp *RaftProcess, peer etf.Pid, serial uint64) bool { + c, exist := qc.candidates[peer] + if exist == false { + return false + } + mon := rp.MonitorProcess(peer) + c.serial = serial + c.monitor = mon + c.joined = true + c.heartbeat = time.Now().Unix() + c.failures = 0 + return true +} + +func (qc *quorumCandidates) SetOffline(rp *RaftProcess, peer etf.Pid) { + c, exist := qc.candidates[peer] + if exist == false { + return + } + // QUODBG fmt.Println(rp.Self(), "peer", peer, "has left") + emptyRef := etf.Ref{} + if c.monitor != emptyRef { + rp.DemonitorProcess(c.monitor) + c.monitor = emptyRef + } + c.joined = false + c.failures = 0 + c.heartbeat = time.Now().Unix() + return +} + +func (qc *quorumCandidates) GetOnline(peer etf.Pid) *candidate { + c, exist := qc.candidates[peer] + if exist && c.joined == false { + return nil + } + return c +} +func (qc *quorumCandidates) Get(peer etf.Pid) *candidate { + c, exist := qc.candidates[peer] + if exist == false { + return nil + } + return c +} + +// List returns list of online peers +func (qc *quorumCandidates) List() []etf.Pid { + type c struct { + pid etf.Pid + serial uint64 + } + list := []c{} + for k, v := range qc.candidates { + if v.joined == false { + continue + } + list = append(list, c{pid: k, serial: v.serial}) + } + + // sort candidates by serial number in desc order + sort.Slice(list, func(a, b int) bool { return list[a].serial > list[b].serial }) + pids := []etf.Pid{} + for i := range list { + pids = append(pids, list[i].pid) + } + return pids +} + +func (qc *quorumCandidates) ListOffline() []etf.Pid { + list := []etf.Pid{} + for pid, c := range qc.candidates { + if c.joined == true { + continue + } + list = append(list, pid) + } + return list +} diff --git a/gen/saga.go b/gen/saga.go index 747b9562..5fdf57bb 100644 --- a/gen/saga.go +++ b/gen/saga.go @@ -8,6 +8,7 @@ import ( "time" "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/lib" ) // SagaBehavior interface @@ -80,6 +81,7 @@ const ( defaultLifespan = 60 ) +// SagaStatus type SagaStatus error var ( @@ -87,7 +89,6 @@ var ( SagaStatusStop SagaStatus = fmt.Errorf("stop") // internal - sagaStatusUnsupported SagaStatus = fmt.Errorf("unsupported") ErrSagaTxEndOfLifespan = fmt.Errorf("End of TX lifespan") ErrSagaTxNextTimeout = fmt.Errorf("Next saga timeout") @@ -100,10 +101,12 @@ var ( ErrSagaNotAllowed = fmt.Errorf("Operation is not allowed") ) +// Saga type Saga struct { Server } +// SagaTransactionOptions type SagaTransactionOptions struct { // HopLimit defines a number of hop within the transaction. Default limit // is 0 (no limit). @@ -117,6 +120,7 @@ type SagaTransactionOptions struct { TwoPhaseCommit bool } +// SagaOptions type SagaOptions struct { // MaxTransactions defines the limit for the number of active transactions. Default: 0 (unlimited) MaxTransactions uint @@ -124,6 +128,7 @@ type SagaOptions struct { Worker SagaWorkerBehavior } +// SagaProcess type SagaProcess struct { ServerProcess options SagaOptions @@ -142,13 +147,16 @@ type SagaProcess struct { mutexJobs sync.Mutex } +// SagaTransactionID type SagaTransactionID etf.Ref +// String func (id SagaTransactionID) String() string { r := etf.Ref(id) return fmt.Sprintf("TX#%d.%d.%d", r.ID[0], r.ID[1], r.ID[2]) } +// SagaTransaction type SagaTransaction struct { sync.Mutex id SagaTransactionID @@ -164,13 +172,16 @@ type SagaTransaction struct { cancelTimer context.CancelFunc } +// SagaNextID type SagaNextID etf.Ref +// String func (id SagaNextID) String() string { r := etf.Ref(id) return fmt.Sprintf("Next#%d.%d.%d", r.ID[0], r.ID[1], r.ID[2]) } +// SagaNext type SagaNext struct { // Saga etf.Pid, string (for the locally registered process), gen.ProcessID{process, node} (for the remote process) Saga interface{} @@ -186,13 +197,16 @@ type SagaNext struct { cancelTimer context.CancelFunc } +// SagaJobID type SagaJobID etf.Ref +// String func (id SagaJobID) String() string { r := etf.Ref(id) return fmt.Sprintf("Job#%d.%d.%d", r.ID[0], r.ID[1], r.ID[2]) } +// SagaJob type SagaJob struct { ID SagaJobID TransactionID SagaTransactionID @@ -207,6 +221,7 @@ type SagaJob struct { cancelTimer context.CancelFunc } +// SagaJobOptions type SagaJobOptions struct { Timeout uint } @@ -243,12 +258,14 @@ type messageSagaCommit struct { Final interface{} } +// MessageSagaCancel type MessageSagaCancel struct { TransactionID SagaTransactionID NextID SagaNextID Reason string } +// MessageSagaError type MessageSagaError struct { TransactionID SagaTransactionID NextID SagaNextID @@ -280,6 +297,7 @@ func (gs *Saga) SetMaxTransactions(process Process, max uint) error { // SagaProcess methods // +// StartTransaction func (sp *SagaProcess) StartTransaction(options SagaTransactionOptions, value interface{}) SagaTransactionID { id := sp.MakeRef() @@ -310,6 +328,7 @@ func (sp *SagaProcess) StartTransaction(options SagaTransactionOptions, value in return SagaTransactionID(id) } +// Next func (sp *SagaProcess) Next(id SagaTransactionID, next SagaNext) (SagaNextID, error) { sp.mutexTXS.Lock() tx, ok := sp.txs[id] @@ -379,6 +398,7 @@ func (sp *SagaProcess) Next(id SagaTransactionID, next SagaNext) (SagaNextID, er return next_id, nil } +// StartJob func (sp *SagaProcess) StartJob(id SagaTransactionID, options SagaJobOptions, value interface{}) (SagaJobID, error) { if sp.options.Worker == nil { @@ -441,6 +461,7 @@ func (sp *SagaProcess) StartJob(id SagaTransactionID, options SagaJobOptions, va return job.ID, nil } +// SendResult func (sp *SagaProcess) SendResult(id SagaTransactionID, result interface{}) error { sp.mutexTXS.Lock() tx, ok := sp.txs[id] @@ -472,6 +493,8 @@ func (sp *SagaProcess) SendResult(id SagaTransactionID, result interface{}) erro }, } + //fmt.Printf("SAGA RESULT %#v\n", message) + // send message to the parent saga if err := sp.Send(tx.parents[0], message); err != nil { return err @@ -497,6 +520,7 @@ func (sp *SagaProcess) SendResult(id SagaTransactionID, result interface{}) erro return nil } +// SendInterim func (sp *SagaProcess) SendInterim(id SagaTransactionID, interim interface{}) error { sp.mutexTXS.Lock() tx, ok := sp.txs[id] @@ -523,6 +547,7 @@ func (sp *SagaProcess) SendInterim(id SagaTransactionID, interim interface{}) er return nil } +// CancelTransaction func (sp *SagaProcess) CancelTransaction(id SagaTransactionID, reason string) error { sp.mutexTXS.Lock() tx, ok := sp.txs[id] @@ -540,6 +565,7 @@ func (sp *SagaProcess) CancelTransaction(id SagaTransactionID, reason string) er return nil } +// CancelJob func (sp *SagaProcess) CancelJob(id SagaTransactionID, job SagaJobID, reason string) error { sp.mutexTXS.Lock() tx, ok := sp.txs[id] @@ -845,7 +871,7 @@ func (sp *SagaProcess) handleSagaRequest(m messageSaga) error { } return SagaStatusOK } - return sagaStatusUnsupported + return ErrUnsupportedRequest } func (sp *SagaProcess) cancelTX(from etf.Pid, cancel messageSagaCancel, tx *SagaTransaction) { @@ -1090,6 +1116,7 @@ func (sp *SagaProcess) handleSagaDown(down MessageDown) error { // Server callbacks // +// Init func (gs *Saga) Init(process *ServerProcess, args ...etf.Term) error { var options SagaOptions @@ -1124,11 +1151,13 @@ func (gs *Saga) Init(process *ServerProcess, args ...etf.Term) error { return nil } +// HandleCall func (gs *Saga) HandleCall(process *ServerProcess, from ServerFrom, message etf.Term) (etf.Term, ServerStatus) { sp := process.State.(*SagaProcess) return sp.behavior.HandleSagaCall(sp, from, message) } +// HandleDirect func (gs *Saga) HandleDirect(process *ServerProcess, message interface{}) (interface{}, error) { sp := process.State.(*SagaProcess) switch m := message.(type) { @@ -1140,6 +1169,7 @@ func (gs *Saga) HandleDirect(process *ServerProcess, message interface{}) (inter } } +// HandleCast func (gs *Saga) HandleCast(process *ServerProcess, message etf.Term) ServerStatus { var status SagaStatus @@ -1215,6 +1245,7 @@ func (gs *Saga) HandleCast(process *ServerProcess, message etf.Term) ServerStatu } } +// HandleInfo func (gs *Saga) HandleInfo(process *ServerProcess, message etf.Term) ServerStatus { var mSaga messageSaga @@ -1247,7 +1278,7 @@ func (gs *Saga) HandleInfo(process *ServerProcess, message etf.Term) ServerStatu return ServerStatusOK case SagaStatusStop: return ServerStatusStop - case sagaStatusUnsupported: + case ErrUnsupportedRequest: return sp.behavior.HandleSagaInfo(sp, message) default: return ServerStatus(status) @@ -1258,43 +1289,60 @@ func (gs *Saga) HandleInfo(process *ServerProcess, message etf.Term) ServerStatu // default Saga callbacks // +// HandleTxInterim func (gs *Saga) HandleTxInterim(process *SagaProcess, id SagaTransactionID, from SagaNextID, interim interface{}) SagaStatus { - fmt.Printf("HandleTxInterim: [%v %v] unhandled message %#v\n", id, from, interim) + lib.Warning("HandleTxInterim: [%v %v] unhandled message %#v", id, from, interim) return ServerStatusOK } + +// HandleTxCommit func (gs *Saga) HandleTxCommit(process *SagaProcess, id SagaTransactionID, final interface{}) SagaStatus { - fmt.Printf("HandleTxCommit: [%v] unhandled message\n", id) + lib.Warning("HandleTxCommit: [%v] unhandled message", id) return ServerStatusOK } + +// HandleTxDone func (gs *Saga) HandleTxDone(process *SagaProcess, id SagaTransactionID, result interface{}) (interface{}, SagaStatus) { - return nil, fmt.Errorf("Saga [%v:%v] has no implementaion of HandleTxDone method", process.Self(), process.Name()) + return nil, fmt.Errorf("Saga [%v:%v] has no implementation of HandleTxDone method", process.Self(), process.Name()) } +// HandleSagaCall func (gs *Saga) HandleSagaCall(process *SagaProcess, from ServerFrom, message etf.Term) (etf.Term, ServerStatus) { - fmt.Printf("HandleSagaCall: unhandled message (from %#v) %#v\n", from, message) + lib.Warning("HandleSagaCall: unhandled message (from %#v) %#v", from, message) return etf.Atom("ok"), ServerStatusOK } + +// HandleSagaCast func (gs *Saga) HandleSagaCast(process *SagaProcess, message etf.Term) ServerStatus { - fmt.Printf("HandleSagaCast: unhandled message %#v\n", message) + lib.Warning("HandleSagaCast: unhandled message %#v", message) return ServerStatusOK } + +// HandleSagaInfo func (gs *Saga) HandleSagaInfo(process *SagaProcess, message etf.Term) ServerStatus { - fmt.Printf("HandleSagaInfo: unhandled message %#v\n", message) + lib.Warning("HandleSagaInfo: unhandled message %#v", message) return ServerStatusOK } + +// HandleSagaDirect func (gs *Saga) HandleSagaDirect(process *SagaProcess, message interface{}) (interface{}, error) { return nil, ErrUnsupportedRequest } +// HandleJobResult func (gs *Saga) HandleJobResult(process *SagaProcess, id SagaTransactionID, from SagaJobID, result interface{}) SagaStatus { - fmt.Printf("HandleJobResult: [%v %v] unhandled message %#v\n", id, from, result) + lib.Warning("HandleJobResult: [%v %v] unhandled message %#v", id, from, result) return SagaStatusOK } + +// HandleJobInterim func (gs *Saga) HandleJobInterim(process *SagaProcess, id SagaTransactionID, from SagaJobID, interim interface{}) SagaStatus { - fmt.Printf("HandleJobInterim: [%v %v] unhandled message %#v\n", id, from, interim) + lib.Warning("HandleJobInterim: [%v %v] unhandled message %#v", id, from, interim) return SagaStatusOK } + +// HandleJobFailed func (gs *Saga) HandleJobFailed(process *SagaProcess, id SagaTransactionID, from SagaJobID, reason string) SagaStatus { - fmt.Printf("HandleJobFailed: [%v %v] unhandled message. reason %q\n", id, from, reason) + lib.Warning("HandleJobFailed: [%v %v] unhandled message. reason %q", id, from, reason) return nil } diff --git a/gen/saga_worker.go b/gen/saga_worker.go index d99c1626..316cec1f 100644 --- a/gen/saga_worker.go +++ b/gen/saga_worker.go @@ -4,8 +4,10 @@ import ( "fmt" "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/lib" ) +// SagaWorkerBehavior type SagaWorkerBehavior interface { ServerBehavior // Mandatory callbacks @@ -39,10 +41,12 @@ type SagaWorkerBehavior interface { HandleWorkerTerminate(process *SagaWorkerProcess, reason string) } +// SagaWorker type SagaWorker struct { Server } +// SagaWorkerProcess type SagaWorkerProcess struct { ServerProcess @@ -123,6 +127,7 @@ func (wp *SagaWorkerProcess) SendInterim(interim interface{}) error { // Server callbacks +// Init func (w *SagaWorker) Init(process *ServerProcess, args ...etf.Term) error { behavior, ok := process.Behavior().(SagaWorkerBehavior) if !ok { @@ -136,6 +141,7 @@ func (w *SagaWorker) Init(process *ServerProcess, args ...etf.Term) error { return nil } +// HandleCast func (w *SagaWorker) HandleCast(process *ServerProcess, message etf.Term) ServerStatus { wp := process.State.(*SagaWorkerProcess) switch m := message.(type) { @@ -166,20 +172,25 @@ func (w *SagaWorker) HandleCast(process *ServerProcess, message etf.Term) Server } } +// HandleCall func (w *SagaWorker) HandleCall(process *ServerProcess, from ServerFrom, message etf.Term) (etf.Term, ServerStatus) { p := process.State.(*SagaWorkerProcess) return p.behavior.HandleWorkerCall(p, from, message) } +// HandleDirect func (w *SagaWorker) HandleDirect(process *ServerProcess, message interface{}) (interface{}, error) { p := process.State.(*SagaWorkerProcess) return p.behavior.HandleWorkerDirect(p, message) } + +// HandleInfo func (w *SagaWorker) HandleInfo(process *ServerProcess, message etf.Term) ServerStatus { p := process.State.(*SagaWorkerProcess) return p.behavior.HandleWorkerInfo(p, message) } +// Terminate func (w *SagaWorker) Terminate(process *ServerProcess, reason string) { p := process.State.(*SagaWorkerProcess) p.behavior.HandleWorkerTerminate(p, reason) @@ -187,27 +198,38 @@ func (w *SagaWorker) Terminate(process *ServerProcess, reason string) { } // default callbacks + +// HandleJobCommit func (w *SagaWorker) HandleJobCommit(process *SagaWorkerProcess, final interface{}) { - fmt.Printf("HandleJobCommit: unhandled message %#v\n", final) + lib.Warning("HandleJobCommit: unhandled message %#v", final) return } + +// HandleWorkerInfo func (w *SagaWorker) HandleWorkerInfo(process *SagaWorkerProcess, message etf.Term) ServerStatus { - fmt.Printf("HandleWorkerInfo: unhandled message %#v\n", message) + lib.Warning("HandleWorkerInfo: unhandled message %#v", message) return ServerStatusOK } + +// HandleWorkerCast func (w *SagaWorker) HandleWorkerCast(process *SagaWorkerProcess, message etf.Term) ServerStatus { - fmt.Printf("HandleWorkerCast: unhandled message %#v\n", message) + lib.Warning("HandleWorkerCast: unhandled message %#v", message) return ServerStatusOK } + +// HandleWorkerCall func (w *SagaWorker) HandleWorkerCall(process *SagaWorkerProcess, from ServerFrom, message etf.Term) (etf.Term, ServerStatus) { - fmt.Printf("HandleWorkerCall: unhandled message (from %#v) %#v\n", from, message) + lib.Warning("HandleWorkerCall: unhandled message (from %#v) %#v", from, message) return etf.Atom("ok"), ServerStatusOK } + +// HandleWorkerDirect func (w *SagaWorker) HandleWorkerDirect(process *SagaWorkerProcess, message interface{}) (interface{}, error) { - fmt.Printf("HandleWorkerDirect: unhandled message %#v\n", message) + lib.Warning("HandleWorkerDirect: unhandled message %#v", message) return nil, nil } +// HandleWorkerTerminate func (w *SagaWorker) HandleWorkerTerminate(process *SagaWorkerProcess, reason string) { return } diff --git a/gen/server.go b/gen/server.go index 2e735f36..9e6063f5 100644 --- a/gen/server.go +++ b/gen/server.go @@ -40,6 +40,7 @@ type ServerBehavior interface { Terminate(process *ServerProcess, reason string) } +// ServerStatus type ServerStatus error var ( @@ -48,6 +49,7 @@ var ( ServerStatusIgnore ServerStatus = fmt.Errorf("ignore") ) +// ServerStatusStopWithReason func ServerStatusStopWithReason(s string) ServerStatus { return ServerStatus(fmt.Errorf(s)) } @@ -67,7 +69,7 @@ type ServerProcess struct { ProcessState behavior ServerBehavior - reductions uint64 // total number of processed messages from mailBox + counter uint64 // total number of processed messages from mailBox currentFunction string trapExit bool @@ -108,18 +110,19 @@ func (sp *ServerProcess) Cast(to interface{}, message etf.Term) error { // Call makes outgoing sync request in fashion of 'gen_server:call'. // 'to' can be Pid, registered local name or gen.ProcessID{RegisteredName, NodeName}. -// This method shouldn't be used outside of the actor. Use Direct method instead. func (sp *ServerProcess) Call(to interface{}, message etf.Term) (etf.Term, error) { return sp.CallWithTimeout(to, message, DefaultCallTimeout) } // CallWithTimeout makes outgoing sync request in fashiod of 'gen_server:call' with given timeout. -// This method shouldn't be used outside of the actor. Use DirectWithTimeout method instead. func (sp *ServerProcess) CallWithTimeout(to interface{}, message etf.Term, timeout int) (etf.Term, error) { ref := sp.MakeRef() from := etf.Tuple{sp.Self(), ref} msg := etf.Term(etf.Tuple{etf.Atom("$gen_call"), from, message}) - if err := sp.SendSyncRequest(ref, to, msg); err != nil { + + sp.PutSyncRequest(ref) + if err := sp.Send(to, msg); err != nil { + sp.CancelSyncRequest(ref) return nil, err } sp.callbackWaitReply <- &ref @@ -187,6 +190,13 @@ func (sp *ServerProcess) SendReply(from ServerFrom, reply etf.Term) error { return sp.Send(to, rep) } +// MessageCounter returns the total number of messages handled by Server callbacks: HandleCall, +// HandleCast, HandleInfo, HandleDirect +func (sp *ServerProcess) MessageCounter() uint64 { + return sp.counter +} + +// ProcessInit func (gs *Server) ProcessInit(p Process, args ...etf.Term) (ProcessState, error) { behavior, ok := p.Behavior().(ServerBehavior) if !ok { @@ -196,7 +206,7 @@ func (gs *Server) ProcessInit(p Process, args ...etf.Term) (ProcessState, error) Process: p, } - gsp := &ServerProcess{ + sp := &ServerProcess{ ProcessState: ps, behavior: behavior, @@ -206,36 +216,37 @@ func (gs *Server) ProcessInit(p Process, args ...etf.Term) (ProcessState, error) callbackWaitReply: make(chan *etf.Ref), } - err := behavior.Init(gsp, args...) + err := behavior.Init(sp, args...) if err != nil { return ProcessState{}, err } - ps.State = gsp + ps.State = sp return ps, nil } +// ProcessLoop func (gs *Server) ProcessLoop(ps ProcessState, started chan<- bool) string { - gsp, ok := ps.State.(*ServerProcess) + sp, ok := ps.State.(*ServerProcess) if !ok { return "ProcessLoop: not a ServerBehavior" } channels := ps.ProcessChannels() - gsp.mailbox = channels.Mailbox - gsp.original = channels.Mailbox - gsp.deferred = make(chan ProcessMailboxMessage, cap(channels.Mailbox)) - gsp.currentFunction = "Server:loop" - gsp.stop = make(chan string, 2) + sp.mailbox = channels.Mailbox + sp.original = channels.Mailbox + sp.deferred = make(chan ProcessMailboxMessage, cap(channels.Mailbox)) + sp.currentFunction = "Server:loop" + sp.stop = make(chan string, 2) defer func() { - if gsp.waitReply == nil { + if sp.waitReply == nil { return } - // there is runnig callback goroutine waiting for reply. to get rid + // there is running callback goroutine that waiting for a reply. to get rid // of infinity lock (of this callback goroutine) we must provide a reader // for the callbackWaitReply channel (it writes a nil value to this channel // on exit) - go gsp.waitCallbackOrDeferr(nil) + go sp.waitCallbackOrDeferr(nil) }() started <- true @@ -245,8 +256,8 @@ func (gs *Server) ProcessLoop(ps ProcessState, started chan<- bool) string { select { case ex := <-channels.GracefulExit: - if !gsp.TrapExit() { - gsp.behavior.Terminate(gsp, ex.Reason) + if !sp.TrapExit() { + sp.behavior.Terminate(sp, ex.Reason) return ex.Reason } // Enabled trap exit message. Transform exit signal @@ -263,29 +274,27 @@ func (gs *Server) ProcessLoop(ps ProcessState, started chan<- bool) string { ps.Send(ps.Self(), message) continue - case reason := <-gsp.stop: - gsp.behavior.Terminate(gsp, reason) + case reason := <-sp.stop: + sp.behavior.Terminate(sp, reason) return reason - case msg := <-gsp.mailbox: - gsp.mailbox = gsp.original + case msg := <-sp.mailbox: + sp.mailbox = sp.original fromPid = msg.From message = msg.Message - case <-gsp.Context().Done(): - gsp.behavior.Terminate(gsp, "kill") + case <-sp.Context().Done(): + sp.behavior.Terminate(sp, "kill") return "kill" case direct := <-channels.Direct: - gsp.waitCallbackOrDeferr(direct) + sp.waitCallbackOrDeferr(direct) continue - case gsp.waitReply = <-gsp.callbackWaitReply: + case sp.waitReply = <-sp.callbackWaitReply: continue } - lib.Log("[%s] GEN_SERVER %s got message from %s", gsp.NodeName(), gsp.Self(), fromPid) - - gsp.reductions++ + lib.Log("[%s] GEN_SERVER %s got message from %s", sp.NodeName(), sp.Self(), fromPid) switch m := message.(type) { case etf.Tuple: @@ -296,12 +305,12 @@ func (gs *Server) ProcessLoop(ps ProcessState, started chan<- bool) string { if len(m) != 2 { break } - gsp.PutSyncReply(mtag, m.Element(2)) - if gsp.waitReply != nil && *gsp.waitReply == mtag { - gsp.waitReply = nil - // continue read gsp.callbackWaitReply channel + sp.PutSyncReply(mtag, m.Element(2)) + if sp.waitReply != nil && *sp.waitReply == mtag { + sp.waitReply = nil + // continue read sp.callbackWaitReply channel // to wait for the exit from the callback call - gsp.waitCallbackOrDeferr(nil) + sp.waitCallbackOrDeferr(nil) continue } @@ -361,7 +370,7 @@ func (gs *Server) ProcessLoop(ps ProcessState, started chan<- bool) string { from: from, message: m.Element(3), } - gsp.waitCallbackOrDeferr(callMessage) + sp.waitCallbackOrDeferr(callMessage) continue case etf.Atom("$gen_cast"): @@ -372,110 +381,119 @@ func (gs *Server) ProcessLoop(ps ProcessState, started chan<- bool) string { castMessage := handleCastMessage{ message: m.Element(2), } - gsp.waitCallbackOrDeferr(castMessage) + sp.waitCallbackOrDeferr(castMessage) continue } } - lib.Log("[%s] GEN_SERVER %#v got simple message %#v", gsp.NodeName(), gsp.Self(), message) + lib.Log("[%s] GEN_SERVER %#v got simple message %#v", sp.NodeName(), sp.Self(), message) infoMessage := handleInfoMessage{ message: message, } - gsp.waitCallbackOrDeferr(infoMessage) + sp.waitCallbackOrDeferr(infoMessage) case handleCallMessage: - gsp.waitCallbackOrDeferr(message) + sp.waitCallbackOrDeferr(message) case handleCastMessage: - gsp.waitCallbackOrDeferr(message) + sp.waitCallbackOrDeferr(message) case handleInfoMessage: - gsp.waitCallbackOrDeferr(message) + sp.waitCallbackOrDeferr(message) case ProcessDirectMessage: - gsp.waitCallbackOrDeferr(message) + sp.waitCallbackOrDeferr(message) default: lib.Log("m: %#v", m) infoMessage := handleInfoMessage{ message: m, } - gsp.waitCallbackOrDeferr(infoMessage) + sp.waitCallbackOrDeferr(infoMessage) } } } // ServerProcess handlers -func (gsp *ServerProcess) waitCallbackOrDeferr(message interface{}) { - if gsp.waitReply != nil { +func (sp *ServerProcess) waitCallbackOrDeferr(message interface{}) { + if sp.waitReply != nil { // already waiting for reply. deferr this message deferred := ProcessMailboxMessage{ Message: message, } select { - case gsp.deferred <- deferred: + case sp.deferred <- deferred: // do nothing default: - fmt.Printf("WARNING! deferred mailbox of %s[%q] is full. dropped message %v", - gsp.Self(), gsp.Name(), message) + lib.Warning("deferred mailbox of %s[%q] is full. dropped message %v", + sp.Self(), sp.Name(), message) } + return + } - } else { - switch m := message.(type) { - case handleCallMessage: - go func() { - gsp.handleCall(m) - gsp.callbackWaitReply <- nil - }() - case handleCastMessage: - go func() { - gsp.handleCast(m) - gsp.callbackWaitReply <- nil - }() - case handleInfoMessage: - go func() { - gsp.handleInfo(m) - gsp.callbackWaitReply <- nil - }() - case ProcessDirectMessage: - go func() { - gsp.handleDirect(m) - gsp.callbackWaitReply <- nil - }() + switch m := message.(type) { + case handleCallMessage: + go func() { + sp.counter++ + sp.handleCall(m) + sp.callbackWaitReply <- nil + }() + case handleCastMessage: + go func() { + sp.counter++ + sp.handleCast(m) + sp.callbackWaitReply <- nil + }() + case handleInfoMessage: + go func() { + sp.counter++ + sp.handleInfo(m) + sp.callbackWaitReply <- nil + }() + case ProcessDirectMessage: + go func() { + sp.counter++ + sp.handleDirect(m) + sp.callbackWaitReply <- nil + }() + case nil: + // it was called just to read the channel sp.callbackWaitReply - } + default: + lib.Warning("unknown message type in waitCallbackOrDeferr: %#v", message) + return } select { - //case <-gsp.Context().Done(): + //case <-sp.Context().Done(): // do not read the context state. otherwise the goroutine with running callback // might lock forever on exit (or on making a Call request) as nobody read // the callbackWaitReply channel. - case gsp.waitReply = <-gsp.callbackWaitReply: + case sp.waitReply = <-sp.callbackWaitReply: // not nil value means callback made a Call request and waiting for reply - if gsp.waitReply == nil && len(gsp.deferred) > 0 { - gsp.mailbox = gsp.deferred + if sp.waitReply == nil && len(sp.deferred) > 0 { + sp.mailbox = sp.deferred } return } } -func (gsp *ServerProcess) panicHandler() { +func (sp *ServerProcess) panicHandler() { if r := recover(); r != nil { pc, fn, line, _ := runtime.Caller(2) - fmt.Printf("Warning: Server terminated %s[%q]. Panic reason: %#v at %s[%s:%d]\n", - gsp.Self(), gsp.Name(), r, runtime.FuncForPC(pc).Name(), fn, line) - gsp.stop <- "panic" + lib.Warning("Server terminated %s[%q]. Panic reason: %#v at %s[%s:%d]", + sp.Self(), sp.Name(), r, runtime.FuncForPC(pc).Name(), fn, line) + sp.stop <- "panic" } } -func (gsp *ServerProcess) handleDirect(direct ProcessDirectMessage) { +func (sp *ServerProcess) handleDirect(direct ProcessDirectMessage) { if lib.CatchPanic() { - defer gsp.panicHandler() + defer sp.panicHandler() } - reply, err := gsp.behavior.HandleDirect(gsp, direct.Message) + reply, err := sp.behavior.HandleDirect(sp, direct.Message) if err != nil { direct.Message = nil direct.Err = err @@ -489,93 +507,100 @@ func (gsp *ServerProcess) handleDirect(direct ProcessDirectMessage) { return } -func (gsp *ServerProcess) handleCall(m handleCallMessage) { +func (sp *ServerProcess) handleCall(m handleCallMessage) { if lib.CatchPanic() { - defer gsp.panicHandler() + defer sp.panicHandler() } - cf := gsp.currentFunction - gsp.currentFunction = "Server:HandleCall" - reply, status := gsp.behavior.HandleCall(gsp, m.from, m.message) - gsp.currentFunction = cf + cf := sp.currentFunction + sp.currentFunction = "Server:HandleCall" + reply, status := sp.behavior.HandleCall(sp, m.from, m.message) + sp.currentFunction = cf switch status { case ServerStatusOK: - gsp.SendReply(m.from, reply) + sp.SendReply(m.from, reply) case ServerStatusIgnore: return case ServerStatusStop: - gsp.stop <- "normal" + sp.stop <- "normal" default: - gsp.stop <- status.Error() + sp.stop <- status.Error() } } -func (gsp *ServerProcess) handleCast(m handleCastMessage) { +func (sp *ServerProcess) handleCast(m handleCastMessage) { if lib.CatchPanic() { - defer gsp.panicHandler() + defer sp.panicHandler() } - cf := gsp.currentFunction - gsp.currentFunction = "Server:HandleCast" - status := gsp.behavior.HandleCast(gsp, m.message) - gsp.currentFunction = cf + cf := sp.currentFunction + sp.currentFunction = "Server:HandleCast" + status := sp.behavior.HandleCast(sp, m.message) + sp.currentFunction = cf switch status { case ServerStatusOK, ServerStatusIgnore: return case ServerStatusStop: - gsp.stop <- "normal" + sp.stop <- "normal" default: - gsp.stop <- status.Error() + sp.stop <- status.Error() } } -func (gsp *ServerProcess) handleInfo(m handleInfoMessage) { +func (sp *ServerProcess) handleInfo(m handleInfoMessage) { if lib.CatchPanic() { - defer gsp.panicHandler() + defer sp.panicHandler() } - cf := gsp.currentFunction - gsp.currentFunction = "Server:HandleInfo" - status := gsp.behavior.HandleInfo(gsp, m.message) - gsp.currentFunction = cf + cf := sp.currentFunction + sp.currentFunction = "Server:HandleInfo" + status := sp.behavior.HandleInfo(sp, m.message) + sp.currentFunction = cf switch status { case ServerStatusOK, ServerStatusIgnore: return case ServerStatusStop: - gsp.stop <- "normal" + sp.stop <- "normal" default: - gsp.stop <- status.Error() + sp.stop <- status.Error() } } // // default callbacks for Server interface // + +// Init func (gs *Server) Init(process *ServerProcess, args ...etf.Term) error { return nil } +// HanldeCast func (gs *Server) HandleCast(process *ServerProcess, message etf.Term) ServerStatus { - fmt.Printf("Server [%s] HandleCast: unhandled message %#v \n", process.Name(), message) + lib.Warning("Server [%s] HandleCast: unhandled message %#v", process.Name(), message) return ServerStatusOK } +// HandleInfo func (gs *Server) HandleCall(process *ServerProcess, from ServerFrom, message etf.Term) (etf.Term, ServerStatus) { - fmt.Printf("Server [%s] HandleCall: unhandled message %#v from %#v \n", process.Name(), message, from) + lib.Warning("Server [%s] HandleCall: unhandled message %#v from %#v", process.Name(), message, from) return "ok", ServerStatusOK } +// HandleDirect func (gs *Server) HandleDirect(process *ServerProcess, message interface{}) (interface{}, error) { return nil, ErrUnsupportedRequest } +// HandleInfo func (gs *Server) HandleInfo(process *ServerProcess, message etf.Term) ServerStatus { - fmt.Printf("Server [%s] HandleInfo: unhandled message %#v \n", process.Name(), message) + lib.Warning("Server [%s] HandleInfo: unhandled message %#v", process.Name(), message) return ServerStatusOK } +// Terminate func (gs *Server) Terminate(process *ServerProcess, reason string) { return } diff --git a/gen/stage.go b/gen/stage.go index b87ddb71..ce099628 100644 --- a/gen/stage.go +++ b/gen/stage.go @@ -5,6 +5,7 @@ import ( "time" "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/lib" //"github.com/ergo-services/ergo/lib" ) @@ -524,57 +525,69 @@ func (gst *Stage) HandleInfo(process *ServerProcess, message etf.Term) ServerSta // default callbacks +// InitStage func (gst *Stage) InitStage(process *StageProcess, args ...etf.Term) error { return nil } +// HandleSagaCall func (gst *Stage) HandleStageCall(process *StageProcess, from ServerFrom, message etf.Term) (etf.Term, ServerStatus) { // default callback if it wasn't implemented - fmt.Printf("HandleStageCall: unhandled message (from %#v) %#v\n", from, message) + lib.Warning("HandleStageCall: unhandled message (from %#v) %#v", from, message) return etf.Atom("ok"), ServerStatusOK } +// HandleStageDirect func (gst *Stage) HandleStageDirect(process *StageProcess, message interface{}) (interface{}, error) { // default callback if it wasn't implemented return nil, ErrUnsupportedRequest } +// HandleStageCast func (gst *Stage) HandleStageCast(process *StageProcess, message etf.Term) ServerStatus { // default callback if it wasn't implemented - fmt.Printf("HandleStageCast: unhandled message %#v\n", message) + lib.Warning("HandleStageCast: unhandled message %#v", message) return ServerStatusOK } + +// HandleStageInfo func (gst *Stage) HandleStageInfo(process *StageProcess, message etf.Term) ServerStatus { // default callback if it wasn't implemnted - fmt.Printf("HandleStageInfo: unhandled message %#v\n", message) + lib.Warning("HandleStageInfo: unhandled message %#v", message) return ServerStatusOK } +// HandleSubscribe func (gst *Stage) HandleSubscribe(process *StageProcess, subscription StageSubscription, options StageSubscribeOptions) StageStatus { return StageStatusNotAProducer } +// HandleSubscribed func (gst *Stage) HandleSubscribed(process *StageProcess, subscription StageSubscription, opts StageSubscribeOptions) (bool, StageStatus) { return opts.ManualDemand, StageStatusOK } +// HandleCancel func (gst *Stage) HandleCancel(process *StageProcess, subscription StageSubscription, reason string) StageStatus { // default callback if it wasn't implemented return StageStatusOK } +// HandleCanceled func (gst *Stage) HandleCanceled(process *StageProcess, subscription StageSubscription, reason string) StageStatus { // default callback if it wasn't implemented return StageStatusOK } +// HanndleEvents func (gst *Stage) HandleEvents(process *StageProcess, subscription StageSubscription, events etf.List) StageStatus { - fmt.Printf("Stage HandleEvents: unhandled subscription (%#v) events %#v\n", subscription, events) + lib.Warning("Stage HandleEvents: unhandled subscription (%#v) events %#v", subscription, events) return StageStatusOK } +// HandleDemand func (gst *Stage) HandleDemand(process *StageProcess, subscription StageSubscription, count uint) (etf.List, StageStatus) { - fmt.Printf("Stage HandleDemand: unhandled subscription (%#v) demand %#v\n", subscription, count) + lib.Warning("Stage HandleDemand: unhandled subscription (%#v) demand %#v", subscription, count) return nil, StageStatusOK } @@ -617,7 +630,7 @@ func handleConsumer(process *StageProcess, subscription StageSubscription, cmd s subInternal, ok := process.producers[subscription.ID] if !ok { - fmt.Printf("Warning! got %d events for unknown subscription %#v\n", numEvents, subscription) + lib.Warning("consumer got %d events for unknown subscription %#v", numEvents, subscription) return etf.Atom("ok"), nil } subInternal.count-- diff --git a/gen/stage_dispatcher.go b/gen/stage_dispatcher.go index bf06a234..d6d66b67 100644 --- a/gen/stage_dispatcher.go +++ b/gen/stage_dispatcher.go @@ -5,6 +5,7 @@ import ( "math/rand" "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/lib" ) // StageDispatcherBehavior defined interface for the dispatcher @@ -27,6 +28,7 @@ type StageDispatcherBehavior interface { Subscribe(state interface{}, subscription StageSubscription, opts StageSubscribeOptions) error } +// StageDispatcher type StageDispatcher int type dispatcherDemand struct{} type dispatcherBroadcast struct{} @@ -126,6 +128,7 @@ type broadcastState struct { bufferKeepLast bool } +// Init func (dd *dispatcherDemand) Init(opts StageOptions) interface{} { state := &demandState{ demands: make(map[etf.Pid]*demand), @@ -137,6 +140,7 @@ func (dd *dispatcherDemand) Init(opts StageOptions) interface{} { return state } +// Ask func (dd *dispatcherDemand) Ask(state interface{}, subscription StageSubscription, count uint) { st := state.(*demandState) demand, ok := st.demands[subscription.Pid] @@ -147,6 +151,7 @@ func (dd *dispatcherDemand) Ask(state interface{}, subscription StageSubscriptio return } +// Cancel func (dd *dispatcherDemand) Cancel(state interface{}, subscription StageSubscription) { st := state.(*demandState) delete(st.demands, subscription.Pid) @@ -161,6 +166,7 @@ func (dd *dispatcherDemand) Cancel(state interface{}, subscription StageSubscrip return } +// Dispatch func (dd *dispatcherDemand) Dispatch(state interface{}, events etf.List) []StageDispatchItem { st := state.(*demandState) // put events into the buffer before we start dispatching @@ -227,6 +233,7 @@ func (dd *dispatcherDemand) Dispatch(state interface{}, events etf.List) []Stage return dispatchItems } +// Subscribe func (dd *dispatcherDemand) Subscribe(state interface{}, subscription StageSubscription, opts StageSubscribeOptions) error { st := state.(*demandState) newDemand := &demand{ @@ -243,6 +250,7 @@ func (dd *dispatcherDemand) Subscribe(state interface{}, subscription StageSubsc // Dispatcher Broadcast implementation // +// Init func (db *dispatcherBroadcast) Init(opts StageOptions) interface{} { state := &broadcastState{ demands: make(map[etf.Pid]*demand), @@ -253,6 +261,7 @@ func (db *dispatcherBroadcast) Init(opts StageOptions) interface{} { return state } +// Ask func (db *dispatcherBroadcast) Ask(state interface{}, subscription StageSubscription, count uint) { st := state.(*broadcastState) demand, ok := st.demands[subscription.Pid] @@ -264,6 +273,7 @@ func (db *dispatcherBroadcast) Ask(state interface{}, subscription StageSubscrip return } +// Cancel func (db *dispatcherBroadcast) Cancel(state interface{}, subscription StageSubscription) { st := state.(*broadcastState) delete(st.demands, subscription.Pid) @@ -271,6 +281,7 @@ func (db *dispatcherBroadcast) Cancel(state interface{}, subscription StageSubsc return } +// Dispatch func (db *dispatcherBroadcast) Dispatch(state interface{}, events etf.List) []StageDispatchItem { st := state.(*broadcastState) // put events into the buffer before we start dispatching @@ -318,6 +329,7 @@ func (db *dispatcherBroadcast) Dispatch(state interface{}, events etf.List) []St return dispatchItems } +// Subscribe func (db *dispatcherBroadcast) Subscribe(state interface{}, subscription StageSubscription, opts StageSubscribeOptions) error { st := state.(*broadcastState) newDemand := &demand{ @@ -357,6 +369,8 @@ func (db *dispatcherBroadcast) Subscribe(state interface{}, subscription StageSu // // Dispatcher Partition implementation // + +// Init func (dp *dispatcherPartition) Init(opts StageOptions) interface{} { state := &partitionState{ demands: make(map[etf.Pid]*demand), @@ -372,6 +386,7 @@ func (dp *dispatcherPartition) Init(opts StageOptions) interface{} { return state } +// Ask func (dp *dispatcherPartition) Ask(state interface{}, subscription StageSubscription, count uint) { st := state.(*partitionState) demand, ok := st.demands[subscription.Pid] @@ -382,6 +397,7 @@ func (dp *dispatcherPartition) Ask(state interface{}, subscription StageSubscrip return } +// Cancel func (dp *dispatcherPartition) Cancel(state interface{}, subscription StageSubscription) { st := state.(*partitionState) demand, ok := st.demands[subscription.Pid] @@ -400,6 +416,7 @@ func (dp *dispatcherPartition) Cancel(state interface{}, subscription StageSubsc return } +// Dispatch func (dp *dispatcherPartition) Dispatch(state interface{}, events etf.List) []StageDispatchItem { st := state.(*partitionState) // put events into the buffer before we start dispatching @@ -421,7 +438,7 @@ func (dp *dispatcherPartition) Dispatch(state interface{}, events etf.List) []St } } // seems we dont have enough space to keep these events. discard the rest of them. - fmt.Println("Warning: dispatcherPartition. Event buffer is full. Discarding event: ", events[e]) + lib.Warning("DispatcherPartition. Event buffer is full. Discarding event: ", events[e]) break } @@ -473,6 +490,7 @@ func (dp *dispatcherPartition) Dispatch(state interface{}, events etf.List) []St return dispatchItems } +// Subscribe func (dp *dispatcherPartition) Subscribe(state interface{}, subscription StageSubscription, opts StageSubscribeOptions) error { st := state.(*partitionState) if opts.Partition > dp.n-1 { diff --git a/gen/supervisor.go b/gen/supervisor.go index 47ed5225..d912a441 100644 --- a/gen/supervisor.go +++ b/gen/supervisor.go @@ -14,6 +14,7 @@ type SupervisorBehavior interface { Init(args ...etf.Term) (SupervisorSpec, error) } +// SupervisorStrategy type SupervisorStrategy struct { Type SupervisorStrategyType Intensity uint16 @@ -21,7 +22,10 @@ type SupervisorStrategy struct { Restart SupervisorStrategyRestart } +// SupervisorStrategyType type SupervisorStrategyType = string + +// SupervisorStrategyRestart type SupervisorStrategyRestart = string const ( @@ -76,6 +80,7 @@ const ( type supervisorChildState int +// SupervisorSpec type SupervisorSpec struct { Name string Children []SupervisorChildSpec @@ -83,12 +88,13 @@ type SupervisorSpec struct { restarts []int64 } +// SupervisorChildSpec type SupervisorChildSpec struct { - // Node to run child on remote node - Node string Name string Child ProcessBehavior + Options ProcessOptions Args []etf.Term + state supervisorChildState // for internal usage process Process } @@ -101,6 +107,7 @@ type messageStartChild struct { args []etf.Term } +// ProcessInit func (sv *Supervisor) ProcessInit(p Process, args ...etf.Term) (ProcessState, error) { behavior, ok := p.Behavior().(SupervisorBehavior) if !ok { @@ -110,7 +117,7 @@ func (sv *Supervisor) ProcessInit(p Process, args ...etf.Term) (ProcessState, er if err != nil { return ProcessState{}, err } - lib.Log("Supervisor spec %#v\n", spec) + lib.Log("[%s] SUPERVISOR %q with restart strategy: %s[%s] ", p.NodeName(), p.Name(), spec.Strategy.Type, spec.Strategy.Restart) p.SetTrapExit(true) return ProcessState{ @@ -119,6 +126,7 @@ func (sv *Supervisor) ProcessInit(p Process, args ...etf.Term) (ProcessState, er }, nil } +// ProcessLoop func (sv *Supervisor) ProcessLoop(ps ProcessState, started chan<- bool) string { spec := ps.State.(*SupervisorSpec) if spec.Strategy.Type != SupervisorStrategySimpleOneForOne { @@ -187,8 +195,8 @@ func startChildren(supervisor Process, spec *SupervisorSpec) { if len(spec.restarts) > int(spec.Strategy.Intensity) { period := time.Now().Unix() - spec.restarts[0] if period <= int64(spec.Strategy.Period) { - fmt.Printf("ERROR: Restart intensity is exceeded (%d restarts for %d seconds)\n", - spec.Strategy.Intensity, spec.Strategy.Period) + lib.Warning("Supervisor %q. Restart intensity is exceeded (%d restarts for %d seconds)", + spec.Name, spec.Strategy.Intensity, spec.Strategy.Period) supervisor.Kill() return } @@ -203,7 +211,7 @@ func startChildren(supervisor Process, spec *SupervisorSpec) { continue case supervisorChildStateStart: spec.Children[i].state = supervisorChildStateRunning - process := startChild(supervisor, spec.Children[i].Name, spec.Children[i].Child, spec.Children[i].Args...) + process := startChild(supervisor, spec.Children[i].Name, spec.Children[i].Child, spec.Children[i].Options, spec.Children[i].Args...) spec.Children[i].process = process default: panic("Incorrect supervisorChildState") @@ -211,14 +219,11 @@ func startChildren(supervisor Process, spec *SupervisorSpec) { } } -func startChild(supervisor Process, name string, child ProcessBehavior, args ...etf.Term) Process { - opts := ProcessOptions{} +func startChild(supervisor Process, name string, child ProcessBehavior, opts ProcessOptions, args ...etf.Term) Process { + opts.GroupLeader = supervisor if leader := supervisor.GroupLeader(); leader != nil { opts.GroupLeader = leader - } else { - // leader is not set - opts.GroupLeader = supervisor } process, err := supervisor.Spawn(name, opts, child, args...) @@ -254,7 +259,7 @@ func handleDirect(supervisor Process, spec *SupervisorSpec, message interface{}) } // Dinamically started child can't be registered with a name. childSpec.Name = "" - process := startChild(supervisor, childSpec.Name, childSpec.Child, childSpec.Args...) + process := startChild(supervisor, childSpec.Name, childSpec.Child, childSpec.Options, childSpec.Args...) childSpec.process = process spec.Children = append(spec.Children, childSpec) return process, nil @@ -413,7 +418,7 @@ func handleMessageExit(p Process, exit ProcessGracefulExitRequest, spec *Supervi break } - process := startChild(p, spec.Children[i].Name, spec.Children[i].Child, spec.Children[i].Args...) + process := startChild(p, spec.Children[i].Name, spec.Children[i].Child, spec.Children[i].Options, spec.Children[i].Args...) spec.Children[i].process = process break } diff --git a/gen/types.go b/gen/types.go index 2da6c68a..d2f12fe3 100644 --- a/gen/types.go +++ b/gen/types.go @@ -9,17 +9,26 @@ import ( ) var ( - ErrUnsupportedRequest = fmt.Errorf("Unsupported request") - ErrServerTerminated = fmt.Errorf("Server terminated") + ErrUnsupportedRequest = fmt.Errorf("unsupported request") + ErrServerTerminated = fmt.Errorf("server terminated") ) +// EnvKey +type EnvKey string + +// Process type Process interface { - Registrar + Core // Spawn create a new process with parent Spawn(name string, opts ProcessOptions, object ProcessBehavior, args ...etf.Term) (Process, error) - // RemoteSpawn creates a new process at a remote node. The object name is a regitered behavior on a remote name using RegisterBehavior(...). Init callback of the started remote process will receive gen.RemoteSpawnRequest as an argument. + + // RemoteSpawn creates a new process at a remote node. The object name is a regitered + // behavior on a remote name using RegisterBehavior(...). The given options will stored + // in the process environment using node.EnvKeyRemoteSpawn as a key RemoteSpawn(node string, object string, opts RemoteSpawnOptions, args ...etf.Term) (etf.Pid, error) + RemoteSpawnWithTimeout(timeout int, node string, object string, opts RemoteSpawnOptions, args ...etf.Term) (etf.Pid, error) + // Name returns process name used on starting. Name() string @@ -29,6 +38,15 @@ type Process interface { // UnregisterName unregister named process. Unregistering name is allowed to the owner only UnregisterName(name string) error + // NodeName returns node name + NodeName() string + + // NodeStop stops the node + NodeStop() + + // NodeUptime returns node lifespan + NodeUptime() int64 + // Info returns process details Info() ProcessInfo @@ -55,7 +73,7 @@ type Process interface { // Exit initiate a graceful stopping process Exit(reason string) error - // Kill immidiately stops process + // Kill immediately stops process Kill() // CreateAlias creates a new alias for the Process @@ -65,15 +83,15 @@ type Process interface { DeleteAlias(alias etf.Alias) error // ListEnv returns a map of configured environment variables. - // It also includes environment variables from the GroupLeader and Parent. - // which are overlapped by priority: Process(Parent(GroupLeader)) - ListEnv() map[string]interface{} + // It also includes environment variables from the GroupLeader, Parent and Node. + // which are overlapped by priority: Process(Parent(GroupLeader(Node))) + ListEnv() map[EnvKey]interface{} // SetEnv set environment variable with given name. Use nil value to remove variable with given name. - SetEnv(name string, value interface{}) + SetEnv(name EnvKey, value interface{}) // Env returns value associated with given environment name. - Env(name string) interface{} + Env(name EnvKey) interface{} // Wait waits until process stopped Wait() @@ -86,12 +104,12 @@ type Process interface { // Links are bidirectional and there can only be one link between two processes. // Repeated calls to Process.Link(Pid) have no effect. If one of the participants // of a link terminates, it will send an exit signal to the other participant and caused - // termination of the last one (if this process hasn't set a trap using Process.SetTrapExit(true)). - Link(with etf.Pid) + // termination of the last one. If process set a trap using Process.SetTrapExit(true) the exit signal transorms into the MessageExit and delivers as a regular message. + Link(with etf.Pid) error // Unlink removes the link, if there is one, between the calling process and // the process referred to by Pid. - Unlink(with etf.Pid) + Unlink(with etf.Pid) error // IsAlive returns whether the process is alive IsAlive() bool @@ -104,8 +122,28 @@ type Process interface { // TrapExit returns whether the trap was enabled on this process TrapExit() bool + // Compression returns true if compression is enabled for this process + Compression() bool + + // SetCompression enables/disables compression for the messages sent outside of this node + SetCompression(enabled bool) + + // CompressionLevel returns comression level for the process + CompressionLevel() int + + // SetCompressionLevel defines compression level. Value must be in range: + // 1 (best speed) ... 9 (best compression), or -1 for the default compression level + SetCompressionLevel(level int) bool + + // CompressionThreshold returns compression threshold for the process + CompressionThreshold() int + + // SetCompressionThreshold defines the minimal size for the message that must be compressed + // Value must be greater than DefaultCompressionThreshold (1024) + SetCompressionThreshold(threshold int) bool + // MonitorNode creates monitor between the current process and node. If Node fails or does not exist, - // the message {nodedown, Node} is delivered to the process. + // the message MessageNodeDown is delivered to the process. MonitorNode(name string) etf.Ref // DemonitorNode removes monitor. Returns false if the given reference wasn't found @@ -144,12 +182,10 @@ type Process interface { // Aliases returns list of aliases of this process. Aliases() []etf.Alias - // Methods below are intended to be used for the ProcessBehavior implementation - - SendSyncRequestRaw(ref etf.Ref, node etf.Atom, messages ...etf.Term) error - PutSyncReply(ref etf.Ref, term etf.Term) error - SendSyncRequest(ref etf.Ref, to interface{}, message etf.Term) error + PutSyncRequest(ref etf.Ref) + CancelSyncRequest(ref etf.Ref) WaitSyncReply(ref etf.Ref, timeout int) (etf.Term, error) + PutSyncReply(ref etf.Ref, term etf.Term) error ProcessChannels() ProcessChannels } @@ -168,67 +204,80 @@ type ProcessInfo struct { Dictionary etf.Map TrapExit bool GroupLeader etf.Pid - Reductions uint64 + Compression bool } +// ProcessOptions type ProcessOptions struct { // Context allows mix the system context with the custom one. E.g. to limit // the lifespan using context.WithTimeout Context context.Context - // MailboxSize defines the lenght of message queue for the process + // MailboxSize defines the length of message queue for the process MailboxSize uint16 // GroupLeader GroupLeader Process // Env set the process environment variables - Env map[string]interface{} + Env map[EnvKey]interface{} + + // Fallback defines the process to where messages will be forwarded + // if the mailbox is overflowed. The tag value could be used to + // differentiate the source processes. Forwarded messages are wrapped + // into the MessageFallback struct. + Fallback ProcessFallback +} + +// ProcessFallback +type ProcessFallback struct { + Name string + Tag string +} + +// RemoteSpawnRequest +type RemoteSpawnRequest struct { + From etf.Pid + Ref etf.Ref + Options RemoteSpawnOptions } // RemoteSpawnOptions defines options for RemoteSpawn method type RemoteSpawnOptions struct { - // RegisterName - RegisterName string + // Name register associated name with spawned process + Name string // Monitor enables monitor on the spawned process using provided reference Monitor etf.Ref // Link enables link between the calling and spawned processes Link bool // Function in order to support {M,F,A} request to the Erlang node Function string - // Timeout - Timeout int -} - -// RemoteSpawnRequest stores in process environment ("ergo:RemoteSpawnRequest") if it was spawned by RemoteSpawn request -type RemoteSpawnRequest struct { - // Ref request id - Ref etf.Ref - // PID of the process made RemoteSpawn request - From etf.Pid - // Function provided via RemoteSpawnOptions.Function - Function string } +// ProcessChannels type ProcessChannels struct { Mailbox <-chan ProcessMailboxMessage Direct <-chan ProcessDirectMessage GracefulExit <-chan ProcessGracefulExitRequest } +// ProcessMailboxMessage type ProcessMailboxMessage struct { From etf.Pid Message interface{} } +// ProcessDirectMessage type ProcessDirectMessage struct { Message interface{} Err error Reply chan ProcessDirectMessage } +// ProcessGracefulExitRequest type ProcessGracefulExitRequest struct { From etf.Pid Reason string } +// ProcessState type ProcessState struct { Process State interface{} @@ -239,42 +288,50 @@ type ProcessBehavior interface { ProcessInit(Process, ...etf.Term) (ProcessState, error) ProcessLoop(ProcessState, chan<- bool) string // method which implements control flow of process } -type Registrar interface { - Monitor - NodeName() string - NodeStop() +// Core the common set of methods provided by Process and node.Node interfaces +type Core interface { - // ProcessByName returns Process struct for the given name. - // Returns nil if it doesn't exist (not found) + // ProcessByName returns Process for the given name. + // Returns nil if it doesn't exist (not found) or terminated. ProcessByName(name string) Process - // ProcessByPid returns Process struct for the given Pid. - // Returns nil if it doesn't exist (not found) + + // ProcessByPid returns Process for the given Pid. + // Returns nil if it doesn't exist (not found) or terminated. ProcessByPid(pid etf.Pid) Process - // ProcessByAlias returns Process struct for the given alias. - // Returns nil if it doesn't exist (not found) + // ProcessByAlias returns Process for the given alias. + // Returns nil if it doesn't exist (not found) or terminated ProcessByAlias(alias etf.Alias) Process // ProcessInfo returns the details about given Pid ProcessInfo(pid etf.Pid) (ProcessInfo, error) + + // ProcessList returns the list of running processes ProcessList() []Process - IsAlias(etf.Alias) bool + + // MakeRef creates an unique reference within this node MakeRef() etf.Ref - // IsProcessAlive returns true if the process with given pid is alive - IsProcessAlive(process Process) bool + // IsAlias checks whether the given alias is belongs to the alive process on this node. + // If the process died all aliases are cleaned up and this function returns + // false for the given alias. For alias from the remote node always returns false. + IsAlias(etf.Alias) bool + + // IsMonitor returns true if the given references is a monitor + IsMonitor(ref etf.Ref) bool + // RegisterBehavior RegisterBehavior(group, name string, behavior ProcessBehavior, data interface{}) error + // RegisteredBehavior RegisteredBehavior(group, name string) (RegisteredBehavior, error) + // RegisteredBehaviorGroup RegisteredBehaviorGroup(group string) []RegisteredBehavior + // UnregisterBehavior UnregisterBehavior(group, name string) error } -type Monitor interface { - IsMonitor(ref etf.Ref) bool -} - +// RegisteredBehavior type RegisteredBehavior struct { Behavior ProcessBehavior Data interface{} @@ -286,12 +343,18 @@ type ProcessID struct { Node string } +// String string representaion of ProcessID value +func (p ProcessID) String() string { + return fmt.Sprintf("<%s:%s>", p.Name, p.Node) +} + // MessageDown delivers as a message to Server's HandleInfo callback of the process // that created monitor using MonitorProcess. // Reason values: // - the exit reason of the process // - 'noproc' (process did not exist at the time of monitor creation) // - 'noconnection' (no connection to the node where the monitored process resides) +// - 'noproxy' (no connection to the proxy this node had has a connection through. monitored process could be still alive) type MessageDown struct { Ref etf.Ref // a monitor reference ProcessID ProcessID // if monitor was created by name @@ -302,15 +365,38 @@ type MessageDown struct { // MessageNodeDown delivers as a message to Server's HandleInfo callback of the process // that created monitor using MonitorNode type MessageNodeDown struct { + Ref etf.Ref Name string } +// MessageProxyDown delivers as a message to Server's HandleInfo callback of the process +// that created monitor using MonitorNode if the connection to the node was through the proxy +// nodes and one of them went down. +type MessageProxyDown struct { + Ref etf.Ref + Node string + Proxy string + Reason string +} + // MessageExit delievers to Server's HandleInfo callback on enabled trap exit using SetTrapExit(true) +// Reason values: +// - the exit reason of the process +// - 'noproc' (process did not exist at the time of link creation) +// - 'noconnection' (no connection to the node where the linked process resides) +// - 'noproxy' (no connection to the proxy this node had has a connection through. linked process could be still alive) type MessageExit struct { Pid etf.Pid Reason string } +// MessageFallback delivers to the process specified as a fallback process in ProcessOptions.Fallback.Name if the mailbox has been overflowed +type MessageFallback struct { + Process etf.Pid + Tag string + Message etf.Term +} + // RPC defines rpc function type type RPC func(...etf.Term) etf.Term @@ -322,8 +408,12 @@ type MessageManageRPC struct { Fun RPC } +// MessageDirectChildren type intended to be used in Process.Children which returns []etf.Pid +// You can handle this type of message in your HandleDirect callback to enable Process.Children +// support for your gen.Server actor. type MessageDirectChildren struct{} +// IsMessageDown func IsMessageDown(message etf.Term) (MessageDown, bool) { var md MessageDown switch m := message.(type) { @@ -333,6 +423,7 @@ func IsMessageDown(message etf.Term) (MessageDown, bool) { return md, false } +// IsMessageExit func IsMessageExit(message etf.Term) (MessageExit, bool) { var me MessageExit switch m := message.(type) { @@ -341,3 +432,23 @@ func IsMessageExit(message etf.Term) (MessageExit, bool) { } return me, false } + +// IsMessageProxyDown +func IsMessageProxyDown(message etf.Term) (MessageProxyDown, bool) { + var mpd MessageProxyDown + switch m := message.(type) { + case MessageProxyDown: + return m, true + } + return mpd, false +} + +// IsMessageFallback +func IsMessageFallback(message etf.Term) (MessageFallback, bool) { + var mf MessageFallback + switch m := message.(type) { + case MessageFallback: + return m, true + } + return mf, false +} diff --git a/go.mod b/go.mod index 5487720e..ee76f0a4 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/ergo-services/ergo -go 1.15 +go 1.17 diff --git a/lib/osdep/bsd.go b/lib/osdep/bsd.go index 89d4fecd..dc8c2278 100644 --- a/lib/osdep/bsd.go +++ b/lib/osdep/bsd.go @@ -1,3 +1,4 @@ +//go:build freebsd || openbsd || netbsd || dragonfly // +build freebsd openbsd netbsd dragonfly package osdep @@ -6,6 +7,7 @@ import ( "syscall" ) +// ResourceUsage func ResourceUsage() (int64, int64) { var usage syscall.Rusage var utime, stime int64 diff --git a/lib/osdep/darwin.go b/lib/osdep/darwin.go index 8b4a7dbf..30742c70 100644 --- a/lib/osdep/darwin.go +++ b/lib/osdep/darwin.go @@ -1,3 +1,4 @@ +//go:build darwin // +build darwin package osdep @@ -6,6 +7,7 @@ import ( "syscall" ) +// ResourceUsage func ResourceUsage() (int64, int64) { var usage syscall.Rusage var utime, stime int64 diff --git a/lib/osdep/linux.go b/lib/osdep/linux.go index 091968f1..1c2e1cdc 100644 --- a/lib/osdep/linux.go +++ b/lib/osdep/linux.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package osdep @@ -6,6 +7,7 @@ import ( "syscall" ) +// ResourceUsage func ResourceUsage() (int64, int64) { var usage syscall.Rusage var utime, stime int64 diff --git a/lib/osdep/windows.go b/lib/osdep/windows.go index 337d7a80..37050775 100644 --- a/lib/osdep/windows.go +++ b/lib/osdep/windows.go @@ -1,7 +1,9 @@ +//go:build windows // +build windows package osdep +// ResourceUsage func ResourceUsage() (int64, int64) { // FIXME Windows doesn't support syscall.Rusage. There should be another // way to get this kind of data from the OS diff --git a/lib/tools.go b/lib/tools.go index 32d63258..06a3f2d9 100644 --- a/lib/tools.go +++ b/lib/tools.go @@ -11,6 +11,7 @@ import ( "time" ) +// Buffer type Buffer struct { B []byte original []byte @@ -18,6 +19,7 @@ type Buffer struct { var ( ergoTrace = false + ergoWarning = false ergoNoRecover = false DefaultBufferLength = 16384 @@ -41,69 +43,97 @@ var ( ) func init() { - flag.BoolVar(&ergoTrace, "ergo.trace", false, "enable extended debug info") + flag.BoolVar(&ergoTrace, "ergo.trace", false, "enable/disable extended debug info") + flag.BoolVar(&ergoWarning, "ergo.warning", true, "enable/disable warning messages") flag.BoolVar(&ergoNoRecover, "ergo.norecover", false, "disable panic catching") } +// Log func Log(f string, a ...interface{}) { if ergoTrace { log.Printf(f, a...) } } +// Warning +func Warning(f string, a ...interface{}) { + if ergoWarning { + log.Printf("WARNING! "+f, a...) + } +} + +// CatchPanic func CatchPanic() bool { return ergoNoRecover == false } +// TakeTimer func TakeTimer() *time.Timer { return timers.Get().(*time.Timer) } +// ReleaseTimer func ReleaseTimer(t *time.Timer) { t.Stop() timers.Put(t) } +// TakeBuffer func TakeBuffer() *Buffer { return buffers.Get().(*Buffer) } +// ReleaseBuffer func ReleaseBuffer(b *Buffer) { - // do not return it to the pool if its grew up too big - if cap(b.B) > 65536 { - b.B = nil // for GC - b.original = nil - return + c := cap(b.B) + // cO := cap(b.original) + // overlaps := c > 0 && cO > 0 && &(x[:c][c-1]) == &(y[:cO][cO-1]) + if c > DefaultBufferLength && c < 65536 { + // reallocation happened. keep reallocated buffer as an original + // if it doesn't exceed the size of 65K (we don't want to keep + // too big slices) + b.original = b.B[:0] } b.B = b.original[:0] buffers.Put(b) } +// Reset func (b *Buffer) Reset() { + c := cap(b.B) + if c > DefaultBufferLength && c < 65536 { + b.original = b.B[:0] + } // use the original start point of the slice b.B = b.original[:0] } +// Set func (b *Buffer) Set(v []byte) { b.B = append(b.B[:0], v...) } +// AppendByte func (b *Buffer) AppendByte(v byte) { b.B = append(b.B, v) } +// Append func (b *Buffer) Append(v []byte) { b.B = append(b.B, v...) } +// String func (b *Buffer) String() string { return string(b.B) } +// Len func (b *Buffer) Len() int { return len(b.B) } +// WriteDataTo func (b *Buffer) WriteDataTo(w io.Writer) error { l := len(b.B) if l == 0 { @@ -128,6 +158,7 @@ func (b *Buffer) WriteDataTo(w io.Writer) error { return nil } +// ReadDataFrom func (b *Buffer) ReadDataFrom(r io.Reader, limit int) (int, error) { capB := cap(b.B) lenB := len(b.B) @@ -149,6 +180,16 @@ func (b *Buffer) ReadDataFrom(r io.Reader, limit int) (int, error) { return n, e } +func (b *Buffer) Write(v []byte) (n int, err error) { + b.B = append(b.B, v...) + return len(v), nil +} + +func (b *Buffer) Read(v []byte) (n int, err error) { + copy(v, b.B) + return len(b.B), io.EOF +} + func (b *Buffer) increase() { cap1 := cap(b.B) * 8 b1 := make([]byte, cap(b.B), cap1) @@ -156,6 +197,7 @@ func (b *Buffer) increase() { b.B = b1 } +// Allocate func (b *Buffer) Allocate(n int) { for { if cap(b.B) < n { @@ -167,6 +209,7 @@ func (b *Buffer) Allocate(n int) { } } +// Extend func (b *Buffer) Extend(n int) []byte { l := len(b.B) e := l + n @@ -180,6 +223,7 @@ func (b *Buffer) Extend(n int) []byte { } } +// RandomString func RandomString(length int) string { buff := make([]byte, length/2) rand.Read(buff) diff --git a/node/core.go b/node/core.go new file mode 100644 index 00000000..aeb57011 --- /dev/null +++ b/node/core.go @@ -0,0 +1,904 @@ +package node + +import ( + "context" + "crypto/rsa" + "fmt" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/gen" + "github.com/ergo-services/ergo/lib" +) + +const ( + startPID = 1000 +) + +type core struct { + monitorInternal + networkInternal + + ctx context.Context + stop context.CancelFunc + + env map[gen.EnvKey]interface{} + mutexEnv sync.RWMutex + + tls TLS + compression Compression + + nextPID uint64 + uniqID uint64 + nodename string + creation uint32 + + names map[string]etf.Pid + mutexNames sync.RWMutex + aliases map[etf.Alias]*process + mutexAliases sync.RWMutex + processes map[uint64]*process + mutexProcesses sync.RWMutex + + behaviors map[string]map[string]gen.RegisteredBehavior + mutexBehaviors sync.Mutex +} + +type coreInternal interface { + gen.Core + CoreRouter + + // core environment + ListEnv() map[gen.EnvKey]interface{} + SetEnv(name gen.EnvKey, value interface{}) + Env(name gen.EnvKey) interface{} + + monitorInternal + networkInternal + + spawn(name string, opts processOptions, behavior gen.ProcessBehavior, args ...etf.Term) (gen.Process, error) + + registerName(name string, pid etf.Pid) error + unregisterName(name string) error + + newAlias(p *process) (etf.Alias, error) + deleteAlias(owner *process, alias etf.Alias) error + + coreNodeName() string + coreStop() + coreUptime() int64 + coreIsAlive() bool + + coreWait() + coreWaitWithTimeout(d time.Duration) error +} + +type coreRouterInternal interface { + CoreRouter + MakeRef() etf.Ref + + ProcessByPid(pid etf.Pid) gen.Process + ProcessByName(name string) gen.Process + ProcessByAlias(alias etf.Alias) gen.Process + + processByPid(pid etf.Pid) *process + getConnection(nodename string) (ConnectionInterface, error) +} + +// transit proxy session +type proxyTransitSession struct { + a ConnectionInterface + b ConnectionInterface +} + +type proxyConnectRequest struct { + privateKey *rsa.PrivateKey + request ProxyConnectRequest + connection chan ConnectionInterface + cancel chan ProxyConnectCancel +} + +func newCore(ctx context.Context, nodename string, cookie string, options Options) (coreInternal, error) { + if options.Compression.Level < 1 || options.Compression.Level > 9 { + options.Compression.Level = DefaultCompressionLevel + } + if options.Compression.Threshold < DefaultCompressionThreshold { + options.Compression.Threshold = DefaultCompressionThreshold + } + c := &core{ + ctx: ctx, + env: options.Env, + nextPID: startPID, + uniqID: uint64(time.Now().UnixNano()), + // keep node to get the process to access to the node's methods + nodename: nodename, + compression: options.Compression, + creation: options.Creation, + names: make(map[string]etf.Pid), + aliases: make(map[etf.Alias]*process), + processes: make(map[uint64]*process), + behaviors: make(map[string]map[string]gen.RegisteredBehavior), + } + + corectx, corestop := context.WithCancel(ctx) + c.stop = corestop + c.ctx = corectx + + c.monitorInternal = newMonitor(nodename, coreRouterInternal(c)) + network, err := newNetwork(c.ctx, nodename, cookie, options, coreRouterInternal(c)) + if err != nil { + corestop() + return nil, err + } + c.networkInternal = network + + return c, nil +} + +func (c *core) coreNodeName() string { + return c.nodename +} + +func (c *core) coreStop() { + c.stop() + c.stopNetwork() +} + +func (c *core) coreUptime() int64 { + return time.Now().Unix() - int64(c.creation) +} + +func (c *core) coreWait() { + <-c.ctx.Done() +} + +// WaitWithTimeout waits until node stopped. Return ErrTimeout +// if given timeout is exceeded +func (c *core) coreWaitWithTimeout(d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + + select { + case <-timer.C: + return ErrTimeout + case <-c.ctx.Done(): + return nil + } +} + +// IsAlive returns true if node is running +func (c *core) coreIsAlive() bool { + return c.ctx.Err() == nil +} + +func (c *core) newPID() etf.Pid { + // http://erlang.org/doc/apps/erts/erl_ext_dist.html#pid_ext + // https://stackoverflow.com/questions/243363/can-someone-explain-the-structure-of-a-pid-in-erlang + i := atomic.AddUint64(&c.nextPID, 1) + return etf.Pid{ + Node: etf.Atom(c.nodename), + ID: i, + Creation: c.creation, + } + +} + +// MakeRef returns atomic reference etf.Ref within this node +func (c *core) MakeRef() (ref etf.Ref) { + ref.Node = etf.Atom(c.nodename) + ref.Creation = c.creation + nt := atomic.AddUint64(&c.uniqID, 1) + ref.ID[0] = uint32(uint64(nt) & ((2 << 17) - 1)) + ref.ID[1] = uint32(uint64(nt) >> 46) + return +} + +// IsAlias +func (c *core) IsAlias(alias etf.Alias) bool { + c.mutexAliases.RLock() + _, ok := c.aliases[alias] + c.mutexAliases.RUnlock() + return ok +} + +func (c *core) newAlias(p *process) (etf.Alias, error) { + var alias etf.Alias + + // chech if its alive + c.mutexProcesses.RLock() + _, exist := c.processes[p.self.ID] + c.mutexProcesses.RUnlock() + if !exist { + return alias, ErrProcessUnknown + } + + alias = etf.Alias(c.MakeRef()) + lib.Log("[%s] CORE create process alias for %v: %s", c.nodename, p.self, alias) + + c.mutexAliases.Lock() + c.aliases[alias] = p + c.mutexAliases.Unlock() + + p.Lock() + p.aliases = append(p.aliases, alias) + p.Unlock() + return alias, nil +} + +func (c *core) deleteAlias(owner *process, alias etf.Alias) error { + lib.Log("[%s] CORE delete process alias %v for %v", c.nodename, alias, owner.self) + + c.mutexAliases.Lock() + p, alias_exist := c.aliases[alias] + c.mutexAliases.Unlock() + + if alias_exist == false { + return ErrAliasUnknown + } + + c.mutexProcesses.RLock() + _, process_exist := c.processes[owner.self.ID] + c.mutexProcesses.RUnlock() + + if process_exist == false { + return ErrProcessUnknown + } + if p.self != owner.self { + return ErrAliasOwner + } + + p.Lock() + for i := range p.aliases { + if alias != p.aliases[i] { + continue + } + // remove it from the global alias list + c.mutexAliases.Lock() + delete(c.aliases, alias) + c.mutexAliases.Unlock() + // remove it from the process alias list + p.aliases[i] = p.aliases[0] + p.aliases = p.aliases[1:] + p.Unlock() + return nil + } + p.Unlock() + + // shouldn't reach this code. seems we got a bug + c.mutexAliases.Lock() + delete(c.aliases, alias) + c.mutexAliases.Unlock() + lib.Warning("Bug: Process lost its alias. Please, report this issue") + + return ErrAliasUnknown +} + +func (c *core) newProcess(name string, behavior gen.ProcessBehavior, opts processOptions) (*process, error) { + + var processContext context.Context + var kill context.CancelFunc + + mailboxSize := DefaultProcessMailboxSize + if opts.MailboxSize > 0 { + mailboxSize = int(opts.MailboxSize) + } + + processContext, kill = context.WithCancel(c.ctx) + if opts.Context != nil { + processContext = context.WithValue(processContext, "context", processContext) + } + + pid := c.newPID() + + env := make(map[gen.EnvKey]interface{}) + // inherite the node environment + c.mutexEnv.RLock() + for k, v := range c.env { + env[k] = v + } + c.mutexEnv.RUnlock() + + // merge the custom ones + for k, v := range opts.Env { + env[k] = v + } + + process := &process{ + coreInternal: c, + + self: pid, + name: name, + behavior: behavior, + env: env, + compression: c.compression, + + parent: opts.parent, + groupLeader: opts.GroupLeader, + + mailBox: make(chan gen.ProcessMailboxMessage, mailboxSize), + gracefulExit: make(chan gen.ProcessGracefulExitRequest, mailboxSize), + direct: make(chan gen.ProcessDirectMessage), + + context: processContext, + kill: kill, + + reply: make(map[etf.Ref]chan etf.Term), + fallback: opts.Fallback, + } + + process.exit = func(from etf.Pid, reason string) error { + lib.Log("[%s] EXIT from %s to %s with reason: %s", c.nodename, from, pid, reason) + if processContext.Err() != nil { + // process is already died + return ErrProcessUnknown + } + + ex := gen.ProcessGracefulExitRequest{ + From: from, + Reason: reason, + } + + // use select just in case if this process isn't been started yet + // or ProcessLoop is already exited (has been set to nil) + // otherwise it cause infinity lock + select { + case process.gracefulExit <- ex: + default: + return ErrProcessBusy + } + + // let the process decide whether to stop itself, otherwise its going to be killed + if !process.trapExit { + process.kill() + } + return nil + } + + if name != "" { + lib.Log("[%s] CORE registering name (%s): %s", c.nodename, pid, name) + c.mutexNames.Lock() + if _, exist := c.names[name]; exist { + c.mutexNames.Unlock() + return nil, ErrTaken + } + c.names[name] = process.self + c.mutexNames.Unlock() + } + + lib.Log("[%s] CORE registering process: %s", c.nodename, pid) + c.mutexProcesses.Lock() + c.processes[process.self.ID] = process + c.mutexProcesses.Unlock() + + return process, nil +} + +func (c *core) deleteProcess(pid etf.Pid) { + c.mutexProcesses.Lock() + p, exist := c.processes[pid.ID] + if !exist { + c.mutexProcesses.Unlock() + return + } + lib.Log("[%s] CORE unregistering process: %s", c.nodename, p.self) + delete(c.processes, pid.ID) + c.mutexProcesses.Unlock() + + c.mutexNames.Lock() + if (p.name) != "" { + lib.Log("[%s] CORE unregistering name (%s): %s", c.nodename, p.self, p.name) + delete(c.names, p.name) + } + + // delete names registered with this pid + for name, pid := range c.names { + if p.self == pid { + delete(c.names, name) + } + } + c.mutexNames.Unlock() + + c.mutexAliases.Lock() + for alias := range c.aliases { + delete(c.aliases, alias) + } + c.mutexAliases.Unlock() + + return +} + +func (c *core) spawn(name string, opts processOptions, behavior gen.ProcessBehavior, args ...etf.Term) (gen.Process, error) { + + process, err := c.newProcess(name, behavior, opts) + if err != nil { + return nil, err + } + lib.Log("[%s] CORE spawn a new process %s (registered name: %q)", c.nodename, process.self, name) + + initProcess := func() (ps gen.ProcessState, err error) { + if lib.CatchPanic() { + defer func() { + if rcv := recover(); rcv != nil { + pc, fn, line, _ := runtime.Caller(2) + lib.Warning("initialization process failed %s[%q] %#v at %s[%s:%d]", + process.self, name, rcv, runtime.FuncForPC(pc).Name(), fn, line) + c.deleteProcess(process.self) + err = fmt.Errorf("panic") + } + }() + } + + ps, err = behavior.ProcessInit(process, args...) + return + } + + processState, err := initProcess() + if err != nil { + return nil, err + } + + started := make(chan bool) + defer close(started) + + cleanProcess := func(reason string) { + // set gracefulExit to nil before we start termination handling + process.gracefulExit = nil + c.deleteProcess(process.self) + // invoke cancel context to prevent memory leaks + // and propagate context canelation + process.Kill() + // notify all the linked process and monitors + c.handleTerminated(process.self, name, reason) + // make the rest empty + process.Lock() + process.aliases = []etf.Alias{} + + // Do not clean self and name. Sometimes its good to know what pid + // (and what name) was used by the dead process. (gen.Applications is using it) + // process.name = "" + // process.self = etf.Pid{} + + process.behavior = nil + process.parent = nil + process.groupLeader = nil + process.exit = nil + process.kill = nil + process.mailBox = nil + process.direct = nil + process.env = nil + process.reply = nil + process.Unlock() + } + + go func(ps gen.ProcessState) { + if lib.CatchPanic() { + defer func() { + if rcv := recover(); rcv != nil { + pc, fn, line, _ := runtime.Caller(2) + lib.Warning("process terminated %s[%q] %#v at %s[%s:%d]", + process.self, name, rcv, runtime.FuncForPC(pc).Name(), fn, line) + cleanProcess("panic") + } + }() + } + + // start process loop + reason := behavior.ProcessLoop(ps, started) + // process stopped + cleanProcess(reason) + + }(processState) + + // wait for the starting process loop + <-started + return process, nil +} + +func (c *core) registerName(name string, pid etf.Pid) error { + lib.Log("[%s] CORE registering name %s", c.nodename, name) + c.mutexNames.Lock() + defer c.mutexNames.Unlock() + if _, ok := c.names[name]; ok { + // already registered + return ErrTaken + } + c.names[name] = pid + return nil +} + +func (c *core) unregisterName(name string) error { + lib.Log("[%s] CORE unregistering name %s", c.nodename, name) + c.mutexNames.Lock() + defer c.mutexNames.Unlock() + if _, ok := c.names[name]; ok { + delete(c.names, name) + return nil + } + return ErrNameUnknown +} + +// ListEnv +func (c *core) ListEnv() map[gen.EnvKey]interface{} { + c.mutexEnv.RLock() + defer c.mutexEnv.RUnlock() + + env := make(map[gen.EnvKey]interface{}) + for key, value := range c.env { + env[key] = value + } + + return env +} + +// SetEnv +func (c *core) SetEnv(name gen.EnvKey, value interface{}) { + c.mutexEnv.Lock() + defer c.mutexEnv.Unlock() + if strings.HasPrefix(string(name), "ergo:") { + return + } + c.env[name] = value +} + +// Env +func (c *core) Env(name gen.EnvKey) interface{} { + c.mutexEnv.RLock() + defer c.mutexEnv.RUnlock() + if value, ok := c.env[name]; ok { + return value + } + return nil +} + +// RegisterBehavior +func (c *core) RegisterBehavior(group, name string, behavior gen.ProcessBehavior, data interface{}) error { + lib.Log("[%s] CORE registering behavior %q in group %q ", c.nodename, name, group) + var groupBehaviors map[string]gen.RegisteredBehavior + var exist bool + + c.mutexBehaviors.Lock() + defer c.mutexBehaviors.Unlock() + + groupBehaviors, exist = c.behaviors[group] + if !exist { + groupBehaviors = make(map[string]gen.RegisteredBehavior) + c.behaviors[group] = groupBehaviors + } + + _, exist = groupBehaviors[name] + if exist { + return ErrTaken + } + + rb := gen.RegisteredBehavior{ + Behavior: behavior, + Data: data, + } + groupBehaviors[name] = rb + return nil +} + +// RegisteredBehavior +func (c *core) RegisteredBehavior(group, name string) (gen.RegisteredBehavior, error) { + var groupBehaviors map[string]gen.RegisteredBehavior + var rb gen.RegisteredBehavior + var exist bool + + c.mutexBehaviors.Lock() + defer c.mutexBehaviors.Unlock() + + groupBehaviors, exist = c.behaviors[group] + if !exist { + return rb, ErrBehaviorGroupUnknown + } + + rb, exist = groupBehaviors[name] + if !exist { + return rb, ErrBehaviorUnknown + } + return rb, nil +} + +// RegisteredBehaviorGroup +func (c *core) RegisteredBehaviorGroup(group string) []gen.RegisteredBehavior { + var groupBehaviors map[string]gen.RegisteredBehavior + var exist bool + var listrb []gen.RegisteredBehavior + + c.mutexBehaviors.Lock() + defer c.mutexBehaviors.Unlock() + + groupBehaviors, exist = c.behaviors[group] + if !exist { + return listrb + } + + for _, v := range groupBehaviors { + listrb = append(listrb, v) + } + return listrb +} + +// UnregisterBehavior +func (c *core) UnregisterBehavior(group, name string) error { + lib.Log("[%s] CORE unregistering behavior %s in group %s ", c.nodename, name, group) + var groupBehaviors map[string]gen.RegisteredBehavior + var exist bool + + c.mutexBehaviors.Lock() + defer c.mutexBehaviors.Unlock() + + groupBehaviors, exist = c.behaviors[group] + if !exist { + return ErrBehaviorUnknown + } + delete(groupBehaviors, name) + + // remove group if its empty + if len(groupBehaviors) == 0 { + delete(c.behaviors, group) + } + return nil +} + +// ProcessInfo +func (c *core) ProcessInfo(pid etf.Pid) (gen.ProcessInfo, error) { + p := c.processByPid(pid) + if p == nil { + return gen.ProcessInfo{}, fmt.Errorf("undefined") + } + + return p.Info(), nil +} + +// ProcessByPid +func (c *core) ProcessByPid(pid etf.Pid) gen.Process { + p := c.processByPid(pid) + if p == nil { + return nil + } + return p +} + +// ProcessByAlias +func (c *core) ProcessByAlias(alias etf.Alias) gen.Process { + c.mutexAliases.RLock() + defer c.mutexAliases.RUnlock() + if p, ok := c.aliases[alias]; ok && p.IsAlive() { + return p + } + // unknown process + return nil +} + +// ProcessByName +func (c *core) ProcessByName(name string) gen.Process { + var pid etf.Pid + if name != "" { + // requesting Process by name + c.mutexNames.RLock() + + if p, ok := c.names[name]; ok { + pid = p + } else { + c.mutexNames.RUnlock() + return nil + } + c.mutexNames.RUnlock() + } + + return c.ProcessByPid(pid) +} + +// ProcessList +func (c *core) ProcessList() []gen.Process { + list := []gen.Process{} + c.mutexProcesses.RLock() + for _, p := range c.processes { + list = append(list, p) + } + c.mutexProcesses.RUnlock() + return list +} + +// +// implementation of CoreRouter interface: +// RouteSend +// RouteSendReg +// RouteSendAlias +// + +// RouteSend implements RouteSend method of Router interface +func (c *core) RouteSend(from etf.Pid, to etf.Pid, message etf.Term) error { + if string(to.Node) == c.nodename { + if to.Creation != c.creation { + // message is addressed to the previous incarnation of this PID + lib.Warning("message from %s is addressed to the previous incarnation of this PID %s", from, to) + return ErrProcessIncarnation + } + // local route + c.mutexProcesses.RLock() + p, exist := c.processes[to.ID] + c.mutexProcesses.RUnlock() + if !exist { + lib.Log("[%s] CORE route message by pid (local) %s failed. Unknown process", c.nodename, to) + return ErrProcessUnknown + } + lib.Log("[%s] CORE route message by pid (local) %s", c.nodename, to) + select { + case p.mailBox <- gen.ProcessMailboxMessage{From: from, Message: message}: + default: + c.mutexNames.RLock() + pid, found := c.names[p.fallback.Name] + c.mutexNames.RUnlock() + if found == false { + //lib.Warning("mailbox of %s[%q] is full. dropped message from %s", p.self, p.name, from) + //FIXME + lib.Warning("mailbox of %s[%q] is full. dropped message from %s %#v", p.self, p.name, from, message) + return ErrProcessBusy + } + fbm := gen.MessageFallback{ + Process: p.self, + Tag: p.fallback.Tag, + Message: message, + } + return c.RouteSend(from, pid, fbm) + } + return nil + } + + // do not allow to send from the alien node. + if string(from.Node) != c.nodename { + return ErrSenderUnknown + } + + // sending to remote node + c.mutexProcesses.RLock() + p_from, exist := c.processes[from.ID] + c.mutexProcesses.RUnlock() + if !exist { + lib.Log("[%s] CORE route message by pid (remote) %s failed. Unknown sender", c.nodename, to) + return ErrSenderUnknown + } + connection, err := c.getConnection(string(to.Node)) + if err != nil { + return err + } + + lib.Log("[%s] CORE route message by pid (remote) %s", c.nodename, to) + return connection.Send(p_from, to, message) +} + +// RouteSendReg implements RouteSendReg method of Router interface +func (c *core) RouteSendReg(from etf.Pid, to gen.ProcessID, message etf.Term) error { + if to.Node == c.nodename { + // local route + c.mutexNames.RLock() + pid, ok := c.names[to.Name] + c.mutexNames.RUnlock() + if !ok { + lib.Log("[%s] CORE route message by gen.ProcessID (local) %s failed. Unknown process", c.nodename, to) + return ErrProcessUnknown + } + lib.Log("[%s] CORE route message by gen.ProcessID (local) %s", c.nodename, to) + return c.RouteSend(from, pid, message) + } + + // do not allow to send from the alien node. + if string(from.Node) != c.nodename { + return ErrSenderUnknown + } + + // send to remote node + c.mutexProcesses.RLock() + p_from, exist := c.processes[from.ID] + c.mutexProcesses.RUnlock() + if !exist { + lib.Log("[%s] CORE route message by gen.ProcessID (remote) %s failed. Unknown sender", c.nodename, to) + return ErrSenderUnknown + } + connection, err := c.getConnection(string(to.Node)) + if err != nil { + return err + } + + lib.Log("[%s] CORE route message by gen.ProcessID (remote) %s", c.nodename, to) + return connection.SendReg(p_from, to, message) +} + +// RouteSendAlias implements RouteSendAlias method of Router interface +func (c *core) RouteSendAlias(from etf.Pid, to etf.Alias, message etf.Term) error { + + if string(to.Node) == c.nodename { + // local route by alias + c.mutexAliases.RLock() + process, ok := c.aliases[to] + c.mutexAliases.RUnlock() + if !ok { + lib.Log("[%s] CORE route message by alias (local) %s failed. Unknown process", c.nodename, to) + return ErrProcessUnknown + } + lib.Log("[%s] CORE route message by alias (local) %s", c.nodename, to) + return c.RouteSend(from, process.self, message) + } + + // do not allow to send from the alien node. Proxy request must be used. + if string(from.Node) != c.nodename { + return ErrSenderUnknown + } + + // send to remote node + c.mutexProcesses.RLock() + p_from, exist := c.processes[from.ID] + c.mutexProcesses.RUnlock() + if !exist { + lib.Log("[%s] CORE route message by alias (remote) %s failed. Unknown sender", c.nodename, to) + return ErrSenderUnknown + } + connection, err := c.getConnection(string(to.Node)) + if err != nil { + return err + } + + lib.Log("[%s] CORE route message by alias (remote) %s", c.nodename, to) + return connection.SendAlias(p_from, to, message) +} + +// RouteSpawnRequest +func (c *core) RouteSpawnRequest(node string, behaviorName string, request gen.RemoteSpawnRequest, args ...etf.Term) error { + if node == c.nodename { + // get connection for reply + connection, err := c.getConnection(string(request.From.Node)) + if err != nil { + return err + } + + // check if we have registered behavior with given name + b, err := c.RegisteredBehavior(remoteBehaviorGroup, behaviorName) + if err != nil { + return connection.SpawnReplyError(request.From, request.Ref, err) + } + + // spawn new process + process_opts := processOptions{} + process_opts.Env = map[gen.EnvKey]interface{}{EnvKeyRemoteSpawn: request.Options} + process, err_spawn := c.spawn(request.Options.Name, process_opts, b.Behavior, args...) + + // reply + if err_spawn != nil { + return connection.SpawnReplyError(request.From, request.Ref, err_spawn) + } + return connection.SpawnReply(request.From, request.Ref, process.Self()) + } + + connection, err := c.getConnection(node) + if err != nil { + return err + } + return connection.SpawnRequest(node, behaviorName, request, args...) +} + +// RouteSpawnReply +func (c *core) RouteSpawnReply(to etf.Pid, ref etf.Ref, result etf.Term) error { + process := c.processByPid(to) + if process == nil { + // seems process terminated + return ErrProcessTerminated + } + process.PutSyncReply(ref, result) + return nil +} + +func (c *core) processByPid(pid etf.Pid) *process { + c.mutexProcesses.RLock() + defer c.mutexProcesses.RUnlock() + if p, ok := c.processes[pid.ID]; ok && p.IsAlive() { + return p + } + // unknown process + return nil +} diff --git a/node/dist/dist.go b/node/dist/dist.go deleted file mode 100644 index 18896186..00000000 --- a/node/dist/dist.go +++ /dev/null @@ -1,1552 +0,0 @@ -package dist - -import ( - "bufio" - "bytes" - "context" - "crypto/md5" - "encoding/binary" - "fmt" - "io" - "math/rand" - "net" - "sync" - "sync/atomic" - "time" - - "github.com/ergo-services/ergo/etf" - "github.com/ergo-services/ergo/lib" -) - -var ( - ErrMissingInCache = fmt.Errorf("Missing in cache") - ErrMalformed = fmt.Errorf("Malformed") -) - -func init() { - rand.Seed(time.Now().UTC().UnixNano()) -} - -type flagId uint64 -type nodeFlag flagId - -const ( - defaultLatency = 200 * time.Nanosecond // for linkFlusher - - defaultCleanTimeout = 5 * time.Second // for checkClean - defaultCleanDeadline = 30 * time.Second // for checkClean - - // http://erlang.org/doc/apps/erts/erl_ext_dist.html#distribution_header - protoDist = 131 - protoDistCompressed = 80 - protoDistMessage = 68 - protoDistFragment1 = 69 - protoDistFragmentN = 70 - - ProtoHandshake5 = 5 - ProtoHandshake6 = 6 - - // distribution flags are defined here https://erlang.org/doc/apps/erts/erl_dist_protocol.html#distribution-flags - PUBLISHED flagId = 0x1 - ATOM_CACHE = 0x2 - EXTENDED_REFERENCES = 0x4 - DIST_MONITOR = 0x8 - FUN_TAGS = 0x10 - DIST_MONITOR_NAME = 0x20 - HIDDEN_ATOM_CACHE = 0x40 - NEW_FUN_TAGS = 0x80 - EXTENDED_PIDS_PORTS = 0x100 - EXPORT_PTR_TAG = 0x200 - BIT_BINARIES = 0x400 - NEW_FLOATS = 0x800 - UNICODE_IO = 0x1000 - DIST_HDR_ATOM_CACHE = 0x2000 - SMALL_ATOM_TAGS = 0x4000 - UTF8_ATOMS = 0x10000 - MAP_TAG = 0x20000 - BIG_CREATION = 0x40000 - SEND_SENDER = 0x80000 // since OTP.21 enable replacement for SEND (distProtoSEND by distProtoSEND_SENDER) - BIG_SEQTRACE_LABELS = 0x100000 - EXIT_PAYLOAD = 0x400000 // since OTP.22 enable replacement for EXIT, EXIT2, MONITOR_P_EXIT - FRAGMENTS = 0x800000 - HANDSHAKE23 = 0x1000000 // new connection setup handshake (version 6) introduced in OTP 23 - UNLINK_ID = 0x2000000 - // for 64bit flags - SPAWN = 1 << 32 - NAME_ME = 1 << 33 - V4_NC = 1 << 34 - ALIAS = 1 << 35 -) - -type HandshakeOptions struct { - Version int // 5 or 6 - Name string - Cookie string - TLS bool - Hidden bool - Creation uint32 -} - -func (nf nodeFlag) toUint32() uint32 { - return uint32(nf) -} - -func (nf nodeFlag) toUint64() uint64 { - return uint64(nf) -} - -func (nf nodeFlag) isSet(f flagId) bool { - return (uint64(nf) & uint64(f)) != 0 -} - -func toNodeFlag(f ...flagId) nodeFlag { - var flags uint64 - for _, v := range f { - flags |= uint64(v) - } - return nodeFlag(flags) -} - -type fragmentedPacket struct { - buffer *lib.Buffer - disordered *lib.Buffer - disorderedSlices map[uint64][]byte - fragmentID uint64 - lastUpdate time.Time -} - -type Link struct { - Name string - Cookie string - Hidden bool - peer *Link - conn net.Conn - challenge uint32 - flags nodeFlag - version uint16 - creation uint32 - digest []byte - - // writer - flusher *linkFlusher - - // atom cache for incomming messages - cacheIn [2048]*etf.Atom - cacheInMutex sync.Mutex - - // atom cache for outgoing messages - cacheOut *etf.AtomCache - - // fragmentation sequence ID - sequenceID int64 - fragments map[uint64]*fragmentedPacket - fragmentsMutex sync.Mutex - - // check and clean lost fragments - checkCleanPending bool - checkCleanTimer *time.Timer - checkCleanTimeout time.Duration // default is 5 seconds - checkCleanDeadline time.Duration // how long we wait for the next fragment of the certain sequenceID. Default is 30 seconds -} - -func (l *Link) GetPeerName() string { - if l.peer == nil { - return "" - } - - return l.peer.Name -} - -func newLinkFlusher(w io.Writer, latency time.Duration) *linkFlusher { - return &linkFlusher{ - latency: latency, - writer: bufio.NewWriter(w), - w: w, // in case if we skip buffering - } -} - -type linkFlusher struct { - mutex sync.Mutex - latency time.Duration - writer *bufio.Writer - w io.Writer - - timer *time.Timer - pending bool -} - -func (lf *linkFlusher) Write(b []byte) (int, error) { - lf.mutex.Lock() - defer lf.mutex.Unlock() - - l := len(b) - lenB := l - - // long data write directly to the socket. - if l > 64000 { - for { - n, e := lf.w.Write(b[lenB-l:]) - if e != nil { - return n, e - } - // check if something left - l -= n - if l > 0 { - continue - } - return lenB, nil - } - } - - // write data to the buffer - for { - n, e := lf.writer.Write(b) - if e != nil { - return n, e - } - // check if something left - l -= n - if l > 0 { - continue - } - break - } - - if lf.pending { - return lenB, nil - } - - lf.pending = true - - if lf.timer != nil { - lf.timer.Reset(lf.latency) - return lenB, nil - } - - lf.timer = time.AfterFunc(lf.latency, func() { - // KeepAlive packet is just 4 bytes with zero value - var keepAlivePacket = []byte{0, 0, 0, 0} - - lf.mutex.Lock() - defer lf.mutex.Unlock() - - // if we have no pending data to send we should - // send a KeepAlive packet - if !lf.pending { - lf.w.Write(keepAlivePacket) - return - } - - lf.writer.Flush() - lf.pending = false - }) - - return lenB, nil - -} - -func Handshake(conn net.Conn, options HandshakeOptions) (*Link, error) { - - link := &Link{ - Name: options.Name, - Cookie: options.Cookie, - Hidden: options.Hidden, - - flags: toNodeFlag(PUBLISHED, UNICODE_IO, DIST_MONITOR, DIST_MONITOR_NAME, - EXTENDED_PIDS_PORTS, EXTENDED_REFERENCES, ATOM_CACHE, - DIST_HDR_ATOM_CACHE, HIDDEN_ATOM_CACHE, NEW_FUN_TAGS, - SMALL_ATOM_TAGS, UTF8_ATOMS, MAP_TAG, - FRAGMENTS, HANDSHAKE23, BIG_CREATION, SPAWN, V4_NC, ALIAS, - ), - - conn: conn, - sequenceID: time.Now().UnixNano(), - version: uint16(options.Version), - creation: options.Creation, - } - - b := lib.TakeBuffer() - defer lib.ReleaseBuffer(b) - - var await []byte - - if options.Version == ProtoHandshake5 { - link.composeName(b, options.TLS) - // the next message must be send_status 's' or send_challenge 'n' (for - // handshake version 5) or 'N' (for handshake version 6) - await = []byte{'s', 'n', 'N'} - } else { - link.composeNameVersion6(b, options.TLS) - await = []byte{'s', 'N'} - } - if e := b.WriteDataTo(conn); e != nil { - return nil, e - } - - // define timeout for the handshaking - timer := time.NewTimer(5 * time.Second) - defer timer.Stop() - - asyncReadChannel := make(chan error, 2) - asyncRead := func() { - _, e := b.ReadDataFrom(conn, 512) - asyncReadChannel <- e - } - - // http://erlang.org/doc/apps/erts/erl_dist_protocol.html#distribution-handshake - // Every message in the handshake starts with a 16-bit big-endian integer, - // which contains the message length (not counting the two initial bytes). - // In Erlang this corresponds to option {packet, 2} in gen_tcp(3). Notice - // that after the handshake, the distribution switches to 4 byte packet headers. - expectingBytes := 2 - if options.TLS { - // TLS connection has 4 bytes packet length header - expectingBytes = 4 - } - - for { - go asyncRead() - - select { - case <-timer.C: - return nil, fmt.Errorf("handshake timeout") - - case e := <-asyncReadChannel: - if e != nil { - return nil, e - } - - next: - l := binary.BigEndian.Uint16(b.B[expectingBytes-2 : expectingBytes]) - buffer := b.B[expectingBytes:] - - if len(buffer) < int(l) { - return nil, fmt.Errorf("malformed handshake (wrong packet length)") - } - - // chech if we got correct message type regarding to 'await' value - if bytes.Count(await, buffer[0:1]) == 0 { - return nil, fmt.Errorf("malformed handshake (wrong response)") - } - - switch buffer[0] { - case 'n': - // 'n' + 2 (version) + 4 (flags) + 4 (challenge) + name... - if len(b.B) < 12 { - return nil, fmt.Errorf("malformed handshake ('n')") - } - - challenge := link.readChallenge(b.B[1:]) - if challenge == 0 { - return nil, fmt.Errorf("malformed handshake (mismatch handshake version") - } - b.Reset() - - link.composeChallengeReply(b, challenge, options.TLS) - - if e := b.WriteDataTo(conn); e != nil { - return nil, e - } - // add 's' status for the case if we got it after 'n' or 'N' message - await = []byte{'s', 'a'} - - case 'N': - // Peer support version 6. - - // The new challenge message format (version 6) - // 8 (flags) + 4 (Creation) + 2 (NameLen) + Name - if len(buffer) < 16 { - return nil, fmt.Errorf("malformed handshake ('N' length)") - } - challenge := link.readChallengeVersion6(buffer[1:]) - b.Reset() - - if link.version == ProtoHandshake5 { - // send complement message - link.composeComplement(b, options.TLS) - if e := b.WriteDataTo(conn); e != nil { - return nil, e - } - link.version = ProtoHandshake6 - } - - link.composeChallengeReply(b, challenge, options.TLS) - - if e := b.WriteDataTo(conn); e != nil { - return nil, e - } - - // add 's' (send_status message) for the case if we got it after 'n' or 'N' message - await = []byte{'s', 'a'} - - case 'a': - // 'a' + 16 (digest) - if len(buffer) != 17 { - return nil, fmt.Errorf("malformed handshake ('a' length of digest)") - } - - // 'a' + 16 (digest) - digest := genDigest(link.peer.challenge, link.Cookie) - if bytes.Compare(buffer[1:17], digest) != 0 { - return nil, fmt.Errorf("malformed handshake ('a' digest)") - } - - // handshaked - link.flusher = newLinkFlusher(link.conn, defaultLatency) - return link, nil - - case 's': - if link.readStatus(buffer[1:]) == false { - return nil, fmt.Errorf("handshake negotiation failed") - } - - await = []byte{'n', 'N'} - // "sok" - if len(buffer) > 4 { - b.B = b.B[expectingBytes+3:] - goto next - } - b.Reset() - - default: - return nil, fmt.Errorf("malformed handshake ('%c' digest)", buffer[0]) - } - - } - - } - -} - -func HandshakeAccept(conn net.Conn, options HandshakeOptions) (*Link, error) { - link := &Link{ - Name: options.Name, - Cookie: options.Cookie, - Hidden: options.Hidden, - - flags: toNodeFlag(PUBLISHED, UNICODE_IO, DIST_MONITOR, DIST_MONITOR_NAME, - EXTENDED_PIDS_PORTS, EXTENDED_REFERENCES, ATOM_CACHE, - DIST_HDR_ATOM_CACHE, HIDDEN_ATOM_CACHE, NEW_FUN_TAGS, - SMALL_ATOM_TAGS, UTF8_ATOMS, MAP_TAG, - FRAGMENTS, HANDSHAKE23, BIG_CREATION, SPAWN, V4_NC, ALIAS, - ), - - conn: conn, - sequenceID: time.Now().UnixNano(), - challenge: rand.Uint32(), - version: ProtoHandshake6, - creation: options.Creation, - } - - b := lib.TakeBuffer() - defer lib.ReleaseBuffer(b) - - var await []byte - - // define timeout for the handshaking - timer := time.NewTimer(5 * time.Second) - defer timer.Stop() - - asyncReadChannel := make(chan error, 2) - asyncRead := func() { - _, e := b.ReadDataFrom(conn, 512) - asyncReadChannel <- e - } - - // http://erlang.org/doc/apps/erts/erl_dist_protocol.html#distribution-handshake - // Every message in the handshake starts with a 16-bit big-endian integer, - // which contains the message length (not counting the two initial bytes). - // In Erlang this corresponds to option {packet, 2} in gen_tcp(3). Notice - // that after the handshake, the distribution switches to 4 byte packet headers. - expectingBytes := 2 - if options.TLS { - // TLS connection has 4 bytes packet length header - expectingBytes = 4 - } - - // the comming message must be 'receive_name' as an answer for the - // 'send_name' message request we just sent - await = []byte{'n', 'N'} - - for { - go asyncRead() - - select { - case <-timer.C: - return nil, fmt.Errorf("handshake accept timeout") - case e := <-asyncReadChannel: - if e != nil { - return nil, e - } - - if b.Len() < expectingBytes+1 { - return nil, fmt.Errorf("malformed handshake (too short packet)") - } - - next: - l := binary.BigEndian.Uint16(b.B[expectingBytes-2 : expectingBytes]) - buffer := b.B[expectingBytes:] - - if len(buffer) < int(l) { - return nil, fmt.Errorf("malformed handshake (wrong packet length)") - } - - if bytes.Count(await, buffer[0:1]) == 0 { - return nil, fmt.Errorf("malformed handshake (wrong response %d)", buffer[0]) - } - - switch buffer[0] { - case 'n': - if len(buffer) < 8 { - return nil, fmt.Errorf("malformed handshake ('n' length)") - } - - link.peer = link.readName(buffer[1:]) - b.Reset() - link.composeStatus(b, options.TLS) - if e := b.WriteDataTo(conn); e != nil { - return nil, fmt.Errorf("malformed handshake ('n' accept name)") - } - - b.Reset() - if link.peer.flags.isSet(HANDSHAKE23) { - link.composeChallengeVersion6(b, options.TLS) - await = []byte{'s', 'r', 'c'} - } else { - link.version = ProtoHandshake5 - link.composeChallenge(b, options.TLS) - await = []byte{'s', 'r'} - } - if e := b.WriteDataTo(conn); e != nil { - return nil, e - } - - case 'N': - // The new challenge message format (version 6) - // 8 (flags) + 4 (Creation) + 2 (NameLen) + Name - if len(buffer) < 16 { - return nil, fmt.Errorf("malformed handshake ('N' length)") - } - link.peer = link.readNameVersion6(buffer[1:]) - b.Reset() - link.composeStatus(b, options.TLS) - if e := b.WriteDataTo(conn); e != nil { - return nil, fmt.Errorf("malformed handshake ('N' accept name)") - } - - b.Reset() - link.composeChallengeVersion6(b, options.TLS) - if e := b.WriteDataTo(conn); e != nil { - return nil, e - } - - await = []byte{'s', 'r'} - - case 'c': - if len(buffer) < 9 { - return nil, fmt.Errorf("malformed handshake ('c' length)") - } - link.readComplement(buffer[1:]) - - await = []byte{'r'} - - if len(buffer) > 9 { - b.B = b.B[expectingBytes+9:] - goto next - } - b.Reset() - - case 'r': - if len(buffer) < 19 { - return nil, fmt.Errorf("malformed handshake ('r' length)") - } - - if !link.validateChallengeReply(buffer[1:]) { - return nil, fmt.Errorf("malformed handshake ('r' invalid reply)") - } - b.Reset() - - link.composeChallengeAck(b, options.TLS) - if e := b.WriteDataTo(conn); e != nil { - return nil, e - } - - // handshaked - link.flusher = newLinkFlusher(link.conn, defaultLatency) - - return link, nil - - case 's': - if link.readStatus(buffer[1:]) == false { - return nil, fmt.Errorf("link status !ok") - } - - await = []byte{'c', 'r'} - if len(buffer) > 4 { - b.B = b.B[expectingBytes+3:] - goto next - } - b.Reset() - - default: - return nil, fmt.Errorf("malformed handshake (unknown code %d)", b.B[0]) - } - - } - - } -} - -func (l *Link) Close() { - if l.conn != nil { - l.conn.Close() - } -} - -func (l *Link) PeerName() string { - if l.peer != nil { - return l.peer.Name - } - return "" -} - -func (l *Link) Read(b *lib.Buffer) (int, error) { - // http://erlang.org/doc/apps/erts/erl_dist_protocol.html#protocol-between-connected-nodes - expectingBytes := 4 - - for { - if b.Len() < expectingBytes { - n, e := b.ReadDataFrom(l.conn, 0) - if n == 0 { - // link was closed - return 0, nil - } - - if e != nil && e != io.EOF { - // something went wrong - return 0, e - } - - // check onemore time if we should read more data - continue - } - - packetLength := binary.BigEndian.Uint32(b.B[:4]) - if packetLength == 0 { - // keepalive - l.conn.Write(b.B[:4]) - b.Set(b.B[4:]) - - expectingBytes = 4 - continue - } - - if b.Len() < int(packetLength)+4 { - expectingBytes = int(packetLength) + 4 - continue - } - - return int(packetLength) + 4, nil - } - -} - -type deferrMissing struct { - b *lib.Buffer - c int -} - -func (l *Link) ReadHandlePacket(ctx context.Context, recv chan *lib.Buffer, - handler func(string, etf.Term, etf.Term) error) { - var b *lib.Buffer - var missing deferrMissing - var Timeout <-chan time.Time - - deferrChannel := make(chan deferrMissing, 100) - defer close(deferrChannel) - - timer := lib.TakeTimer() - defer lib.ReleaseTimer(timer) - - dChannel := deferrChannel - - for { - select { - case missing = <-dChannel: - b = missing.b - default: - if len(deferrChannel) > 0 { - timer.Reset(150 * time.Millisecond) - Timeout = timer.C - } else { - Timeout = nil - } - select { - case b = <-recv: - if b == nil { - // channel was closed - return - } - case <-Timeout: - dChannel = deferrChannel - continue - } - } - - // read and decode received packet - control, message, err := l.ReadPacket(b.B) - - if err == ErrMissingInCache { - if b == missing.b && missing.c > 100 { - fmt.Println("Error: Disordered data at the link with", l.PeerName(), ". Close connection") - l.Close() - lib.ReleaseBuffer(b) - return - } - - if b == missing.b { - missing.c++ - } else { - missing.b = b - missing.c = 0 - } - - select { - case deferrChannel <- missing: - // read recv channel - dChannel = nil - continue - default: - fmt.Println("Error: Mess at the link with", l.PeerName(), ". Close connection") - l.Close() - lib.ReleaseBuffer(b) - return - } - } - - dChannel = deferrChannel - - if err != nil { - fmt.Println("Malformed Dist proto at the link with", l.PeerName(), err) - l.Close() - lib.ReleaseBuffer(b) - return - } - - if control == nil { - // fragment - continue - } - - // handle message - if err := handler(l.peer.Name, control, message); err != nil { - fmt.Printf("Malformed Control packet at the link with %s: %#v\n", l.PeerName(), control) - l.Close() - lib.ReleaseBuffer(b) - return - } - - // we have to release this buffer - lib.ReleaseBuffer(b) - - } -} - -func (l *Link) ReadPacket(packet []byte) (etf.Term, etf.Term, error) { - if len(packet) < 5 { - return nil, nil, fmt.Errorf("malformed packet") - } - - // [:3] length - switch packet[4] { - case protoDist: - return l.ReadDist(packet[5:]) - default: - // unknown proto - return nil, nil, fmt.Errorf("unknown/unsupported proto") - } - -} - -func (l *Link) ReadDist(packet []byte) (etf.Term, etf.Term, error) { - switch packet[0] { - case protoDistCompressed: - // do we need it? - // zip.NewReader(...) - // ...unzipping to the new buffer b (lib.TakeBuffer) - // just in case: if b[0] == protoDistCompressed return error - // otherwise it will cause recursive call and im not sure if its ok - // return l.ReadDist(b) - - case protoDistMessage: - var control, message etf.Term - var cache []etf.Atom - var err error - - cache, packet, err = l.decodeDistHeaderAtomCache(packet[1:]) - - if err != nil { - return nil, nil, err - } - - decodeOptions := etf.DecodeOptions{ - FlagV4NC: l.peer.flags.isSet(V4_NC), - FlagBigCreation: l.peer.flags.isSet(BIG_CREATION), - } - - control, packet, err = etf.Decode(packet, cache, decodeOptions) - if err != nil { - return nil, nil, err - } - - if len(packet) == 0 { - return control, nil, nil - } - - message, packet, err = etf.Decode(packet, cache, decodeOptions) - if err != nil { - return nil, nil, err - } - - if len(packet) != 0 { - return nil, nil, fmt.Errorf("packet has extra %d byte(s)", len(packet)) - } - - return control, message, nil - - case protoDistFragment1, protoDistFragmentN: - first := packet[0] == protoDistFragment1 - if len(packet) < 18 { - return nil, nil, fmt.Errorf("malformed fragment") - } - - // We should decode first fragment in order to process Atom Cache Header - // to get rid the case when we get the first fragment of the packet - // and the next packet is not the part of the fragmented packet, but with - // the ids were encoded in the first fragment - if first { - l.decodeDistHeaderAtomCache(packet[1:]) - } - - if assembled, err := l.decodeFragment(packet[1:], first); assembled != nil { - if err != nil { - return nil, nil, err - } - defer lib.ReleaseBuffer(assembled) - return l.ReadDist(assembled.B) - } else { - if err != nil { - return nil, nil, err - } - } - - return nil, nil, nil - } - - return nil, nil, fmt.Errorf("unknown packet type %d", packet[0]) -} - -func (l *Link) decodeFragment(packet []byte, first bool) (*lib.Buffer, error) { - l.fragmentsMutex.Lock() - defer l.fragmentsMutex.Unlock() - - if l.fragments == nil { - l.fragments = make(map[uint64]*fragmentedPacket) - } - - sequenceID := binary.BigEndian.Uint64(packet) - fragmentID := binary.BigEndian.Uint64(packet[8:]) - if fragmentID == 0 { - return nil, fmt.Errorf("fragmentID can't be 0") - } - - fragmented, ok := l.fragments[sequenceID] - if !ok { - fragmented = &fragmentedPacket{ - buffer: lib.TakeBuffer(), - disordered: lib.TakeBuffer(), - disorderedSlices: make(map[uint64][]byte), - lastUpdate: time.Now(), - } - fragmented.buffer.AppendByte(protoDistMessage) - l.fragments[sequenceID] = fragmented - } - - // until we get the first item everything will be treated as disordered - if first { - fragmented.fragmentID = fragmentID + 1 - } - - if fragmented.fragmentID-fragmentID != 1 { - // got the next fragment. disordered - slice := fragmented.disordered.Extend(len(packet) - 16) - copy(slice, packet[16:]) - fragmented.disorderedSlices[fragmentID] = slice - } else { - // order is correct. just append - fragmented.buffer.Append(packet[16:]) - fragmented.fragmentID = fragmentID - } - - // check whether we have disordered slices and try - // to append them if it does fit - if fragmented.fragmentID > 0 && len(fragmented.disorderedSlices) > 0 { - for i := fragmented.fragmentID - 1; i > 0; i-- { - if slice, ok := fragmented.disorderedSlices[i]; ok { - fragmented.buffer.Append(slice) - delete(fragmented.disorderedSlices, i) - fragmented.fragmentID = i - continue - } - break - } - } - - fragmented.lastUpdate = time.Now() - - if fragmented.fragmentID == 1 && len(fragmented.disorderedSlices) == 0 { - // it was the last fragment - delete(l.fragments, sequenceID) - lib.ReleaseBuffer(fragmented.disordered) - return fragmented.buffer, nil - } - - if l.checkCleanPending { - return nil, nil - } - - if l.checkCleanTimer != nil { - l.checkCleanTimer.Reset(l.checkCleanTimeout) - return nil, nil - } - - l.checkCleanTimer = time.AfterFunc(l.checkCleanTimeout, func() { - l.fragmentsMutex.Lock() - defer l.fragmentsMutex.Unlock() - - if l.checkCleanTimeout == 0 { - l.checkCleanTimeout = defaultCleanTimeout - } - if l.checkCleanDeadline == 0 { - l.checkCleanDeadline = defaultCleanDeadline - } - - valid := time.Now().Add(-l.checkCleanDeadline) - for sequenceID, fragmented := range l.fragments { - if fragmented.lastUpdate.Before(valid) { - // dropping due to excided deadline - delete(l.fragments, sequenceID) - } - } - if len(l.fragments) == 0 { - l.checkCleanPending = false - return - } - - l.checkCleanPending = true - l.checkCleanTimer.Reset(l.checkCleanTimeout) - }) - - return nil, nil -} - -func (l *Link) decodeDistHeaderAtomCache(packet []byte) ([]etf.Atom, []byte, error) { - // all the details are here https://erlang.org/doc/apps/erts/erl_ext_dist.html#normal-distribution-header - - // number of atom references are present in package - references := int(packet[0]) - if references == 0 { - return nil, packet[1:], nil - } - - cache := make([]etf.Atom, references) - flagsLen := references/2 + 1 - if len(packet) < 1+flagsLen { - // malformed - return nil, nil, ErrMalformed - } - flags := packet[1 : flagsLen+1] - - // The least significant bit in a half byte is flag LongAtoms. - // If it is set, 2 bytes are used for atom lengths instead of 1 byte - // in the distribution header. - headerAtomLength := 1 // if 'LongAtom' is not set - - // extract this bit. just increase headereAtomLength if this flag is set - lastByte := flags[len(flags)-1] - shift := uint((references & 0x01) * 4) - headerAtomLength += int((lastByte >> shift) & 0x01) - - // 1 (number of references) + references/2+1 (length of flags) - packet = packet[1+flagsLen:] - - for i := 0; i < references; i++ { - if len(packet) < 1+headerAtomLength { - // malformed - return nil, nil, ErrMalformed - } - shift = uint((i & 0x01) * 4) - flag := (flags[i/2] >> shift) & 0x0F - isNewReference := flag&0x08 == 0x08 - idxReference := uint16(flag & 0x07) - idxInternal := uint16(packet[0]) - idx := (idxReference << 8) | idxInternal - - if isNewReference { - atomLen := uint16(packet[1]) - if headerAtomLength == 2 { - atomLen = binary.BigEndian.Uint16(packet[1:3]) - } - // extract atom - packet = packet[1+headerAtomLength:] - if len(packet) < int(atomLen) { - // malformed - return nil, nil, ErrMalformed - } - atom := etf.Atom(packet[:atomLen]) - // store in temporary cache for decoding - cache[i] = atom - - // store in link' cache - l.cacheInMutex.Lock() - l.cacheIn[idx] = &atom - l.cacheInMutex.Unlock() - packet = packet[atomLen:] - continue - } - - l.cacheInMutex.Lock() - c := l.cacheIn[idx] - l.cacheInMutex.Unlock() - if c == nil { - return cache, packet, ErrMissingInCache - } - cache[i] = *c - packet = packet[1:] - } - - return cache, packet, nil -} - -func (l *Link) SetAtomCache(cache *etf.AtomCache) { - l.cacheOut = cache -} - -func (l *Link) encodeDistHeaderAtomCache(b *lib.Buffer, - writerAtomCache map[etf.Atom]etf.CacheItem, - encodingAtomCache *etf.ListAtomCache) { - - n := encodingAtomCache.Len() - if n == 0 { - b.AppendByte(0) - return - } - - b.AppendByte(byte(n)) // write NumberOfAtomCache - - lenFlags := n/2 + 1 - b.Extend(lenFlags) - - flags := b.B[1 : lenFlags+1] - flags[lenFlags-1] = 0 // clear last byte to make sure we have valid LongAtom flag - - for i := 0; i < len(encodingAtomCache.L); i++ { - shift := uint((i & 0x01) * 4) - idxReference := byte(encodingAtomCache.L[i].ID >> 8) // SegmentIndex - idxInternal := byte(encodingAtomCache.L[i].ID & 255) // InternalSegmentIndex - - cachedItem := writerAtomCache[encodingAtomCache.L[i].Name] - if !cachedItem.Encoded { - idxReference |= 8 // set NewCacheEntryFlag - } - - // we have to clear before reuse - if shift == 0 { - flags[i/2] = 0 - } - flags[i/2] |= idxReference << shift - - if cachedItem.Encoded { - b.AppendByte(idxInternal) - continue - } - - if encodingAtomCache.HasLongAtom { - // 1 (InternalSegmentIndex) + 2 (length) + name - allocLen := 1 + 2 + len(encodingAtomCache.L[i].Name) - buf := b.Extend(allocLen) - buf[0] = idxInternal - binary.BigEndian.PutUint16(buf[1:3], uint16(len(encodingAtomCache.L[i].Name))) - copy(buf[3:], encodingAtomCache.L[i].Name) - } else { - - // 1 (InternalSegmentIndex) + 1 (length) + name - allocLen := 1 + 1 + len(encodingAtomCache.L[i].Name) - buf := b.Extend(allocLen) - buf[0] = idxInternal - buf[1] = byte(len(encodingAtomCache.L[i].Name)) - copy(buf[2:], encodingAtomCache.L[i].Name) - } - - cachedItem.Encoded = true - writerAtomCache[encodingAtomCache.L[i].Name] = cachedItem - } - - if encodingAtomCache.HasLongAtom { - shift := uint((n & 0x01) * 4) - flags[lenFlags-1] |= 1 << shift // set LongAtom = 1 - } -} - -func (l *Link) Writer(send <-chan []etf.Term, fragmentationUnit int) { - var terms []etf.Term - - var encodingAtomCache *etf.ListAtomCache - var writerAtomCache map[etf.Atom]etf.CacheItem - var linkAtomCache *etf.AtomCache - var lastCacheID int16 = -1 - - var lenControl, lenMessage, lenAtomCache, lenPacket, startDataPosition int - var atomCacheBuffer, packetBuffer *lib.Buffer - var err error - - cacheEnabled := l.peer.flags.isSet(DIST_HDR_ATOM_CACHE) && l.cacheOut != nil - fragmentationEnabled := l.peer.flags.isSet(FRAGMENTS) && fragmentationUnit > 0 - - // Header atom cache is encoded right after the control/message encoding process - // but should be stored as a first item in the packet. - // Thats why we do reserve some space for it in order to get rid - // of reallocation packetBuffer data - reserveHeaderAtomCache := 8192 - - if cacheEnabled { - encodingAtomCache = etf.TakeListAtomCache() - defer etf.ReleaseListAtomCache(encodingAtomCache) - writerAtomCache = make(map[etf.Atom]etf.CacheItem) - linkAtomCache = l.cacheOut - } - - encodeOptions := etf.EncodeOptions{ - LinkAtomCache: linkAtomCache, - WriterAtomCache: writerAtomCache, - EncodingAtomCache: encodingAtomCache, - FlagBigCreation: l.peer.flags.isSet(BIG_CREATION), - FlagV4NC: l.peer.flags.isSet(V4_NC), - } - - for { - terms = nil - terms = <-send - - if terms == nil { - // channel was closed - return - } - - packetBuffer = lib.TakeBuffer() - lenControl, lenMessage, lenAtomCache, lenPacket, startDataPosition = 0, 0, 0, 0, reserveHeaderAtomCache - - // do reserve for the header 8K, should be enough - packetBuffer.Allocate(reserveHeaderAtomCache) - - // clear encoding cache - if cacheEnabled { - encodingAtomCache.Reset() - } - - // encode Control - err = etf.Encode(terms[0], packetBuffer, encodeOptions) - if err != nil { - fmt.Println(err) - lib.ReleaseBuffer(packetBuffer) - continue - } - lenControl = packetBuffer.Len() - reserveHeaderAtomCache - - // encode Message if present - if len(terms) == 2 { - err = etf.Encode(terms[1], packetBuffer, encodeOptions) - if err != nil { - fmt.Println(err) - lib.ReleaseBuffer(packetBuffer) - continue - } - - } - lenMessage = packetBuffer.Len() - reserveHeaderAtomCache - lenControl - - // encode Header Atom Cache if its enabled - if cacheEnabled && encodingAtomCache.Len() > 0 { - atomCacheBuffer = lib.TakeBuffer() - l.encodeDistHeaderAtomCache(atomCacheBuffer, writerAtomCache, encodingAtomCache) - lenAtomCache = atomCacheBuffer.Len() - - if lenAtomCache > reserveHeaderAtomCache-22 { - // are you serious? ))) what da hell you just sent? - // FIXME i'm gonna fix it if someone report about this issue :) - panic("exceed atom header cache size limit. please report about this issue") - } - - startDataPosition -= lenAtomCache - copy(packetBuffer.B[startDataPosition:], atomCacheBuffer.B) - lib.ReleaseBuffer(atomCacheBuffer) - - } else { - lenAtomCache = 1 - startDataPosition -= lenAtomCache - packetBuffer.B[startDataPosition] = byte(0) - } - - for { - - // 4 (packet len) + 1 (dist header: 131) + 1 (dist header: protoDistMessage) + lenAtomCache - lenPacket = 1 + 1 + lenAtomCache + lenControl + lenMessage - - if !fragmentationEnabled || lenPacket < fragmentationUnit { - // send as a single packet - startDataPosition -= 6 - - binary.BigEndian.PutUint32(packetBuffer.B[startDataPosition:], uint32(lenPacket)) - packetBuffer.B[startDataPosition+4] = protoDist // 131 - packetBuffer.B[startDataPosition+5] = protoDistMessage // 68 - if _, err := l.flusher.Write(packetBuffer.B[startDataPosition:]); err != nil { - return - } - break - } - - // Message should be fragmented - - // https://erlang.org/doc/apps/erts/erl_ext_dist.html#distribution-header-for-fragmented-messages - // "The entire atom cache and control message has to be part of the starting fragment" - - sequenceID := uint64(atomic.AddInt64(&l.sequenceID, 1)) - numFragments := lenMessage/fragmentationUnit + 1 - - // 1 (dist header: 131) + 1 (dist header: protoDistFragment) + 8 (sequenceID) + 8 (fragmentID) + ... - lenPacket = 1 + 1 + 8 + 8 + lenAtomCache + lenControl + fragmentationUnit - - // 4 (packet len) + 1 (dist header: 131) + 1 (dist header: protoDistFragment) + 8 (sequenceID) + 8 (fragmentID) - startDataPosition -= 22 - - binary.BigEndian.PutUint32(packetBuffer.B[startDataPosition:], uint32(lenPacket)) - packetBuffer.B[startDataPosition+4] = protoDist // 131 - packetBuffer.B[startDataPosition+5] = protoDistFragment1 // 69 - - binary.BigEndian.PutUint64(packetBuffer.B[startDataPosition+6:], uint64(sequenceID)) - binary.BigEndian.PutUint64(packetBuffer.B[startDataPosition+14:], uint64(numFragments)) - if _, err := l.flusher.Write(packetBuffer.B[startDataPosition : startDataPosition+4+lenPacket]); err != nil { - return - } - - startDataPosition += 4 + lenPacket - numFragments-- - - nextFragment: - - if len(packetBuffer.B[startDataPosition:]) > fragmentationUnit { - lenPacket = 1 + 1 + 8 + 8 + fragmentationUnit - // reuse the previous 22 bytes for the next frame header - startDataPosition -= 22 - - } else { - // the last one - lenPacket = 1 + 1 + 8 + 8 + len(packetBuffer.B[startDataPosition:]) - startDataPosition -= 22 - } - - binary.BigEndian.PutUint32(packetBuffer.B[startDataPosition:], uint32(lenPacket)) - packetBuffer.B[startDataPosition+4] = protoDist // 131 - packetBuffer.B[startDataPosition+5] = protoDistFragmentN // 70 - - binary.BigEndian.PutUint64(packetBuffer.B[startDataPosition+6:], uint64(sequenceID)) - binary.BigEndian.PutUint64(packetBuffer.B[startDataPosition+14:], uint64(numFragments)) - - if _, err := l.flusher.Write(packetBuffer.B[startDataPosition : startDataPosition+4+lenPacket]); err != nil { - return - } - - startDataPosition += 4 + lenPacket - numFragments-- - if numFragments > 0 { - goto nextFragment - } - - // done - break - } - - lib.ReleaseBuffer(packetBuffer) - - if !cacheEnabled { - continue - } - - // get updates from link AtomCache and update the local one (map writerAtomCache) - id := linkAtomCache.GetLastID() - if lastCacheID < id { - linkAtomCache.Lock() - for _, a := range linkAtomCache.ListSince(lastCacheID + 1) { - writerAtomCache[a] = etf.CacheItem{ID: lastCacheID + 1, Name: a, Encoded: false} - lastCacheID++ - } - linkAtomCache.Unlock() - } - - } - -} - -func (l *Link) GetRemoteName() string { - return l.peer.Name -} - -func (l *Link) composeName(b *lib.Buffer, tls bool) { - if tls { - b.Allocate(11) - dataLength := 7 + len(l.Name) // byte + uint16 + uint32 + len(l.Name) - binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) - b.B[4] = 'n' - binary.BigEndian.PutUint16(b.B[5:7], l.version) // uint16 - binary.BigEndian.PutUint32(b.B[7:11], l.flags.toUint32()) // uint32 - b.Append([]byte(l.Name)) - return - } - - b.Allocate(9) - dataLength := 7 + len(l.Name) // byte + uint16 + uint32 + len(l.Name) - binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) - b.B[2] = 'n' - binary.BigEndian.PutUint16(b.B[3:5], l.version) // uint16 - binary.BigEndian.PutUint32(b.B[5:9], l.flags.toUint32()) // uint32 - b.Append([]byte(l.Name)) -} - -func (l *Link) composeNameVersion6(b *lib.Buffer, tls bool) { - if tls { - b.Allocate(19) - dataLength := 15 + len(l.Name) // 1 + 8 (flags) + 4 (creation) + 2 (len l.Name) - binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) - b.B[4] = 'N' - binary.BigEndian.PutUint64(b.B[5:13], l.flags.toUint64()) // uint64 - binary.BigEndian.PutUint32(b.B[13:17], l.creation) //uint32 - binary.BigEndian.PutUint16(b.B[17:19], uint16(len(l.Name))) // uint16 - b.Append([]byte(l.Name)) - return - } - - b.Allocate(17) - dataLength := 15 + len(l.Name) // 1 + 8 (flags) + 4 (creation) + 2 (len l.Name) - binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) - b.B[2] = 'N' - binary.BigEndian.PutUint64(b.B[3:11], l.flags.toUint64()) // uint64 - binary.BigEndian.PutUint32(b.B[11:15], l.creation) // uint32 - binary.BigEndian.PutUint16(b.B[15:17], uint16(len(l.Name))) // uint16 - b.Append([]byte(l.Name)) -} - -func (l *Link) readName(b []byte) *Link { - peer := &Link{ - Name: string(b[6:]), - version: binary.BigEndian.Uint16(b[0:2]), - flags: nodeFlag(binary.BigEndian.Uint32(b[2:6])), - } - return peer -} - -func (l *Link) readNameVersion6(b []byte) *Link { - nameLen := int(binary.BigEndian.Uint16(b[12:14])) - peer := &Link{ - flags: nodeFlag(binary.BigEndian.Uint64(b[0:8])), - creation: binary.BigEndian.Uint32(b[8:12]), - Name: string(b[14 : 14+nameLen]), - version: ProtoHandshake6, - } - return peer -} - -func (l *Link) composeStatus(b *lib.Buffer, tls bool) { - //FIXME: there are few options for the status: - // ok, ok_simultaneous, nok, not_allowed, alive - // More details here: https://erlang.org/doc/apps/erts/erl_dist_protocol.html#the-handshake-in-detail - if tls { - b.Allocate(4) - dataLength := 3 // 's' + "ok" - binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) - b.Append([]byte("sok")) - return - } - - b.Allocate(2) - dataLength := 3 // 's' + "ok" - binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) - b.Append([]byte("sok")) - -} - -func (l *Link) readStatus(msg []byte) bool { - if string(msg[:2]) == "ok" { - return true - } - - return false -} - -func (l *Link) composeChallenge(b *lib.Buffer, tls bool) { - if tls { - b.Allocate(15) - dataLength := uint32(11 + len(l.Name)) - binary.BigEndian.PutUint32(b.B[0:4], dataLength) - b.B[4] = 'n' - binary.BigEndian.PutUint16(b.B[5:7], l.version) // uint16 - binary.BigEndian.PutUint32(b.B[7:11], l.flags.toUint32()) // uint32 - binary.BigEndian.PutUint32(b.B[11:15], l.challenge) // uint32 - b.Append([]byte(l.Name)) - return - } - - b.Allocate(13) - dataLength := 11 + len(l.Name) - binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) - b.B[2] = 'n' - binary.BigEndian.PutUint16(b.B[3:5], l.version) // uint16 - binary.BigEndian.PutUint32(b.B[5:9], l.flags.toUint32()) // uint32 - binary.BigEndian.PutUint32(b.B[9:13], l.challenge) // uint32 - b.Append([]byte(l.Name)) -} - -func (l *Link) composeChallengeVersion6(b *lib.Buffer, tls bool) { - if tls { - // 1 ('N') + 8 (flags) + 4 (chalange) + 4 (creation) + 2 (len(l.Name)) - b.Allocate(23) - dataLength := 19 + len(l.Name) - binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) - b.B[4] = 'N' - binary.BigEndian.PutUint64(b.B[5:13], uint64(l.flags)) // uint64 - binary.BigEndian.PutUint32(b.B[13:17], l.challenge) // uint32 - binary.BigEndian.PutUint32(b.B[17:21], l.creation) // uint32 - binary.BigEndian.PutUint16(b.B[21:23], uint16(len(l.Name))) // uint16 - b.Append([]byte(l.Name)) - return - } - - // 1 ('N') + 8 (flags) + 4 (chalange) + 4 (creation) + 2 (len(l.Name)) - b.Allocate(21) - dataLength := 19 + len(l.Name) - binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) - b.B[2] = 'N' - binary.BigEndian.PutUint64(b.B[3:11], uint64(l.flags)) // uint64 - binary.BigEndian.PutUint32(b.B[11:15], l.challenge) // uint32 - binary.BigEndian.PutUint32(b.B[15:19], l.creation) // uint32 - binary.BigEndian.PutUint16(b.B[19:21], uint16(len(l.Name))) // uint16 - b.Append([]byte(l.Name)) -} - -func (l *Link) readChallenge(msg []byte) (challenge uint32) { - version := binary.BigEndian.Uint16(msg[0:2]) - if version != ProtoHandshake5 { - return 0 - } - - link := &Link{ - Name: string(msg[10:]), - version: version, - flags: nodeFlag(binary.BigEndian.Uint32(msg[2:6])), - } - l.peer = link - return binary.BigEndian.Uint32(msg[6:10]) -} - -func (l *Link) readChallengeVersion6(msg []byte) (challenge uint32) { - lenName := int(binary.BigEndian.Uint16(msg[16:18])) - link := &Link{ - Name: string(msg[18 : 18+lenName]), - version: ProtoHandshake6, - flags: nodeFlag(binary.BigEndian.Uint64(msg[0:8])), - creation: binary.BigEndian.Uint32(msg[12:16]), - } - l.peer = link - return binary.BigEndian.Uint32(msg[8:12]) -} - -func (l *Link) readComplement(msg []byte) { - flags := uint64(binary.BigEndian.Uint32(msg[0:4])) << 32 - l.peer.flags = nodeFlag(l.peer.flags.toUint64() | flags) - l.peer.creation = binary.BigEndian.Uint32(msg[4:8]) - return -} - -func (l *Link) validateChallengeReply(b []byte) bool { - l.peer.challenge = binary.BigEndian.Uint32(b[:4]) - digestB := b[4:] - - digestA := genDigest(l.challenge, l.Cookie) - return bytes.Equal(digestA[:], digestB) -} - -func (l *Link) composeChallengeAck(b *lib.Buffer, tls bool) { - if tls { - b.Allocate(5) - dataLength := uint32(17) // 'a' + 16 (digest) - binary.BigEndian.PutUint32(b.B[0:4], dataLength) - b.B[4] = 'a' - digest := genDigest(l.peer.challenge, l.Cookie) - b.Append(digest) - return - } - - b.Allocate(3) - dataLength := uint16(17) // 'a' + 16 (digest) - binary.BigEndian.PutUint16(b.B[0:2], dataLength) - b.B[2] = 'a' - digest := genDigest(l.peer.challenge, l.Cookie) - b.Append(digest) -} - -func (l *Link) composeChallengeReply(b *lib.Buffer, challenge uint32, tls bool) { - if tls { - l.digest = genDigest(challenge, l.Cookie) - b.Allocate(9) - dataLength := 5 + len(l.digest) // 1 (byte) + 4 (challenge) + 16 (digest) - binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) - b.B[4] = 'r' - binary.BigEndian.PutUint32(b.B[5:9], l.challenge) // uint32 - b.Append(l.digest[:]) - return - } - - b.Allocate(7) - l.digest = genDigest(challenge, l.Cookie) - dataLength := 5 + len(l.digest) // 1 (byte) + 4 (challenge) + 16 (digest) - binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) - b.B[2] = 'r' - binary.BigEndian.PutUint32(b.B[3:7], l.challenge) // uint32 - b.Append(l.digest) -} - -func (l *Link) composeComplement(b *lib.Buffer, tls bool) { - flags := uint32(l.flags.toUint64() >> 32) - if tls { - b.Allocate(13) - dataLength := 9 // 1 + 4 (flag high) + 4 (creation) - binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) - b.B[4] = 'c' - binary.BigEndian.PutUint32(b.B[5:9], flags) - binary.BigEndian.PutUint32(b.B[9:13], l.creation) - return - } - - dataLength := 9 // 1 + 4 (flag high) + 4 (creation) - b.Allocate(11) - binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) - b.B[2] = 'c' - binary.BigEndian.PutUint32(b.B[3:7], flags) - binary.BigEndian.PutUint32(b.B[7:11], l.creation) - return -} - -func genDigest(challenge uint32, cookie string) []byte { - s := fmt.Sprintf("%s%d", cookie, challenge) - digest := md5.Sum([]byte(s)) - return digest[:] -} diff --git a/node/epmd.go b/node/epmd.go deleted file mode 100644 index 7488c543..00000000 --- a/node/epmd.go +++ /dev/null @@ -1,476 +0,0 @@ -package node - -import ( - "context" - "encoding/binary" - "fmt" - "io" - "net" - "strconv" - "strings" - "sync" - "time" - - "github.com/ergo-services/ergo/lib" -) - -const ( - EPMD_ALIVE2_REQ = 120 - EPMD_ALIVE2_RESP = 121 - - EPMD_PORT_PLEASE2_REQ = 122 - EPMD_PORT2_RESP = 119 - - EPMD_NAMES_REQ = 110 // $n - - EPMD_DUMP_REQ = 100 // $d - EPMD_KILL_REQ = 107 // $k - EPMD_STOP_REQ = 115 // $s -) - -type epmd struct { - Name string - Domain string - - // Listening port for incoming connections - NodePort uint16 - - // EPMD port for the cluster - Port uint16 - Type uint8 - - Protocol uint8 - HighVsn uint16 - LowVsn uint16 - Extra []byte - Creation uint16 - - staticOnly bool - staticRoutes map[string]NetworkRoute - mtx sync.RWMutex - - response chan interface{} -} - -func (e *epmd) Init(ctx context.Context, name string, port uint16, opts Options) error { - ns := strings.Split(name, "@") - if len(ns) != 2 { - return fmt.Errorf("(EMPD) FQDN for node name is required (example: node@hostname)") - } - - e.Name = ns[0] - e.Domain = ns[1] - e.NodePort = port - e.Port = opts.EPMDPort - - // http://erlang.org/doc/reference_manual/distributed.html (section 13.5) - // // 77 — regular public node, 72 — hidden - if opts.Hidden { - e.Type = 72 - } else { - e.Type = 77 - } - - e.Protocol = 0 - e.HighVsn = uint16(opts.HandshakeVersion) - e.LowVsn = 5 - // FIXME overflows value opts.creation is uint32 - e.Creation = uint16(opts.creation) - - e.staticOnly = opts.DisableEPMD - e.staticRoutes = make(map[string]NetworkRoute) - - ready := make(chan error) - - go func(e *epmd) { - defer close(ready) - for { - if !opts.DisableEPMDServer { - // trying to start embedded EPMD before we go further - Server(ctx, e.Port) - } - dialer := net.Dialer{ - KeepAlive: 15 * time.Second, - } - dsn := net.JoinHostPort("", strconv.Itoa(int(e.Port))) - conn, err := dialer.Dial("tcp", dsn) - if err != nil { - ready <- err - return - } - - conn.Write(compose_ALIVE2_REQ(e)) - - for { - buf := make([]byte, 1024) - _, err := conn.Read(buf) - if err != nil { - lib.Log("EPMD: closing connection") - conn.Close() - break - } - - if buf[0] == EPMD_ALIVE2_RESP { - creation := read_ALIVE2_RESP(buf) - switch creation { - case false: - ready <- fmt.Errorf("Duplicate name '%s'", e.Name) - return - default: - e.Creation = creation.(uint16) - } - ready <- nil - } else { - lib.Log("Malformed EPMD reply") - conn.Close() - break - } - } - - } - }(e) - - return <-ready -} - -func (e *epmd) AddStaticRoute(name string, port uint16, cookie string, tls bool) error { - ns := strings.Split(name, "@") - if len(ns) == 1 { - ns = append(ns, "localhost") - } - if len(ns) != 2 { - return fmt.Errorf("wrong FQDN") - } - if _, err := net.LookupHost(ns[1]); err != nil { - return err - } - - if e.staticOnly && port == 0 { - return fmt.Errorf("EMPD is disabled. Port must be > 0") - } - - e.mtx.Lock() - defer e.mtx.Unlock() - if _, ok := e.staticRoutes[name]; ok { - // already exist - return fmt.Errorf("already exist") - } - e.staticRoutes[name] = NetworkRoute{int(port), cookie, tls} - - return nil -} - -func (e *epmd) RemoveStaticRoute(name string) { - e.mtx.Lock() - defer e.mtx.Unlock() - delete(e.staticRoutes, name) - return -} - -func (e *epmd) resolve(name string) (NetworkRoute, error) { - // chech static routes first - e.mtx.RLock() - defer e.mtx.RUnlock() - nr, ok := e.staticRoutes[name] - if ok && nr.Port > 0 { - return nr, nil - } - - if e.staticOnly { - return nr, fmt.Errorf("Can't resolve %s", name) - } - - // no static route for the given name. go the regular way - port, err := e.resolvePort(name) - if err != nil { - return nr, err - } - return NetworkRoute{port, nr.Cookie, nr.TLS}, nil -} - -func (e *epmd) resolvePort(name string) (int, error) { - ns := strings.Split(name, "@") - if len(ns) != 2 { - return 0, fmt.Errorf("incorrect FQDN node name (example: node@localhost)") - } - conn, err := net.Dial("tcp", net.JoinHostPort(ns[1], fmt.Sprintf("%d", e.Port))) - if err != nil { - return 0, err - } - - defer conn.Close() - - buf := compose_PORT_PLEASE2_REQ(ns[0]) - _, err = conn.Write(buf) - if err != nil { - return -1, fmt.Errorf("initiate connection - %s", err) - } - - buf = make([]byte, 1024) - _, err = conn.Read(buf) - if err != nil && err != io.EOF { - return -1, fmt.Errorf("reading from link - %s", err) - } - - if buf[0] == EPMD_PORT2_RESP && buf[1] == 0 { - p := binary.BigEndian.Uint16(buf[2:4]) - // we don't use all the extra info for a while. FIXME (do we need it?) - return int(p), nil - } else if buf[1] > 0 { - return -1, fmt.Errorf("desired node not found") - } else { - return -1, fmt.Errorf("malformed reply - %#v", buf) - } -} - -func compose_ALIVE2_REQ(e *epmd) (reply []byte) { - reply = make([]byte, 2+14+len(e.Name)+len(e.Extra)) - binary.BigEndian.PutUint16(reply[0:2], uint16(len(reply)-2)) - reply[2] = byte(EPMD_ALIVE2_REQ) - binary.BigEndian.PutUint16(reply[3:5], e.NodePort) - reply[5] = e.Type - reply[6] = e.Protocol - binary.BigEndian.PutUint16(reply[7:9], e.HighVsn) - binary.BigEndian.PutUint16(reply[9:11], e.LowVsn) - nLen := len(e.Name) - binary.BigEndian.PutUint16(reply[11:13], uint16(nLen)) - offset := (13 + nLen) - copy(reply[13:offset], e.Name) - nELen := len(e.Extra) - binary.BigEndian.PutUint16(reply[offset:offset+2], uint16(nELen)) - copy(reply[offset+2:offset+2+nELen], e.Extra) - return -} - -func read_ALIVE2_RESP(reply []byte) interface{} { - if reply[1] == 0 { - return binary.BigEndian.Uint16(reply[2:4]) - } - return false -} - -func compose_PORT_PLEASE2_REQ(name string) (reply []byte) { - replylen := uint16(2 + len(name) + 1) - reply = make([]byte, replylen) - binary.BigEndian.PutUint16(reply[0:2], uint16(len(reply)-2)) - reply[2] = byte(EPMD_PORT_PLEASE2_REQ) - copy(reply[3:replylen], name) - return -} - -/// empd server implementation - -type nodeinfo struct { - Port uint16 - Hidden bool - HiVersion uint16 - LoVersion uint16 - Extra []byte -} - -type embeddedEPMDserver struct { - portmap map[string]*nodeinfo - mtx sync.RWMutex -} - -func (e *embeddedEPMDserver) Join(name string, info *nodeinfo) bool { - - e.mtx.Lock() - defer e.mtx.Unlock() - if _, ok := e.portmap[name]; ok { - // already registered - return false - } - lib.Log("EPMD registering node: '%s' port:%d hidden:%t", name, info.Port, info.Hidden) - e.portmap[name] = info - - return true -} - -func (e *embeddedEPMDserver) Get(name string) *nodeinfo { - e.mtx.RLock() - defer e.mtx.RUnlock() - if info, ok := e.portmap[name]; ok { - return info - } - return nil -} - -func (e *embeddedEPMDserver) Leave(name string) { - lib.Log("EPMD unregistering node: '%s'", name) - - e.mtx.Lock() - delete(e.portmap, name) - e.mtx.Unlock() -} - -func (e *embeddedEPMDserver) ListAll() map[string]uint16 { - e.mtx.Lock() - lst := make(map[string]uint16) - for k, v := range e.portmap { - lst[k] = v.Port - } - e.mtx.Unlock() - return lst -} - -func Server(ctx context.Context, port uint16) error { - - lc := net.ListenConfig{} - epmd, err := lc.Listen(ctx, "tcp", net.JoinHostPort("", strconv.Itoa(int(port)))) - if err != nil { - lib.Log("Can't start embedded EPMD service: %s", err) - return fmt.Errorf("Can't start embedded EPMD service: %s", err) - - } - - epmdServer := &embeddedEPMDserver{ - portmap: make(map[string]*nodeinfo), - } - - lib.Log("Started embedded EMPD service and listen port: %d", port) - - go func() { - for { - c, err := epmd.Accept() - if err != nil { - lib.Log(err.Error()) - continue - } - - lib.Log("EPMD accepted new connection from %s", c.RemoteAddr().String()) - - //epmd connection handler loop - go func(c net.Conn) { - defer c.Close() - buf := make([]byte, 1024) - name := "" - for { - n, err := c.Read(buf) - lib.Log("Request from EPMD client: %v", buf[:n]) - if err != nil { - if name != "" { - epmdServer.Leave(name) - } - return - } - // buf[0:1] - length - if uint16(n-2) != binary.BigEndian.Uint16(buf[0:2]) { - continue - } - - switch buf[2] { - case EPMD_ALIVE2_REQ: - reply, registered := epmdServer.compose_ALIVE2_RESP(buf[3:n]) - c.Write(reply) - if registered == "" { - return - } - name = registered - if tcp, ok := c.(*net.TCPConn); !ok { - tcp.SetKeepAlive(true) - tcp.SetKeepAlivePeriod(15 * time.Second) - tcp.SetNoDelay(true) - } - continue - case EPMD_PORT_PLEASE2_REQ: - c.Write(epmdServer.compose_EPMD_PORT2_RESP(buf[3:n])) - return - case EPMD_NAMES_REQ: - c.Write(epmdServer.compose_EPMD_NAMES_RESP(port, buf[3:n])) - return - default: - lib.Log("unknown EPMD request") - return - } - - } - }(c) - - } - }() - - return nil -} - -func (e *embeddedEPMDserver) compose_ALIVE2_RESP(req []byte) ([]byte, string) { - - hidden := false // - if req[2] == 72 { - hidden = true - } - - namelen := binary.BigEndian.Uint16(req[8:10]) - name := string(req[10 : 10+namelen]) - - info := nodeinfo{ - Port: binary.BigEndian.Uint16(req[0:2]), - Hidden: hidden, - HiVersion: binary.BigEndian.Uint16(req[4:6]), - LoVersion: binary.BigEndian.Uint16(req[6:8]), - } - - reply := make([]byte, 4) - reply[0] = EPMD_ALIVE2_RESP - - registered := "" - if e.Join(name, &info) { - reply[1] = 0 - registered = name - } else { - reply[1] = 1 - } - - binary.BigEndian.PutUint16(reply[2:], uint16(1)) - lib.Log("Made reply for ALIVE2_REQ: (%s) %#v", name, reply) - return reply, registered -} - -func (e *embeddedEPMDserver) compose_EPMD_PORT2_RESP(req []byte) []byte { - name := string(req) - info := e.Get(name) - - if info == nil { - // not found - lib.Log("EPMD: looking for '%s'. Not found", name) - return []byte{EPMD_PORT2_RESP, 1} - } - - reply := make([]byte, 12+len(name)+2+len(info.Extra)) - reply[0] = EPMD_PORT2_RESP - reply[1] = 0 - binary.BigEndian.PutUint16(reply[2:4], uint16(info.Port)) - if info.Hidden { - reply[4] = 72 - } else { - reply[4] = 77 - } - reply[5] = 0 // protocol tcp - binary.BigEndian.PutUint16(reply[6:8], uint16(info.HiVersion)) - binary.BigEndian.PutUint16(reply[8:10], uint16(info.LoVersion)) - binary.BigEndian.PutUint16(reply[10:12], uint16(len(name))) - offset := 12 + len(name) - copy(reply[12:offset], name) - nELen := len(info.Extra) - binary.BigEndian.PutUint16(reply[offset:offset+2], uint16(nELen)) - copy(reply[offset+2:offset+2+nELen], info.Extra) - - lib.Log("Made reply for EPMD_PORT_PLEASE2_REQ: %#v", reply) - - return reply -} - -func (e *embeddedEPMDserver) compose_EPMD_NAMES_RESP(port uint16, req []byte) []byte { - // io:format("name ~ts at port ~p~n", [NodeName, Port]). - var str strings.Builder - var s string - var portbuf [4]byte - binary.BigEndian.PutUint32(portbuf[0:4], uint32(port)) - str.WriteString(string(portbuf[0:])) - for h, p := range e.ListAll() { - s = fmt.Sprintf("name %s at port %d\n", h, p) - str.WriteString(s) - } - - return []byte(str.String()) -} diff --git a/node/monitor.go b/node/monitor.go index ba89c4c1..cf283eff 100644 --- a/node/monitor.go +++ b/node/monitor.go @@ -3,8 +3,7 @@ package node // http://erlang.org/doc/reference_manual/processes.html import ( - "math" - "strings" + "fmt" "sync" "github.com/ergo-services/ergo/etf" @@ -17,22 +16,33 @@ type monitorItem struct { ref etf.Ref } -type linkProcessRequest struct { - pidA etf.Pid - pidB etf.Pid -} - type monitorInternal interface { - monitorProcess(by etf.Pid, process interface{}, ref etf.Ref) - demonitorProcess(ref etf.Ref) bool - monitorNode(by etf.Pid, node string) etf.Ref + // RouteLink + RouteLink(pidA etf.Pid, pidB etf.Pid) error + // RouteUnlink + RouteUnlink(pidA etf.Pid, pidB etf.Pid) error + // RouteExit + RouteExit(to etf.Pid, terminated etf.Pid, reason string) error + // RouteMonitorReg + RouteMonitorReg(by etf.Pid, process gen.ProcessID, ref etf.Ref) error + // RouteMonitor + RouteMonitor(by etf.Pid, process etf.Pid, ref etf.Ref) error + // RouteDemonitor + RouteDemonitor(by etf.Pid, ref etf.Ref) error + // RouteMonitorExitReg + RouteMonitorExitReg(terminated gen.ProcessID, reason string, ref etf.Ref) error + // RouteMonitorExit + RouteMonitorExit(terminated etf.Pid, reason string, ref etf.Ref) error + // RouteNodeDown + RouteNodeDown(name string, disconnect *ProxyDisconnect) + + // IsMonitor + IsMonitor(ref etf.Ref) bool + + monitorNode(by etf.Pid, node string, ref etf.Ref) demonitorNode(ref etf.Ref) bool - nodeDown(name string) - processTerminated(terminated etf.Pid, name, reason string) - - link(pidA, pidB etf.Pid) - unlink(pidA, pidB etf.Pid) + handleTerminated(terminated etf.Pid, name, reason string) processLinks(process etf.Pid) []etf.Pid processMonitors(process etf.Pid) []etf.Pid @@ -41,297 +51,48 @@ type monitorInternal interface { } type monitor struct { + // monitors by pid processes map[etf.Pid][]monitorItem ref2pid map[etf.Ref]etf.Pid - mutexProcesses sync.Mutex - links map[etf.Pid][]etf.Pid - mutexLinks sync.Mutex - nodes map[string][]monitorItem - ref2node map[etf.Ref]string - mutexNodes sync.Mutex - - registrar registrarInternal + mutexProcesses sync.RWMutex + // monitors by name + names map[gen.ProcessID][]monitorItem + ref2name map[etf.Ref]gen.ProcessID + mutexNames sync.RWMutex + + // links + links map[etf.Pid][]etf.Pid + mutexLinks sync.RWMutex + + // monitors of nodes + nodes map[string][]monitorItem + ref2node map[etf.Ref]string + mutexNodes sync.RWMutex + + nodename string + router coreRouterInternal } -func newMonitor(registrar registrarInternal) monitor { - return monitor{ +func newMonitor(nodename string, router coreRouterInternal) monitorInternal { + return &monitor{ processes: make(map[etf.Pid][]monitorItem), + names: make(map[gen.ProcessID][]monitorItem), links: make(map[etf.Pid][]etf.Pid), nodes: make(map[string][]monitorItem), ref2pid: make(map[etf.Ref]etf.Pid), + ref2name: make(map[etf.Ref]gen.ProcessID), ref2node: make(map[etf.Ref]string), - registrar: registrar, - } -} - -func (m *monitor) monitorProcess(by etf.Pid, process interface{}, ref etf.Ref) { - if by.Node != ref.Node { - lib.Log("[%s] Incorrect monitor request by Pid = %v and Ref = %v", m.registrar.NodeName(), by, ref) - return - } - -next: - switch t := process.(type) { - case etf.Pid: - lib.Log("[%s] MONITOR process: %s => %s", m.registrar.NodeName(), by, t) - - // If 'process' belongs to this node we should make sure if its alive. - // http://erlang.org/doc/reference_manual/processes.html#monitors - // If Pid does not exist a gen.MessageDown must be - // send immediately with Reason set to noproc. - if p := m.registrar.ProcessByPid(t); string(t.Node) == m.registrar.NodeName() && p == nil { - m.notifyProcessTerminated(ref, by, t, "noproc") - return - } - - m.mutexProcesses.Lock() - l := m.processes[t] - item := monitorItem{ - pid: by, - ref: ref, - } - m.processes[t] = append(l, item) - m.ref2pid[ref] = t - m.mutexProcesses.Unlock() - - if isVirtualPid(t) { - // this Pid was created as a virtual. we use virtual pids for the - // monitoring process by the registered name. - return - } - - if string(t.Node) == m.registrar.NodeName() { - // this is the local process so we have nothing to do - return - } - - // request monitoring the remote process - message := etf.Tuple{distProtoMONITOR, by, t, ref} - if err := m.registrar.routeRaw(t.Node, message); err != nil { - m.notifyProcessTerminated(ref, by, t, "noconnection") - m.mutexProcesses.Lock() - delete(m.ref2pid, ref) - m.mutexProcesses.Unlock() - } - - case string: - // requesting monitor of local process - vPid := virtualPid(gen.ProcessID{t, m.registrar.NodeName()}) - // If Pid does not exist a gen.MessageDown must be - // send immediately with Reason set to noproc. - if p := m.registrar.ProcessByName(t); p == nil { - m.notifyProcessTerminated(ref, by, vPid, "noproc") - return - } - process = vPid - goto next - - case etf.Atom: - // the same as 'string' - vPid := virtualPid(gen.ProcessID{string(t), m.registrar.NodeName()}) - if p := m.registrar.ProcessByName(string(t)); p == nil { - m.notifyProcessTerminated(ref, by, vPid, "noproc") - return - } - process = vPid - goto next - - case gen.ProcessID: - // requesting monitor of remote process by the local one using registered process name - vPid := virtualPid(t) - process = vPid - - if t.Node == m.registrar.NodeName() { - // If Pid does not exist a gen.MessageDown must be - // send immediately with Reason set to noproc. - if p := m.registrar.ProcessByName(t.Name); p == nil { - m.notifyProcessTerminated(ref, by, vPid, "noproc") - return - } - goto next - } - - message := etf.Tuple{distProtoMONITOR, by, etf.Atom(t.Name), ref} - if err := m.registrar.routeRaw(etf.Atom(t.Node), message); err != nil { - m.notifyProcessTerminated(ref, by, vPid, "noconnection") - return - } - - // in order to handle 'nodedown' event we create a local monitor on a virtual pid - goto next - } -} - -func (m *monitor) demonitorProcess(ref etf.Ref) bool { - var process interface{} - var node etf.Atom - - m.mutexProcesses.Lock() - defer m.mutexProcesses.Unlock() - - pid, knownRef := m.ref2pid[ref] - if !knownRef { - // unknown monitor reference - return false - } - - // cheching for monitorItem list - items := m.processes[pid] - - // remove PID from monitoring processes list - for i := range items { - if items[i].ref != ref { - continue - } - process = pid - node = pid.Node - if isVirtualPid(pid) { - processID := virtualPidToProcessID(pid) - process = etf.Atom(processID.Name) - node = etf.Atom(processID.Node) - } - - if string(node) != m.registrar.NodeName() { - message := etf.Tuple{distProtoDEMONITOR, items[i].pid, process, ref} - m.registrar.routeRaw(node, message) - } - - items[i] = items[0] - items = items[1:] - delete(m.ref2pid, ref) - break - - } - - if len(items) == 0 { - delete(m.processes, pid) - } else { - m.processes[pid] = items - } - - return true -} - -func (m *monitor) link(pidA, pidB etf.Pid) { - lib.Log("[%s] LINK process: %v => %v", m.registrar.NodeName(), pidA, pidB) - - // http://erlang.org/doc/reference_manual/processes.html#links - // Links are bidirectional and there can only be one link between - // two processes. Repeated calls to link(Pid) have no effect. - - // If the link already exists or a process attempts to create - // a link to itself, nothing is done. - if pidA == pidB { - return - } - - m.mutexLinks.Lock() - defer m.mutexLinks.Unlock() - - linksA := m.links[pidA] - if pidA.Node == etf.Atom(m.registrar.NodeName()) { - // check if these processes are linked already (source) - for i := range linksA { - if linksA[i] == pidB { - return - } - } - - m.links[pidA] = append(linksA, pidB) - } - - // check if these processes are linked already (destination) - linksB := m.links[pidB] - for i := range linksB { - if linksB[i] == pidA { - return - } - } - - if pidB.Node == etf.Atom(m.registrar.NodeName()) { - // for the local process we should make sure if its alive - // otherwise send 'EXIT' message with 'noproc' as a reason - if p := m.registrar.ProcessByPid(pidB); p == nil { - m.notifyProcessExit(pidA, pidB, "noproc") - if len(linksA) > 0 { - m.links[pidA] = linksA - } else { - delete(m.links, pidA) - } - return - } - } else { - // linking with remote process - message := etf.Tuple{distProtoLINK, pidA, pidB} - if err := m.registrar.routeRaw(pidB.Node, message); err != nil { - // seems we have no connection with this node. notify the sender - // with 'EXIT' message and 'noconnection' as a reason - m.notifyProcessExit(pidA, pidB, "noconnection") - if len(linksA) > 0 { - m.links[pidA] = linksA - } else { - delete(m.links, pidA) - } - return - } + nodename: nodename, + router: router, } - - m.links[pidB] = append(linksB, pidA) } -func (m *monitor) unlink(pidA, pidB etf.Pid) { - m.mutexLinks.Lock() - defer m.mutexLinks.Unlock() +func (m *monitor) monitorNode(by etf.Pid, node string, ref etf.Ref) { + lib.Log("[%s] MONITOR NODE : %v => %s", m.nodename, by, node) - if pidB.Node != etf.Atom(m.registrar.NodeName()) { - message := etf.Tuple{distProtoUNLINK, pidA, pidB} - m.registrar.routeRaw(pidB.Node, message) - } - - if pidA.Node == etf.Atom(m.registrar.NodeName()) { - linksA := m.links[pidA] - for i := range linksA { - if linksA[i] != pidB { - continue - } - - linksA[i] = linksA[0] - linksA = linksA[1:] - if len(linksA) > 0 { - m.links[pidA] = linksA - } else { - delete(m.links, pidA) - } - break - - } - } - - linksB := m.links[pidB] - for i := range linksB { - if linksB[i] != pidA { - continue - } - linksB[i] = linksB[0] - linksB = linksB[1:] - if len(linksB) > 0 { - m.links[pidB] = linksB - } else { - delete(m.links, pidB) - } - break - - } -} - -func (m *monitor) monitorNode(by etf.Pid, node string) etf.Ref { - lib.Log("[%s] MONITOR NODE : %v => %s", m.registrar.NodeName(), by, node) - - ref := m.registrar.MakeRef() m.mutexNodes.Lock() - defer m.mutexNodes.Unlock() l := m.nodes[node] item := monitorItem{ @@ -340,8 +101,12 @@ func (m *monitor) monitorNode(by etf.Pid, node string) etf.Ref { } m.nodes[node] = append(l, item) m.ref2node[ref] = node + m.mutexNodes.Unlock() - return ref + _, err := m.router.getConnection(node) + if err != nil { + m.RouteNodeDown(node, nil) + } } func (m *monitor) demonitorNode(ref etf.Ref) bool { @@ -365,47 +130,83 @@ func (m *monitor) demonitorNode(ref etf.Ref) bool { l[i] = l[0] l = l[1:] - m.mutexProcesses.Lock() - delete(m.ref2pid, ref) - m.mutexProcesses.Unlock() break - } - m.nodes[name] = l delete(m.ref2node, ref) + + if len(l) == 0 { + delete(m.nodes, name) + } else { + m.nodes[name] = l + } + return true } -func (m *monitor) nodeDown(name string) { - lib.Log("[%s] MONITOR NODE down: %v", m.registrar.NodeName(), name) +func (m *monitor) RouteNodeDown(name string, disconnect *ProxyDisconnect) { + lib.Log("[%s] MONITOR NODE down: %v", m.nodename, name) - m.mutexNodes.Lock() + // notify node monitors + m.mutexNodes.RLock() if pids, ok := m.nodes[name]; ok { for i := range pids { - lib.Log("[%s] MONITOR node down: %v. send notify to: %s", m.registrar.NodeName(), name, pids[i].pid) - m.notifyNodeDown(pids[i].pid, name) - delete(m.nodes, name) + lib.Log("[%s] MONITOR node down: %v. send notify to: %s", m.nodename, name, pids[i].pid) + if disconnect == nil { + message := gen.MessageNodeDown{Ref: pids[i].ref, Name: name} + m.router.RouteSend(etf.Pid{}, pids[i].pid, message) + continue + } + message := gen.MessageProxyDown{ + Ref: pids[i].ref, + Node: disconnect.Node, + Proxy: disconnect.Proxy, + Reason: disconnect.Reason, + } + m.router.RouteSend(etf.Pid{}, pids[i].pid, message) + } + delete(m.nodes, name) } - m.mutexNodes.Unlock() + m.mutexNodes.RUnlock() - // notify process monitors + // notify processes created monitors by pid m.mutexProcesses.Lock() for pid, ps := range m.processes { - if isVirtualPid(pid) { - processID := virtualPidToProcessID(pid) - if processID.Node != name { - continue - } + if string(pid.Node) != name { + continue } for i := range ps { - m.notifyProcessTerminated(ps[i].ref, ps[i].pid, pid, "noconnection") + // args: (to, terminated, reason, ref) delete(m.ref2pid, ps[i].ref) + if disconnect == nil || disconnect.Node == name { + m.sendMonitorExit(ps[i].pid, pid, "noconnection", ps[i].ref) + continue + } + m.sendMonitorExit(ps[i].pid, pid, "noproxy", ps[i].ref) } delete(m.processes, pid) } m.mutexProcesses.Unlock() + // notify processes created monitors by name + m.mutexNames.Lock() + for processID, ps := range m.names { + if processID.Node != name { + continue + } + for i := range ps { + // args: (to, terminated, reason, ref) + delete(m.ref2name, ps[i].ref) + if disconnect == nil || disconnect.Node == name { + m.sendMonitorExitReg(ps[i].pid, processID, "noconnection", ps[i].ref) + continue + } + m.sendMonitorExitReg(ps[i].pid, processID, "noproxy", ps[i].ref) + } + delete(m.names, processID) + } + m.mutexNames.Unlock() + // notify linked processes m.mutexLinks.Lock() for link, pids := range m.links { @@ -414,7 +215,11 @@ func (m *monitor) nodeDown(name string) { } for i := range pids { - m.notifyProcessExit(pids[i], link, "noconnection") + if disconnect == nil || disconnect.Node == name { + m.sendExit(pids[i], link, "noconnection") + } else { + m.sendExit(pids[i], link, "noproxy") + } p, ok := m.links[pids[i]] if !ok { @@ -444,39 +249,41 @@ func (m *monitor) nodeDown(name string) { m.mutexLinks.Unlock() } -func (m *monitor) processTerminated(terminated etf.Pid, name, reason string) { - lib.Log("[%s] MONITOR process terminated: %v", m.registrar.NodeName(), terminated) +func (m *monitor) handleTerminated(terminated etf.Pid, name string, reason string) { + lib.Log("[%s] MONITOR process terminated: %v", m.nodename, terminated) - // just wrapper for the iterating through monitors list - handleMonitors := func(terminatedPid etf.Pid, items []monitorItem) { - for i := range items { - lib.Log("[%s] MONITOR process terminated: %s. send notify to: %s", m.registrar.NodeName(), terminated, items[i].pid) - m.notifyProcessTerminated(items[i].ref, items[i].pid, terminatedPid, reason) - delete(m.ref2pid, items[i].ref) - } - delete(m.processes, terminatedPid) - } - - m.mutexProcesses.Lock() // if terminated process had a name we should make shure to clean up them all + m.mutexNames.Lock() if name != "" { - // monitor was created by the name so we should look up using virtual pid - terminatedPid := virtualPid(gen.ProcessID{name, m.registrar.NodeName()}) - if items, ok := m.processes[terminatedPid]; ok { - handleMonitors(terminatedPid, items) + terminatedProcessID := gen.ProcessID{Name: name, Node: m.nodename} + if items, ok := m.names[terminatedProcessID]; ok { + for i := range items { + lib.Log("[%s] MONITOR process terminated: %s. send notify to: %s", m.nodename, terminatedProcessID, items[i].pid) + m.sendMonitorExitReg(items[i].pid, terminatedProcessID, reason, items[i].ref) + delete(m.ref2name, items[i].ref) + } + delete(m.names, terminatedProcessID) } } + m.mutexNames.Unlock() // check whether we have monitorItem on this process by Pid (terminated) + m.mutexProcesses.Lock() if items, ok := m.processes[terminated]; ok { - handleMonitors(terminated, items) + + for i := range items { + lib.Log("[%s] MONITOR process terminated: %s. send notify to: %s", m.nodename, terminated, items[i].pid) + m.sendMonitorExit(items[i].pid, terminated, reason, items[i].ref) + delete(m.ref2pid, items[i].ref) + } + delete(m.processes, terminated) } m.mutexProcesses.Unlock() m.mutexLinks.Lock() if pidLinks, ok := m.links[terminated]; ok { for i := range pidLinks { - lib.Log("[%s] LINK process exited: %s. send notify to: %s", m.registrar.NodeName(), terminated, pidLinks[i]) - m.notifyProcessExit(pidLinks[i], terminated, reason) + lib.Log("[%s] LINK process exited: %s. send notify to: %s", m.nodename, terminated, pidLinks[i]) + m.sendExit(pidLinks[i], terminated, reason) // remove A link pids, ok := m.links[pidLinks[i]] @@ -501,13 +308,13 @@ func (m *monitor) processTerminated(terminated etf.Pid, name, reason string) { // remove link delete(m.links, terminated) } - defer m.mutexLinks.Unlock() + m.mutexLinks.Unlock() } func (m *monitor) processLinks(process etf.Pid) []etf.Pid { - m.mutexLinks.Lock() - defer m.mutexLinks.Unlock() + m.mutexLinks.RLock() + defer m.mutexLinks.RUnlock() if l, ok := m.links[process]; ok { return l @@ -517,13 +324,10 @@ func (m *monitor) processLinks(process etf.Pid) []etf.Pid { func (m *monitor) processMonitors(process etf.Pid) []etf.Pid { monitors := []etf.Pid{} - m.mutexProcesses.Lock() - defer m.mutexProcesses.Unlock() + m.mutexProcesses.RLock() + defer m.mutexProcesses.RUnlock() for p, by := range m.processes { - if isVirtualPid(p) { - continue - } for b := range by { if by[b].pid == process { monitors = append(monitors, p) @@ -535,16 +339,12 @@ func (m *monitor) processMonitors(process etf.Pid) []etf.Pid { func (m *monitor) processMonitorsByName(process etf.Pid) []gen.ProcessID { monitors := []gen.ProcessID{} - m.mutexProcesses.Lock() - defer m.mutexProcesses.Unlock() + m.mutexProcesses.RLock() + defer m.mutexProcesses.RUnlock() - for p, by := range m.processes { - if !isVirtualPid(p) { - continue - } + for processID, by := range m.names { for b := range by { if by[b].pid == process { - processID := virtualPidToProcessID(p) monitors = append(monitors, processID) } } @@ -554,11 +354,9 @@ func (m *monitor) processMonitorsByName(process etf.Pid) []gen.ProcessID { func (m *monitor) processMonitoredBy(process etf.Pid) []etf.Pid { monitors := []etf.Pid{} - m.mutexProcesses.Lock() - defer m.mutexProcesses.Unlock() - + m.mutexProcesses.RLock() + defer m.mutexProcesses.RUnlock() if m, ok := m.processes[process]; ok { - monitors := []etf.Pid{} for i := range m { monitors = append(monitors, m[i].pid) } @@ -568,94 +366,489 @@ func (m *monitor) processMonitoredBy(process etf.Pid) []etf.Pid { } func (m *monitor) IsMonitor(ref etf.Ref) bool { - m.mutexProcesses.Lock() - defer m.mutexProcesses.Unlock() + m.mutexProcesses.RLock() + defer m.mutexProcesses.RUnlock() if _, ok := m.ref2pid[ref]; ok { return true } + if _, ok := m.ref2name[ref]; ok { + return true + } return false } -func (m *monitor) notifyNodeDown(to etf.Pid, node string) { - message := gen.MessageNodeDown{node} - m.registrar.route(etf.Pid{}, to, message) +// +// implementation of CoreRouter interface: +// +// RouteLink +// RouteUnlink +// RouteExit +// RouteMonitor +// RouteMonitorReg +// RouteDemonitor +// RouteMonitorExit +// RouteMonitorExitReg +// + +func (m *monitor) RouteLink(pidA etf.Pid, pidB etf.Pid) error { + lib.Log("[%s] LINK process: %v => %v", m.nodename, pidA, pidB) + + // http://erlang.org/doc/reference_manual/processes.html#links + // Links are bidirectional and there can only be one link between + // two processes. Repeated calls to link(Pid) have no effect. + + // Returns error if link is already exist or a process attempts to create + // a link to itself + + if pidA == pidB { + return fmt.Errorf("Can not link to itself") + } + + m.mutexLinks.RLock() + linksA := m.links[pidA] + if pidA.Node == etf.Atom(m.nodename) { + // check if these processes are linked already (source) + for i := range linksA { + if linksA[i] == pidB { + m.mutexLinks.RUnlock() + return fmt.Errorf("Already linked") + } + } + + } + m.mutexLinks.RUnlock() + + // check if these processes are linked already (destination) + m.mutexLinks.RLock() + linksB := m.links[pidB] + + for i := range linksB { + if linksB[i] == pidA { + m.mutexLinks.RUnlock() + return fmt.Errorf("Already linked") + } + } + m.mutexLinks.RUnlock() + + if pidB.Node == etf.Atom(m.nodename) { + // for the local process we should make sure if its alive + // otherwise send 'EXIT' message with 'noproc' as a reason + if p := m.router.processByPid(pidB); p == nil { + m.sendExit(pidA, pidB, "noproc") + return ErrProcessUnknown + } + m.mutexLinks.Lock() + m.links[pidA] = append(linksA, pidB) + m.links[pidB] = append(linksB, pidA) + m.mutexLinks.Unlock() + return nil + } + + // linking with remote process + connection, err := m.router.getConnection(string(pidB.Node)) + if err != nil { + m.sendExit(pidA, pidB, "noconnection") + return nil + } + + if err := connection.Link(pidA, pidB); err != nil { + m.sendExit(pidA, pidB, err.Error()) + return nil + } + + m.mutexLinks.Lock() + m.links[pidA] = append(linksA, pidB) + m.links[pidB] = append(linksB, pidA) + m.mutexLinks.Unlock() + return nil +} + +func (m *monitor) RouteUnlink(pidA etf.Pid, pidB etf.Pid) error { + m.mutexLinks.Lock() + defer m.mutexLinks.Unlock() + + if pidA.Node == etf.Atom(m.nodename) { + linksA := m.links[pidA] + for i := range linksA { + if linksA[i] != pidB { + continue + } + + linksA[i] = linksA[0] + linksA = linksA[1:] + if len(linksA) > 0 { + m.links[pidA] = linksA + } else { + delete(m.links, pidA) + } + break + } + } + + linksB := m.links[pidB] + for i := range linksB { + if linksB[i] != pidA { + continue + } + linksB[i] = linksB[0] + linksB = linksB[1:] + if len(linksB) > 0 { + m.links[pidB] = linksB + } else { + delete(m.links, pidB) + } + break + + } + + if pidB.Node != etf.Atom(m.nodename) { + connection, err := m.router.getConnection(string(pidB.Node)) + if err != nil { + m.sendExit(pidA, pidB, "noconnection") + return err + } + if err := connection.Unlink(pidA, pidB); err != nil { + m.sendExit(pidA, pidB, err.Error()) + return err + } + } + return nil +} + +func (m *monitor) RouteExit(to etf.Pid, terminated etf.Pid, reason string) error { + m.mutexLinks.Lock() + defer m.mutexLinks.Unlock() + + pidLinks, ok := m.links[terminated] + if !ok { + return nil + } + for i := range pidLinks { + lib.Log("[%s] LINK process exited: %s. send notify to: %s", m.nodename, terminated, pidLinks[i]) + m.sendExit(pidLinks[i], terminated, reason) + + // remove A link + pids, ok := m.links[pidLinks[i]] + if !ok { + continue + } + for k := range pids { + if pids[k] != terminated { + continue + } + pids[k] = pids[0] + pids = pids[1:] + break + } + + if len(pids) > 0 { + m.links[pidLinks[i]] = pids + } else { + delete(m.links, pidLinks[i]) + } + } + // remove link + delete(m.links, terminated) + return nil + } -func (m *monitor) notifyProcessTerminated(ref etf.Ref, to etf.Pid, terminated etf.Pid, reason string) { - // for remote {21, FromProc, ToPid, Ref, Reason}, where FromProc = monitored process - localNode := etf.Atom(m.registrar.NodeName()) - if to.Node != localNode { - // do nothing - if reason == "noconnection" { - return +func (m *monitor) RouteMonitor(by etf.Pid, pid etf.Pid, ref etf.Ref) error { + lib.Log("[%s] MONITOR process: %s => %s", m.nodename, by, pid) + + // If 'process' belongs to this node we should make sure if its alive. + // http://erlang.org/doc/reference_manual/processes.html#monitors + // If Pid does not exist a gen.MessageDown must be + // send immediately with Reason set to noproc. + if p := m.router.processByPid(pid); string(pid.Node) == m.nodename && p == nil { + return m.sendMonitorExit(by, pid, "noproc", ref) + } + + if string(pid.Node) != m.nodename { + connection, err := m.router.getConnection(string(pid.Node)) + if err != nil { + m.sendMonitorExit(by, pid, "noconnection", ref) + return err } - if isVirtualPid(terminated) { - // it was monitored by name and this Pid was created using virtualPid(). - processID := virtualPidToProcessID(terminated) - message := etf.Tuple{distProtoMONITOR_EXIT, etf.Atom(processID.Name), to, ref, etf.Atom(reason)} - m.registrar.routeRaw(to.Node, message) - return + + if err := connection.Monitor(by, pid, ref); err != nil { + switch err { + case ErrPeerUnsupported: + m.sendMonitorExit(by, pid, "unsupported", ref) + case ErrProcessIncarnation: + m.sendMonitorExit(by, pid, "incarnation", ref) + default: + m.sendMonitorExit(by, pid, "noconnection", ref) + } + return err } - // terminated is a real Pid. send it as it is. - message := etf.Tuple{distProtoMONITOR_EXIT, terminated, to, ref, etf.Atom(reason)} - m.registrar.routeRaw(to.Node, message) - return } - if isVirtualPid(terminated) { - // it was monitored by name - down := gen.MessageDown{ - Ref: ref, - ProcessID: virtualPidToProcessID(terminated), - Reason: reason, + m.mutexProcesses.Lock() + l := m.processes[pid] + item := monitorItem{ + pid: by, + ref: ref, + } + m.processes[pid] = append(l, item) + m.ref2pid[ref] = pid + m.mutexProcesses.Unlock() + + return nil +} + +func (m *monitor) RouteMonitorReg(by etf.Pid, process gen.ProcessID, ref etf.Ref) error { + // If 'process' belongs to this node and does not exist a gen.MessageDown must be + // send immediately with Reason set to noproc. + if p := m.router.ProcessByName(process.Name); process.Node == m.nodename && p == nil { + return m.sendMonitorExitReg(by, process, "noproc", ref) + } + if process.Node != m.nodename { + connection, err := m.router.getConnection(process.Node) + if err != nil { + m.sendMonitorExitReg(by, process, "noconnection", ref) + return err + } + + if err := connection.MonitorReg(by, process, ref); err != nil { + if err == ErrPeerUnsupported { + m.sendMonitorExitReg(by, process, "unsupported", ref) + } else { + m.sendMonitorExitReg(by, process, "noconnection", ref) + } + return err } - m.registrar.route(terminated, to, down) - return } + + m.mutexNames.Lock() + l := m.names[process] + item := monitorItem{ + pid: by, + ref: ref, + } + m.names[process] = append(l, item) + m.ref2name[ref] = process + m.mutexNames.Unlock() + + return nil +} + +func (m *monitor) RouteDemonitor(by etf.Pid, ref etf.Ref) error { + m.mutexProcesses.RLock() + pid, knownRefByPid := m.ref2pid[ref] + m.mutexProcesses.RUnlock() + + if knownRefByPid == false { + // monitor was created by process name + m.mutexNames.Lock() + defer m.mutexNames.Unlock() + processID, knownRefByName := m.ref2name[ref] + if knownRefByName == false { + // unknown monitor reference + return ErrMonitorUnknown + } + items := m.names[processID] + + for i := range items { + if items[i].pid != by { + continue + } + if items[i].ref != ref { + continue + } + + items[i] = items[0] + items = items[1:] + + if len(items) == 0 { + delete(m.names, processID) + } else { + m.names[processID] = items + } + delete(m.ref2name, ref) + + if processID.Node != m.nodename { + connection, err := m.router.getConnection(processID.Node) + if err != nil { + return err + } + return connection.DemonitorReg(by, processID, ref) + } + return nil + } + return nil + } + + // monitor was created by pid + + // cheching for monitorItem list + m.mutexProcesses.Lock() + defer m.mutexProcesses.Unlock() + items := m.processes[pid] + + // remove PID from monitoring processes list + for i := range items { + if items[i].pid != by { + continue + } + if items[i].ref != ref { + continue + } + + items[i] = items[0] + items = items[1:] + + if len(items) == 0 { + delete(m.processes, pid) + } else { + m.processes[pid] = items + } + delete(m.ref2pid, ref) + + if string(pid.Node) != m.nodename { + connection, err := m.router.getConnection(string(pid.Node)) + if err != nil { + return err + } + return connection.Demonitor(by, pid, ref) + } + + return nil + } + return nil +} + +func (m *monitor) RouteMonitorExit(terminated etf.Pid, reason string, ref etf.Ref) error { + m.mutexProcesses.Lock() + defer m.mutexProcesses.Unlock() + + items, ok := m.processes[terminated] + if !ok { + return nil + } + + for i := range items { + lib.Log("[%s] MONITOR process terminated: %s. send notify to: %s", m.nodename, terminated, items[i].pid) + if items[i].ref != ref { + continue + } + + delete(m.ref2pid, items[i].ref) + m.sendMonitorExit(items[i].pid, terminated, reason, items[i].ref) + + items[i] = items[0] + items = items[1:] + if len(items) == 0 { + delete(m.processes, terminated) + return nil + } + m.processes[terminated] = items + return nil + } + + return nil +} + +func (m *monitor) RouteMonitorExitReg(terminated gen.ProcessID, reason string, ref etf.Ref) error { + m.mutexNames.Lock() + defer m.mutexNames.Unlock() + + items, ok := m.names[terminated] + if !ok { + return nil + } + + for i := range items { + lib.Log("[%s] MONITOR process terminated: %s. send notify to: %s", m.nodename, terminated, items[i].pid) + if items[i].ref != ref { + continue + } + + delete(m.ref2name, items[i].ref) + m.sendMonitorExitReg(items[i].pid, terminated, reason, items[i].ref) + + items[i] = items[0] + items = items[1:] + if len(items) == 0 { + delete(m.names, terminated) + return nil + } + m.names[terminated] = items + return nil + } + + return nil +} + +func (m *monitor) sendMonitorExit(to etf.Pid, terminated etf.Pid, reason string, ref etf.Ref) error { + if string(to.Node) != m.nodename { + // remote + if reason == "noconnection" { + // do nothing. it was a monitor created by the remote node we lost connection to. + return nil + } + + connection, err := m.router.getConnection(string(to.Node)) + if err != nil { + return err + } + + return connection.MonitorExit(to, terminated, reason, ref) + } + + // local down := gen.MessageDown{ Ref: ref, Pid: terminated, Reason: reason, } - m.registrar.route(terminated, to, down) + from := to + return m.router.RouteSend(from, to, down) } -func (m *monitor) notifyProcessExit(to etf.Pid, terminated etf.Pid, reason string) { - // for remote: {3, FromPid, ToPid, Reason} - if to.Node != etf.Atom(m.registrar.NodeName()) { +func (m *monitor) sendMonitorExitReg(to etf.Pid, terminated gen.ProcessID, reason string, ref etf.Ref) error { + if string(to.Node) != m.nodename { + // remote if reason == "noconnection" { - return + // do nothing + return nil } - message := etf.Tuple{distProtoEXIT, terminated, to, etf.Atom(reason)} - m.registrar.routeRaw(to.Node, message) - return - } - // check if 'to' process is still alive. otherwise ignore this event - if p := m.registrar.getProcessByPid(to); p != nil && p.IsAlive() { - p.exit(terminated, reason) + connection, err := m.router.getConnection(string(to.Node)) + if err != nil { + return err + } + + return connection.MonitorExitReg(to, terminated, reason, ref) } -} -func virtualPid(p gen.ProcessID) etf.Pid { - pid := etf.Pid{} - pid.Node = etf.Atom(p.Name + "|" + p.Node) // registered process name - pid.ID = math.MaxUint64 - pid.Creation = math.MaxUint32 - return pid + // local + down := gen.MessageDown{ + Ref: ref, + ProcessID: terminated, + Reason: reason, + } + from := to + return m.router.RouteSend(from, to, down) } -func virtualPidToProcessID(pid etf.Pid) gen.ProcessID { - s := strings.Split(string(pid.Node), "|") - if len(s) != 2 { - return gen.ProcessID{} +func (m *monitor) sendExit(to etf.Pid, terminated etf.Pid, reason string) error { + // for remote: {3, FromPid, ToPid, Reason} + if to.Node != etf.Atom(m.nodename) { + if reason == "noconnection" { + return nil + } + connection, err := m.router.getConnection(string(to.Node)) + if err != nil { + return err + } + return connection.LinkExit(to, terminated, reason) } - return gen.ProcessID{s[0], s[1]} -} -func isVirtualPid(pid etf.Pid) bool { - if pid.ID == math.MaxUint64 && pid.Creation == math.MaxUint32 { - return true + // check if 'to' process is still alive + if p := m.router.processByPid(to); p != nil { + p.exit(terminated, reason) + return nil } - return false + return ErrProcessUnknown } diff --git a/node/network.go b/node/network.go index ec2ae1a7..ba6aa52d 100644 --- a/node/network.go +++ b/node/network.go @@ -3,559 +3,1399 @@ package node import ( "bytes" "context" + "encoding/pem" + "math/big" + "sync" + "time" + + "crypto/aes" "crypto/ecdsa" "crypto/elliptic" + "crypto/md5" "crypto/rand" - "sync" - - //"crypto/rsa" + "crypto/rsa" + "crypto/sha256" "crypto/tls" "crypto/x509" "crypto/x509/pkix" - "encoding/pem" "fmt" - "runtime" "github.com/ergo-services/ergo/etf" "github.com/ergo-services/ergo/gen" "github.com/ergo-services/ergo/lib" - "github.com/ergo-services/ergo/node/dist" - - "math/big" "net" - // "net/http" "strconv" "strings" - "time" -) - -const ( - remoteBehaviorGroup = "ergo:remote" ) type networkInternal interface { - Network - connect(to string) error + // add/remove static route + AddStaticRoute(node string, host string, port uint16, options RouteOptions) error + AddStaticRoutePort(node string, port uint16, options RouteOptions) error + AddStaticRouteOptions(node string, options RouteOptions) error + RemoveStaticRoute(node string) bool + StaticRoutes() []Route + StaticRoute(name string) (Route, bool) + + // add/remove proxy route + AddProxyRoute(node string, route ProxyRoute) error + RemoveProxyRoute(node string) bool + ProxyRoutes() []ProxyRoute + ProxyRoute(name string) (ProxyRoute, bool) + + Resolve(peername string) (Route, error) + Connect(peername string) error + Disconnect(peername string) error + Nodes() []string + NodesIndirect() []string + + // core router methods + RouteProxyConnectRequest(from ConnectionInterface, request ProxyConnectRequest) error + RouteProxyConnectReply(from ConnectionInterface, reply ProxyConnectReply) error + RouteProxyConnectCancel(from ConnectionInterface, cancel ProxyConnectCancel) error + RouteProxyDisconnect(from ConnectionInterface, disconnect ProxyDisconnect) error + RouteProxy(from ConnectionInterface, sessionID string, packet *lib.Buffer) error + + getConnection(peername string) (ConnectionInterface, error) + stopNetwork() +} + +type connectionInternal struct { + // conn. has nil value for the proxy connection + conn net.Conn + // connection interface of the network connection + connection ConnectionInterface + // + proxySessionID string } type network struct { - registrar registrarInternal - name string - opts Options - ctx context.Context - remoteSpawnMutex sync.Mutex + nodename string + cookie string + ctx context.Context + listener net.Listener + + resolver Resolver + staticOnly bool + staticRoutes map[string]Route + staticRoutesMutex sync.RWMutex + + proxyRoutes map[string]ProxyRoute + proxyRoutesMutex sync.RWMutex + + connections map[string]connectionInternal + connectionsProxy map[ConnectionInterface][]string // peers via proxy + connectionsTransit map[ConnectionInterface][]string // transit session IDs + connectionsMutex sync.RWMutex + + proxyTransitSessions map[string]proxyTransitSession + proxyTransitSessionsMutex sync.RWMutex + + proxyConnectRequest map[etf.Ref]proxyConnectRequest + proxyConnectRequestMutex sync.RWMutex + + tls TLS + proxy Proxy + version Version + creation uint32 + + router coreRouterInternal + handshake HandshakeInterface + proto ProtoInterface + remoteSpawn map[string]gen.ProcessBehavior - epmd *epmd - tlscertServer tls.Certificate - tlscertClient tls.Certificate + remoteSpawnMutex sync.Mutex } -func newNetwork(ctx context.Context, name string, opts Options, r registrarInternal) (networkInternal, error) { +func newNetwork(ctx context.Context, nodename string, cookie string, options Options, router coreRouterInternal) (networkInternal, error) { n := &network{ - name: name, - opts: opts, - ctx: ctx, - registrar: r, + nodename: nodename, + cookie: cookie, + ctx: ctx, + staticOnly: options.StaticRoutesOnly, + staticRoutes: make(map[string]Route), + proxyRoutes: make(map[string]ProxyRoute), + connections: make(map[string]connectionInternal), + connectionsProxy: make(map[ConnectionInterface][]string), + connectionsTransit: make(map[ConnectionInterface][]string), + proxyTransitSessions: make(map[string]proxyTransitSession), + proxyConnectRequest: make(map[etf.Ref]proxyConnectRequest), + remoteSpawn: make(map[string]gen.ProcessBehavior), + proxy: options.Proxy, + resolver: options.Resolver, + handshake: options.Handshake, + proto: options.Proto, + router: router, + creation: options.Creation, } - ns := strings.Split(name, "@") - if len(ns) != 2 { + + nn := strings.Split(nodename, "@") + if len(nn) != 2 { return nil, fmt.Errorf("(EMPD) FQDN for node name is required (example: node@hostname)") } - port, err := n.listen(ctx, ns[1]) + n.version, _ = options.Env[EnvKeyVersion].(Version) + if n.proxy.Flags.Enable == false { + n.proxy.Flags = DefaultProxyFlags() + } + + n.tls = options.TLS + selfSignedCert, err := generateSelfSignedCert(n.version) + if n.tls.Server.Certificate == nil { + n.tls.Server = selfSignedCert + n.tls.SkipVerify = true + } + if n.tls.Client.Certificate == nil { + n.tls.Client = selfSignedCert + } + + err = n.handshake.Init(n.nodename, n.creation, options.Flags) if err != nil { return nil, err } - n.epmd = &epmd{} - if err := n.epmd.Init(ctx, name, port, opts); err != nil { + + port, err := n.listen(ctx, nn[1], options.ListenBegin, options.ListenEnd) + if err != nil { return nil, err } + + resolverOptions := ResolverOptions{ + NodeVersion: n.version, + HandshakeVersion: n.handshake.Version(), + EnableTLS: n.tls.Enable, + EnableProxy: options.Flags.EnableProxy, + EnableCompression: options.Flags.EnableCompression, + } + if err := n.resolver.Register(n.ctx, nodename, port, resolverOptions); err != nil { + return nil, err + } + return n, nil } -// AddStaticRoute adds static route record into the EPMD client -func (n *network) AddStaticRoute(name string, port uint16) error { - tlsEnabled := n.opts.TLSMode != TLSModeDisabled - return n.epmd.AddStaticRoute(name, port, n.opts.cookie, tlsEnabled) +func (n *network) stopNetwork() { + if n.listener != nil { + n.listener.Close() + } + n.connectionsMutex.RLock() + defer n.connectionsMutex.RUnlock() + for _, ci := range n.connections { + if ci.conn == nil { + continue + } + ci.conn.Close() + } } -func (n *network) AddStaticRouteExt(name string, port uint16, cookie string, tls bool) error { - return n.epmd.AddStaticRoute(name, port, cookie, tls) +// AddStaticRouteOptions adds static options for the given node. +func (n *network) AddStaticRouteOptions(node string, options RouteOptions) error { + if n.staticOnly { + return fmt.Errorf("can't be used if enabled StaticRoutesOnly") + } + return n.AddStaticRoute(node, "", 0, options) } -// RemoveStaticRoute removes static route record from the EPMD client -func (n *network) RemoveStaticRoute(name string) { - n.epmd.RemoveStaticRoute(name) +// AddStaticRoutePort adds a static route to the node with the given name +func (n *network) AddStaticRoutePort(node string, port uint16, options RouteOptions) error { + ns := strings.Split(node, "@") + if port < 1 { + return fmt.Errorf("port must be greater 0") + } + if len(ns) != 2 { + return fmt.Errorf("wrong FQDN") + } + return n.AddStaticRoute(node, ns[1], port, options) + } -func (n *network) listen(ctx context.Context, name string) (uint16, error) { - var TLSenabled bool = true - var version Version - version, _ = ctx.Value("version").(Version) +// AddStaticRoute adds a static route to the node with the given name +func (n *network) AddStaticRoute(node string, host string, port uint16, options RouteOptions) error { + if len(strings.Split(node, "@")) != 2 { + return fmt.Errorf("wrong FQDN") + } - lc := net.ListenConfig{} - for p := n.opts.ListenRangeBegin; p <= n.opts.ListenRangeEnd; p++ { - l, err := lc.Listen(ctx, "tcp", net.JoinHostPort(name, strconv.Itoa(int(p)))) - if err != nil { - continue + if port > 0 { + if _, err := net.LookupHost(host); err != nil { + return err } + } - switch n.opts.TLSMode { - case TLSModeAuto: - cert, err := generateSelfSignedCert(version) - if err != nil { - return 0, fmt.Errorf("Can't generate certificate: %s\n", err) + route := Route{ + Node: node, + Host: host, + Port: port, + Options: options, + } + + n.staticRoutesMutex.Lock() + defer n.staticRoutesMutex.Unlock() + + _, exist := n.staticRoutes[node] + if exist { + return ErrTaken + } + n.staticRoutes[node] = route + + return nil +} + +// RemoveStaticRoute removes static route record. Returns false if it doesn't exist. +func (n *network) RemoveStaticRoute(node string) bool { + n.staticRoutesMutex.Lock() + defer n.staticRoutesMutex.Unlock() + _, exist := n.staticRoutes[node] + if exist { + delete(n.staticRoutes, node) + return true + } + return false +} + +// StaticRoutes returns list of static routes added with AddStaticRoute +func (n *network) StaticRoutes() []Route { + var routes []Route + + n.staticRoutesMutex.RLock() + defer n.staticRoutesMutex.RUnlock() + for _, v := range n.staticRoutes { + routes = append(routes, v) + } + + return routes +} + +func (n *network) StaticRoute(name string) (Route, bool) { + n.staticRoutesMutex.RLock() + defer n.staticRoutesMutex.RUnlock() + route, exist := n.staticRoutes[name] + return route, exist +} + +func (n *network) getConnectionDirect(peername string, connect bool) (ConnectionInterface, error) { + n.connectionsMutex.RLock() + ci, ok := n.connections[peername] + n.connectionsMutex.RUnlock() + if ok { + return ci.connection, nil + } + + if connect == false { + return nil, ErrNoRoute + } + + connection, err := n.connect(peername) + if err != nil { + lib.Log("[%s] CORE no route to node %q: %s", n.nodename, peername, err) + return nil, ErrNoRoute + } + return connection, nil + +} + +// getConnection +func (n *network) getConnection(peername string) (ConnectionInterface, error) { + if peername == n.nodename { + // can't connect to itself + return nil, ErrNoRoute + } + n.connectionsMutex.RLock() + ci, ok := n.connections[peername] + n.connectionsMutex.RUnlock() + if ok { + return ci.connection, nil + } + + // try to connect via proxy if there ProxyRoute was presented for this peer + request := ProxyConnectRequest{ + ID: n.router.MakeRef(), + To: peername, + Creation: n.creation, + } + + if err := n.RouteProxyConnectRequest(nil, request); err != nil { + if err != ErrProxyNoRoute { + return nil, err + } + + // there wasn't proxy presented. try to connect directly. + connection, err := n.getConnectionDirect(peername, true) + return connection, err + } + + connection, err := n.waitProxyConnection(request.ID, 5) + if err != nil { + return nil, err + } + + return connection, nil +} + +// Resolve +func (n *network) Resolve(node string) (Route, error) { + n.staticRoutesMutex.Lock() + defer n.staticRoutesMutex.Unlock() + + if r, ok := n.staticRoutes[node]; ok { + if r.Port == 0 { + // use static option for this route + route, err := n.resolver.Resolve(node) + route.Options = r.Options + return route, err + } + return r, nil + } + + if n.staticOnly { + return Route{}, ErrNoRoute + } + + return n.resolver.Resolve(node) +} + +// Connect +func (n *network) Connect(node string) error { + _, err := n.getConnection(node) + return err +} + +// Disconnect +func (n *network) Disconnect(node string) error { + n.connectionsMutex.RLock() + ci, ok := n.connections[node] + n.connectionsMutex.RUnlock() + if !ok { + return ErrNoRoute + } + + if ci.conn == nil { + // this is proxy connection + disconnect := ProxyDisconnect{ + Node: n.nodename, + Proxy: n.nodename, + SessionID: ci.proxySessionID, + Reason: "normal", + } + n.unregisterConnection(node, &disconnect) + return ci.connection.ProxyDisconnect(disconnect) + } + + ci.conn.Close() + return nil +} + +// Nodes +func (n *network) Nodes() []string { + list := []string{} + n.connectionsMutex.RLock() + defer n.connectionsMutex.RUnlock() + + for node := range n.connections { + list = append(list, node) + } + return list +} + +func (n *network) NodesIndirect() []string { + list := []string{} + n.connectionsMutex.RLock() + defer n.connectionsMutex.RUnlock() + + for node, ci := range n.connections { + if ci.conn == nil { + list = append(list, node) + } + } + return list + +} + +// RouteProxyConnectRequest +func (n *network) RouteProxyConnectRequest(from ConnectionInterface, request ProxyConnectRequest) error { + // check if we have proxy route + n.proxyRoutesMutex.RLock() + route, has_route := n.proxyRoutes[request.To] + n.proxyRoutesMutex.RUnlock() + + if request.To != n.nodename { + var connection ConnectionInterface + var err error + + if from != nil { + // + // transit request + // + + lib.Log("[%s] NETWORK transit proxy connection to %q via %q", n.nodename, request.To, route.Proxy) + // proxy feature must be enabled explicitly for the transitional requests + if n.proxy.Transit == false { + lib.Log("[%s] NETWORK proxy. Proxy feature is disabled on this node", n.nodename) + return ErrProxyTransitDisabled + } + if request.Hop < 1 { + lib.Log("[%s] NETWORK proxy. Error: exceeded hop limit", n.nodename) + return ErrProxyHopExceeded } + request.Hop-- - n.tlscertServer = cert - n.tlscertClient = cert + if len(request.Path) > defaultProxyPathLimit { + return ErrProxyPathTooLong + } - TLSconfig := &tls.Config{ - Certificates: []tls.Certificate{cert}, - InsecureSkipVerify: true, + for i := range request.Path { + if n.nodename != request.Path[i] { + continue + } + lib.Log("[%s] NETWORK proxy. Error: loop detected in proxy path %#v", n.nodename, request.Path) + return ErrProxyLoopDetected } - l = tls.NewListener(l, TLSconfig) - case TLSModeStrict: - certServer, err := tls.LoadX509KeyPair(n.opts.TLScrtServer, n.opts.TLSkeyServer) - if err != nil { - return 0, fmt.Errorf("Can't load server certificate: %s\n", err) + // try to connect to the next-hop node + if has_route == false { + connection, err = n.getConnectionDirect(request.To, true) + } else { + connection, err = n.getConnectionDirect(route.Proxy, true) } - certClient, err := tls.LoadX509KeyPair(n.opts.TLScrtServer, n.opts.TLSkeyServer) + if err != nil { - return 0, fmt.Errorf("Can't load client certificate: %s\n", err) + return err } - n.tlscertServer = certServer - n.tlscertClient = certClient - - TLSconfig := &tls.Config{ - Certificates: []tls.Certificate{certServer}, - ServerName: "localhost", + if from == connection { + lib.Log("[%s] NETWORK proxy. Error: proxy route points to the connection this request came from", n.nodename) + return ErrProxyLoopDetected } - l = tls.NewListener(l, TLSconfig) + request.Path = append([]string{n.nodename}, request.Path...) + return connection.ProxyConnectRequest(request) + } - default: - TLSenabled = false + if has_route == false { + // if it was invoked from getConnection ('from' == nil) there will + // be attempt to make direct connection using getConnectionDirect + return ErrProxyNoRoute } - go func() { - for { - c, err := l.Accept() - lib.Log("[%s] Accepted new connection from %s", n.name, c.RemoteAddr().String()) + // + // initiating proxy connection + // + lib.Log("[%s] NETWORK initiate proxy connection to %q via %q", n.nodename, request.To, route.Proxy) + connection, err = n.getConnectionDirect(route.Proxy, true) + if err != nil { + return err + } - if ctx.Err() != nil { - // Context was canceled - c.Close() - return - } + privKey, _ := rsa.GenerateKey(rand.Reader, 2048) + pubKey := x509.MarshalPKCS1PublicKey(&privKey.PublicKey) + request.PublicKey = pubKey - if err != nil { - lib.Log(err.Error()) - continue - } - handshakeOptions := dist.HandshakeOptions{ - Name: n.name, - Cookie: n.opts.cookie, - TLS: TLSenabled, - Hidden: n.opts.Hidden, - Creation: n.opts.creation, - Version: n.opts.HandshakeVersion, - } + // create digest using nodename, cookie, peername and pubKey + request.Digest = generateProxyDigest(n.nodename, route.Cookie, request.To, pubKey) - link, e := dist.HandshakeAccept(c, handshakeOptions) - if e != nil { - lib.Log("[%s] Can't handshake with %s: %s", n.name, c.RemoteAddr().String(), e) - c.Close() - continue - } + request.Flags = route.Flags + if request.Flags.Enable == false { + request.Flags = n.proxy.Flags + } - // start serving this link - if err := n.serve(ctx, link); err != nil { - lib.Log("Can't serve connection link due to: %s", err) - c.Close() - } + request.Hop = route.MaxHop + if request.Hop < 1 { + request.Hop = DefaultProxyMaxHop + } + request.Creation = n.creation + connectRequest := proxyConnectRequest{ + privateKey: privKey, + request: request, + connection: make(chan ConnectionInterface), + cancel: make(chan ProxyConnectCancel), + } + request.Path = []string{n.nodename} + if err := connection.ProxyConnectRequest(request); err != nil { + return err + } + n.putProxyConnectRequest(connectRequest) + return nil + } - } - }() + // + // handle proxy connect request + // - // return port number this node listenig on for the incoming connections - return p, nil + // check digest + // use the last item in the request.Path as a peername + if len(request.Path) < 2 { + // reply error. there must be atleast 2 nodes - initiating and transit nodes + lib.Log("[%s] NETWORK proxy. Proxy connect request has wrong path (too short)", n.nodename) + return ErrProxyConnect + } + peername := request.Path[len(request.Path)-1] + cookie := n.proxy.Cookie + flags := n.proxy.Flags + if has_route { + cookie = route.Cookie + if route.Flags.Enable == true { + flags = route.Flags + } + } + checkDigest := generateProxyDigest(peername, cookie, n.nodename, request.PublicKey) + if bytes.Equal(request.Digest, checkDigest) == false { + // reply error. digest mismatch + lib.Log("[%s] NETWORK proxy. Proxy connect request has wrong digest", n.nodename) + return ErrProxyConnect } - // all the ports within a given range are taken - return 0, fmt.Errorf("Can't start listener. Port range is taken") -} + // do some encryption magic + pk, err := x509.ParsePKCS1PublicKey(request.PublicKey) + if err != nil { + lib.Log("[%s] NETWORK proxy. Proxy connect request has wrong public key", n.nodename) + return ErrProxyConnect + } + hash := sha256.New() + key := make([]byte, 32) + rand.Read(key) + cipherkey, err := rsa.EncryptOAEP(hash, rand.Reader, pk, key, nil) + if err != nil { + lib.Log("[%s] NETWORK proxy. Proxy connect request. Can't encrypt: %s ", n.nodename, err) + return ErrProxyConnect + } + block, err := aes.NewCipher(key) + if err != nil { + return err + } -func (n *network) ProvideRemoteSpawn(name string, behavior gen.ProcessBehavior) error { - return n.registrar.RegisterBehavior(remoteBehaviorGroup, name, behavior, nil) -} + sessionID := lib.RandomString(32) + digest := generateProxyDigest(n.nodename, n.proxy.Cookie, peername, key) + if flags.Enable == false { + flags = DefaultProxyFlags() + } -func (n *network) RevokeRemoteSpawn(name string) error { - return n.registrar.UnregisterBehavior(remoteBehaviorGroup, name) -} + // if one of the nodes want to use encryption then it must be used by both nodes + if request.Flags.EnableEncryption || flags.EnableEncryption { + request.Flags.EnableEncryption = true + flags.EnableEncryption = true + } -func (n *network) Resolve(name string) (NetworkRoute, error) { - return n.epmd.resolve(name) -} + cInternal := connectionInternal{ + connection: from, + proxySessionID: sessionID, + } + if _, err := n.registerConnection(peername, cInternal); err != nil { + return ErrProxySessionDuplicate + } -func (n *network) serve(ctx context.Context, link *dist.Link) error { - // define the total number of reader/writer goroutines - numHandlers := runtime.GOMAXPROCS(n.opts.ConnectionHandlers) + reply := ProxyConnectReply{ + ID: request.ID, + To: peername, + Digest: digest, + Cipher: cipherkey, + Flags: flags, + Creation: n.creation, + SessionID: sessionID, + Path: request.Path[1:], + } - // do not use shared channels within intencive code parts, impacts on a performance - receivers := struct { - recv []chan *lib.Buffer - n int - i int - }{ - recv: make([]chan *lib.Buffer, numHandlers), - n: numHandlers, + if err := from.ProxyConnectReply(reply); err != nil { + // can't send reply. ignore this connection request + lib.Log("[%s] NETWORK proxy. Proxy connect request. Can't send reply: %s ", n.nodename, err) + n.unregisterConnection(peername, nil) + return ErrProxyConnect } - p := &peer{ - name: link.GetRemoteName(), - send: make([]chan []etf.Term, numHandlers), - n: numHandlers, + session := ProxySession{ + ID: sessionID, + NodeFlags: reply.Flags, + PeerFlags: request.Flags, + PeerName: peername, + Creation: request.Creation, + Block: block, } - if err := n.registrar.registerPeer(p); err != nil { - // duplicate link? - return err + // register proxy session + from.ProxyRegisterSession(session) + return nil +} + +func (n *network) RouteProxyConnectReply(from ConnectionInterface, reply ProxyConnectReply) error { + + n.proxyTransitSessionsMutex.RLock() + _, duplicate := n.proxyTransitSessions[reply.SessionID] + n.proxyTransitSessionsMutex.RUnlock() + + if duplicate { + return ErrProxySessionDuplicate } - // run readers for incoming messages - for i := 0; i < numHandlers; i++ { - // run packet reader/handler routines (decoder) - recv := make(chan *lib.Buffer, n.opts.RecvQueueLength) - receivers.recv[i] = recv - go link.ReadHandlePacket(ctx, recv, n.handleMessage) + if from == nil { + // from value can't be nil + return ErrProxyUnknownRequest } - cacheIsReady := make(chan bool) + if reply.To != n.nodename { + // send this reply further and register this session + if n.proxy.Transit == false { + return ErrProxyTransitDisabled + } - // run link reader routine - go func() { - var err error - var packetLength int - var recv chan *lib.Buffer + if len(reply.Path) == 0 { + return ErrProxyUnknownRequest + } + if len(reply.Path) > defaultProxyPathLimit { + return ErrProxyPathTooLong + } - linkctx, cancel := context.WithCancel(ctx) - defer cancel() + next := reply.Path[0] + connection, err := n.getConnectionDirect(next, false) + if err != nil { + return err + } + if connection == from { + return ErrProxyLoopDetected + } - go func() { - select { - case <-linkctx.Done(): - // if node's context is done - link.Close() + reply.Path = reply.Path[1:] + // check for the looping + for i := range reply.Path { + if reply.Path[i] == next { + return ErrProxyLoopDetected } - }() + } - // initializing atom cache if its enabled - if !n.opts.DisableHeaderAtomCache { - link.SetAtomCache(etf.NewAtomCache(linkctx)) + if err := connection.ProxyConnectReply(reply); err != nil { + return err } - cacheIsReady <- true - defer func() { - link.Close() - n.registrar.unregisterPeer(link.GetRemoteName()) + // register transit proxy session + n.proxyTransitSessionsMutex.Lock() + session := proxyTransitSession{ + a: from, + b: connection, + } + n.proxyTransitSessions[reply.SessionID] = session + n.proxyTransitSessionsMutex.Unlock() + + // keep session id for both connections in order + // to handle connection closing (we should + // send ProxyDisconnect if one of the connection + // was closed) + n.connectionsMutex.Lock() + sessions, _ := n.connectionsTransit[session.a] + sessions = append(sessions, reply.SessionID) + n.connectionsTransit[session.a] = sessions + sessions, _ = n.connectionsTransit[session.b] + sessions = append(sessions, reply.SessionID) + n.connectionsTransit[session.b] = sessions + n.connectionsMutex.Unlock() + return nil + } - // close handlers channel - p.mutex.Lock() - for i := 0; i < numHandlers; i++ { - if p.send[i] != nil { - close(p.send[i]) - } - if receivers.recv[i] != nil { - close(receivers.recv[i]) - } - } - p.mutex.Unlock() - }() + // look up for the request we made earlier + r, found := n.getProxyConnectRequest(reply.ID) + if found == false { + return ErrProxyUnknownRequest + } - b := lib.TakeBuffer() - for { - packetLength, err = link.Read(b) - if err != nil || packetLength == 0 { - // link was closed or got malformed data - if err != nil { - fmt.Println("link was closed", link.GetPeerName(), "error:", err) - } - lib.ReleaseBuffer(b) - return - } + // decrypt cipher key using private key + hash := sha256.New() + key, err := rsa.DecryptOAEP(hash, rand.Reader, r.privateKey, reply.Cipher, nil) + if err != nil { + lib.Log("[%s] CORE route proxy. Proxy connect reply has invalid cipher", n.nodename) + return ErrProxyConnect + } - // take new buffer for the next reading and append the tail (part of the next packet) - b1 := lib.TakeBuffer() - b1.Set(b.B[packetLength:]) - // cut the tail and send it further for handling. - // buffer b has to be released by the reader of - // recv channel (link.ReadHandlePacket) - b.B = b.B[:packetLength] - recv = receivers.recv[receivers.i] + cookie := n.proxy.Cookie + // check if we should use proxy route cookie + n.proxyRoutesMutex.RLock() + route, has_route := n.proxyRoutes[r.request.To] + n.proxyRoutesMutex.RUnlock() + if has_route { + cookie = route.Cookie + } + // check digest + checkDigest := generateProxyDigest(r.request.To, cookie, n.nodename, key) + if bytes.Equal(checkDigest, reply.Digest) == false { + lib.Log("[%s] CORE route proxy. Proxy connect reply has wrong digest", n.nodename) + return ErrProxyConnect + } - recv <- b + block, err := aes.NewCipher(key) + if err != nil { + return err + } + cInternal := connectionInternal{ + connection: from, + proxySessionID: reply.SessionID, + } + if registered, err := n.registerConnection(r.request.To, cInternal); err != nil { + select { + case r.connection <- registered: + } + return ErrProxySessionDuplicate + } + // if one of the nodes want to use encryption then it must be used by both nodes + if r.request.Flags.EnableEncryption || reply.Flags.EnableEncryption { + r.request.Flags.EnableEncryption = true + reply.Flags.EnableEncryption = true + } + + session := ProxySession{ + ID: reply.SessionID, + NodeFlags: r.request.Flags, + PeerFlags: reply.Flags, + PeerName: r.request.To, + Creation: reply.Creation, + Block: block, + } + + // register proxy session + from.ProxyRegisterSession(session) - // set new buffer as a current for the next reading - b = b1 + select { + case r.connection <- from: + } + + return nil +} + +func (n *network) RouteProxyConnectCancel(from ConnectionInterface, cancel ProxyConnectCancel) error { + if from == nil { + // from value can not be nil + return ErrProxyConnect + } + if len(cancel.Path) == 0 { + n.cancelProxyConnectRequest(cancel) + return nil + } - // round-robin switch to the next receiver - receivers.i++ - if receivers.i < receivers.n { + next := cancel.Path[0] + if next != n.nodename { + if len(cancel.Path) > defaultProxyPathLimit { + return ErrProxyPathTooLong + } + connection, err := n.getConnectionDirect(next, false) + if err != nil { + return err + } + + if connection == from { + return ErrProxyLoopDetected + } + + cancel.Path = cancel.Path[1:] + // check for the looping + for i := range cancel.Path { + if cancel.Path[i] == next { + return ErrProxyLoopDetected + } + } + + if err := connection.ProxyConnectCancel(cancel); err != nil { + return err + } + return nil + } + + return ErrProxyUnknownRequest +} + +func (n *network) RouteProxyDisconnect(from ConnectionInterface, disconnect ProxyDisconnect) error { + + n.proxyTransitSessionsMutex.RLock() + session, isTransitSession := n.proxyTransitSessions[disconnect.SessionID] + n.proxyTransitSessionsMutex.RUnlock() + if isTransitSession == false { + // check for the proxy connection endpoint + var peername string + var found bool + var ci connectionInternal + + // get peername by session id + n.connectionsMutex.RLock() + for p, c := range n.connections { + if c.proxySessionID != disconnect.SessionID { continue } - receivers.i = 0 + found = true + peername = p + ci = c + break + } + if found == false { + n.connectionsMutex.RUnlock() + return ErrProxySessionUnknown + } + n.connectionsMutex.RUnlock() + if ci.proxySessionID != disconnect.SessionID || ci.connection != from { + return ErrProxySessionUnknown } - }() - // we should make sure if the cache is ready before we start writers - <-cacheIsReady + n.unregisterConnection(peername, &disconnect) + return nil + } + + n.proxyTransitSessionsMutex.Lock() + delete(n.proxyTransitSessions, disconnect.SessionID) + n.proxyTransitSessionsMutex.Unlock() + + // remove this session from the connections + n.connectionsMutex.Lock() + sessions, ok := n.connectionsTransit[session.a] + if ok { + for i := range sessions { + if sessions[i] == disconnect.SessionID { + sessions[i] = sessions[0] + sessions = sessions[1:] + n.connectionsTransit[session.a] = sessions + break + } + } + } + sessions, ok = n.connectionsTransit[session.b] + if ok { + for i := range sessions { + if sessions[i] == disconnect.SessionID { + sessions[i] = sessions[0] + sessions = sessions[1:] + n.connectionsTransit[session.b] = sessions + break + } + } + } + n.connectionsMutex.Unlock() + + // send this message further + switch from { + case session.b: + return session.a.ProxyDisconnect(disconnect) + case session.a: + return session.b.ProxyDisconnect(disconnect) + default: + // shouldn't happen + panic("internal error") + } +} + +func (n *network) RouteProxy(from ConnectionInterface, sessionID string, packet *lib.Buffer) error { + // check if this session is present on this node + n.proxyTransitSessionsMutex.RLock() + session, ok := n.proxyTransitSessions[sessionID] + n.proxyTransitSessionsMutex.RUnlock() + + if !ok { + return ErrProxySessionUnknown + } + + switch from { + case session.b: + return session.a.ProxyPacket(packet) + case session.a: + return session.b.ProxyPacket(packet) + default: + // shouldn't happen + panic("internal error") + } +} + +func (n *network) AddProxyRoute(node string, route ProxyRoute) error { + n.proxyRoutesMutex.Lock() + defer n.proxyRoutesMutex.Unlock() + if route.MaxHop > defaultProxyPathLimit { + return ErrProxyPathTooLong + } + if route.MaxHop < 1 { + route.MaxHop = DefaultProxyMaxHop + } + + if route.Flags.Enable == false { + route.Flags = n.proxy.Flags + } - // run readers/writers for incoming/outgoing messages - for i := 0; i < numHandlers; i++ { - // run writer routines (encoder) - send := make(chan []etf.Term, n.opts.SendQueueLength) - p.mutex.Lock() - p.send[i] = send - p.mutex.Unlock() - go link.Writer(send, n.opts.FragmentationUnit) + if _, exist := n.proxyRoutes[node]; exist { + return ErrTaken } + n.proxyRoutes[node] = route return nil } -func (n *network) handleMessage(fromNode string, control, message etf.Term) (err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("%s", r) - } - }() - - switch t := control.(type) { - case etf.Tuple: - switch act := t.Element(1).(type) { - case int: - switch act { - case distProtoREG_SEND: - // {6, FromPid, Unused, ToName} - lib.Log("[%s] CONTROL REG_SEND [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - n.registrar.route(t.Element(2).(etf.Pid), t.Element(4), message) - - case distProtoSEND: - // {2, Unused, ToPid} - // SEND has no sender pid - lib.Log("[%s] CONTROL SEND [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - n.registrar.route(etf.Pid{}, t.Element(3), message) - - case distProtoLINK: - // {1, FromPid, ToPid} - lib.Log("[%s] CONTROL LINK [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - n.registrar.link(t.Element(2).(etf.Pid), t.Element(3).(etf.Pid)) - - case distProtoUNLINK: - // {4, FromPid, ToPid} - lib.Log("[%s] CONTROL UNLINK [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - n.registrar.unlink(t.Element(2).(etf.Pid), t.Element(3).(etf.Pid)) - - case distProtoNODE_LINK: - lib.Log("[%s] CONTROL NODE_LINK [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - - case distProtoEXIT: - // {3, FromPid, ToPid, Reason} - lib.Log("[%s] CONTROL EXIT [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - terminated := t.Element(2).(etf.Pid) - reason := fmt.Sprint(t.Element(4)) - n.registrar.processTerminated(terminated, "", string(reason)) - - case distProtoEXIT2: - lib.Log("[%s] CONTROL EXIT2 [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - - case distProtoMONITOR: - // {19, FromPid, ToProc, Ref}, where FromPid = monitoring process - // and ToProc = monitored process pid or name (atom) - lib.Log("[%s] CONTROL MONITOR [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - n.registrar.monitorProcess(t.Element(2).(etf.Pid), t.Element(3), t.Element(4).(etf.Ref)) - - case distProtoDEMONITOR: - // {20, FromPid, ToProc, Ref}, where FromPid = monitoring process - // and ToProc = monitored process pid or name (atom) - lib.Log("[%s] CONTROL DEMONITOR [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - n.registrar.demonitorProcess(t.Element(4).(etf.Ref)) - - case distProtoMONITOR_EXIT: - // {21, FromProc, ToPid, Ref, Reason}, where FromProc = monitored process - // pid or name (atom), ToPid = monitoring process, and Reason = exit reason for the monitored process - lib.Log("[%s] CONTROL MONITOR_EXIT [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - reason := fmt.Sprint(t.Element(5)) - switch terminated := t.Element(2).(type) { - case etf.Pid: - n.registrar.processTerminated(terminated, "", string(reason)) - case etf.Atom: - vpid := virtualPid(gen.ProcessID{string(terminated), fromNode}) - n.registrar.processTerminated(vpid, "", string(reason)) - } +func (n *network) RemoveProxyRoute(node string) bool { + n.proxyRoutesMutex.Lock() + defer n.proxyRoutesMutex.Unlock() + if _, exist := n.proxyRoutes[node]; exist == false { + return false + } + delete(n.proxyRoutes, node) + return true +} - // Not implemented yet, just stubs. TODO. - case distProtoSEND_SENDER: - lib.Log("[%s] CONTROL SEND_SENDER unsupported [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - case distProtoPAYLOAD_EXIT: - lib.Log("[%s] CONTROL PAYLOAD_EXIT unsupported [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - case distProtoPAYLOAD_EXIT2: - lib.Log("[%s] CONTROL PAYLOAD_EXIT2 unsupported [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - case distProtoPAYLOAD_MONITOR_P_EXIT: - lib.Log("[%s] CONTROL PAYLOAD_MONITOR_P_EXIT unsupported [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - - // alias support - case distProtoALIAS_SEND: - // {33, FromPid, Alias} - lib.Log("[%s] CONTROL ALIAS_SEND [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - alias := etf.Alias(t.Element(3).(etf.Ref)) - n.registrar.route(t.Element(2).(etf.Pid), alias, message) - - case distProtoSPAWN_REQUEST: - // {29, ReqId, From, GroupLeader, {Module, Function, Arity}, OptList} - lib.Log("[%s] CONTROL SPAWN_REQUEST [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - registerName := "" - for _, option := range t.Element(6).(etf.List) { - name, ok := option.(etf.Tuple) - if !ok { - break - } - if name.Element(1).(etf.Atom) == etf.Atom("name") { - registerName = string(name.Element(2).(etf.Atom)) - } - } +func (n *network) ProxyRoutes() []ProxyRoute { + var routes []ProxyRoute + n.proxyRoutesMutex.RLock() + defer n.proxyRoutesMutex.RUnlock() + for _, v := range n.proxyRoutes { + routes = append(routes, v) + } + return routes +} + +func (n *network) ProxyRoute(name string) (ProxyRoute, bool) { + n.proxyRoutesMutex.RLock() + defer n.proxyRoutesMutex.RUnlock() + route, exist := n.proxyRoutes[name] + return route, exist +} + +func (n *network) listen(ctx context.Context, hostname string, begin uint16, end uint16) (uint16, error) { - from := t.Element(3).(etf.Pid) - ref := t.Element(2).(etf.Ref) - - mfa := t.Element(5).(etf.Tuple) - module := mfa.Element(1).(etf.Atom) - function := mfa.Element(2).(etf.Atom) - var args etf.List - if str, ok := message.(string); !ok { - args, _ = message.(etf.List) - } else { - // stupid Erlang's strings :). [1,2,3,4,5] sends as a string. - // args can't be anything but etf.List. - for i := range []byte(str) { - args = append(args, str[i]) + lc := net.ListenConfig{ + KeepAlive: defaultKeepAlivePeriod * time.Second, + } + for port := begin; port <= end; port++ { + hostPort := net.JoinHostPort(hostname, strconv.Itoa(int(port))) + listener, err := lc.Listen(ctx, "tcp", hostPort) + if err != nil { + continue + } + if n.tls.Enable { + config := tls.Config{ + Certificates: []tls.Certificate{n.tls.Server}, + InsecureSkipVerify: n.tls.SkipVerify, + } + listener = tls.NewListener(listener, &config) + } + n.listener = listener + + go func() { + for { + c, err := listener.Accept() + if err != nil { + if ctx.Err() == nil { + continue } + lib.Log(err.Error()) + return } + lib.Log("[%s] NETWORK accepted new connection from %s", n.nodename, c.RemoteAddr().String()) - rb, err_behavior := n.registrar.RegisteredBehavior(remoteBehaviorGroup, string(module)) - if err_behavior != nil { - message := etf.Tuple{distProtoSPAWN_REPLY, ref, from, 0, etf.Atom("not_provided")} - n.registrar.routeRaw(from.Node, message) - return + details, err := n.handshake.Accept(c, n.tls.Enable, n.cookie) + if err != nil { + lib.Log("[%s] Can't handshake with %s: %s", n.nodename, c.RemoteAddr().String(), err) + c.Close() + continue } - remote_request := gen.RemoteSpawnRequest{ - Ref: ref, - From: from, - Function: string(function), + // TODO we need to detect somehow whether to enable software keepalive. + // Erlang nodes are required to be receiving keepalive messages, + // but Ergo doesn't need it. + details.Flags.EnableSoftwareKeepAlive = true + connection, err := n.proto.Init(n.ctx, c, n.nodename, details) + if err != nil { + c.Close() + continue } - process_opts := processOptions{} - process_opts.Env = map[string]interface{}{"ergo:RemoteSpawnRequest": remote_request} - process, err_spawn := n.registrar.spawn(registerName, process_opts, rb.Behavior, args...) - if err_spawn != nil { - message := etf.Tuple{distProtoSPAWN_REPLY, ref, from, 0, etf.Atom(err_spawn.Error())} - n.registrar.routeRaw(from.Node, message) - return + cInternal := connectionInternal{ + conn: c, + connection: connection, } - message := etf.Tuple{distProtoSPAWN_REPLY, ref, from, 0, process.Self()} - n.registrar.routeRaw(from.Node, message) - case distProtoSPAWN_REPLY: - // {31, ReqId, To, Flags, Result} - lib.Log("[%s] CONTROL SPAWN_REPLY [from %s]: %#v", n.registrar.NodeName(), fromNode, control) - - to := t.Element(3).(etf.Pid) - process := n.registrar.ProcessByPid(to) - if process == nil { - return + if _, err := n.registerConnection(details.Name, cInternal); err != nil { + // Race condition: + // There must be another goroutine which already created and registered + // connection to this node. + // Close this connection and use the already registered connection + c.Close() + continue } - ref := t.Element(2).(etf.Ref) - //flags := t.Element(4) - process.PutSyncReply(ref, t.Element(5)) - default: - lib.Log("[%s] CONTROL unknown command [from %s]: %#v", n.registrar.NodeName(), fromNode, control) + // run serving connection + go func(ctx context.Context, ci connectionInternal) { + n.proto.Serve(ci.connection, n.router) + n.unregisterConnection(details.Name, nil) + n.proto.Terminate(ci.connection) + ci.conn.Close() + }(ctx, cInternal) + } - default: - err = fmt.Errorf("unsupported message %#v", control) - } + }() + + // return port number this node listenig on for the incoming connections + return port, nil } - return + // all ports within a given range are taken + return 0, fmt.Errorf("Can't start listener. Port range is taken") } -func (n *network) connect(to string) error { - var nr NetworkRoute - var err error +func (n *network) connect(node string) (ConnectionInterface, error) { + var route Route var c net.Conn - if nr, err = n.epmd.resolve(string(to)); err != nil { - return fmt.Errorf("Can't resolve port for %s: %s", to, err) + var err error + var enabledTLS bool + lib.Log("[%s] NETWORK trying to connect to %#v", n.nodename, node) + + // resolve the route + route, err = n.Resolve(node) + if err != nil { + return nil, err } - if nr.Cookie == "" { - nr.Cookie = n.opts.cookie + + HostPort := net.JoinHostPort(route.Host, strconv.Itoa(int(route.Port))) + dialer := net.Dialer{ + KeepAlive: defaultKeepAlivePeriod * time.Second, } - ns := strings.Split(to, "@") - TLSenabled := false + if route.Options.IsErgo == true { + // rely on the route TLS settings if they were defined + if route.Options.EnableTLS { + if route.Options.Cert.Certificate == nil { + // use the local TLS settings + config := tls.Config{ + Certificates: []tls.Certificate{n.tls.Client}, + InsecureSkipVerify: n.tls.SkipVerify, + } + tlsdialer := tls.Dialer{ + NetDialer: &dialer, + Config: &config, + } + c, err = tlsdialer.DialContext(n.ctx, "tcp", HostPort) + } else { + // use the route TLS settings + config := tls.Config{ + Certificates: []tls.Certificate{route.Options.Cert}, + } + tlsdialer := tls.Dialer{ + NetDialer: &dialer, + Config: &config, + } + c, err = tlsdialer.DialContext(n.ctx, "tcp", HostPort) + } + enabledTLS = true - switch n.opts.TLSMode { - case TLSModeAuto: - tlsdialer := tls.Dialer{ - Config: &tls.Config{ - Certificates: []tls.Certificate{n.tlscertClient}, - InsecureSkipVerify: true, - }, + } else { + // TLS disabled on a remote node + c, err = dialer.DialContext(n.ctx, "tcp", HostPort) } - c, err = tlsdialer.DialContext(n.ctx, "tcp", net.JoinHostPort(ns[1], strconv.Itoa(nr.Port))) - TLSenabled = true - case TLSModeStrict: - tlsdialer := tls.Dialer{ - Config: &tls.Config{ - Certificates: []tls.Certificate{n.tlscertClient}, - }, + } else { + // rely on the local TLS settings + if n.tls.Enable { + config := tls.Config{ + Certificates: []tls.Certificate{n.tls.Client}, + InsecureSkipVerify: n.tls.SkipVerify, + } + tlsdialer := tls.Dialer{ + NetDialer: &dialer, + Config: &config, + } + c, err = tlsdialer.DialContext(n.ctx, "tcp", HostPort) + enabledTLS = true + + } else { + c, err = dialer.DialContext(n.ctx, "tcp", HostPort) } - c, err = tlsdialer.DialContext(n.ctx, "tcp", net.JoinHostPort(ns[1], strconv.Itoa(nr.Port))) - TLSenabled = true + } - default: - dialer := net.Dialer{} - c, err = dialer.DialContext(n.ctx, "tcp", net.JoinHostPort(ns[1], strconv.Itoa(nr.Port))) + // check if we couldn't establish a connection with the node + if err != nil { + return nil, err } + // handshake + handshake := route.Options.Handshake + if handshake == nil { + // use default handshake + handshake = n.handshake + } + + cookie := n.cookie + if route.Options.Cookie != "" { + cookie = route.Options.Cookie + } + + details, err := n.handshake.Start(c, enabledTLS, cookie) if err != nil { - lib.Log("Error calling net.Dialer.DialerContext : %s", err.Error()) - return err + c.Close() + return nil, err + } + if details.Name != node { + err := fmt.Errorf("node %q introduced itself as %q", node, details.Name) + lib.Warning("%s", err) + return nil, err + } + + // proto + proto := route.Options.Proto + if proto == nil { + // use default proto + proto = n.proto } - handshakeOptions := dist.HandshakeOptions{ - Name: n.name, - Cookie: nr.Cookie, - TLS: TLSenabled, - Hidden: false, - Creation: n.opts.creation, - Version: n.opts.HandshakeVersion, + // TODO we need to detect somehow whether to enable software keepalive. + // Erlang nodes are required to be receiving keepalive messages, + // but Ergo doesn't need it. + details.Flags.EnableSoftwareKeepAlive = true + connection, err := n.proto.Init(n.ctx, c, n.nodename, details) + if err != nil { + c.Close() + return nil, err } - link, e := dist.Handshake(c, handshakeOptions) - if e != nil { - return e + cInternal := connectionInternal{ + conn: c, + connection: connection, } - if err := n.serve(n.ctx, link); err != nil { + if registered, err := n.registerConnection(details.Name, cInternal); err != nil { + // Race condition: + // There must be another goroutine which already created and registered + // connection to this node. + // Close this connection and use the already registered one c.Close() - return err + if err == ErrTaken { + return registered, nil + } + return nil, err } - return nil + + // run serving connection + go func(ctx context.Context, ci connectionInternal) { + n.proto.Serve(ci.connection, n.router) + n.unregisterConnection(details.Name, nil) + n.proto.Terminate(ci.connection) + ci.conn.Close() + }(n.ctx, cInternal) + + return connection, nil +} + +func (n *network) registerConnection(peername string, ci connectionInternal) (ConnectionInterface, error) { + lib.Log("[%s] NETWORK registering peer %#v", n.nodename, peername) + n.connectionsMutex.Lock() + defer n.connectionsMutex.Unlock() + + if registered, exist := n.connections[peername]; exist { + // already registered + return registered.connection, ErrTaken + } + n.connections[peername] = ci + if ci.conn == nil { + // this is proxy connection + p, _ := n.connectionsProxy[ci.connection] + p = append(p, peername) + n.connectionsProxy[ci.connection] = p + } + return ci.connection, nil +} + +func (n *network) unregisterConnection(peername string, disconnect *ProxyDisconnect) { + lib.Log("[%s] NETWORK unregistering peer %v", n.nodename, peername) + + n.connectionsMutex.Lock() + ci, exist := n.connections[peername] + if exist == false { + n.connectionsMutex.Unlock() + return + } + delete(n.connections, peername) + n.connectionsMutex.Unlock() + + n.router.RouteNodeDown(peername, disconnect) + + if ci.conn == nil { + // it was proxy connection + ci.connection.ProxyUnregisterSession(ci.proxySessionID) + return + } + + n.connectionsMutex.Lock() + cp, _ := n.connectionsProxy[ci.connection] + for _, p := range cp { + lib.Log("[%s] NETWORK unregistering peer (via proxy) %v", n.nodename, p) + delete(n.connections, p) + } + + ct, _ := n.connectionsTransit[ci.connection] + delete(n.connectionsTransit, ci.connection) + n.connectionsMutex.Unlock() + + // send disconnect for the proxy sessions + for _, p := range cp { + disconnect := ProxyDisconnect{ + Node: peername, + Proxy: n.nodename, + Reason: "noconnection", + } + n.router.RouteNodeDown(p, &disconnect) + } + + // disconnect for the transit proxy sessions + for i := range ct { + disconnect := ProxyDisconnect{ + Node: peername, + Proxy: n.nodename, + SessionID: ct[i], + Reason: "noconnection", + } + n.RouteProxyDisconnect(ci.connection, disconnect) + } + +} + +// +// Connection interface default callbacks +// +func (c *Connection) Send(from gen.Process, to etf.Pid, message etf.Term) error { + return ErrUnsupported +} +func (c *Connection) SendReg(from gen.Process, to gen.ProcessID, message etf.Term) error { + return ErrUnsupported +} +func (c *Connection) SendAlias(from gen.Process, to etf.Alias, message etf.Term) error { + return ErrUnsupported +} +func (c *Connection) Link(local gen.Process, remote etf.Pid) error { + return ErrUnsupported +} +func (c *Connection) Unlink(local gen.Process, remote etf.Pid) error { + return ErrUnsupported +} +func (c *Connection) LinkExit(local etf.Pid, remote etf.Pid, reason string) error { + return ErrUnsupported +} +func (c *Connection) Monitor(local gen.Process, remote etf.Pid, ref etf.Ref) error { + return ErrUnsupported +} +func (c *Connection) MonitorReg(local gen.Process, remote gen.ProcessID, ref etf.Ref) error { + return ErrUnsupported +} +func (c *Connection) Demonitor(by etf.Pid, process etf.Pid, ref etf.Ref) error { + return ErrUnsupported +} +func (c *Connection) DemonitorReg(by etf.Pid, process gen.ProcessID, ref etf.Ref) error { + return ErrUnsupported +} +func (c *Connection) MonitorExitReg(process gen.Process, reason string, ref etf.Ref) error { + return ErrUnsupported +} +func (c *Connection) MonitorExit(to etf.Pid, terminated etf.Pid, reason string, ref etf.Ref) error { + return ErrUnsupported +} +func (c *Connection) SpawnRequest(nodeName string, behaviorName string, request gen.RemoteSpawnRequest, args ...etf.Term) error { + return ErrUnsupported +} +func (c *Connection) SpawnReply(to etf.Pid, ref etf.Ref, pid etf.Pid) error { + return ErrUnsupported +} +func (c *Connection) SpawnReplyError(to etf.Pid, ref etf.Ref, err error) error { + return ErrUnsupported +} +func (c *Connection) ProxyConnectRequest(connect ProxyConnectRequest) error { + return ErrUnsupported +} +func (c *Connection) ProxyConnectReply(reply ProxyConnectReply) error { + return ErrUnsupported +} +func (c *Connection) ProxyDisconnect(disconnect ProxyDisconnect) error { + return ErrUnsupported +} +func (c *Connection) ProxyRegisterSession(session ProxySession) error { + return ErrUnsupported +} +func (c *Connection) ProxyUnregisterSession(id string) error { + return ErrUnsupported +} +func (c *Connection) ProxyPacket(packet *lib.Buffer) error { + return ErrUnsupported +} + +// +// Handshake interface default callbacks +// +func (h *Handshake) Start(c net.Conn) (Flags, error) { + return Flags{}, ErrUnsupported +} +func (h *Handshake) Accept(c net.Conn) (string, Flags, error) { + return "", Flags{}, ErrUnsupported +} +func (h *Handshake) Version() HandshakeVersion { + var v HandshakeVersion + return v +} + +func (n *network) putProxyConnectRequest(r proxyConnectRequest) { + n.proxyConnectRequestMutex.Lock() + defer n.proxyConnectRequestMutex.Unlock() + n.proxyConnectRequest[r.request.ID] = r +} + +func (n *network) cancelProxyConnectRequest(cancel ProxyConnectCancel) { + n.proxyConnectRequestMutex.Lock() + defer n.proxyConnectRequestMutex.Unlock() + + r, found := n.proxyConnectRequest[cancel.ID] + if found == false { + return + } + + delete(n.proxyConnectRequest, cancel.ID) + select { + case r.cancel <- cancel: + default: + } + return +} + +func (n *network) waitProxyConnection(id etf.Ref, timeout int) (ConnectionInterface, error) { + n.proxyConnectRequestMutex.RLock() + r, found := n.proxyConnectRequest[id] + n.proxyConnectRequestMutex.RUnlock() + + if found == false { + return nil, ErrProxyUnknownRequest + } + + defer func(id etf.Ref) { + n.proxyConnectRequestMutex.Lock() + delete(n.proxyConnectRequest, id) + n.proxyConnectRequestMutex.Unlock() + }(id) + + timer := lib.TakeTimer() + defer lib.ReleaseTimer(timer) + timer.Reset(time.Second * time.Duration(timeout)) + + for { + select { + case connection := <-r.connection: + return connection, nil + case err := <-r.cancel: + return nil, fmt.Errorf("[%s] %s", err.From, err.Reason) + case <-timer.C: + return nil, ErrTimeout + case <-n.ctx.Done(): + // node is on the way to terminate, it means connection is closed + // so it doesn't matter what kind of error will be returned + return nil, ErrProxyUnknownRequest + } + } +} + +func (n *network) getProxyConnectRequest(id etf.Ref) (proxyConnectRequest, bool) { + n.proxyConnectRequestMutex.RLock() + defer n.proxyConnectRequestMutex.RUnlock() + r, found := n.proxyConnectRequest[id] + return r, found +} + +// +// internals +// + +func generateProxyDigest(node string, cookie string, peer string, pubkey []byte) []byte { + // md5(md5(md5(md5(node)+cookie)+peer)+pubkey) + digest1 := md5.Sum([]byte(node)) + digest2 := md5.Sum(append(digest1[:], []byte(cookie)...)) + digest3 := md5.Sum(append(digest2[:], []byte(peer)...)) + digest4 := md5.Sum(append(digest3[:], pubkey...)) + return digest4[:] } func generateSelfSignedCert(version Version) (tls.Certificate, error) { @@ -608,27 +1448,3 @@ func generateSelfSignedCert(version Version) (tls.Certificate, error) { return tls.X509KeyPair(certPEM.Bytes(), certPrivKeyPEM.Bytes()) } - -type peer struct { - name string - send []chan []etf.Term - i int - n int - - mutex sync.Mutex -} - -func (p *peer) getChannel() chan []etf.Term { - p.mutex.Lock() - defer p.mutex.Unlock() - - c := p.send[p.i] - - p.i++ - if p.i < p.n { - return c - } - - p.i = 0 - return c -} diff --git a/node/node.go b/node/node.go index 4dfd4e28..3d3c3730 100644 --- a/node/node.go +++ b/node/node.go @@ -3,6 +3,7 @@ package node import ( "context" "fmt" + "runtime" "strings" "time" @@ -12,109 +13,94 @@ import ( ) const ( - appBehaviorGroup = "ergo:applications" + appBehaviorGroup = "ergo:applications" + remoteBehaviorGroup = "ergo:remote" ) -type nodeInternal interface { - Node - registrarInternal -} - // node instance of created node using CreateNode type node struct { - registrarInternal - networkInternal + coreInternal name string - cookie string creation uint32 - opts Options context context.Context stop context.CancelFunc version Version } // StartWithContext create new node with specified context, name and cookie string -func StartWithContext(ctx context.Context, name string, cookie string, opts Options) (nodeInternal, error) { - - lib.Log("Start with name '%s' and cookie '%s'", name, cookie) - nodectx, nodestop := context.WithCancel(ctx) - - // Creation must be > 0 so make 'or 0x1' - creation := uint32(time.Now().Unix()) | 1 +func StartWithContext(ctx context.Context, name string, cookie string, opts Options) (Node, error) { - node := &node{ - cookie: cookie, - context: nodectx, - stop: nodestop, - creation: creation, - } + lib.Log("Start node with name %q and cookie %q", name, cookie) - if name == "" { - return nil, fmt.Errorf("Node name must be defined") - } - // set defaults - if opts.ListenRangeBegin == 0 { - opts.ListenRangeBegin = defaultListenRangeBegin + if len(strings.Split(name, "@")) != 2 { + return nil, fmt.Errorf("incorrect FQDN node name (example: node@localhost)") } - if opts.ListenRangeEnd == 0 { - opts.ListenRangeEnd = defaultListenRangeEnd + if opts.Creation == 0 { + opts.Creation = uint32(time.Now().Unix()) } - lib.Log("Listening range: %d...%d", opts.ListenRangeBegin, opts.ListenRangeEnd) - if opts.EPMDPort == 0 { - opts.EPMDPort = defaultEPMDPort - } - if opts.EPMDPort != 4369 { - lib.Log("Using custom EPMD port: %d", opts.EPMDPort) + if opts.Flags.Enable == false { + opts.Flags = DefaultFlags() } - if opts.SendQueueLength == 0 { - opts.SendQueueLength = defaultSendQueueLength + // set defaults listening port range + if opts.Listen > 0 { + opts.ListenBegin = opts.Listen + opts.ListenEnd = opts.Listen + lib.Log("Node listening port: %d", opts.Listen) + } else { + if opts.ListenBegin == 0 { + opts.ListenBegin = defaultListenBegin + } + if opts.ListenEnd == 0 { + opts.ListenEnd = defaultListenEnd + } + lib.Log("Node listening range: %d...%d", opts.ListenBegin, opts.ListenEnd) } - if opts.RecvQueueLength == 0 { - opts.RecvQueueLength = defaultRecvQueueLength + if opts.Handshake == nil { + return nil, fmt.Errorf("Handshake must be defined") } - - if opts.FragmentationUnit < 1500 { - opts.FragmentationUnit = defaultFragmentationUnit + if opts.Proto == nil { + return nil, fmt.Errorf("Proto must be defined") } - - // must be 5 or 6 - if opts.HandshakeVersion != 5 && opts.HandshakeVersion != 6 { - opts.HandshakeVersion = defaultHandshakeVersion + if opts.StaticRoutesOnly == false && opts.Resolver == nil { + return nil, fmt.Errorf("Resolver must be defined if StaticRoutesOnly == false") } - if opts.Hidden { - lib.Log("Running as hidden node") + nodectx, nodestop := context.WithCancel(ctx) + node := &node{ + name: name, + context: nodectx, + stop: nodestop, + creation: opts.Creation, } - if len(strings.Split(name, "@")) != 2 { - return nil, fmt.Errorf("incorrect FQDN node name (example: node@localhost)") + // create a copy of envs + copyEnv := make(map[gen.EnvKey]interface{}) + for k, v := range opts.Env { + copyEnv[k] = v } - opts.cookie = cookie - opts.creation = creation - node.opts = opts - node.name = name + // set global variable 'ergo:Node' + copyEnv[EnvKeyNode] = Node(node) + opts.Env = copyEnv - registrar := newRegistrar(nodectx, name, creation, node) - network, err := newNetwork(nodectx, name, opts, registrar) + core, err := newCore(nodectx, name, cookie, opts) if err != nil { return nil, err } + node.coreInternal = core - node.registrarInternal = registrar - node.networkInternal = network - - // load applications for _, app := range opts.Applications { + // load applications name, err := node.ApplicationLoad(app) if err != nil { nodestop() return nil, err } + // start applications _, err = node.ApplicationStart(name) if err != nil { nodestop() @@ -125,40 +111,12 @@ func StartWithContext(ctx context.Context, name string, cookie string, opts Opti return node, nil } -// IsAlive returns true if node is running -func (n *node) IsAlive() bool { - return n.context.Err() == nil -} - -// Wait waits until node stopped -func (n *node) Wait() { - <-n.context.Done() -} - -// Uptime return uptime in seconds -func (n *node) Uptime() int64 { - return time.Now().Unix() - int64(n.creation) -} - // Version returns version of the node func (n *node) Version() Version { return n.version } -// WaitWithTimeout waits until node stopped. Return ErrTimeout -// if given timeout is exceeded -func (n *node) WaitWithTimeout(d time.Duration) error { - - timer := time.NewTimer(d) - defer timer.Stop() - - select { - case <-timer.C: - return ErrTimeout - case <-n.context.Done(): - return nil - } -} +// Spawn func (n *node) Spawn(name string, opts gen.ProcessOptions, object gen.ProcessBehavior, args ...etf.Term) (gen.Process, error) { // process started by node has no parent options := processOptions{ @@ -167,19 +125,44 @@ func (n *node) Spawn(name string, opts gen.ProcessOptions, object gen.ProcessBeh return n.spawn(name, options, object, args...) } +// RegisterName +func (n *node) RegisterName(name string, pid etf.Pid) error { + return n.registerName(name, pid) +} + +// UnregisterName +func (n *node) UnregisterName(name string) error { + return n.unregisterName(name) +} + +// Stop func (n *node) Stop() { - n.stop() + n.coreStop() } +// Name func (n *node) Name() string { return n.name } -func (n *node) RegisterName(name string, pid etf.Pid) error { - return n.registerName(name, pid) +// IsAlive +func (n *node) IsAlive() bool { + return n.coreIsAlive() } -func (n *node) UnregisterName(name string) error { - return n.unregisterName(name) + +// Uptime +func (n *node) Uptime() int64 { + return n.coreUptime() +} + +// Wait +func (n *node) Wait() { + n.coreWait() +} + +// WaitWithTimeout +func (n *node) WaitWithTimeout(d time.Duration) error { + return n.coreWaitWithTimeout(d) } // LoadedApplications returns a list of loaded applications (including running applications) @@ -328,8 +311,8 @@ func (n *node) applicationStart(startType, appName string, args ...etf.Term) (ge } } - env := map[string]interface{}{ - "spec": spec, + env := map[gen.EnvKey]interface{}{ + gen.EnvKeySpec: spec, } options := gen.ProcessOptions{ Env: env, @@ -369,15 +352,23 @@ func (n *node) ApplicationStop(name string) error { } return nil } + +// Links func (n *node) Links(process etf.Pid) []etf.Pid { return n.processLinks(process) } + +// Monitors func (n *node) Monitors(process etf.Pid) []etf.Pid { return n.processMonitors(process) } + +// MonitorsByName func (n *node) MonitorsByName(process etf.Pid) []gen.ProcessID { return n.processMonitorsByName(process) } + +// MonitoredBy func (n *node) MonitoredBy(process etf.Pid) []etf.Pid { return n.processMonitoredBy(process) } @@ -424,3 +415,50 @@ func (n *node) RevokeRPC(module, function string) error { return nil } + +// ProvideRemoteSpawn +func (n *node) ProvideRemoteSpawn(name string, behavior gen.ProcessBehavior) error { + return n.RegisterBehavior(remoteBehaviorGroup, name, behavior, nil) +} + +// RevokeRemoteSpawn +func (n *node) RevokeRemoteSpawn(name string) error { + return n.UnregisterBehavior(remoteBehaviorGroup, name) +} + +// DefaultFlags +func DefaultFlags() Flags { + // all features are enabled by default + return Flags{ + Enable: true, + EnableHeaderAtomCache: true, + EnableBigCreation: true, + EnableBigPidRef: true, + EnableFragmentation: true, + EnableAlias: true, + EnableRemoteSpawn: true, + EnableCompression: true, + EnableProxy: true, + } +} + +func DefaultProxyFlags() ProxyFlags { + return ProxyFlags{ + Enable: true, + EnableLink: true, + EnableMonitor: true, + EnableRemoteSpawn: true, + EnableEncryption: false, + } +} + +// DefaultProtoOptions +func DefaultProtoOptions() ProtoOptions { + return ProtoOptions{ + NumHandlers: runtime.NumCPU(), + MaxMessageSize: 0, // no limit + SendQueueLength: DefaultProtoSendQueueLength, + RecvQueueLength: DefaultProtoRecvQueueLength, + FragmentationUnit: DefaultProroFragmentationUnit, + } +} diff --git a/node/process.go b/node/process.go index 0c81a3a6..4e238bb9 100644 --- a/node/process.go +++ b/node/process.go @@ -12,17 +12,32 @@ import ( ) const ( + // DefaultProcessMailboxSize DefaultProcessMailboxSize = 100 ) +var ( + syncReplyChannels = &sync.Pool{ + New: func() interface{} { + return make(chan etf.Term, 2) + }, + } + + directChannels = &sync.Pool{ + New: func() interface{} { + return make(chan gen.ProcessDirectMessage, 1) + }, + } +) + type process struct { - registrarInternal + coreInternal sync.RWMutex name string self etf.Pid behavior gen.ProcessBehavior - env map[string]interface{} + env map[gen.EnvKey]interface{} parent *process groupLeader gen.Process @@ -36,10 +51,13 @@ type process struct { kill context.CancelFunc exit processExitFunc - replyMutex sync.Mutex + replyMutex sync.RWMutex reply map[etf.Ref]chan etf.Term - trapExit bool + trapExit bool + compression Compression + + fallback gen.ProcessFallback } type processOptions struct { @@ -49,14 +67,17 @@ type processOptions struct { type processExitFunc func(from etf.Pid, reason string) error +// Self func (p *process) Self() etf.Pid { return p.self } +// Name func (p *process) Name() string { return p.name } +// RegisterName func (p *process) RegisterName(name string) error { if p.behavior == nil { return ErrProcessTerminated @@ -64,6 +85,7 @@ func (p *process) RegisterName(name string) error { return p.registerName(name, p.self) } +// UnregisterName func (p *process) UnregisterName(name string) error { if p.behavior == nil { return ErrProcessTerminated @@ -78,6 +100,7 @@ func (p *process) UnregisterName(name string) error { return p.unregisterName(name) } +// Kill func (p *process) Kill() { if p.behavior == nil { return @@ -85,6 +108,7 @@ func (p *process) Kill() { p.kill() } +// Exit func (p *process) Exit(reason string) error { if p.behavior == nil { return ErrProcessTerminated @@ -92,10 +116,12 @@ func (p *process) Exit(reason string) error { return p.exit(p.self, reason) } +// Context func (p *process) Context() context.Context { return p.context } +// Parent func (p *process) Parent() gen.Process { if p.parent == nil { return nil @@ -103,6 +129,7 @@ func (p *process) Parent() gen.Process { return p.parent } +// GroupLeader func (p *process) GroupLeader() gen.Process { if p.groupLeader == nil { return nil @@ -110,22 +137,32 @@ func (p *process) GroupLeader() gen.Process { return p.groupLeader } +// Links func (p *process) Links() []etf.Pid { return p.processLinks(p.self) } + +// Monitors func (p *process) Monitors() []etf.Pid { return p.processMonitors(p.self) } + +// MonitorsByName func (p *process) MonitorsByName() []gen.ProcessID { return p.processMonitorsByName(p.self) } + +// MonitoredBy func (p *process) MonitoredBy() []etf.Pid { return p.processMonitoredBy(p.self) } + +// Aliases func (p *process) Aliases() []etf.Alias { return p.aliases } +// Info func (p *process) Info() gen.ProcessInfo { if p.behavior == nil { return gen.ProcessInfo{} @@ -151,16 +188,31 @@ func (p *process) Info() gen.ProcessInfo { Status: "running", MessageQueueLen: len(p.mailBox), TrapExit: p.trapExit, + Compression: p.compression.Enable, } } +// Send func (p *process) Send(to interface{}, message etf.Term) error { if p.behavior == nil { return ErrProcessTerminated } - return p.route(p.self, to, message) + switch receiver := to.(type) { + case etf.Pid: + return p.RouteSend(p.self, receiver, message) + case string: + return p.RouteSendReg(p.self, gen.ProcessID{Name: receiver, Node: string(p.self.Node)}, message) + case etf.Atom: + return p.RouteSendReg(p.self, gen.ProcessID{Name: string(receiver), Node: string(p.self.Node)}, message) + case gen.ProcessID: + return p.RouteSendReg(p.self, receiver, message) + case etf.Alias: + return p.RouteSendAlias(p.self, receiver, message) + } + return fmt.Errorf("Unknown receiver type") } +// SendAfter func (p *process) SendAfter(to interface{}, message etf.Term, after time.Duration) context.CancelFunc { //TODO: should we control the number of timers/goroutines have been created this way? ctx, cancel := context.WithCancel(p.context) @@ -175,13 +227,14 @@ func (p *process) SendAfter(to interface{}, message etf.Term, after time.Duratio return case <-timer.C: if p.IsAlive() { - p.route(p.self, to, message) + p.Send(to, message) } } }() return cancel } +// CreateAlias func (p *process) CreateAlias() (etf.Alias, error) { if p.behavior == nil { return etf.Alias{}, ErrProcessTerminated @@ -189,6 +242,7 @@ func (p *process) CreateAlias() (etf.Alias, error) { return p.newAlias(p) } +// DeleteAlias func (p *process) DeleteAlias(alias etf.Alias) error { if p.behavior == nil { return ErrProcessTerminated @@ -196,11 +250,12 @@ func (p *process) DeleteAlias(alias etf.Alias) error { return p.deleteAlias(p, alias) } -func (p *process) ListEnv() map[string]interface{} { +// ListEnv +func (p *process) ListEnv() map[gen.EnvKey]interface{} { p.RLock() defer p.RUnlock() - env := make(map[string]interface{}) + env := make(map[gen.EnvKey]interface{}) if p.groupLeader != nil { for key, value := range p.groupLeader.ListEnv() { @@ -219,7 +274,8 @@ func (p *process) ListEnv() map[string]interface{} { return env } -func (p *process) SetEnv(name string, value interface{}) { +// SetEnv +func (p *process) SetEnv(name gen.EnvKey, value interface{}) { p.Lock() defer p.Unlock() if value == nil { @@ -229,7 +285,8 @@ func (p *process) SetEnv(name string, value interface{}) { p.env[name] = value } -func (p *process) Env(name string) interface{} { +// Env +func (p *process) Env(name gen.EnvKey) interface{} { p.RLock() defer p.RUnlock() @@ -244,12 +301,14 @@ func (p *process) Env(name string) interface{} { return nil } +// Wait func (p *process) Wait() { if p.IsAlive() { <-p.context.Done() } } +// WaitWithTimeout func (p *process) WaitWithTimeout(d time.Duration) error { if !p.IsAlive() { return nil @@ -266,47 +325,107 @@ func (p *process) WaitWithTimeout(d time.Duration) error { } } -func (p *process) Link(with etf.Pid) { +// Link +func (p *process) Link(with etf.Pid) error { if p.behavior == nil { - return + return ErrProcessTerminated } - p.link(p.self, with) + return p.RouteLink(p.self, with) } -func (p *process) Unlink(with etf.Pid) { - p.Lock() - defer p.Unlock() +// Unlink +func (p *process) Unlink(with etf.Pid) error { if p.behavior == nil { - return + return ErrProcessTerminated } - p.unlink(p.self, with) + return p.RouteUnlink(p.self, with) } +// IsAlive func (p *process) IsAlive() bool { - p.Lock() - defer p.Unlock() if p.behavior == nil { return false } return p.context.Err() == nil } +// NodeName +func (p *process) NodeName() string { + return p.coreNodeName() +} + +// NodeStop +func (p *process) NodeStop() { + p.coreStop() +} + +// NodeUptime +func (p *process) NodeUptime() int64 { + return p.coreUptime() +} + +// Children func (p *process) Children() ([]etf.Pid, error) { c, err := p.directRequest(gen.MessageDirectChildren{}, 5) - if err == nil { - return c.([]etf.Pid), nil + if err != nil { + return []etf.Pid{}, err } - return []etf.Pid{}, err + children, correct := c.([]etf.Pid) + if correct == false { + return []etf.Pid{}, err + } + return children, nil } +// SetTrapExit func (p *process) SetTrapExit(trap bool) { p.trapExit = trap } +// TrapExit func (p *process) TrapExit() bool { return p.trapExit } +// SetCompression +func (p *process) SetCompression(enable bool) { + p.compression.Enable = enable +} + +// Compression +func (p *process) Compression() bool { + return p.compression.Enable +} + +// CompressionLevel +func (p *process) CompressionLevel() int { + return p.compression.Level +} + +// SetCompressionLevel +func (p *process) SetCompressionLevel(level int) bool { + if level < 1 || level > 9 { + return false + } + p.compression.Level = level + return true +} + +// CompressionThreshold +func (p *process) CompressionThreshold() int { + return p.compression.Threshold +} + +// SetCompressionThreshold +func (p *process) SetCompressionThreshold(threshold int) bool { + if threshold < DefaultCompressionThreshold { + return false + } + p.compression.Threshold = threshold + return true +} + +// Behavior func (p *process) Behavior() gen.ProcessBehavior { p.Lock() defer p.Unlock() @@ -316,10 +435,12 @@ func (p *process) Behavior() gen.ProcessBehavior { return p.behavior } +// Direct func (p *process) Direct(request interface{}) (interface{}, error) { return p.directRequest(request, gen.DefaultCallTimeout) } +// DirectWithTimeout func (p *process) DirectWithTimeout(request interface{}, timeout int) (interface{}, error) { if timeout < 1 { timeout = 5 @@ -327,41 +448,69 @@ func (p *process) DirectWithTimeout(request interface{}, timeout int) (interface return p.directRequest(request, timeout) } +// MonitorNode func (p *process) MonitorNode(name string) etf.Ref { - return p.monitorNode(p.self, name) + ref := p.MakeRef() + p.monitorNode(p.self, name, ref) + return ref } +// DemonitorNode func (p *process) DemonitorNode(ref etf.Ref) bool { return p.demonitorNode(ref) } +// MonitorProcess func (p *process) MonitorProcess(process interface{}) etf.Ref { ref := p.MakeRef() - p.monitorProcess(p.self, process, ref) + switch mp := process.(type) { + case etf.Pid: + p.RouteMonitor(p.self, mp, ref) + return ref + case gen.ProcessID: + p.RouteMonitorReg(p.self, mp, ref) + return ref + case string: + p.RouteMonitorReg(p.self, gen.ProcessID{Name: mp, Node: string(p.self.Node)}, ref) + return ref + case etf.Atom: + p.RouteMonitorReg(p.self, gen.ProcessID{Name: string(mp), Node: string(p.self.Node)}, ref) + return ref + } + + // create fake gen.ProcessID. Monitor will send MessageDown with "noproc" as a reason + p.RouteMonitorReg(p.self, gen.ProcessID{Node: string(p.self.Node)}, ref) return ref } +// DemonitorProcess func (p *process) DemonitorProcess(ref etf.Ref) bool { - return p.demonitorProcess(ref) + if err := p.RouteDemonitor(p.self, ref); err != nil { + return false + } + return true } +// RemoteSpawn makes request to spawn new process on a remote node func (p *process) RemoteSpawn(node string, object string, opts gen.RemoteSpawnOptions, args ...etf.Term) (etf.Pid, error) { - ref := p.MakeRef() - optlist := etf.List{} - if opts.RegisterName != "" { - optlist = append(optlist, etf.Tuple{etf.Atom("name"), etf.Atom(opts.RegisterName)}) + return p.RemoteSpawnWithTimeout(gen.DefaultCallTimeout, node, object, opts, args...) +} +// RemoteSpawnWithTimeout makes request to spawn new process on a remote node with given timeout +func (p *process) RemoteSpawnWithTimeout(timeout int, node string, object string, opts gen.RemoteSpawnOptions, args ...etf.Term) (etf.Pid, error) { + ref := p.MakeRef() + p.PutSyncRequest(ref) + request := gen.RemoteSpawnRequest{ + From: p.self, + Ref: ref, + Options: opts, + } + if err := p.RouteSpawnRequest(node, object, request, args...); err != nil { + p.CancelSyncRequest(ref) + return etf.Pid{}, err } - if opts.Timeout == 0 { - opts.Timeout = gen.DefaultCallTimeout - } - control := etf.Tuple{distProtoSPAWN_REQUEST, ref, p.self, p.self, - // {M,F,A} - etf.Tuple{etf.Atom(object), etf.Atom(opts.Function), len(args)}, - optlist, - } - p.SendSyncRequestRaw(ref, etf.Atom(node), append([]etf.Term{control}, args)...) - reply, err := p.WaitSyncReply(ref, opts.Timeout) + + reply, err := p.WaitSyncReply(ref, timeout) if err != nil { return etf.Pid{}, err } @@ -374,17 +523,18 @@ func (p *process) RemoteSpawn(node string, object string, opts gen.RemoteSpawnOp case etf.Pid: m := etf.Ref{} // empty reference if opts.Monitor != m { - p.monitorProcess(p.self, r, opts.Monitor) + p.RouteMonitor(p.self, r, opts.Monitor) } if opts.Link { - p.Link(r) + p.RouteLink(p.self, r) } return r, nil case etf.Atom: switch string(r) { case ErrTaken.Error(): return etf.Pid{}, ErrTaken - + case ErrBehaviorUnknown.Error(): + return etf.Pid{}, ErrBehaviorUnknown } return etf.Pid{}, fmt.Errorf(string(r)) } @@ -392,6 +542,7 @@ func (p *process) RemoteSpawn(node string, object string, opts gen.RemoteSpawnOp return etf.Pid{}, fmt.Errorf("unknown result: %#v", reply) } +// Spawn func (p *process) Spawn(name string, opts gen.ProcessOptions, behavior gen.ProcessBehavior, args ...etf.Term) (gen.Process, error) { options := processOptions{ ProcessOptions: opts, @@ -410,7 +561,7 @@ func (p *process) directRequest(request interface{}, timeout int) (interface{}, direct := gen.ProcessDirectMessage{ Message: request, - Reply: make(chan gen.ProcessDirectMessage, 1), + Reply: directChannels.Get().(chan gen.ProcessDirectMessage), } // sending request @@ -424,6 +575,7 @@ func (p *process) directRequest(request interface{}, timeout int) (interface{}, // receiving response select { case response := <-direct.Reply: + directChannels.Put(direct.Reply) if response.Err != nil { return nil, response.Err } @@ -434,51 +586,53 @@ func (p *process) directRequest(request interface{}, timeout int) (interface{}, } } -func (p *process) SendSyncRequestRaw(ref etf.Ref, node etf.Atom, messages ...etf.Term) error { +// PutSyncRequest +func (p *process) PutSyncRequest(ref etf.Ref) { if p.reply == nil { - return ErrProcessTerminated - } - reply := make(chan etf.Term, 2) - p.replyMutex.Lock() - defer p.replyMutex.Unlock() - p.reply[ref] = reply - return p.routeRaw(node, messages...) -} -func (p *process) SendSyncRequest(ref etf.Ref, to interface{}, message etf.Term) error { - if p.reply == nil { - return ErrProcessTerminated + return } + reply := syncReplyChannels.Get().(chan etf.Term) p.replyMutex.Lock() - defer p.replyMutex.Unlock() - - reply := make(chan etf.Term, 2) p.reply[ref] = reply - - return p.Send(to, message) + p.replyMutex.Unlock() } +// PutSyncReply func (p *process) PutSyncReply(ref etf.Ref, reply etf.Term) error { if p.reply == nil { return ErrProcessTerminated } - p.replyMutex.Lock() + + p.replyMutex.RLock() rep, ok := p.reply[ref] - p.replyMutex.Unlock() + p.replyMutex.RUnlock() + if !ok { - // ignored, no process waiting for the reply + // ignore this reply, no process waiting for it return nil } select { case rep <- reply: } - return nil } -func (p *process) WaitSyncReply(ref etf.Ref, timeout int) (etf.Term, error) { +// CancelSyncRequest +func (p *process) CancelSyncRequest(ref etf.Ref) { p.replyMutex.Lock() - reply, wait_for_reply := p.reply[ref] + delete(p.reply, ref) p.replyMutex.Unlock() +} + +// WaitSyncReply +func (p *process) WaitSyncReply(ref etf.Ref, timeout int) (etf.Term, error) { + if p.reply == nil { + return nil, ErrProcessTerminated + } + + p.replyMutex.RLock() + reply, wait_for_reply := p.reply[ref] + p.replyMutex.RUnlock() if !wait_for_reply { return nil, fmt.Errorf("Unknown request") @@ -497,6 +651,7 @@ func (p *process) WaitSyncReply(ref etf.Ref, timeout int) (etf.Term, error) { for { select { case m := <-reply: + syncReplyChannels.Put(reply) return m, nil case <-timer.C: return nil, ErrTimeout @@ -507,6 +662,7 @@ func (p *process) WaitSyncReply(ref etf.Ref, timeout int) (etf.Term, error) { } +// ProcessChannels func (p *process) ProcessChannels() gen.ProcessChannels { return gen.ProcessChannels{ Mailbox: p.mailBox, diff --git a/node/registrar.go b/node/registrar.go deleted file mode 100644 index 6bb64746..00000000 --- a/node/registrar.go +++ /dev/null @@ -1,765 +0,0 @@ -package node - -import ( - "context" - "fmt" - "runtime" - "sync" - "sync/atomic" - "time" - - "github.com/ergo-services/ergo/etf" - "github.com/ergo-services/ergo/gen" - "github.com/ergo-services/ergo/lib" -) - -const ( - startPID = 1000 -) - -type registrar struct { - monitor - ctx context.Context - - nextPID uint64 - uniqID uint64 - nodename string - creation uint32 - - net networkInternal - node nodeInternal - - names map[string]etf.Pid - mutexNames sync.Mutex - aliases map[etf.Alias]*process - mutexAliases sync.Mutex - processes map[uint64]*process - mutexProcesses sync.Mutex - peers map[string]*peer - mutexPeers sync.Mutex - - behaviors map[string]map[string]gen.RegisteredBehavior - mutexBehaviors sync.Mutex -} - -type registrarInternal interface { - gen.Registrar - monitorInternal - - spawn(name string, opts processOptions, behavior gen.ProcessBehavior, args ...etf.Term) (gen.Process, error) - registerName(name string, pid etf.Pid) error - unregisterName(name string) error - registerPeer(peer *peer) error - unregisterPeer(name string) - newAlias(p *process) (etf.Alias, error) - deleteAlias(owner *process, alias etf.Alias) error - getProcessByPid(etf.Pid) *process - - route(from etf.Pid, to etf.Term, message etf.Term) error - routeRaw(nodename etf.Atom, messages ...etf.Term) error -} - -func newRegistrar(ctx context.Context, nodename string, creation uint32, node nodeInternal) registrarInternal { - r := ®istrar{ - ctx: ctx, - nextPID: startPID, - uniqID: uint64(time.Now().UnixNano()), - net: node.(networkInternal), - node: node.(nodeInternal), - nodename: nodename, - creation: creation, - names: make(map[string]etf.Pid), - aliases: make(map[etf.Alias]*process), - processes: make(map[uint64]*process), - peers: make(map[string]*peer), - behaviors: make(map[string]map[string]gen.RegisteredBehavior), - } - r.monitor = newMonitor(r) - return r -} - -func (r *registrar) NodeName() string { - return r.node.Name() -} - -func (r *registrar) NodeStop() { - r.node.Stop() -} - -func (r *registrar) newPID() etf.Pid { - // http://erlang.org/doc/apps/erts/erl_ext_dist.html#pid_ext - // https://stackoverflow.com/questions/243363/can-someone-explain-the-structure-of-a-pid-in-erlang - i := atomic.AddUint64(&r.nextPID, 1) - return etf.Pid{ - Node: etf.Atom(r.nodename), - ID: i, - Creation: r.creation, - } - -} - -// MakeRef returns atomic reference etf.Ref within this node -func (r *registrar) MakeRef() (ref etf.Ref) { - ref.Node = etf.Atom(r.nodename) - ref.Creation = r.creation - nt := atomic.AddUint64(&r.uniqID, 1) - ref.ID[0] = uint32(uint64(nt) & ((2 << 17) - 1)) - ref.ID[1] = uint32(uint64(nt) >> 46) - - return -} - -func (r *registrar) IsAlias(alias etf.Alias) bool { - r.mutexAliases.Lock() - _, ok := r.aliases[alias] - r.mutexAliases.Unlock() - return ok -} - -func (r *registrar) newAlias(p *process) (etf.Alias, error) { - var alias etf.Alias - - // chech if its alive - r.mutexProcesses.Lock() - _, exist := r.processes[p.self.ID] - r.mutexProcesses.Unlock() - if !exist { - return alias, ErrProcessUnknown - } - - alias = etf.Alias(r.MakeRef()) - lib.Log("[%s] REGISTRAR create process alias for %v: %s", r.nodename, p.self, alias) - - r.mutexAliases.Lock() - r.aliases[alias] = p - r.mutexAliases.Unlock() - - p.Lock() - p.aliases = append(p.aliases, alias) - p.Unlock() - return alias, nil -} - -func (r *registrar) deleteAlias(owner *process, alias etf.Alias) error { - lib.Log("[%s] REGISTRAR delete process alias %v for %v", r.nodename, alias, owner.self) - - r.mutexAliases.Lock() - p, alias_exist := r.aliases[alias] - r.mutexAliases.Unlock() - - if !alias_exist { - return ErrAliasUnknown - } - - r.mutexProcesses.Lock() - _, process_exist := r.processes[owner.self.ID] - r.mutexProcesses.Unlock() - - if !process_exist { - return ErrProcessUnknown - } - if p.self != owner.self { - return ErrAliasOwner - } - - p.Lock() - for i := range p.aliases { - if alias != p.aliases[i] { - continue - } - delete(r.aliases, alias) - p.aliases[i] = p.aliases[0] - p.aliases = p.aliases[1:] - p.Unlock() - return nil - } - - p.Unlock() - fmt.Println("Bug: Process lost its alias. Please, report this issue") - r.mutexAliases.Lock() - delete(r.aliases, alias) - r.mutexAliases.Unlock() - - return ErrAliasUnknown -} - -func (r *registrar) newProcess(name string, behavior gen.ProcessBehavior, opts processOptions) (*process, error) { - - var parentContext context.Context - - mailboxSize := DefaultProcessMailboxSize - if opts.MailboxSize > 0 { - mailboxSize = int(opts.MailboxSize) - } - - parentContext = r.ctx - - processContext, kill := context.WithCancel(parentContext) - if opts.Context != nil { - processContext, _ = context.WithCancel(opts.Context) - } - - pid := r.newPID() - - // set global variable 'node' - if opts.Env == nil { - opts.Env = make(map[string]interface{}) - } - opts.Env["ergo:Node"] = r.node.(Node) - - process := &process{ - registrarInternal: r, - - self: pid, - name: name, - behavior: behavior, - env: opts.Env, - - parent: opts.parent, - groupLeader: opts.GroupLeader, - - mailBox: make(chan gen.ProcessMailboxMessage, mailboxSize), - gracefulExit: make(chan gen.ProcessGracefulExitRequest, mailboxSize), - direct: make(chan gen.ProcessDirectMessage), - - context: processContext, - kill: kill, - - reply: make(map[etf.Ref]chan etf.Term), - } - - process.exit = func(from etf.Pid, reason string) error { - lib.Log("[%s] EXIT from %s to %s with reason: %s", r.nodename, from, pid, reason) - if processContext.Err() != nil { - // process is already died - return ErrProcessUnknown - } - - ex := gen.ProcessGracefulExitRequest{ - From: from, - Reason: reason, - } - - // use select just in case if this process isn't been started yet - // or ProcessLoop is already exited (has been set to nil) - // otherwise it cause infinity lock - select { - case process.gracefulExit <- ex: - default: - return ErrProcessBusy - } - - // let the process decide whether to stop itself, otherwise its going to be killed - if !process.trapExit { - process.kill() - } - return nil - } - - if name != "" { - lib.Log("[%s] REGISTRAR registering name (%s): %s", r.nodename, pid, name) - r.mutexNames.Lock() - if _, exist := r.names[name]; exist { - r.mutexNames.Unlock() - return nil, ErrTaken - } - r.names[name] = process.self - r.mutexNames.Unlock() - } - - lib.Log("[%s] REGISTRAR registering process: %s", r.nodename, pid) - r.mutexProcesses.Lock() - r.processes[process.self.ID] = process - r.mutexProcesses.Unlock() - - return process, nil -} - -func (r *registrar) deleteProcess(pid etf.Pid) { - r.mutexProcesses.Lock() - p, exist := r.processes[pid.ID] - if !exist { - r.mutexProcesses.Unlock() - return - } - lib.Log("[%s] REGISTRAR unregistering process: %s", r.nodename, p.self) - delete(r.processes, pid.ID) - r.mutexProcesses.Unlock() - - r.mutexNames.Lock() - if (p.name) != "" { - lib.Log("[%s] REGISTRAR unregistering name (%s): %s", r.nodename, p.self, p.name) - delete(r.names, p.name) - } - - // delete names registered with this pid - for name, pid := range r.names { - if p.self == pid { - delete(r.names, name) - } - } - r.mutexNames.Unlock() - - r.mutexAliases.Lock() - for alias := range r.aliases { - delete(r.aliases, alias) - } - r.mutexAliases.Unlock() - - return -} - -func (r *registrar) spawn(name string, opts processOptions, behavior gen.ProcessBehavior, args ...etf.Term) (gen.Process, error) { - - process, err := r.newProcess(name, behavior, opts) - if err != nil { - return nil, err - } - - initProcess := func() (ps gen.ProcessState, err error) { - if lib.CatchPanic() { - defer func() { - if rcv := recover(); rcv != nil { - pc, fn, line, _ := runtime.Caller(2) - fmt.Printf("Warning: initialization process failed %s[%q] %#v at %s[%s:%d]\n", - process.self, name, rcv, runtime.FuncForPC(pc).Name(), fn, line) - r.deleteProcess(process.self) - err = fmt.Errorf("panic") - } - }() - } - - ps, err = behavior.ProcessInit(process, args...) - return - } - - processState, err := initProcess() - if err != nil { - return nil, err - } - - started := make(chan bool) - defer close(started) - - cleanProcess := func(reason string) { - // set gracefulExit to nil before we start termination handling - process.gracefulExit = nil - r.deleteProcess(process.self) - // invoke cancel context to prevent memory leaks - // and propagate context canelation - process.Kill() - // notify all the linked process and monitors - r.processTerminated(process.self, name, reason) - // make the rest empty - process.Lock() - process.aliases = []etf.Alias{} - - // Do not clean self and name. Sometimes its good to know what pid - // (and what name) was used by the dead process. (gen.Applications is using it) - // process.name = "" - // process.self = etf.Pid{} - - process.behavior = nil - process.parent = nil - process.groupLeader = nil - process.exit = nil - process.kill = nil - process.mailBox = nil - process.direct = nil - process.env = nil - process.reply = nil - process.Unlock() - } - - go func(ps gen.ProcessState) { - if lib.CatchPanic() { - defer func() { - if rcv := recover(); rcv != nil { - pc, fn, line, _ := runtime.Caller(2) - fmt.Printf("Warning: process terminated %s[%q] %#v at %s[%s:%d]\n", - process.self, name, rcv, runtime.FuncForPC(pc).Name(), fn, line) - cleanProcess("panic") - } - }() - } - - // start process loop - reason := behavior.ProcessLoop(ps, started) - // process stopped - cleanProcess(reason) - - }(processState) - - // wait for the starting process loop - <-started - return process, nil -} - -func (r *registrar) registerName(name string, pid etf.Pid) error { - lib.Log("[%s] REGISTRAR registering name %s", r.nodename, name) - r.mutexNames.Lock() - defer r.mutexNames.Unlock() - if _, ok := r.names[name]; ok { - // already registered - return ErrTaken - } - r.names[name] = pid - return nil -} - -func (r *registrar) unregisterName(name string) error { - lib.Log("[%s] REGISTRAR unregistering name %s", r.nodename, name) - r.mutexNames.Lock() - defer r.mutexNames.Unlock() - if _, ok := r.names[name]; ok { - delete(r.names, name) - return nil - } - return ErrNameUnknown -} - -func (r *registrar) registerPeer(peer *peer) error { - lib.Log("[%s] REGISTRAR registering peer %#v", r.nodename, peer.name) - r.mutexPeers.Lock() - defer r.mutexPeers.Unlock() - - if _, ok := r.peers[peer.name]; ok { - // already registered - return ErrTaken - } - r.peers[peer.name] = peer - return nil -} - -func (r *registrar) unregisterPeer(name string) { - lib.Log("[%s] REGISTRAR unregistering peer %v", r.nodename, name) - r.mutexPeers.Lock() - if _, ok := r.peers[name]; ok { - delete(r.peers, name) - // mutex must be unlocked before we call nodeDown - r.mutexPeers.Unlock() - r.nodeDown(name) - return - } - r.mutexPeers.Unlock() -} - -func (r *registrar) RegisterBehavior(group, name string, behavior gen.ProcessBehavior, data interface{}) error { - lib.Log("[%s] REGISTRAR registering behavior %q in group %q ", r.nodename, name, group) - var groupBehaviors map[string]gen.RegisteredBehavior - var exist bool - - r.mutexBehaviors.Lock() - defer r.mutexBehaviors.Unlock() - - groupBehaviors, exist = r.behaviors[group] - if !exist { - groupBehaviors = make(map[string]gen.RegisteredBehavior) - r.behaviors[group] = groupBehaviors - } - - _, exist = groupBehaviors[name] - if exist { - return ErrTaken - } - - rb := gen.RegisteredBehavior{ - Behavior: behavior, - Data: data, - } - groupBehaviors[name] = rb - return nil -} - -func (r *registrar) RegisteredBehavior(group, name string) (gen.RegisteredBehavior, error) { - var groupBehaviors map[string]gen.RegisteredBehavior - var rb gen.RegisteredBehavior - var exist bool - - r.mutexBehaviors.Lock() - defer r.mutexBehaviors.Unlock() - - groupBehaviors, exist = r.behaviors[group] - if !exist { - return rb, ErrBehaviorGroupUnknown - } - - rb, exist = groupBehaviors[name] - if !exist { - return rb, ErrBehaviorUnknown - } - return rb, nil -} - -func (r *registrar) RegisteredBehaviorGroup(group string) []gen.RegisteredBehavior { - var groupBehaviors map[string]gen.RegisteredBehavior - var exist bool - var listrb []gen.RegisteredBehavior - - r.mutexBehaviors.Lock() - defer r.mutexBehaviors.Unlock() - - groupBehaviors, exist = r.behaviors[group] - if !exist { - return listrb - } - - for _, v := range groupBehaviors { - listrb = append(listrb, v) - } - return listrb -} - -func (r *registrar) UnregisterBehavior(group, name string) error { - lib.Log("[%s] REGISTRAR unregistering behavior %s in group %s ", r.nodename, name, group) - var groupBehaviors map[string]gen.RegisteredBehavior - var exist bool - - r.mutexBehaviors.Lock() - defer r.mutexBehaviors.Unlock() - - groupBehaviors, exist = r.behaviors[group] - if !exist { - return ErrBehaviorUnknown - } - delete(groupBehaviors, name) - - // remove group if its empty - if len(groupBehaviors) == 0 { - delete(r.behaviors, group) - } - return nil -} - -func (r *registrar) IsProcessAlive(process gen.Process) bool { - pid := process.Self() - p := r.ProcessByPid(pid) - if p == nil { - return false - } - - return p.IsAlive() -} - -func (r *registrar) ProcessInfo(pid etf.Pid) (gen.ProcessInfo, error) { - p := r.ProcessByPid(pid) - if p == nil { - return gen.ProcessInfo{}, fmt.Errorf("undefined") - } - - return p.Info(), nil -} - -func (r *registrar) ProcessByPid(pid etf.Pid) gen.Process { - if p := r.getProcessByPid(pid); p != nil { - return p - } - // we must return nil explicitly, otherwise returning value is not nil - // even for the nil(*process) due to the nature of interface type - return nil -} - -func (r *registrar) getProcessByPid(pid etf.Pid) *process { - r.mutexProcesses.Lock() - defer r.mutexProcesses.Unlock() - if p, ok := r.processes[pid.ID]; ok { - return p - } - // unknown process - return nil -} - -func (r *registrar) ProcessByAlias(alias etf.Alias) gen.Process { - r.mutexAliases.Lock() - defer r.mutexAliases.Unlock() - if p, ok := r.aliases[alias]; ok { - return p - } - // unknown process - return nil -} - -func (r *registrar) ProcessByName(name string) gen.Process { - var pid etf.Pid - if name != "" { - // requesting Process by name - r.mutexNames.Lock() - - if p, ok := r.names[name]; ok { - pid = p - } else { - r.mutexNames.Unlock() - return nil - } - r.mutexNames.Unlock() - } - - return r.ProcessByPid(pid) -} - -func (r *registrar) ProcessList() []gen.Process { - list := []gen.Process{} - r.mutexProcesses.Lock() - for _, p := range r.processes { - list = append(list, p) - } - r.mutexProcesses.Unlock() - return list -} - -func (r *registrar) PeerList() []string { - list := []string{} - for n, _ := range r.peers { - list = append(list, n) - } - return list -} - -// route message to a local/remote process -func (r *registrar) route(from etf.Pid, to etf.Term, message etf.Term) error { -next: - switch tto := to.(type) { - case etf.Pid: - lib.Log("[%s] REGISTRAR sending message by pid %s", r.nodename, tto) - if string(tto.Node) == r.nodename { - // local route - r.mutexProcesses.Lock() - p, exist := r.processes[tto.ID] - r.mutexProcesses.Unlock() - if !exist { - return ErrProcessUnknown - } - select { - case p.mailBox <- gen.ProcessMailboxMessage{from, message}: - default: - return fmt.Errorf("WARNING! mailbox of %s is full. dropped message from %s", p.Self(), from) - } - return nil - } - - r.mutexPeers.Lock() - peer, ok := r.peers[string(tto.Node)] - r.mutexPeers.Unlock() - if !ok { - if err := r.net.connect(string(tto.Node)); err != nil { - lib.Log("[%s] Can't connect to %v: %s", r.nodename, tto.Node, err) - return fmt.Errorf("Can't connect to %s: %s", tto.Node, err) - } - - r.mutexPeers.Lock() - peer, _ = r.peers[string(tto.Node)] - r.mutexPeers.Unlock() - } - - send := peer.getChannel() - send <- []etf.Term{etf.Tuple{distProtoSEND, etf.Atom(""), tto}, message} - - case gen.ProcessID: - lib.Log("[%s] REGISTRAR sending message by gen.ProcessID %#v", r.nodename, tto) - - if tto.Node == r.nodename { - // local route - to = tto.Name - goto next - } - - // sending to remote node - r.mutexPeers.Lock() - peer, ok := r.peers[tto.Node] - r.mutexPeers.Unlock() - if !ok { - // initiate connection and make yet another attempt to deliver this message - if err := r.net.connect(tto.Node); err != nil { - lib.Log("[%s] Can't connect to %v: %s", r.nodename, tto.Node, err) - return fmt.Errorf("Can't connect to %s: %s", tto.Node, err) - } - - r.mutexPeers.Lock() - peer, _ = r.peers[tto.Node] - r.mutexPeers.Unlock() - } - - send := peer.getChannel() - send <- []etf.Term{etf.Tuple{distProtoREG_SEND, from, etf.Atom(""), etf.Atom(tto.Name)}, message} - - case string: - lib.Log("[%s] REGISTRAR sending message by name %#v", r.nodename, tto) - r.mutexNames.Lock() - if pid, ok := r.names[tto]; ok { - to = pid - r.mutexNames.Unlock() - goto next - } - r.mutexNames.Unlock() - - case etf.Atom: - lib.Log("[%s] REGISTRAR sending message by name %#v", r.nodename, tto) - r.mutexNames.Lock() - if pid, ok := r.names[string(tto)]; ok { - to = pid - r.mutexNames.Unlock() - goto next - } - r.mutexNames.Unlock() - - case etf.Alias: - lib.Log("[%s] REGISTRAR sending message by alias %s", r.nodename, tto) - r.mutexAliases.Lock() - if string(tto.Node) == r.nodename { - // local route by alias - if p, ok := r.aliases[tto]; ok { - to = p.self - r.mutexAliases.Unlock() - goto next - } - } - r.mutexAliases.Unlock() - - r.mutexPeers.Lock() - peer, ok := r.peers[string(tto.Node)] - r.mutexPeers.Unlock() - if !ok { - if err := r.net.connect(string(tto.Node)); err != nil { - lib.Log("[%s] Can't connect to %v: %s", r.nodename, tto.Node, err) - return fmt.Errorf("Can't connect to %s: %s", tto.Node, err) - } - - r.mutexPeers.Lock() - peer, _ = r.peers[string(tto.Node)] - r.mutexPeers.Unlock() - } - - send := peer.getChannel() - send <- []etf.Term{etf.Tuple{distProtoALIAS_SEND, from, tto}, message} - - default: - lib.Log("[%s] unsupported receiver type %#v", r.nodename, tto) - return fmt.Errorf("unsupported receiver type %#v", tto) - } - - return nil -} - -func (r *registrar) routeRaw(nodename etf.Atom, messages ...etf.Term) error { - r.mutexPeers.Lock() - peer, ok := r.peers[string(nodename)] - r.mutexPeers.Unlock() - if len(messages) == 0 { - return fmt.Errorf("nothing to send") - } - if !ok { - // initiate connection and make yet another attempt to deliver this message - if err := r.net.connect(string(nodename)); err != nil { - lib.Log("[%s] Can't connect to %v: %s", r.nodename, nodename, err) - return err - } - - r.mutexPeers.Lock() - peer, _ = r.peers[string(nodename)] - r.mutexPeers.Unlock() - } - - send := peer.getChannel() - send <- messages - return nil -} diff --git a/node/types.go b/node/types.go index 7cd3d708..a4e97915 100644 --- a/node/types.go +++ b/node/types.go @@ -1,85 +1,100 @@ package node import ( + "context" + "crypto/cipher" + "crypto/tls" "fmt" + "io" "time" "github.com/ergo-services/ergo/etf" "github.com/ergo-services/ergo/gen" + "github.com/ergo-services/ergo/lib" ) var ( - ErrAppAlreadyLoaded = fmt.Errorf("Application is already loaded") - ErrAppAlreadyStarted = fmt.Errorf("Application is already started") - ErrAppUnknown = fmt.Errorf("Unknown application name") - ErrAppIsNotRunning = fmt.Errorf("Application is not running") - ErrNameUnknown = fmt.Errorf("Unknown name") - ErrNameOwner = fmt.Errorf("Not an owner") - ErrProcessBusy = fmt.Errorf("Process is busy") - ErrProcessUnknown = fmt.Errorf("Unknown process") - ErrProcessTerminated = fmt.Errorf("Process terminated") - ErrBehaviorUnknown = fmt.Errorf("Unknown behavior") - ErrBehaviorGroupUnknown = fmt.Errorf("Unknown behavior group") - ErrAliasUnknown = fmt.Errorf("Unknown alias") - ErrAliasOwner = fmt.Errorf("Not an owner") - ErrTaken = fmt.Errorf("Resource is taken") - ErrTimeout = fmt.Errorf("Timed out") - ErrFragmented = fmt.Errorf("Fragmented data") + ErrAppAlreadyLoaded = fmt.Errorf("application is already loaded") + ErrAppAlreadyStarted = fmt.Errorf("application is already started") + ErrAppUnknown = fmt.Errorf("unknown application name") + ErrAppIsNotRunning = fmt.Errorf("application is not running") + ErrNameUnknown = fmt.Errorf("unknown name") + ErrNameOwner = fmt.Errorf("not an owner") + ErrProcessBusy = fmt.Errorf("process is busy") + ErrProcessUnknown = fmt.Errorf("unknown process") + ErrProcessIncarnation = fmt.Errorf("process ID belongs to the previous incarnation") + ErrProcessTerminated = fmt.Errorf("process terminated") + ErrMonitorUnknown = fmt.Errorf("unknown monitor reference") + ErrSenderUnknown = fmt.Errorf("unknown sender") + ErrBehaviorUnknown = fmt.Errorf("unknown behavior") + ErrBehaviorGroupUnknown = fmt.Errorf("unknown behavior group") + ErrAliasUnknown = fmt.Errorf("unknown alias") + ErrAliasOwner = fmt.Errorf("not an owner") + ErrNoRoute = fmt.Errorf("no route to node") + ErrTaken = fmt.Errorf("resource is taken") + ErrTimeout = fmt.Errorf("timed out") + ErrFragmented = fmt.Errorf("fragmented data") + + ErrUnsupported = fmt.Errorf("not supported") + ErrPeerUnsupported = fmt.Errorf("peer does not support this feature") + + ErrProxyUnknownRequest = fmt.Errorf("unknown proxy request") + ErrProxyTransitDisabled = fmt.Errorf("proxy feature disabled") + ErrProxyNoRoute = fmt.Errorf("no proxy route to node") + ErrProxyConnect = fmt.Errorf("can't establish proxy connection") + ErrProxyHopExceeded = fmt.Errorf("proxy hop is exceeded") + ErrProxyLoopDetected = fmt.Errorf("proxy loop detected") + ErrProxyPathTooLong = fmt.Errorf("proxy path too long") + ErrProxySessionUnknown = fmt.Errorf("unknown session id") + ErrProxySessionDuplicate = fmt.Errorf("session is already exist") ) -// Distributed operations codes (http://www.erlang.org/doc/apps/erts/erl_dist_protocol.html) const ( - distProtoLINK = 1 - distProtoSEND = 2 - distProtoEXIT = 3 - distProtoUNLINK = 4 - distProtoNODE_LINK = 5 - distProtoREG_SEND = 6 - distProtoGROUP_LEADER = 7 - distProtoEXIT2 = 8 - distProtoSEND_TT = 12 - distProtoEXIT_TT = 13 - distProtoREG_SEND_TT = 16 - distProtoEXIT2_TT = 18 - distProtoMONITOR = 19 - distProtoDEMONITOR = 20 - distProtoMONITOR_EXIT = 21 - distProtoSEND_SENDER = 22 - distProtoSEND_SENDER_TT = 23 - distProtoPAYLOAD_EXIT = 24 - distProtoPAYLOAD_EXIT_TT = 25 - distProtoPAYLOAD_EXIT2 = 26 - distProtoPAYLOAD_EXIT2_TT = 27 - distProtoPAYLOAD_MONITOR_P_EXIT = 28 - distProtoSPAWN_REQUEST = 29 - distProtoSPAWN_REQUEST_TT = 30 - distProtoSPAWN_REPLY = 31 - distProtoSPAWN_REPLY_TT = 32 - distProtoALIAS_SEND = 33 - distProtoALIAS_SEND_TT = 34 - distProtoUNLINK_ID = 35 - distProtoUNLINK_ID_ACK = 36 - - defaultListenRangeBegin uint16 = 15000 - defaultListenRangeEnd uint16 = 65000 - defaultEPMDPort uint16 = 4369 - defaultSendQueueLength int = 100 - defaultRecvQueueLength int = 100 - defaultFragmentationUnit = 65000 - defaultHandshakeVersion = 5 + // node options + defaultListenBegin uint16 = 15000 + defaultListenEnd uint16 = 65000 + defaultKeepAlivePeriod time.Duration = 15 + defaultProxyPathLimit int = 32 + + EnvKeyVersion gen.EnvKey = "ergo:Version" + EnvKeyNode gen.EnvKey = "ergo:Node" + EnvKeyRemoteSpawn gen.EnvKey = "ergo:RemoteSpawn" + + DefaultProtoRecvQueueLength int = 100 + DefaultProtoSendQueueLength int = 100 + DefaultProroFragmentationUnit int = 65000 + + DefaultCompressionLevel int = -1 + DefaultCompressionThreshold int = 1024 + + DefaultProxyMaxHop int = 8 ) type Node interface { - gen.Registrar - Network + gen.Core + // Name returns node name Name() string + // IsAlive returns true if node is still alive IsAlive() bool + // Uptime returns node uptime in seconds Uptime() int64 + // Version return node version Version() Version + // ListEnv returns a map of configured Node environment variables. + ListEnv() map[gen.EnvKey]interface{} + // SetEnv set node environment variable with given name. Use nil value to remove variable with given name. Ignores names with "ergo:" as a prefix. + SetEnv(name gen.EnvKey, value interface{}) + // Env returns value associated with given environment name. + Env(name gen.EnvKey) interface{} + + // Spawn spawns a new process Spawn(name string, opts gen.ProcessOptions, object gen.ProcessBehavior, args ...etf.Term) (gen.Process, error) + // RegisterName RegisterName(name string, pid etf.Pid) error + // UnregisterName UnregisterName(name string) error + LoadedApplications() []gen.ApplicationInfo WhichApplications() []gen.ApplicationInfo ApplicationInfo(name string) (gen.ApplicationInfo, error) @@ -89,8 +104,41 @@ type Node interface { ApplicationStartPermanent(appName string, args ...etf.Term) (gen.Process, error) ApplicationStartTransient(appName string, args ...etf.Term) (gen.Process, error) ApplicationStop(appName string) error + ProvideRPC(module string, function string, fun gen.RPC) error RevokeRPC(module, function string) error + ProvideRemoteSpawn(name string, object gen.ProcessBehavior) error + RevokeRemoteSpawn(name string) error + + // AddStaticRoute adds static route for the given name + AddStaticRoute(node string, host string, port uint16, options RouteOptions) error + // AddStaticRoutePort adds static route for the given node name which makes node skip resolving port process + AddStaticRoutePort(node string, port uint16, options RouteOptions) error + // AddStaticRouteOptions adds static route options for the given node name which does regular port resolving but applies static options + AddStaticRouteOptions(node string, options RouteOptions) error + // Remove static route removes static route with given name + RemoveStaticRoute(name string) bool + // StaticRoutes returns list of routes added using AddStaticRoute + StaticRoutes() []Route + // StaticRoute returns Route for the given name. Returns false if it doesn't exist. + StaticRoute(name string) (Route, bool) + + AddProxyRoute(name string, proxy ProxyRoute) error + RemoveProxyRoute(name string) bool + ProxyRoutes() []ProxyRoute + ProxyRoute(name string) (ProxyRoute, bool) + + // Resolve + Resolve(peername string) (Route, error) + + // Connect sets up a connection to node + Connect(nodename string) error + // Disconnect close connection to the node + Disconnect(nodename string) error + // Nodes returns the list of connected nodes + Nodes() []string + // NodesIndirect returns the list of nodes connected via proxies + NodesIndirect() []string Links(process etf.Pid) []etf.Pid Monitors(process etf.Pid) []etf.Pid @@ -102,66 +150,381 @@ type Node interface { WaitWithTimeout(d time.Duration) error } +// Version type Version struct { Release string Prefix string OTP int } -type Network interface { - AddStaticRoute(name string, port uint16) error - AddStaticRouteExt(name string, port uint16, cookie string, tls bool) error - RemoveStaticRoute(name string) - Resolve(name string) (NetworkRoute, error) +// CoreRouter routes messages from/to remote node +type CoreRouter interface { - ProvideRemoteSpawn(name string, object gen.ProcessBehavior) error - RevokeRemoteSpawn(name string) error + // + // implemented by core + // + + // RouteSend routes message by Pid + RouteSend(from etf.Pid, to etf.Pid, message etf.Term) error + // RouteSendReg routes message by registered process name (gen.ProcessID) + RouteSendReg(from etf.Pid, to gen.ProcessID, message etf.Term) error + // RouteSendAlias routes message by process alias + RouteSendAlias(from etf.Pid, to etf.Alias, message etf.Term) error + + RouteSpawnRequest(node string, behaviorName string, request gen.RemoteSpawnRequest, args ...etf.Term) error + RouteSpawnReply(to etf.Pid, ref etf.Ref, result etf.Term) error + + // + // implemented by monitor + // + + // RouteLink makes linking of the given two processes + RouteLink(pidA etf.Pid, pidB etf.Pid) error + // RouteUnlink makes unlinking of the given two processes + RouteUnlink(pidA etf.Pid, pidB etf.Pid) error + // RouteExit routes MessageExit to the linked process + RouteExit(to etf.Pid, terminated etf.Pid, reason string) error + // RouteMonitorReg makes monitor to the given registered process name (gen.ProcessID) + RouteMonitorReg(by etf.Pid, process gen.ProcessID, ref etf.Ref) error + // RouteMonitor makes monitor to the given Pid + RouteMonitor(by etf.Pid, process etf.Pid, ref etf.Ref) error + RouteDemonitor(by etf.Pid, ref etf.Ref) error + RouteMonitorExitReg(terminated gen.ProcessID, reason string, ref etf.Ref) error + RouteMonitorExit(terminated etf.Pid, reason string, ref etf.Ref) error + // RouteNodeDown + RouteNodeDown(name string, disconnect *ProxyDisconnect) + + // + // implemented by network + // + + // RouteProxyConnectRequest + RouteProxyConnectRequest(from ConnectionInterface, request ProxyConnectRequest) error + // RouteProxyConnectReply + RouteProxyConnectReply(from ConnectionInterface, reply ProxyConnectReply) error + // RouteProxyConnectCancel + RouteProxyConnectCancel(from ConnectionInterface, cancel ProxyConnectCancel) error + // RouteProxyDisconnect + RouteProxyDisconnect(from ConnectionInterface, disconnect ProxyDisconnect) error + // RouteProxy returns ErrProxySessionEndpoint if this node is the endpoint of the + // proxy session. In this case, the packet must be handled on this node with + // provided ProxySession parameters. + RouteProxy(from ConnectionInterface, sessionID string, packet *lib.Buffer) error } -type NetworkRoute struct { - Port int +// Options defines bootstrapping options for the node +type Options struct { + // Applications application list that must be started + Applications []gen.ApplicationBehavior + + // Env node environment + Env map[gen.EnvKey]interface{} + + // Creation. Default value: uint32(time.Now().Unix()) + Creation uint32 + + // Flags defines enabled options for the running node + Flags Flags + + // Listen defines a listening port number for accepting incoming connections. + Listen uint16 + // ListenBegin and ListenEnd define a range of the port numbers where + // the node looking for available free port number for the listening. + // Default values 15000 and 65000 accordingly + ListenBegin uint16 + ListenEnd uint16 + + // TLS settings + TLS TLS + + // StaticRoutesOnly disables resolving service (default is EPMD client) and + // makes resolving localy only for nodes added using gen.AddStaticRoute + StaticRoutesOnly bool + + // Resolver defines a resolving service (default is EPMD service, client and server) + Resolver Resolver + + // Compression enables compression for outgoing messages (if peer node has this feature enabled) + Compression Compression + + // Handshake defines a handshake handler. By default is using + // DIST handshake created with dist.CreateHandshake(...) + Handshake HandshakeInterface + + // Proto defines a proto handler. By default is using + // DIST proto created with dist.CreateProto(...) + Proto ProtoInterface + + // Proxy enable proxy feature on this node. Disabling this option makes + // this node to reject any proxy request. + Proxy Proxy +} + +type TLS struct { + Enable bool + Server tls.Certificate + Client tls.Certificate + SkipVerify bool +} + +type Cloud struct { + Enable bool + ID string Cookie string - TLS bool } -// Options struct with bootstrapping options for CreateNode -type Options struct { - Applications []gen.ApplicationBehavior - ListenRangeBegin uint16 - ListenRangeEnd uint16 - Hidden bool - EPMDPort uint16 - DisableEPMDServer bool - DisableEPMD bool // use static routes only - SendQueueLength int - RecvQueueLength int - FragmentationUnit int - DisableHeaderAtomCache bool - TLSMode TLSModeType - TLScrtServer string - TLSkeyServer string - TLScrtClient string - TLSkeyClient string - // HandshakeVersion. Allowed values 5 or 6. Default version is 5 - HandshakeVersion int - // ConnectionHandlers defines the number of readers/writers per connection. Default is the number of CPU. - ConnectionHandlers int - - cookie string - creation uint32 -} - -// TLSmodeType should be one of TLSmodeDisabled (default), TLSmodeAuto or TLSmodeStrict -type TLSmodeType string - -// TLSmodeType should be one of TLSmodeDisabled (default), TLSmodeAuto or TLSmodeStrict -type TLSModeType string +type Proxy struct { + Transit bool + Flags ProxyFlags + Cookie string // set cookie for incoming connection + Routes map[string]ProxyRoute +} -const ( - // TLSModeDisabled no TLS encryption - TLSModeDisabled TLSModeType = "" - // TLSModeAuto generate self-signed certificate - TLSModeAuto TLSModeType = "auto" - // TLSModeStrict with validation certificate - TLSModeStrict TLSModeType = "strict" -) +type Metrics struct { + Disable bool +} + +type Compression struct { + // Enable enables compression for all outgoing messages having size + // greater than the defined threshold. + Enable bool + // Level defines compression level. Value must be in range 1..9 or -1 for the default level + Level int + // Threshold defines the minimal message size for the compression. + // Messages less of this threshold will not be compressed. + Threshold int +} + +// Connection +type Connection struct { + ConnectionInterface +} + +// ConnectionInterface +type ConnectionInterface interface { + Send(from gen.Process, to etf.Pid, message etf.Term) error + SendReg(from gen.Process, to gen.ProcessID, message etf.Term) error + SendAlias(from gen.Process, to etf.Alias, message etf.Term) error + + Link(local etf.Pid, remote etf.Pid) error + Unlink(local etf.Pid, remote etf.Pid) error + LinkExit(to etf.Pid, terminated etf.Pid, reason string) error + + Monitor(local etf.Pid, remote etf.Pid, ref etf.Ref) error + Demonitor(local etf.Pid, remote etf.Pid, ref etf.Ref) error + MonitorExit(to etf.Pid, terminated etf.Pid, reason string, ref etf.Ref) error + + MonitorReg(local etf.Pid, remote gen.ProcessID, ref etf.Ref) error + DemonitorReg(local etf.Pid, remote gen.ProcessID, ref etf.Ref) error + MonitorExitReg(to etf.Pid, terminated gen.ProcessID, reason string, ref etf.Ref) error + + SpawnRequest(nodeName string, behaviorName string, request gen.RemoteSpawnRequest, args ...etf.Term) error + SpawnReply(to etf.Pid, ref etf.Ref, spawned etf.Pid) error + SpawnReplyError(to etf.Pid, ref etf.Ref, err error) error + + ProxyConnectRequest(connect ProxyConnectRequest) error + ProxyConnectReply(reply ProxyConnectReply) error + ProxyConnectCancel(cancel ProxyConnectCancel) error + ProxyDisconnect(disconnect ProxyDisconnect) error + ProxyRegisterSession(session ProxySession) error + ProxyUnregisterSession(id string) error + ProxyPacket(packet *lib.Buffer) error + + Creation() uint32 +} + +// Handshake template struct for the custom Handshake implementation +type Handshake struct { + HandshakeInterface +} + +// Handshake defines handshake interface +type HandshakeInterface interface { + // Init initialize handshake. + Init(nodename string, creation uint32, flags Flags) error + // Start initiates handshake process. Argument tls means the connection is wrapped by TLS + // Returns the name of connected peer, Flags and Creation wrapped into HandshakeDetails struct + Start(conn io.ReadWriter, tls bool, cookie string) (HandshakeDetails, error) + // Accept accepts handshake process initiated by another side of this connection. + // Returns the name of connected peer, Flags and Creation wrapped into HandshakeDetails struct + Accept(conn io.ReadWriter, tls bool, cookie string) (HandshakeDetails, error) + // Version handshake version. Must be implemented if this handshake is going to be used + // for the accepting connections (this method is used in registration on the Resolver) + Version() HandshakeVersion +} + +// HandshakeDetails +type HandshakeDetails struct { + Name string + Flags Flags + Creation uint32 + Version int + Custom HandshakeCustomDetails +} + +type HandshakeCustomDetails interface{} + +type HandshakeVersion int + +// Proto template struct for the custom Proto implementation +type Proto struct { + ProtoInterface +} + +// Proto defines proto interface for the custom Proto implementation +type ProtoInterface interface { + // Init initialize connection handler + Init(ctx context.Context, conn io.ReadWriter, nodename string, details HandshakeDetails) (ConnectionInterface, error) + // Serve connection + Serve(connection ConnectionInterface, router CoreRouter) + // Terminate invoked once Serve callback is finished + Terminate(connection ConnectionInterface) +} + +// ProtoOptions +type ProtoOptions struct { + // NumHandlers defines the number of readers/writers per connection. Default is the number of CPU + NumHandlers int + // MaxMessageSize limit the message size. Default 0 (no limit) + MaxMessageSize int + // SendQueueLength defines queue size of handler for the outgoing messages. Default 100. + SendQueueLength int + // RecvQueueLength defines queue size of handler for the incoming messages. Default 100. + RecvQueueLength int + // FragmentationUnit defines unit size for the fragmentation feature. Default 65000 + FragmentationUnit int + // Custom brings a custom set of options to the ProtoInterface.Serve handler + Custom CustomProtoOptions +} + +// CustomProtoOptions a custom set of proto options +type CustomProtoOptions interface{} + +// Flags +type Flags struct { + // Enable enable flags customization + Enable bool + // EnableHeaderAtomCache enables header atom cache feature + EnableHeaderAtomCache bool + // EnableBigCreation + EnableBigCreation bool + // EnableBigPidRef accepts a larger amount of data in pids and references + EnableBigPidRef bool + // EnableFragmentation enables fragmentation feature for the sending data + EnableFragmentation bool + // EnableAlias accepts process aliases + EnableAlias bool + // EnableRemoteSpawn accepts remote spawn request + EnableRemoteSpawn bool + // Compression compression support + EnableCompression bool + // Proxy enables support for incoming proxy connection + EnableProxy bool + // Software keepalive enables sending keep alive messages if node doesn't support + // TCPConn.SetKeepAlive(true). For erlang peers this flag is mandatory. + EnableSoftwareKeepAlive bool +} + +// Resolver defines resolving interface +type Resolver interface { + Register(ctx context.Context, nodename string, port uint16, options ResolverOptions) error + Resolve(peername string) (Route, error) +} + +// ResolverOptions defines resolving options +type ResolverOptions struct { + NodeVersion Version + HandshakeVersion HandshakeVersion + EnableTLS bool + EnableProxy bool + EnableCompression bool +} + +// Route +type Route struct { + Node string + Host string + Port uint16 + Options RouteOptions +} + +// RouteOptions +type RouteOptions struct { + Cookie string + EnableTLS bool + IsErgo bool + Cert tls.Certificate + Handshake HandshakeInterface + Proto ProtoInterface + Custom CustomRouteOptions +} + +// ProxyRoute +type ProxyRoute struct { + Proxy string + Cookie string + Flags ProxyFlags + MaxHop int // DefaultProxyMaxHop == 8 +} + +// ProxyFlags +type ProxyFlags struct { + Enable bool + EnableLink bool + EnableMonitor bool + EnableRemoteSpawn bool + EnableEncryption bool +} + +// ProxyConnectRequest +type ProxyConnectRequest struct { + ID etf.Ref + To string // To node + Digest []byte // md5(md5(md5(md5(Node)+Cookie)+To)+PublicKey) + PublicKey []byte + Flags ProxyFlags + Creation uint32 + Hop int + Path []string +} + +// ProxyConnectReply +type ProxyConnectReply struct { + ID etf.Ref + To string + Digest []byte // md5(md5(md5(md5(Node)+Cookie)+To)+symmetric key) + Cipher []byte // encrypted symmetric key using PublicKey from the ProxyConnectRequest + Flags ProxyFlags + Creation uint32 + SessionID string // proxy session ID + Path []string +} + +// ProxyConnectCancel +type ProxyConnectCancel struct { + ID etf.Ref + From string + Reason string + Path []string +} + +// ProxyDisconnect +type ProxyDisconnect struct { + Node string + Proxy string + SessionID string + Reason string +} + +// Proxy session +type ProxySession struct { + ID string + NodeFlags ProxyFlags + PeerFlags ProxyFlags + Creation uint32 + PeerName string + Block cipher.Block // made from symmetric key +} + +// CustomRouteOptions a custom set of route options +type CustomRouteOptions interface{} diff --git a/proto/dist/epmd.go b/proto/dist/epmd.go new file mode 100644 index 00000000..d1b80d7e --- /dev/null +++ b/proto/dist/epmd.go @@ -0,0 +1,234 @@ +package dist + +import ( + "context" + "encoding/binary" + "fmt" + "net" + "strconv" + "strings" + "sync" + "time" + + "github.com/ergo-services/ergo/lib" +) + +type registeredNode struct { + port uint16 + hidden bool + hi uint16 + lo uint16 + extra []byte +} + +type epmd struct { + port uint16 + nodes map[string]registeredNode + nodesMutex sync.Mutex +} + +func startServerEPMD(ctx context.Context, host string, port uint16) error { + lc := net.ListenConfig{} + listener, err := lc.Listen(ctx, "tcp", net.JoinHostPort(host, strconv.Itoa(int(port)))) + if err != nil { + lib.Log("Can't start embedded EPMD service: %s", err) + return err + } + + epmd := epmd{ + port: port, + nodes: make(map[string]registeredNode), + } + go epmd.serve(listener) + lib.Log("Started embedded EMPD service and listen port: %d", port) + + return nil +} + +func (e *epmd) serve(l net.Listener) { + for { + c, err := l.Accept() + if err != nil { + lib.Log("EPMD server stopped: %s", err.Error()) + return + } + lib.Log("EPMD accepted new connection from %s", c.RemoteAddr().String()) + go e.handle(c) + } +} + +func (e *epmd) handle(c net.Conn) { + var name string + var node registeredNode + buf := make([]byte, 1024) + + defer c.Close() + for { + n, err := c.Read(buf) + lib.Log("Request from EPMD client: %v", buf[:n]) + if err != nil { + lib.Log("EPMD unregistering node: '%s'", name) + e.nodesMutex.Lock() + delete(e.nodes, name) + e.nodesMutex.Unlock() + return + } + if len(buf) < 6 { + lib.Log("Too short") + return + } + + buf = buf[:n] + // buf[0:1] - length + if uint16(n-2) != binary.BigEndian.Uint16(buf[0:2]) { + continue + } + + switch buf[2] { + case epmdAliveReq: + name, node, err = e.readAliveReq(buf[3:]) + if err != nil { + // send error and close connection + e.sendAliveResp(c, 1) + return + } + + // check if node with this name is already registered + e.nodesMutex.Lock() + _, exist := e.nodes[name] + e.nodesMutex.Unlock() + if exist { + // send error and close connection + e.sendAliveResp(c, 1) + return + } + + // send alive response + if err := e.sendAliveResp(c, 0); err != nil { + return + } + + // register new node + e.nodesMutex.Lock() + e.nodes[name] = node + e.nodesMutex.Unlock() + + // enable keep alive on this connection + if tcp, ok := c.(*net.TCPConn); ok { + tcp.SetKeepAlive(true) + tcp.SetKeepAlivePeriod(15 * time.Second) + tcp.SetNoDelay(true) + } + continue + case epmdPortPleaseReq: + requestedName := string(buf[3:n]) + + e.nodesMutex.Lock() + node, exist := e.nodes[requestedName] + e.nodesMutex.Unlock() + + if exist == false { + lib.Log("EPMD: looking for '%s'. Not found", name) + c.Write([]byte{epmdPortResp, 1}) + return + } + e.sendPortPleaseResp(c, requestedName, node) + return + case epmdNamesReq: + e.sendNamesResp(c, buf[3:n]) + return + default: + lib.Log("unknown EPMD request") + return + } + + } +} + +func (e *epmd) readAliveReq(req []byte) (string, registeredNode, error) { + if len(req) < 10 { + return "", registeredNode{}, fmt.Errorf("Malformed EPMD request %v", req) + } + // Name length + l := binary.BigEndian.Uint16(req[8:10]) + // Name + name := string(req[10 : 10+l]) + // Hidden + hidden := false + if req[2] == 72 { + hidden = true + } + // node + node := registeredNode{ + port: binary.BigEndian.Uint16(req[0:2]), + hidden: hidden, + hi: binary.BigEndian.Uint16(req[4:6]), + lo: binary.BigEndian.Uint16(req[6:8]), + extra: req[10+l:], + } + + return name, node, nil +} + +func (e *epmd) sendAliveResp(c net.Conn, code int) error { + buf := make([]byte, 4) + buf[0] = epmdAliveResp + buf[1] = byte(code) + + // Creation. Ergo doesn't use it. Just for Erlang nodes. + binary.BigEndian.PutUint16(buf[2:], uint16(1)) + _, err := c.Write(buf) + return err +} + +func (e *epmd) sendPortPleaseResp(c net.Conn, name string, node registeredNode) { + buf := make([]byte, 12+len(name)+2+len(node.extra)) + buf[0] = epmdPortResp + + // Result 0 + buf[1] = 0 + // Port + binary.BigEndian.PutUint16(buf[2:4], uint16(node.port)) + // Hidden + if node.hidden { + buf[4] = 72 + } else { + buf[4] = 77 + } + // Protocol TCP + buf[5] = 0 + // Highest version + binary.BigEndian.PutUint16(buf[6:8], uint16(node.hi)) + // Lowest version + binary.BigEndian.PutUint16(buf[8:10], uint16(node.lo)) + // Name + binary.BigEndian.PutUint16(buf[10:12], uint16(len(name))) + offset := 12 + len(name) + copy(buf[12:offset], name) + // Extra + l := len(node.extra) + copy(buf[offset:offset+l], node.extra) + // send + c.Write(buf) + return +} + +func (e *epmd) sendNamesResp(c net.Conn, req []byte) { + var str strings.Builder + var s string + var buf [4]byte + + binary.BigEndian.PutUint32(buf[0:4], uint32(e.port)) + str.WriteString(string(buf[0:])) + + e.nodesMutex.Lock() + for k, v := range e.nodes { + // io:format("name ~ts at port ~p~n", [NodeName, Port]). + s = fmt.Sprintf("name %s at port %d\n", k, v.port) + str.WriteString(s) + } + e.nodesMutex.Unlock() + + c.Write([]byte(str.String())) + return +} diff --git a/proto/dist/flusher.go b/proto/dist/flusher.go new file mode 100644 index 00000000..2ddbd308 --- /dev/null +++ b/proto/dist/flusher.go @@ -0,0 +1,109 @@ +package dist + +import ( + "bufio" + "io" + "sync" + "time" +) + +var ( + // KeepAlive packet is just 4 bytes with zero value + keepAlivePacket = []byte{0, 0, 0, 0} + keepAlivePeriod = 15 * time.Second +) + +func newLinkFlusher(w io.Writer, latency time.Duration, softwareKeepAlive bool) *linkFlusher { + lf := &linkFlusher{ + latency: latency, + writer: bufio.NewWriter(w), + w: w, // in case if we skip buffering + softwareKeepAlive: softwareKeepAlive, + } + + lf.timer = time.AfterFunc(keepAlivePeriod, func() { + + lf.mutex.Lock() + defer lf.mutex.Unlock() + + // if we have no pending data to send we should + // send a KeepAlive packet + if lf.pending == false && lf.softwareKeepAlive { + lf.w.Write(keepAlivePacket) + lf.timer.Reset(keepAlivePeriod) + return + } + + lf.writer.Flush() + lf.pending = false + if lf.softwareKeepAlive { + lf.timer.Reset(keepAlivePeriod) + } + }) + + return lf +} + +type linkFlusher struct { + mutex sync.Mutex + latency time.Duration + writer *bufio.Writer + w io.Writer + softwareKeepAlive bool + + timer *time.Timer + pending bool +} + +func (lf *linkFlusher) Write(b []byte) (int, error) { + lf.mutex.Lock() + defer lf.mutex.Unlock() + + l := len(b) + lenB := l + + // long data write directly to the socket. + if l > 64000 { + for { + n, e := lf.w.Write(b[lenB-l:]) + if e != nil { + return n, e + } + // check if something left + l -= n + if l > 0 { + continue + } + return lenB, nil + } + } + + // write data to the buffer + for { + n, e := lf.writer.Write(b) + if e != nil { + return n, e + } + // check if something left + l -= n + if l > 0 { + continue + } + break + } + + if lf.pending { + return lenB, nil + } + + lf.pending = true + lf.timer.Reset(lf.latency) + + return lenB, nil +} + +func (lf *linkFlusher) Stop() { + if lf.timer != nil { + lf.timer.Stop() + } +} diff --git a/proto/dist/handshake.go b/proto/dist/handshake.go new file mode 100644 index 00000000..7a411a4f --- /dev/null +++ b/proto/dist/handshake.go @@ -0,0 +1,826 @@ +package dist + +import ( + "bytes" + "crypto/md5" + "encoding/binary" + "fmt" + "io" + "math/rand" + "time" + + "github.com/ergo-services/ergo/lib" + "github.com/ergo-services/ergo/node" +) + +const ( + HandshakeVersion5 node.HandshakeVersion = 5 + HandshakeVersion6 node.HandshakeVersion = 6 + + DefaultHandshakeVersion = HandshakeVersion5 + DefaultHandshakeTimeout = 5 * time.Second + + // distribution flags are defined here https://erlang.org/doc/apps/erts/erl_dist_protocol.html#distribution-flags + flagPublished nodeFlagId = 0x1 + flagAtomCache nodeFlagId = 0x2 + flagExtendedReferences nodeFlagId = 0x4 + flagDistMonitor nodeFlagId = 0x8 + flagFunTags nodeFlagId = 0x10 + flagDistMonitorName nodeFlagId = 0x20 + flagHiddenAtomCache nodeFlagId = 0x40 + flagNewFunTags nodeFlagId = 0x80 + flagExtendedPidsPorts nodeFlagId = 0x100 + flagExportPtrTag nodeFlagId = 0x200 + flagBitBinaries nodeFlagId = 0x400 + flagNewFloats nodeFlagId = 0x800 + flagUnicodeIO nodeFlagId = 0x1000 + flagDistHdrAtomCache nodeFlagId = 0x2000 + flagSmallAtomTags nodeFlagId = 0x4000 + // flagCompressed = 0x8000 // erlang uses this flag for the internal purposes + flagUTF8Atoms nodeFlagId = 0x10000 + flagMapTag nodeFlagId = 0x20000 + flagBigCreation nodeFlagId = 0x40000 + flagSendSender nodeFlagId = 0x80000 // since OTP.21 enable replacement for SEND (distProtoSEND by distProtoSEND_SENDER) + flagBigSeqTraceLabels = 0x100000 + flagExitPayload nodeFlagId = 0x400000 // since OTP.22 enable replacement for EXIT, EXIT2, MONITOR_P_EXIT + flagFragments nodeFlagId = 0x800000 + flagHandshake23 nodeFlagId = 0x1000000 // new connection setup handshake (version 6) introduced in OTP 23 + flagUnlinkID nodeFlagId = 0x2000000 + // for 64bit flags + flagSpawn nodeFlagId = 1 << 32 + flagNameMe nodeFlagId = 1 << 33 + flagV4NC nodeFlagId = 1 << 34 + flagAlias nodeFlagId = 1 << 35 + + // ergo flags + flagCompression = 1 << 63 + flagProxy = 1 << 62 +) + +type nodeFlagId uint64 +type nodeFlags nodeFlagId + +func (nf nodeFlags) toUint32() uint32 { + return uint32(nf) +} + +func (nf nodeFlags) toUint64() uint64 { + return uint64(nf) +} + +func (nf nodeFlags) isSet(f nodeFlagId) bool { + return (uint64(nf) & uint64(f)) != 0 +} + +func toNodeFlags(f ...nodeFlagId) nodeFlags { + var flags uint64 + for _, v := range f { + flags |= uint64(v) + } + return nodeFlags(flags) +} + +// DistHandshake implements Erlang handshake +type DistHandshake struct { + node.Handshake + nodename string + flags node.Flags + creation uint32 + challenge uint32 + options HandshakeOptions +} + +type HandshakeOptions struct { + Timeout time.Duration + Version node.HandshakeVersion // 5 or 6 +} + +func CreateHandshake(options HandshakeOptions) node.HandshakeInterface { + // must be 5 or 6 + if options.Version != HandshakeVersion5 && options.Version != HandshakeVersion6 { + options.Version = DefaultHandshakeVersion + } + + if options.Timeout == 0 { + options.Timeout = DefaultHandshakeTimeout + } + return &DistHandshake{ + options: options, + challenge: rand.Uint32(), + } +} + +// Init implements Handshake interface mothod +func (dh *DistHandshake) Init(nodename string, creation uint32, flags node.Flags) error { + dh.nodename = nodename + dh.creation = creation + dh.flags = flags + return nil +} + +func (dh *DistHandshake) Version() node.HandshakeVersion { + return dh.options.Version +} + +func (dh *DistHandshake) Start(conn io.ReadWriter, tls bool, cookie string) (node.HandshakeDetails, error) { + + var details node.HandshakeDetails + + b := lib.TakeBuffer() + defer lib.ReleaseBuffer(b) + + var await []byte + + if dh.options.Version == HandshakeVersion5 { + dh.composeName(b, tls) + // the next message must be send_status 's' or send_challenge 'n' (for + // handshake version 5) or 'N' (for handshake version 6) + await = []byte{'s', 'n', 'N'} + } else { + dh.composeNameVersion6(b, tls) + await = []byte{'s', 'N'} + } + if e := b.WriteDataTo(conn); e != nil { + return details, e + } + + // define timeout for the handshaking + timer := time.NewTimer(dh.options.Timeout) + defer timer.Stop() + + asyncReadChannel := make(chan error, 2) + asyncRead := func() { + _, e := b.ReadDataFrom(conn, 512) + asyncReadChannel <- e + } + + // http://erlang.org/doc/apps/erts/erl_dist_protocol.html#distribution-handshake + // Every message in the handshake starts with a 16-bit big-endian integer, + // which contains the message length (not counting the two initial bytes). + // In Erlang this corresponds to option {packet, 2} in gen_tcp(3). Notice + // that after the handshake, the distribution switches to 4 byte packet headers. + expectingBytes := 2 + if tls { + // TLS connection has 4 bytes packet length header + expectingBytes = 4 + } + + for { + go asyncRead() + + select { + case <-timer.C: + return details, fmt.Errorf("handshake timeout") + + case e := <-asyncReadChannel: + if e != nil { + return details, e + } + + next: + l := binary.BigEndian.Uint16(b.B[expectingBytes-2 : expectingBytes]) + buffer := b.B[expectingBytes:] + + if len(buffer) < int(l) { + return details, fmt.Errorf("malformed handshake (wrong packet length)") + } + + // chech if we got correct message type regarding to 'await' value + if bytes.Count(await, buffer[0:1]) == 0 { + return details, fmt.Errorf("malformed handshake (wrong response)") + } + + switch buffer[0] { + case 'n': + // 'n' + 2 (version) + 4 (flags) + 4 (challenge) + name... + if len(b.B) < 12 { + return details, fmt.Errorf("malformed handshake ('n')") + } + + challenge, err := dh.readChallenge(buffer[1:], &details) + if err != nil { + return details, err + } + b.Reset() + + dh.composeChallengeReply(b, challenge, tls, cookie) + + if e := b.WriteDataTo(conn); e != nil { + return details, e + } + // add 's' status for the case if we got it after 'n' or 'N' message + // yes, sometime it happens + await = []byte{'s', 'a'} + + case 'N': + // Peer support version 6. + + // The new challenge message format (version 6) + // 8 (flags) + 4 (Creation) + 2 (NameLen) + Name + if len(buffer) < 16 { + return details, fmt.Errorf("malformed handshake ('N' length)") + } + + challenge, err := dh.readChallengeVersion6(buffer[1:], &details) + if err != nil { + return details, err + } + b.Reset() + + if dh.options.Version == HandshakeVersion5 { + // upgrade handshake to version 6 by sending complement message + dh.composeComplement(b, tls) + if e := b.WriteDataTo(conn); e != nil { + return details, e + } + } + + dh.composeChallengeReply(b, challenge, tls, cookie) + + if e := b.WriteDataTo(conn); e != nil { + return details, e + } + + // add 's' (send_status message) for the case if we got it after 'n' or 'N' message + await = []byte{'s', 'a'} + + case 'a': + // 'a' + 16 (digest) + if len(buffer) != 17 { + return details, fmt.Errorf("malformed handshake ('a' length of digest)") + } + + // 'a' + 16 (digest) + digest := genDigest(dh.challenge, cookie) + if bytes.Compare(buffer[1:17], digest) != 0 { + return details, fmt.Errorf("malformed handshake ('a' digest)") + } + + // handshaked + return details, nil + + case 's': + if dh.readStatus(buffer[1:]) == false { + return details, fmt.Errorf("handshake negotiation failed") + } + + await = []byte{'n', 'N'} + // "sok" + if len(buffer) > 4 { + b.B = b.B[expectingBytes+3:] + goto next + } + b.Reset() + + default: + return details, fmt.Errorf("malformed handshake ('%c' digest)", buffer[0]) + } + + } + + } + +} + +func (dh *DistHandshake) Accept(conn io.ReadWriter, tls bool, cookie string) (node.HandshakeDetails, error) { + var details node.HandshakeDetails + + b := lib.TakeBuffer() + defer lib.ReleaseBuffer(b) + + var await []byte + + // define timeout for the handshaking + timer := time.NewTimer(dh.options.Timeout) + defer timer.Stop() + + asyncReadChannel := make(chan error, 2) + asyncRead := func() { + _, e := b.ReadDataFrom(conn, 512) + asyncReadChannel <- e + } + + // http://erlang.org/doc/apps/erts/erl_dist_protocol.html#distribution-handshake + // Every message in the handshake starts with a 16-bit big-endian integer, + // which contains the message length (not counting the two initial bytes). + // In Erlang this corresponds to option {packet, 2} in gen_tcp(3). Notice + // that after the handshake, the distribution switches to 4 byte packet headers. + expectingBytes := 2 + if tls { + // TLS connection has 4 bytes packet length header + expectingBytes = 4 + } + + // the comming message must be 'receive_name' as an answer for the + // 'send_name' message request we just sent + await = []byte{'n', 'N'} + + for { + go asyncRead() + + select { + case <-timer.C: + return details, fmt.Errorf("handshake accept timeout") + case e := <-asyncReadChannel: + if e != nil { + return details, e + } + + if b.Len() < expectingBytes+1 { + return details, fmt.Errorf("malformed handshake (too short packet)") + } + + next: + l := binary.BigEndian.Uint16(b.B[expectingBytes-2 : expectingBytes]) + buffer := b.B[expectingBytes:] + + if len(buffer) < int(l) { + return details, fmt.Errorf("malformed handshake (wrong packet length)") + } + + if bytes.Count(await, buffer[0:1]) == 0 { + return details, fmt.Errorf("malformed handshake (wrong response %d)", buffer[0]) + } + + switch buffer[0] { + case 'n': + if len(buffer) < 8 { + return details, fmt.Errorf("malformed handshake ('n' length)") + } + + if err := dh.readName(buffer[1:], &details); err != nil { + return details, err + } + b.Reset() + dh.composeStatus(b, tls) + if e := b.WriteDataTo(conn); e != nil { + return details, fmt.Errorf("malformed handshake ('n' accept name)") + } + + b.Reset() + if details.Version == 6 { + dh.composeChallengeVersion6(b, tls) + await = []byte{'s', 'r', 'c'} + } else { + dh.composeChallenge(b, tls) + await = []byte{'s', 'r'} + } + if e := b.WriteDataTo(conn); e != nil { + return details, e + } + + case 'N': + // The new challenge message format (version 6) + // 8 (flags) + 4 (Creation) + 2 (NameLen) + Name + if len(buffer) < 16 { + return details, fmt.Errorf("malformed handshake ('N' length)") + } + if err := dh.readNameVersion6(buffer[1:], &details); err != nil { + return details, err + } + b.Reset() + dh.composeStatus(b, tls) + if e := b.WriteDataTo(conn); e != nil { + return details, fmt.Errorf("malformed handshake ('N' accept name)") + } + + b.Reset() + dh.composeChallengeVersion6(b, tls) + if e := b.WriteDataTo(conn); e != nil { + return details, e + } + + await = []byte{'s', 'r'} + + case 'c': + if len(buffer) < 9 { + return details, fmt.Errorf("malformed handshake ('c' length)") + } + dh.readComplement(buffer[1:], &details) + + await = []byte{'r'} + + if len(buffer) > 9 { + b.B = b.B[expectingBytes+9:] + goto next + } + b.Reset() + + case 'r': + if len(buffer) < 19 { + return details, fmt.Errorf("malformed handshake ('r' length)") + } + + challenge, valid := dh.validateChallengeReply(buffer[1:], cookie) + if valid == false { + return details, fmt.Errorf("malformed handshake ('r' invalid reply)") + } + b.Reset() + + dh.composeChallengeAck(b, challenge, tls, cookie) + if e := b.WriteDataTo(conn); e != nil { + return details, e + } + + // handshaked + + return details, nil + + case 's': + if dh.readStatus(buffer[1:]) == false { + return details, fmt.Errorf("link status != ok") + } + + await = []byte{'c', 'r'} + if len(buffer) > 4 { + b.B = b.B[expectingBytes+3:] + goto next + } + b.Reset() + + default: + return details, fmt.Errorf("malformed handshake (unknown code %d)", b.B[0]) + } + + } + + } +} + +// private functions + +func (dh *DistHandshake) composeName(b *lib.Buffer, tls bool) { + flags := composeFlags(dh.flags) + version := uint16(dh.options.Version) + if tls { + b.Allocate(11) + dataLength := 7 + len(dh.nodename) // byte + uint16 + uint32 + len(dh.nodename) + binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) + b.B[4] = 'n' + binary.BigEndian.PutUint16(b.B[5:7], version) // uint16 + binary.BigEndian.PutUint32(b.B[7:11], flags.toUint32()) // uint32 + b.Append([]byte(dh.nodename)) + return + } + + b.Allocate(9) + dataLength := 7 + len(dh.nodename) // byte + uint16 + uint32 + len(dh.nodename) + binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) + b.B[2] = 'n' + binary.BigEndian.PutUint16(b.B[3:5], version) // uint16 + binary.BigEndian.PutUint32(b.B[5:9], flags.toUint32()) // uint32 + b.Append([]byte(dh.nodename)) +} + +func (dh *DistHandshake) composeNameVersion6(b *lib.Buffer, tls bool) { + flags := composeFlags(dh.flags) + creation := uint32(dh.creation) + if tls { + b.Allocate(19) + dataLength := 15 + len(dh.nodename) // 1 + 8 (flags) + 4 (creation) + 2 (len dh.nodename) + binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) + b.B[4] = 'N' + binary.BigEndian.PutUint64(b.B[5:13], flags.toUint64()) // uint64 + binary.BigEndian.PutUint32(b.B[13:17], creation) //uint32 + binary.BigEndian.PutUint16(b.B[17:19], uint16(len(dh.nodename))) // uint16 + b.Append([]byte(dh.nodename)) + return + } + + b.Allocate(17) + dataLength := 15 + len(dh.nodename) // 1 + 8 (flags) + 4 (creation) + 2 (len dh.nodename) + binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) + b.B[2] = 'N' + binary.BigEndian.PutUint64(b.B[3:11], flags.toUint64()) // uint64 + binary.BigEndian.PutUint32(b.B[11:15], creation) // uint32 + binary.BigEndian.PutUint16(b.B[15:17], uint16(len(dh.nodename))) // uint16 + b.Append([]byte(dh.nodename)) +} + +func (dh *DistHandshake) readName(b []byte, details *node.HandshakeDetails) error { + flags := nodeFlags(binary.BigEndian.Uint32(b[2:6])) + details.Flags = node.DefaultFlags() + details.Flags.EnableFragmentation = flags.isSet(flagFragments) + details.Flags.EnableBigCreation = flags.isSet(flagBigCreation) + details.Flags.EnableHeaderAtomCache = flags.isSet(flagDistHdrAtomCache) + details.Flags.EnableAlias = flags.isSet(flagAlias) + details.Flags.EnableRemoteSpawn = flags.isSet(flagSpawn) + details.Flags.EnableBigPidRef = flags.isSet(flagV4NC) + version := int(binary.BigEndian.Uint16(b[0:2])) + if version != 5 { + return fmt.Errorf("Malformed version for handshake 5") + } + + details.Version = 5 + if flags.isSet(flagHandshake23) { + details.Version = 6 + } + + // Erlang node limits the node name length to 256 characters (not bytes). + // I don't think anyone wants to use such a ridiculous name with a length > 250 bytes. + // Report an issue you really want to have a name longer that 255 bytes. + if len(b[6:]) > 255 { + return fmt.Errorf("Malformed node name") + } + details.Name = string(b[6:]) + + return nil +} + +func (dh *DistHandshake) readNameVersion6(b []byte, details *node.HandshakeDetails) error { + details.Creation = binary.BigEndian.Uint32(b[8:12]) + + flags := nodeFlags(binary.BigEndian.Uint64(b[0:8])) + details.Flags = node.DefaultFlags() + details.Flags.EnableFragmentation = flags.isSet(flagFragments) + details.Flags.EnableBigCreation = flags.isSet(flagBigCreation) + details.Flags.EnableHeaderAtomCache = flags.isSet(flagDistHdrAtomCache) + details.Flags.EnableAlias = flags.isSet(flagAlias) + details.Flags.EnableRemoteSpawn = flags.isSet(flagSpawn) + details.Flags.EnableBigPidRef = flags.isSet(flagV4NC) + details.Flags.EnableCompression = flags.isSet(flagCompression) + details.Flags.EnableProxy = flags.isSet(flagProxy) + + // see my prev comment about name len + nameLen := int(binary.BigEndian.Uint16(b[12:14])) + if nameLen > 255 { + return fmt.Errorf("Malformed node name") + } + nodename := string(b[14 : 14+nameLen]) + details.Name = nodename + + return nil +} + +func (dh *DistHandshake) composeStatus(b *lib.Buffer, tls bool) { + // there are few options for the status: ok, ok_simultaneous, nok, not_allowed, alive + // More details here: https://erlang.org/doc/apps/erts/erl_dist_protocol.html#the-handshake-in-detail + // support "ok" only, in any other cases link will be just closed + + if tls { + b.Allocate(4) + dataLength := 3 // 's' + "ok" + binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) + b.Append([]byte("sok")) + return + } + + b.Allocate(2) + dataLength := 3 // 's' + "ok" + binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) + b.Append([]byte("sok")) + +} + +func (dh *DistHandshake) readStatus(msg []byte) bool { + if string(msg[:2]) == "ok" { + return true + } + + return false +} + +func (dh *DistHandshake) composeChallenge(b *lib.Buffer, tls bool) { + flags := composeFlags(dh.flags) + if tls { + b.Allocate(15) + dataLength := uint32(11 + len(dh.nodename)) + binary.BigEndian.PutUint32(b.B[0:4], dataLength) + b.B[4] = 'n' + + //https://www.erlang.org/doc/apps/erts/erl_dist_protocol.html#distribution-handshake + // The Version is a 16-bit big endian integer and must always have the value 5 + binary.BigEndian.PutUint16(b.B[5:7], 5) // uint16 + + binary.BigEndian.PutUint32(b.B[7:11], flags.toUint32()) // uint32 + binary.BigEndian.PutUint32(b.B[11:15], dh.challenge) // uint32 + b.Append([]byte(dh.nodename)) + return + } + + b.Allocate(13) + dataLength := 11 + len(dh.nodename) + binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) + b.B[2] = 'n' + //https://www.erlang.org/doc/apps/erts/erl_dist_protocol.html#distribution-handshake + // The Version is a 16-bit big endian integer and must always have the value 5 + binary.BigEndian.PutUint16(b.B[3:5], 5) // uint16 + binary.BigEndian.PutUint32(b.B[5:9], flags.toUint32()) // uint32 + binary.BigEndian.PutUint32(b.B[9:13], dh.challenge) // uint32 + b.Append([]byte(dh.nodename)) +} + +func (dh *DistHandshake) composeChallengeVersion6(b *lib.Buffer, tls bool) { + + flags := composeFlags(dh.flags) + if tls { + // 1 ('N') + 8 (flags) + 4 (chalange) + 4 (creation) + 2 (len(dh.nodename)) + b.Allocate(23) + dataLength := 19 + len(dh.nodename) + binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) + b.B[4] = 'N' + binary.BigEndian.PutUint64(b.B[5:13], uint64(flags)) // uint64 + binary.BigEndian.PutUint32(b.B[13:17], dh.challenge) // uint32 + binary.BigEndian.PutUint32(b.B[17:21], dh.creation) // uint32 + binary.BigEndian.PutUint16(b.B[21:23], uint16(len(dh.nodename))) // uint16 + b.Append([]byte(dh.nodename)) + return + } + + // 1 ('N') + 8 (flags) + 4 (chalange) + 4 (creation) + 2 (len(dh.nodename)) + b.Allocate(21) + dataLength := 19 + len(dh.nodename) + binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) + b.B[2] = 'N' + binary.BigEndian.PutUint64(b.B[3:11], uint64(flags)) // uint64 + binary.BigEndian.PutUint32(b.B[11:15], dh.challenge) // uint32 + binary.BigEndian.PutUint32(b.B[15:19], dh.creation) // uint32 + binary.BigEndian.PutUint16(b.B[19:21], uint16(len(dh.nodename))) // uint16 + b.Append([]byte(dh.nodename)) +} + +func (dh *DistHandshake) readChallenge(msg []byte, details *node.HandshakeDetails) (uint32, error) { + var challenge uint32 + if len(msg) < 15 { + return challenge, fmt.Errorf("malformed handshake challenge") + } + flags := nodeFlags(binary.BigEndian.Uint32(msg[2:6])) + details.Flags = node.DefaultFlags() + details.Flags.EnableFragmentation = flags.isSet(flagFragments) + details.Flags.EnableBigCreation = flags.isSet(flagBigCreation) + details.Flags.EnableHeaderAtomCache = flags.isSet(flagDistHdrAtomCache) + details.Flags.EnableAlias = flags.isSet(flagAlias) + details.Flags.EnableRemoteSpawn = flags.isSet(flagSpawn) + details.Flags.EnableBigPidRef = flags.isSet(flagV4NC) + + version := binary.BigEndian.Uint16(msg[0:2]) + if version != uint16(HandshakeVersion5) { + return challenge, fmt.Errorf("malformed handshake version %d", version) + } + details.Version = int(version) + + if flags.isSet(flagHandshake23) { + // remote peer does support version 6 + details.Version = 6 + } + + details.Name = string(msg[10:]) + challenge = binary.BigEndian.Uint32(msg[6:10]) + return challenge, nil +} + +func (dh *DistHandshake) readChallengeVersion6(msg []byte, details *node.HandshakeDetails) (uint32, error) { + var challenge uint32 + flags := nodeFlags(binary.BigEndian.Uint64(msg[0:8])) + details.Flags = node.DefaultFlags() + details.Flags.EnableFragmentation = flags.isSet(flagFragments) + details.Flags.EnableBigCreation = flags.isSet(flagBigCreation) + details.Flags.EnableHeaderAtomCache = flags.isSet(flagDistHdrAtomCache) + details.Flags.EnableAlias = flags.isSet(flagAlias) + details.Flags.EnableRemoteSpawn = flags.isSet(flagSpawn) + details.Flags.EnableBigPidRef = flags.isSet(flagV4NC) + details.Flags.EnableCompression = flags.isSet(flagCompression) + details.Flags.EnableProxy = flags.isSet(flagProxy) + + details.Creation = binary.BigEndian.Uint32(msg[12:16]) + details.Version = 6 + + challenge = binary.BigEndian.Uint32(msg[8:12]) + + lenName := int(binary.BigEndian.Uint16(msg[16:18])) + details.Name = string(msg[18 : 18+lenName]) + + return challenge, nil +} + +func (dh *DistHandshake) readComplement(msg []byte, details *node.HandshakeDetails) { + flags := nodeFlags(uint64(binary.BigEndian.Uint32(msg[0:4])) << 32) + + details.Flags.EnableCompression = flags.isSet(flagCompression) + details.Flags.EnableProxy = flags.isSet(flagProxy) + details.Creation = binary.BigEndian.Uint32(msg[4:8]) +} + +func (dh *DistHandshake) validateChallengeReply(b []byte, cookie string) (uint32, bool) { + challenge := binary.BigEndian.Uint32(b[:4]) + digestB := b[4:] + + digestA := genDigest(dh.challenge, cookie) + return challenge, bytes.Equal(digestA[:], digestB) +} + +func (dh *DistHandshake) composeChallengeAck(b *lib.Buffer, challenge uint32, tls bool, cookie string) { + if tls { + b.Allocate(5) + dataLength := uint32(17) // 'a' + 16 (digest) + binary.BigEndian.PutUint32(b.B[0:4], dataLength) + b.B[4] = 'a' + digest := genDigest(challenge, cookie) + b.Append(digest) + return + } + + b.Allocate(3) + dataLength := uint16(17) // 'a' + 16 (digest) + binary.BigEndian.PutUint16(b.B[0:2], dataLength) + b.B[2] = 'a' + digest := genDigest(challenge, cookie) + b.Append(digest) +} + +func (dh *DistHandshake) composeChallengeReply(b *lib.Buffer, challenge uint32, tls bool, cookie string) { + if tls { + digest := genDigest(challenge, cookie) + b.Allocate(9) + dataLength := 5 + len(digest) // 1 (byte) + 4 (challenge) + 16 (digest) + binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) + b.B[4] = 'r' + binary.BigEndian.PutUint32(b.B[5:9], dh.challenge) // uint32 + b.Append(digest) + return + } + + b.Allocate(7) + digest := genDigest(challenge, cookie) + dataLength := 5 + len(digest) // 1 (byte) + 4 (challenge) + 16 (digest) + binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) + b.B[2] = 'r' + binary.BigEndian.PutUint32(b.B[3:7], dh.challenge) // uint32 + b.Append(digest) +} + +func (dh *DistHandshake) composeComplement(b *lib.Buffer, tls bool) { + flags := composeFlags(dh.flags) + // cast must cast creation to int32 in order to follow the + // erlang's handshake. Ergo don't care of it. + node_flags := uint32(flags.toUint64() >> 32) + if tls { + b.Allocate(13) + dataLength := 9 // 1 + 4 (flag high) + 4 (creation) + binary.BigEndian.PutUint32(b.B[0:4], uint32(dataLength)) + b.B[4] = 'c' + binary.BigEndian.PutUint32(b.B[5:9], node_flags) + binary.BigEndian.PutUint32(b.B[9:13], dh.creation) + return + } + + dataLength := 9 // 1 + 4 (flag high) + 4 (creation) + b.Allocate(11) + binary.BigEndian.PutUint16(b.B[0:2], uint16(dataLength)) + b.B[2] = 'c' + binary.BigEndian.PutUint32(b.B[3:7], node_flags) + binary.BigEndian.PutUint32(b.B[7:11], dh.creation) +} + +func genDigest(challenge uint32, cookie string) []byte { + s := fmt.Sprintf("%s%d", cookie, challenge) + digest := md5.Sum([]byte(s)) + return digest[:] +} + +func composeFlags(flags node.Flags) nodeFlags { + + // default flags + enabledFlags := []nodeFlagId{ + flagPublished, + flagUnicodeIO, + flagDistMonitor, + flagDistMonitorName, + flagExtendedPidsPorts, + flagExtendedReferences, + flagAtomCache, + flagHiddenAtomCache, + flagNewFunTags, + flagSmallAtomTags, + flagUTF8Atoms, + flagMapTag, + flagHandshake23, + } + + // optional flags + if flags.EnableHeaderAtomCache { + enabledFlags = append(enabledFlags, flagDistHdrAtomCache) + } + if flags.EnableFragmentation { + enabledFlags = append(enabledFlags, flagFragments) + } + if flags.EnableBigCreation { + enabledFlags = append(enabledFlags, flagBigCreation) + } + if flags.EnableAlias { + enabledFlags = append(enabledFlags, flagAlias) + } + if flags.EnableBigPidRef { + enabledFlags = append(enabledFlags, flagV4NC) + } + if flags.EnableRemoteSpawn { + enabledFlags = append(enabledFlags, flagSpawn) + } + if flags.EnableCompression { + enabledFlags = append(enabledFlags, flagCompression) + } + if flags.EnableProxy { + enabledFlags = append(enabledFlags, flagProxy) + } + return toNodeFlags(enabledFlags...) +} diff --git a/proto/dist/proto.go b/proto/dist/proto.go new file mode 100644 index 00000000..7f42022d --- /dev/null +++ b/proto/dist/proto.go @@ -0,0 +1,2182 @@ +package dist + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/aes" + "crypto/cipher" + crand "crypto/rand" + "encoding/binary" + "fmt" + "io" + "math/rand" + "sync" + "sync/atomic" + "time" + + "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/gen" + "github.com/ergo-services/ergo/lib" + "github.com/ergo-services/ergo/node" +) + +var ( + errMissingInCache = fmt.Errorf("missing in cache") + errMalformed = fmt.Errorf("malformed") + gzipReaders = &sync.Pool{ + New: func() interface{} { + return nil + }, + } + gzipWriters = [10]*sync.Pool{} + sendMessages = &sync.Pool{ + New: func() interface{} { + return &sendMessage{} + }, + } +) + +func init() { + rand.Seed(time.Now().UTC().UnixNano()) + for i := range gzipWriters { + gzipWriters[i] = &sync.Pool{ + New: func() interface{} { + return nil + }, + } + } +} + +const ( + defaultLatency = 200 * time.Nanosecond // for linkFlusher + + defaultCleanTimeout = 5 * time.Second // for checkClean + defaultCleanDeadline = 30 * time.Second // for checkClean + + // ergo proxy message + protoProxy = 141 + // ergo proxy encrypted message + protoProxyX = 142 + + // http://erlang.org/doc/apps/erts/erl_ext_dist.html#distribution_header + protoDist = 131 + protoDistMessage = 68 + protoDistFragment1 = 69 + protoDistFragmentN = 70 + + // ergo gzipped messages + protoDistMessageZ = 200 + protoDistFragment1Z = 201 + protoDistFragmentNZ = 202 +) + +type fragmentedPacket struct { + buffer *lib.Buffer + disordered *lib.Buffer + disorderedSlices map[uint64][]byte + fragmentID uint64 + lastUpdate time.Time +} + +type proxySession struct { + session node.ProxySession + cache etf.AtomCache + senderCache []map[etf.Atom]etf.CacheItem +} + +type distConnection struct { + node.Connection + + nodename string + peername string + ctx context.Context + + // peer flags + flags node.Flags + + creation uint32 + + // socket + conn io.ReadWriter + cancelContext context.CancelFunc + + // proxy session (endpoints) + proxySessionsByID map[string]proxySession + proxySessionsByPeerName map[string]proxySession + proxySessionsMutex sync.RWMutex + + // route incoming messages + router node.CoreRouter + + // writer + flusher *linkFlusher + + // senders list of channels for the sending goroutines + senders senders + // receivers list of channels for the receiving goroutines + receivers receivers + + // atom cache for outgoing messages + cache etf.AtomCache + + // fragmentation sequence ID + sequenceID int64 + fragments map[uint64]*fragmentedPacket + fragmentsMutex sync.Mutex + + // check and clean lost fragments + checkCleanPending bool + checkCleanTimer *time.Timer + checkCleanTimeout time.Duration // default is 5 seconds + checkCleanDeadline time.Duration // how long we wait for the next fragment of the certain sequenceID. Default is 30 seconds +} + +type distProto struct { + node.Proto + nodename string + options node.ProtoOptions +} + +func CreateProto(options node.ProtoOptions) node.ProtoInterface { + return &distProto{ + options: options, + } +} + +// +// node.Proto interface implementation +// + +type senders struct { + sender []*senderChannel + n int32 + i int32 +} + +type senderChannel struct { + sync.Mutex + sendChannel chan *sendMessage +} + +type sendMessage struct { + packet *lib.Buffer + control etf.Term + payload etf.Term + compression bool + compressionLevel int + compressionThreshold int + proxy *proxySession +} + +type receivers struct { + recv []chan *lib.Buffer + n int32 + i int32 +} + +func (dp *distProto) Init(ctx context.Context, conn io.ReadWriter, nodename string, details node.HandshakeDetails) (node.ConnectionInterface, error) { + connection := &distConnection{ + nodename: nodename, + peername: details.Name, + flags: details.Flags, + creation: details.Creation, + conn: conn, + cache: etf.NewAtomCache(), + proxySessionsByID: make(map[string]proxySession), + proxySessionsByPeerName: make(map[string]proxySession), + fragments: make(map[uint64]*fragmentedPacket), + checkCleanTimeout: defaultCleanTimeout, + checkCleanDeadline: defaultCleanDeadline, + } + connection.ctx, connection.cancelContext = context.WithCancel(ctx) + + // create connection buffering + connection.flusher = newLinkFlusher(conn, defaultLatency, details.Flags.EnableSoftwareKeepAlive) + + // do not use shared channels within intencive code parts, impacts on a performance + connection.receivers = receivers{ + recv: make([]chan *lib.Buffer, dp.options.NumHandlers), + n: int32(dp.options.NumHandlers), + } + + // run readers for incoming messages + for i := 0; i < dp.options.NumHandlers; i++ { + // run packet reader routines (decoder) + recv := make(chan *lib.Buffer, dp.options.RecvQueueLength) + connection.receivers.recv[i] = recv + go connection.receiver(recv) + } + + connection.senders = senders{ + sender: make([]*senderChannel, dp.options.NumHandlers), + n: int32(dp.options.NumHandlers), + } + + // run readers/writers for incoming/outgoing messages + for i := 0; i < dp.options.NumHandlers; i++ { + // run writer routines (encoder) + send := make(chan *sendMessage, dp.options.SendQueueLength) + connection.senders.sender[i] = &senderChannel{ + sendChannel: send, + } + go connection.sender(i, send, dp.options, connection.flags) + } + + return connection, nil +} + +func (dp *distProto) Serve(ci node.ConnectionInterface, router node.CoreRouter) { + connection, ok := ci.(*distConnection) + if !ok { + lib.Warning("conn is not a *distConnection type") + return + } + + connection.router = router + + // run read loop + var err error + var packetLength int + + b := lib.TakeBuffer() + for { + packetLength, err = connection.read(b, dp.options.MaxMessageSize) + + // validation + if err != nil || packetLength == 0 { + // link was closed or got malformed data + if err != nil { + lib.Warning("link was closed", connection.peername, "error:", err) + } + lib.ReleaseBuffer(b) + return + } + + // check the context if it was cancelled + if connection.ctx.Err() != nil { + // canceled + lib.ReleaseBuffer(b) + return + } + + // take the new buffer for the next reading and append the tail + // (which is part of the next packet) + b1 := lib.TakeBuffer() + b1.Set(b.B[packetLength:]) + + // cut the tail and send it further for handling. + // buffer b has to be released by the reader of + // recv channel (link.ReadHandlePacket) + b.B = b.B[:packetLength] + connection.receivers.recv[connection.receivers.i] <- b + + // set new buffer as a current for the next reading + b = b1 + + // round-robin switch to the next receiver + connection.receivers.i++ + if connection.receivers.i < connection.receivers.n { + continue + } + connection.receivers.i = 0 + } + +} + +func (dp *distProto) Terminate(ci node.ConnectionInterface) { + connection, ok := ci.(*distConnection) + if !ok { + lib.Warning("conn is not a *distConnection type") + return + } + + for i := 0; i < dp.options.NumHandlers; i++ { + sender := connection.senders.sender[i] + if sender != nil { + sender.Lock() + close(sender.sendChannel) + sender.sendChannel = nil + sender.Unlock() + connection.senders.sender[i] = nil + } + if connection.receivers.recv[i] != nil { + close(connection.receivers.recv[i]) + } + } + connection.flusher.Stop() + connection.cancelContext() +} + +// node.Connection interface implementation + +func (dc *distConnection) Send(from gen.Process, to etf.Pid, message etf.Term) error { + msg := sendMessages.Get().(*sendMessage) + + msg.control = etf.Tuple{distProtoSEND, etf.Atom(""), to} + msg.payload = message + msg.compression = from.Compression() + msg.compressionLevel = from.CompressionLevel() + msg.compressionThreshold = from.CompressionThreshold() + + return dc.send(string(to.Node), to.Creation, msg) +} +func (dc *distConnection) SendReg(from gen.Process, to gen.ProcessID, message etf.Term) error { + msg := sendMessages.Get().(*sendMessage) + + msg.control = etf.Tuple{distProtoREG_SEND, from.Self(), etf.Atom(""), etf.Atom(to.Name)} + msg.payload = message + msg.compression = from.Compression() + msg.compressionLevel = from.CompressionLevel() + msg.compressionThreshold = from.CompressionThreshold() + return dc.send(to.Node, 0, msg) +} +func (dc *distConnection) SendAlias(from gen.Process, to etf.Alias, message etf.Term) error { + if dc.flags.EnableAlias == false { + return node.ErrUnsupported + } + + msg := sendMessages.Get().(*sendMessage) + + msg.control = etf.Tuple{distProtoALIAS_SEND, from.Self(), to} + msg.payload = message + msg.compression = from.Compression() + msg.compressionLevel = from.CompressionLevel() + msg.compressionThreshold = from.CompressionThreshold() + + return dc.send(string(to.Node), to.Creation, msg) +} + +func (dc *distConnection) Link(local etf.Pid, remote etf.Pid) error { + dc.proxySessionsMutex.RLock() + ps, isProxy := dc.proxySessionsByPeerName[string(remote.Node)] + dc.proxySessionsMutex.RUnlock() + if isProxy && ps.session.PeerFlags.EnableLink == false { + return node.ErrPeerUnsupported + } + msg := &sendMessage{ + control: etf.Tuple{distProtoLINK, local, remote}, + } + return dc.send(string(remote.Node), remote.Creation, msg) +} +func (dc *distConnection) Unlink(local etf.Pid, remote etf.Pid) error { + dc.proxySessionsMutex.RLock() + ps, isProxy := dc.proxySessionsByPeerName[string(remote.Node)] + dc.proxySessionsMutex.RUnlock() + if isProxy && ps.session.PeerFlags.EnableLink == false { + return node.ErrPeerUnsupported + } + msg := &sendMessage{ + control: etf.Tuple{distProtoUNLINK, local, remote}, + } + return dc.send(string(remote.Node), remote.Creation, msg) +} +func (dc *distConnection) LinkExit(to etf.Pid, terminated etf.Pid, reason string) error { + msg := &sendMessage{ + control: etf.Tuple{distProtoEXIT, terminated, to, etf.Atom(reason)}, + } + return dc.send(string(to.Node), 0, msg) +} + +func (dc *distConnection) Monitor(local etf.Pid, remote etf.Pid, ref etf.Ref) error { + dc.proxySessionsMutex.RLock() + ps, isProxy := dc.proxySessionsByPeerName[string(remote.Node)] + dc.proxySessionsMutex.RUnlock() + if isProxy && ps.session.PeerFlags.EnableMonitor == false { + return node.ErrPeerUnsupported + } + msg := &sendMessage{ + control: etf.Tuple{distProtoMONITOR, local, remote, ref}, + } + return dc.send(string(remote.Node), remote.Creation, msg) +} +func (dc *distConnection) MonitorReg(local etf.Pid, remote gen.ProcessID, ref etf.Ref) error { + dc.proxySessionsMutex.RLock() + ps, isProxy := dc.proxySessionsByPeerName[remote.Node] + dc.proxySessionsMutex.RUnlock() + if isProxy && ps.session.PeerFlags.EnableMonitor == false { + return node.ErrPeerUnsupported + } + msg := &sendMessage{ + control: etf.Tuple{distProtoMONITOR, local, etf.Atom(remote.Name), ref}, + } + return dc.send(remote.Node, 0, msg) +} +func (dc *distConnection) Demonitor(local etf.Pid, remote etf.Pid, ref etf.Ref) error { + dc.proxySessionsMutex.RLock() + ps, isProxy := dc.proxySessionsByPeerName[string(remote.Node)] + dc.proxySessionsMutex.RUnlock() + if isProxy && ps.session.PeerFlags.EnableMonitor == false { + return node.ErrPeerUnsupported + } + msg := &sendMessage{ + control: etf.Tuple{distProtoDEMONITOR, local, remote, ref}, + } + return dc.send(string(remote.Node), remote.Creation, msg) +} +func (dc *distConnection) DemonitorReg(local etf.Pid, remote gen.ProcessID, ref etf.Ref) error { + dc.proxySessionsMutex.RLock() + ps, isProxy := dc.proxySessionsByPeerName[remote.Node] + dc.proxySessionsMutex.RUnlock() + if isProxy && ps.session.PeerFlags.EnableMonitor == false { + return node.ErrPeerUnsupported + } + msg := &sendMessage{ + control: etf.Tuple{distProtoDEMONITOR, local, etf.Atom(remote.Name), ref}, + } + return dc.send(remote.Node, 0, msg) +} +func (dc *distConnection) MonitorExitReg(to etf.Pid, terminated gen.ProcessID, reason string, ref etf.Ref) error { + msg := &sendMessage{ + control: etf.Tuple{distProtoMONITOR_EXIT, etf.Atom(terminated.Name), to, ref, etf.Atom(reason)}, + } + return dc.send(string(to.Node), to.Creation, msg) +} +func (dc *distConnection) MonitorExit(to etf.Pid, terminated etf.Pid, reason string, ref etf.Ref) error { + msg := &sendMessage{ + control: etf.Tuple{distProtoMONITOR_EXIT, terminated, to, ref, etf.Atom(reason)}, + } + return dc.send(string(to.Node), to.Creation, msg) +} + +func (dc *distConnection) SpawnRequest(nodeName string, behaviorName string, request gen.RemoteSpawnRequest, args ...etf.Term) error { + dc.proxySessionsMutex.RLock() + ps, isProxy := dc.proxySessionsByPeerName[nodeName] + dc.proxySessionsMutex.RUnlock() + if isProxy { + if ps.session.PeerFlags.EnableRemoteSpawn == false { + return node.ErrPeerUnsupported + } + } else { + if dc.flags.EnableRemoteSpawn == false { + return node.ErrPeerUnsupported + } + } + + optlist := etf.List{} + if request.Options.Name != "" { + optlist = append(optlist, etf.Tuple{etf.Atom("name"), etf.Atom(request.Options.Name)}) + + } + msg := &sendMessage{ + control: etf.Tuple{distProtoSPAWN_REQUEST, request.Ref, request.From, request.From, + // {M,F,A} + etf.Tuple{etf.Atom(behaviorName), etf.Atom(request.Options.Function), len(args)}, + optlist, + }, + payload: args, + } + return dc.send(nodeName, 0, msg) +} + +func (dc *distConnection) SpawnReply(to etf.Pid, ref etf.Ref, pid etf.Pid) error { + msg := &sendMessage{ + control: etf.Tuple{distProtoSPAWN_REPLY, ref, to, 0, pid}, + } + return dc.send(string(to.Node), to.Creation, msg) +} + +func (dc *distConnection) SpawnReplyError(to etf.Pid, ref etf.Ref, err error) error { + msg := &sendMessage{ + control: etf.Tuple{distProtoSPAWN_REPLY, ref, to, 0, etf.Atom(err.Error())}, + } + return dc.send(string(to.Node), to.Creation, msg) +} + +func (dc *distConnection) ProxyConnectRequest(request node.ProxyConnectRequest) error { + if dc.flags.EnableProxy == false { + return node.ErrPeerUnsupported + } + + path := []etf.Atom{} + for i := range request.Path { + path = append(path, etf.Atom(request.Path[i])) + } + + msg := &sendMessage{ + control: etf.Tuple{distProtoPROXY_CONNECT_REQUEST, + request.ID, // etf.Ref + etf.Atom(request.To), // to node + request.Digest, // + request.PublicKey, // public key for the sending symmetric key + proxyFlagsToUint64(request.Flags), + request.Creation, + request.Hop, + path, + }, + } + return dc.send(dc.peername, 0, msg) +} + +func (dc *distConnection) ProxyConnectReply(reply node.ProxyConnectReply) error { + if dc.flags.EnableProxy == false { + return node.ErrPeerUnsupported + } + + path := etf.List{} + for i := range reply.Path { + path = append(path, etf.Atom(reply.Path[i])) + } + + msg := &sendMessage{ + control: etf.Tuple{distProtoPROXY_CONNECT_REPLY, + reply.ID, // etf.Ref + etf.Atom(reply.To), // to node + reply.Digest, // + reply.Cipher, // + proxyFlagsToUint64(reply.Flags), + reply.Creation, + reply.SessionID, + path, + }, + } + + return dc.send(dc.peername, 0, msg) +} + +func (dc *distConnection) ProxyConnectCancel(err node.ProxyConnectCancel) error { + if dc.flags.EnableProxy == false { + return node.ErrPeerUnsupported + } + + path := etf.List{} + for i := range err.Path { + path = append(path, etf.Atom(err.Path[i])) + } + + msg := &sendMessage{ + control: etf.Tuple{distProtoPROXY_CONNECT_CANCEL, + err.ID, // etf.Ref + etf.Atom(err.From), // from node + err.Reason, + path, + }, + } + + return dc.send(dc.peername, 0, msg) +} + +func (dc *distConnection) ProxyDisconnect(disconnect node.ProxyDisconnect) error { + if dc.flags.EnableProxy == false { + return node.ErrPeerUnsupported + } + + msg := &sendMessage{ + control: etf.Tuple{distProtoPROXY_DISCONNECT, + etf.Atom(disconnect.Node), + etf.Atom(disconnect.Proxy), + disconnect.SessionID, + disconnect.Reason, + }, + } + + return dc.send(dc.peername, 0, msg) +} + +func (dc *distConnection) ProxyRegisterSession(session node.ProxySession) error { + dc.proxySessionsMutex.Lock() + defer dc.proxySessionsMutex.Unlock() + _, exist := dc.proxySessionsByPeerName[session.PeerName] + if exist { + return node.ErrProxySessionDuplicate + } + _, exist = dc.proxySessionsByID[session.ID] + if exist { + return node.ErrProxySessionDuplicate + } + ps := proxySession{ + session: session, + cache: etf.NewAtomCache(), + // every sender should have its own senderAtomCache in the proxy session + senderCache: make([]map[etf.Atom]etf.CacheItem, len(dc.senders.sender)), + } + dc.proxySessionsByPeerName[session.PeerName] = ps + dc.proxySessionsByID[session.ID] = ps + return nil +} + +func (dc *distConnection) ProxyUnregisterSession(id string) error { + dc.proxySessionsMutex.Lock() + defer dc.proxySessionsMutex.Unlock() + ps, exist := dc.proxySessionsByID[id] + if exist == false { + return node.ErrProxySessionUnknown + } + delete(dc.proxySessionsByPeerName, ps.session.PeerName) + delete(dc.proxySessionsByID, ps.session.ID) + return nil +} + +func (dc *distConnection) ProxyPacket(packet *lib.Buffer) error { + if dc.flags.EnableProxy == false { + return node.ErrPeerUnsupported + } + msg := &sendMessage{ + packet: packet, + } + return dc.send(dc.peername, 0, msg) +} + +// +// internal +// + +func (dc *distConnection) read(b *lib.Buffer, max int) (int, error) { + // http://erlang.org/doc/apps/erts/erl_dist_protocol.html#protocol-between-connected-nodes + expectingBytes := 4 + for { + if b.Len() < expectingBytes { + n, e := b.ReadDataFrom(dc.conn, max) + if n == 0 { + // link was closed + return 0, nil + } + + if e != nil && e != io.EOF { + // something went wrong + return 0, e + } + + // check onemore time if we should read more data + continue + } + + packetLength := binary.BigEndian.Uint32(b.B[:4]) + if packetLength == 0 { + // it was "software" keepalive + expectingBytes = 4 + if len(b.B) == 4 { + b.Reset() + continue + } + b.B = b.B[4:] + continue + } + + if b.Len() < int(packetLength)+4 { + expectingBytes = int(packetLength) + 4 + continue + } + + return int(packetLength) + 4, nil + } + +} + +type deferrMissing struct { + b *lib.Buffer + c int +} + +type distMessage struct { + control etf.Term + payload etf.Term + proxy *proxySession +} + +func (dc *distConnection) receiver(recv <-chan *lib.Buffer) { + var b *lib.Buffer + var missing deferrMissing + var Timeout <-chan time.Time + + // cancel connection context if something went wrong + // it will cause closing connection with stopping all + // goroutines around this connection + defer dc.cancelContext() + + deferrChannel := make(chan deferrMissing, 100) + defer close(deferrChannel) + + timer := lib.TakeTimer() + defer lib.ReleaseTimer(timer) + + dChannel := deferrChannel + + for { + select { + case missing = <-dChannel: + b = missing.b + default: + if len(deferrChannel) > 0 { + timer.Reset(150 * time.Millisecond) + Timeout = timer.C + } else { + Timeout = nil + } + select { + case b = <-recv: + if b == nil { + // channel was closed + return + } + case <-Timeout: + dChannel = deferrChannel + continue + } + } + + // read and decode received packet + message, err := dc.decodePacket(b) + + if err == errMissingInCache { + if b == missing.b && missing.c > 100 { + lib.Warning("Disordered data at the link with %q. Close connection", dc.peername) + dc.cancelContext() + lib.ReleaseBuffer(b) + return + } + + if b == missing.b { + missing.c++ + } else { + missing.b = b + missing.c = 0 + } + + select { + case deferrChannel <- missing: + // read recv channel + dChannel = nil + continue + default: + lib.Warning("Mess at the link with %q. Close connection", dc.peername) + dc.cancelContext() + lib.ReleaseBuffer(b) + return + } + } + + dChannel = deferrChannel + + if err != nil { + lib.Warning("[%s] Malformed Dist proto at the link with %s: %s", dc.nodename, dc.peername, err) + dc.cancelContext() + lib.ReleaseBuffer(b) + return + } + + if message == nil { + // fragment or proxy message + continue + } + + // handle message + if err := dc.handleMessage(message); err != nil { + if message.proxy == nil { + lib.Warning("[%s] Malformed Control packet at the link with %s: %#v", dc.nodename, dc.peername, message.control) + dc.cancelContext() + lib.ReleaseBuffer(b) + return + } + // drop proxy session + lib.Warning("[%s] Malformed Control packet at the proxy link with %s: %#v", dc.nodename, message.proxy.session.PeerName, message.control) + disconnect := node.ProxyDisconnect{ + Node: dc.nodename, + Proxy: dc.nodename, + SessionID: message.proxy.session.ID, + Reason: err.Error(), + } + // route it locally to unregister this session + dc.router.RouteProxyDisconnect(dc, disconnect) + // send it to the peer + dc.ProxyDisconnect(disconnect) + } + + // we have to release this buffer + lib.ReleaseBuffer(b) + + } +} + +func (dc *distConnection) decodePacket(b *lib.Buffer) (*distMessage, error) { + packet := b.B + if len(packet) < 5 { + return nil, fmt.Errorf("malformed packet") + } + + // [:3] length + switch packet[4] { + case protoDist: + // do not check the length. it was checked on the receiving this packet. + control, payload, err := dc.decodeDist(packet[5:], nil) + if control == nil { + return nil, err + } + message := &distMessage{control: control, payload: payload} + return message, err + + case protoProxy: + sessionID := string(packet[5:37]) + dc.proxySessionsMutex.RLock() + ps, exist := dc.proxySessionsByID[sessionID] + dc.proxySessionsMutex.RUnlock() + if exist == false { + // must be send further + if err := dc.router.RouteProxy(dc, sessionID, b); err != nil { + // drop proxy session + disconnect := node.ProxyDisconnect{ + Node: dc.nodename, + Proxy: dc.nodename, + SessionID: sessionID, + Reason: err.Error(), + } + dc.ProxyDisconnect(disconnect) + } + return nil, nil + } + // this node is endpoint of this session + packet = b.B[37:] + control, payload, err := dc.decodeDist(packet, &ps) + if err != nil { + if err == errMissingInCache { + // will be deferred. + // 37 - 5 + // where: + // 37 = packet len (4) + protoProxy (1) + session id (32) + // reserving 5 bytes for: packet len(4) + protoDist (1) + // we don't update packet len value. it was already validated + // and will be ignored on the next dc.decodeDist call + b.B = b.B[32:] + b.B[4] = protoDist + return nil, err + } + // drop this proxy session. send back ProxyDisconnect + disconnect := node.ProxyDisconnect{ + Node: dc.nodename, + Proxy: dc.nodename, + SessionID: sessionID, + Reason: err.Error(), + } + dc.router.RouteProxyDisconnect(dc, disconnect) + dc.ProxyDisconnect(disconnect) + return nil, nil + } + if control == nil { + return nil, nil + } + message := &distMessage{control: control, payload: payload, proxy: &ps} + return message, nil + + case protoProxyX: + sessionID := string(packet[5:37]) + dc.proxySessionsMutex.RLock() + ps, exist := dc.proxySessionsByID[sessionID] + dc.proxySessionsMutex.RUnlock() + if exist == false { + // must be send further + if err := dc.router.RouteProxy(dc, sessionID, b); err != nil { + // drop proxy session + disconnect := node.ProxyDisconnect{ + Node: dc.nodename, + Proxy: dc.nodename, + SessionID: sessionID, + Reason: err.Error(), + } + dc.ProxyDisconnect(disconnect) + } + return nil, nil + } + + packet = b.B[37:] + if (len(packet) % aes.BlockSize) != 0 { + // drop this proxy session. + disconnect := node.ProxyDisconnect{ + Node: dc.nodename, + Proxy: dc.nodename, + SessionID: sessionID, + Reason: "wrong blocksize of the encrypted message", + } + dc.router.RouteProxyDisconnect(dc, disconnect) + dc.ProxyDisconnect(disconnect) + return nil, nil + } + iv := packet[:aes.BlockSize] + msg := packet[aes.BlockSize:] + cfb := cipher.NewCFBDecrypter(ps.session.Block, iv) + cfb.XORKeyStream(msg, msg) + + // check padding + length := len(msg) + unpadding := int(msg[length-1]) + if unpadding > length { + // drop this proxy session. + disconnect := node.ProxyDisconnect{ + Node: dc.nodename, + Proxy: dc.nodename, + SessionID: sessionID, + Reason: "wrong padding of the encrypted message", + } + dc.router.RouteProxyDisconnect(dc, disconnect) + dc.ProxyDisconnect(disconnect) + return nil, nil + } + packet = msg[:(length - unpadding)] + control, payload, err := dc.decodeDist(packet, &ps) + if err != nil { + if err == errMissingInCache { + // will be deferred + // TODO make sure if this shift is correct + b.B = b.B[32+aes.BlockSize:] + b.B[4] = protoDist + return nil, err + } + // drop this proxy session. + disconnect := node.ProxyDisconnect{ + Node: dc.nodename, + Proxy: dc.nodename, + SessionID: sessionID, + Reason: err.Error(), + } + dc.router.RouteProxyDisconnect(dc, disconnect) + dc.ProxyDisconnect(disconnect) + return nil, nil + } + if control == nil { + return nil, nil + } + message := &distMessage{control: control, payload: payload, proxy: &ps} + return message, nil + + default: + // unknown proto + return nil, fmt.Errorf("unknown/unsupported proto") + } + +} + +func (dc *distConnection) decodeDist(packet []byte, proxy *proxySession) (etf.Term, etf.Term, error) { + switch packet[0] { + case protoDistMessage: + var control, payload etf.Term + var err error + var cache []etf.Atom + + cache, packet, err = dc.decodeDistHeaderAtomCache(packet[1:], proxy) + if err != nil { + return nil, nil, err + } + + decodeOptions := etf.DecodeOptions{ + FlagBigPidRef: dc.flags.EnableBigPidRef, + } + if proxy != nil { + decodeOptions.FlagBigPidRef = true + } + + // decode control message + control, packet, err = etf.Decode(packet, cache, decodeOptions) + if err != nil { + return nil, nil, err + } + + if len(packet) == 0 { + return control, nil, nil + } + + // decode payload message + payload, packet, err = etf.Decode(packet, cache, decodeOptions) + if err != nil { + return nil, nil, err + } + + if len(packet) != 0 { + return nil, nil, fmt.Errorf("packet has extra %d byte(s)", len(packet)) + } + + return control, payload, nil + + case protoDistMessageZ: + var control, payload etf.Term + var err error + var cache []etf.Atom + var zReader *gzip.Reader + var total int + // compressed protoDistMessage + + cache, packet, err = dc.decodeDistHeaderAtomCache(packet[1:], proxy) + if err != nil { + return nil, nil, err + } + + // read the length of unpacked data + lenUnpacked := int(binary.BigEndian.Uint32(packet[:4])) + + // take the gzip reader from the pool + if r, ok := gzipReaders.Get().(*gzip.Reader); ok { + zReader = r + zReader.Reset(bytes.NewBuffer(packet[4:])) + } else { + zReader, _ = gzip.NewReader(bytes.NewBuffer(packet[4:])) + } + defer gzipReaders.Put(zReader) + + // take new buffer and allocate space for the unpacked data + zBuffer := lib.TakeBuffer() + zBuffer.Allocate(lenUnpacked) + defer lib.ReleaseBuffer(zBuffer) + + // unzipping and decoding the data + for { + n, e := zReader.Read(zBuffer.B[total:]) + if n == 0 { + return nil, nil, fmt.Errorf("zbuffer too small") + } + total += n + if e == io.EOF { + break + } + if e != nil { + return nil, nil, e + } + } + + packet = zBuffer.B + decodeOptions := etf.DecodeOptions{ + FlagBigPidRef: dc.flags.EnableBigPidRef, + } + if proxy != nil { + decodeOptions.FlagBigPidRef = true + } + + // decode control message + control, packet, err = etf.Decode(packet, cache, decodeOptions) + if err != nil { + return nil, nil, err + } + if len(packet) == 0 { + return control, nil, nil + } + + // decode payload message + payload, packet, err = etf.Decode(packet, cache, decodeOptions) + if err != nil { + return nil, nil, err + } + + if len(packet) != 0 { + return nil, nil, fmt.Errorf("packet has extra %d byte(s)", len(packet)) + } + + return control, payload, nil + + case protoDistFragment1, protoDistFragmentN, protoDistFragment1Z, protoDistFragmentNZ: + if len(packet) < 18 { + return nil, nil, fmt.Errorf("malformed fragment (too small)") + } + + if assembled, err := dc.decodeFragment(packet, proxy); assembled != nil { + if err != nil { + return nil, nil, err + } + control, payload, err := dc.decodeDist(assembled.B, nil) + lib.ReleaseBuffer(assembled) + return control, payload, err + } else { + if err != nil { + return nil, nil, err + } + } + return nil, nil, nil + } + + return nil, nil, fmt.Errorf("unknown packet type %d", packet[0]) +} + +func (dc *distConnection) handleMessage(message *distMessage) (err error) { + defer func() { + if lib.CatchPanic() { + if r := recover(); r != nil { + err = fmt.Errorf("%s", r) + } + } + }() + + switch t := message.control.(type) { + case etf.Tuple: + switch act := t.Element(1).(type) { + case int: + switch act { + case distProtoREG_SEND: + // {6, FromPid, Unused, ToName} + lib.Log("[%s] CONTROL REG_SEND [from %s]: %#v", dc.nodename, dc.peername, message.control) + to := gen.ProcessID{ + Node: dc.nodename, + Name: string(t.Element(4).(etf.Atom)), + } + dc.router.RouteSendReg(t.Element(2).(etf.Pid), to, message.payload) + return nil + + case distProtoSEND: + // {2, Unused, ToPid} + // SEND has no sender pid + lib.Log("[%s] CONTROL SEND [from %s]: %#v", dc.nodename, dc.peername, message.control) + dc.router.RouteSend(etf.Pid{}, t.Element(3).(etf.Pid), message.payload) + return nil + + case distProtoLINK: + // {1, FromPid, ToPid} + lib.Log("[%s] CONTROL LINK [from %s]: %#v", dc.nodename, dc.peername, message.control) + if message.proxy != nil && message.proxy.session.NodeFlags.EnableLink == false { + // we didn't allow this feature. proxy session will be closed due to + // this violation of the contract + return node.ErrPeerUnsupported + } + dc.router.RouteLink(t.Element(2).(etf.Pid), t.Element(3).(etf.Pid)) + return nil + + case distProtoUNLINK: + // {4, FromPid, ToPid} + lib.Log("[%s] CONTROL UNLINK [from %s]: %#v", dc.nodename, dc.peername, message.control) + if message.proxy != nil && message.proxy.session.NodeFlags.EnableLink == false { + // we didn't allow this feature. proxy session will be closed due to + // this violation of the contract + return node.ErrPeerUnsupported + } + dc.router.RouteUnlink(t.Element(2).(etf.Pid), t.Element(3).(etf.Pid)) + return nil + + case distProtoNODE_LINK: + lib.Log("[%s] CONTROL NODE_LINK [from %s]: %#v", dc.nodename, dc.peername, message.control) + return nil + + case distProtoEXIT: + // {3, FromPid, ToPid, Reason} + lib.Log("[%s] CONTROL EXIT [from %s]: %#v", dc.nodename, dc.peername, message.control) + terminated := t.Element(2).(etf.Pid) + to := t.Element(3).(etf.Pid) + reason := fmt.Sprint(t.Element(4)) + dc.router.RouteExit(to, terminated, string(reason)) + return nil + + case distProtoEXIT2: + lib.Log("[%s] CONTROL EXIT2 [from %s]: %#v", dc.nodename, dc.peername, message.control) + return nil + + case distProtoMONITOR: + // {19, FromPid, ToProc, Ref}, where FromPid = monitoring process + // and ToProc = monitored process pid or name (atom) + lib.Log("[%s] CONTROL MONITOR [from %s]: %#v", dc.nodename, dc.peername, message.control) + if message.proxy != nil && message.proxy.session.NodeFlags.EnableMonitor == false { + // we didn't allow this feature. proxy session will be closed due to + // this violation of the contract + return node.ErrPeerUnsupported + } + + fromPid := t.Element(2).(etf.Pid) + ref := t.Element(4).(etf.Ref) + // if monitoring by pid + if to, ok := t.Element(3).(etf.Pid); ok { + dc.router.RouteMonitor(fromPid, to, ref) + return nil + } + + // if monitoring by process name + if to, ok := t.Element(3).(etf.Atom); ok { + processID := gen.ProcessID{ + Node: dc.nodename, + Name: string(to), + } + dc.router.RouteMonitorReg(fromPid, processID, ref) + return nil + } + + return fmt.Errorf("malformed monitor message") + + case distProtoDEMONITOR: + // {20, FromPid, ToProc, Ref}, where FromPid = monitoring process + // and ToProc = monitored process pid or name (atom) + lib.Log("[%s] CONTROL DEMONITOR [from %s]: %#v", dc.nodename, dc.peername, message.control) + if message.proxy != nil && message.proxy.session.NodeFlags.EnableMonitor == false { + // we didn't allow this feature. proxy session will be closed due to + // this violation of the contract + return node.ErrPeerUnsupported + } + ref := t.Element(4).(etf.Ref) + fromPid := t.Element(2).(etf.Pid) + dc.router.RouteDemonitor(fromPid, ref) + return nil + + case distProtoMONITOR_EXIT: + // {21, FromProc, ToPid, Ref, Reason}, where FromProc = monitored process + // pid or name (atom), ToPid = monitoring process, and Reason = exit reason for the monitored process + lib.Log("[%s] CONTROL MONITOR_EXIT [from %s]: %#v", dc.nodename, dc.peername, message.control) + reason := fmt.Sprint(t.Element(5)) + ref := t.Element(4).(etf.Ref) + switch terminated := t.Element(2).(type) { + case etf.Pid: + dc.router.RouteMonitorExit(terminated, reason, ref) + return nil + case etf.Atom: + processID := gen.ProcessID{Name: string(terminated), Node: dc.peername} + if message.proxy != nil { + processID.Node = message.proxy.session.PeerName + } + dc.router.RouteMonitorExitReg(processID, reason, ref) + return nil + } + return fmt.Errorf("malformed monitor exit message") + + // Not implemented yet, just stubs. TODO. + case distProtoSEND_SENDER: + lib.Log("[%s] CONTROL SEND_SENDER unsupported [from %s]: %#v", dc.nodename, dc.peername, message.control) + return nil + case distProtoPAYLOAD_EXIT: + lib.Log("[%s] CONTROL PAYLOAD_EXIT unsupported [from %s]: %#v", dc.nodename, dc.peername, message.control) + return nil + case distProtoPAYLOAD_EXIT2: + lib.Log("[%s] CONTROL PAYLOAD_EXIT2 unsupported [from %s]: %#v", dc.nodename, dc.peername, message.control) + return nil + case distProtoPAYLOAD_MONITOR_P_EXIT: + lib.Log("[%s] CONTROL PAYLOAD_MONITOR_P_EXIT unsupported [from %s]: %#v", dc.nodename, dc.peername, message.control) + return nil + + // alias support + case distProtoALIAS_SEND: + // {33, FromPid, Alias} + lib.Log("[%s] CONTROL ALIAS_SEND [from %s]: %#v", dc.nodename, dc.peername, message.control) + alias := etf.Alias(t.Element(3).(etf.Ref)) + dc.router.RouteSendAlias(t.Element(2).(etf.Pid), alias, message.payload) + return nil + + case distProtoSPAWN_REQUEST: + // {29, ReqId, From, GroupLeader, {Module, Function, Arity}, OptList} + lib.Log("[%s] CONTROL SPAWN_REQUEST [from %s]: %#v", dc.nodename, dc.peername, message.control) + if message.proxy != nil && message.proxy.session.NodeFlags.EnableRemoteSpawn == false { + // we didn't allow this feature. proxy session will be closed due to + // this violation of the contract + return node.ErrPeerUnsupported + } + registerName := "" + for _, option := range t.Element(6).(etf.List) { + name, ok := option.(etf.Tuple) + if !ok || len(name) != 2 { + return fmt.Errorf("malformed spawn request") + } + switch name.Element(1) { + case etf.Atom("name"): + registerName = string(name.Element(2).(etf.Atom)) + } + } + + from := t.Element(3).(etf.Pid) + ref := t.Element(2).(etf.Ref) + + mfa := t.Element(5).(etf.Tuple) + module := mfa.Element(1).(etf.Atom) + function := mfa.Element(2).(etf.Atom) + var args etf.List + if str, ok := message.payload.(string); !ok { + args, _ = message.payload.(etf.List) + } else { + // stupid Erlang's strings :). [1,2,3,4,5] sends as a string. + // args can't be anything but etf.List. + for i := range []byte(str) { + args = append(args, str[i]) + } + } + + spawnRequestOptions := gen.RemoteSpawnOptions{ + Name: registerName, + Function: string(function), + } + spawnRequest := gen.RemoteSpawnRequest{ + From: from, + Ref: ref, + Options: spawnRequestOptions, + } + dc.router.RouteSpawnRequest(dc.nodename, string(module), spawnRequest, args...) + return nil + + case distProtoSPAWN_REPLY: + // {31, ReqId, To, Flags, Result} + lib.Log("[%s] CONTROL SPAWN_REPLY [from %s]: %#v", dc.nodename, dc.peername, message.control) + ref := t.Element(2).(etf.Ref) + to := t.Element(3).(etf.Pid) + dc.router.RouteSpawnReply(to, ref, t.Element(5)) + return nil + + case distProtoPROXY_CONNECT_REQUEST: + // {101, ID, To, Digest, PublicKey, Flags, Hop, Path} + lib.Log("[%s] PROXY CONNECT REQUEST [from %s]: %#v", dc.nodename, dc.peername, message.control) + request := node.ProxyConnectRequest{ + ID: t.Element(2).(etf.Ref), + To: string(t.Element(3).(etf.Atom)), + Digest: t.Element(4).([]byte), + PublicKey: t.Element(5).([]byte), + // FIXME it will be int64 after using more than 32 flags + Flags: proxyFlagsFromUint64(uint64(t.Element(6).(int))), + Creation: uint32(t.Element(7).(int64)), + Hop: t.Element(8).(int), + } + for _, p := range t.Element(9).(etf.List) { + request.Path = append(request.Path, string(p.(etf.Atom))) + } + if err := dc.router.RouteProxyConnectRequest(dc, request); err != nil { + errReply := node.ProxyConnectCancel{ + ID: request.ID, + From: dc.nodename, + Reason: err.Error(), + Path: request.Path[1:], + } + dc.ProxyConnectCancel(errReply) + } + return nil + + case distProtoPROXY_CONNECT_REPLY: + // {102, ID, To, Digest, Cipher, Flags, SessionID, Path} + lib.Log("[%s] PROXY CONNECT REPLY [from %s]: %#v", dc.nodename, dc.peername, message.control) + connectReply := node.ProxyConnectReply{ + ID: t.Element(2).(etf.Ref), + To: string(t.Element(3).(etf.Atom)), + Digest: t.Element(4).([]byte), + Cipher: t.Element(5).([]byte), + // FIXME it will be int64 after using more than 32 flags + Flags: proxyFlagsFromUint64(uint64(t.Element(6).(int))), + Creation: uint32(t.Element(7).(int64)), + SessionID: t.Element(8).(string), + } + for _, p := range t.Element(9).(etf.List) { + connectReply.Path = append(connectReply.Path, string(p.(etf.Atom))) + } + if err := dc.router.RouteProxyConnectReply(dc, connectReply); err != nil { + lib.Log("[%s] PROXY CONNECT REPLY error %s (message: %#v)", dc.nodename, err, connectReply) + // send disconnect to clean up this session all the way to the + // destination node + disconnect := node.ProxyDisconnect{ + Node: dc.nodename, + Proxy: dc.nodename, + SessionID: connectReply.SessionID, + Reason: err.Error(), + } + dc.ProxyDisconnect(disconnect) + if err == node.ErrNoRoute { + return nil + } + + // send cancel message to the source node + cancel := node.ProxyConnectCancel{ + ID: connectReply.ID, + From: dc.nodename, + Reason: err.Error(), + Path: connectReply.Path, + } + dc.router.RouteProxyConnectCancel(dc, cancel) + } + + return nil + + case distProtoPROXY_CONNECT_CANCEL: + lib.Log("[%s] PROXY CONNECT CANCEL [from %s]: %#v", dc.nodename, dc.peername, message.control) + connectError := node.ProxyConnectCancel{ + ID: t.Element(2).(etf.Ref), + From: string(t.Element(3).(etf.Atom)), + Reason: t.Element(4).(string), + } + for _, p := range t.Element(5).(etf.List) { + connectError.Path = append(connectError.Path, string(p.(etf.Atom))) + } + dc.router.RouteProxyConnectCancel(dc, connectError) + return nil + + case distProtoPROXY_DISCONNECT: + // {104, Node, Proxy, SessionID, Reason} + lib.Log("[%s] PROXY DISCONNECT [from %s]: %#v", dc.nodename, dc.peername, message.control) + proxyDisconnect := node.ProxyDisconnect{ + Node: string(t.Element(2).(etf.Atom)), + Proxy: string(t.Element(3).(etf.Atom)), + SessionID: t.Element(4).(string), + Reason: t.Element(5).(string), + } + dc.router.RouteProxyDisconnect(dc, proxyDisconnect) + return nil + + default: + lib.Log("[%s] CONTROL unknown command [from %s]: %#v", dc.nodename, dc.peername, message.control) + return fmt.Errorf("unknown control command %#v", message.control) + } + } + } + + return fmt.Errorf("unsupported control message %#v", message.control) +} + +func (dc *distConnection) decodeFragment(packet []byte, proxy *proxySession) (*lib.Buffer, error) { + var first, compressed bool + var err error + + sequenceID := binary.BigEndian.Uint64(packet[1:9]) + fragmentID := binary.BigEndian.Uint64(packet[9:17]) + if fragmentID == 0 { + return nil, fmt.Errorf("fragmentID can't be 0") + } + + switch packet[0] { + case protoDistFragment1: + // We should decode atom cache from the first fragment in order + // to get rid the case when we get the first fragment of the packet with + // cached atoms and the next packet is not the part of the fragmented packet, + // but with the ids were cached in the first fragment + _, _, err = dc.decodeDistHeaderAtomCache(packet[17:], proxy) + if err != nil { + return nil, err + } + first = true + case protoDistFragment1Z: + _, _, err = dc.decodeDistHeaderAtomCache(packet[17:], proxy) + if err != nil { + return nil, err + } + first = true + compressed = true + case protoDistFragmentNZ: + compressed = true + } + packet = packet[17:] + + dc.fragmentsMutex.Lock() + defer dc.fragmentsMutex.Unlock() + + fragmented, ok := dc.fragments[sequenceID] + if !ok { + fragmented = &fragmentedPacket{ + buffer: lib.TakeBuffer(), + disordered: lib.TakeBuffer(), + disorderedSlices: make(map[uint64][]byte), + lastUpdate: time.Now(), + } + + // append new packet type + if compressed { + fragmented.buffer.AppendByte(protoDistMessageZ) + } else { + fragmented.buffer.AppendByte(protoDistMessage) + } + dc.fragments[sequenceID] = fragmented + } + + // until we get the first item everything will be treated as disordered + if first { + fragmented.fragmentID = fragmentID + 1 + } + + if fragmented.fragmentID-fragmentID != 1 { + // got the next fragment. disordered + slice := fragmented.disordered.Extend(len(packet)) + copy(slice, packet) + fragmented.disorderedSlices[fragmentID] = slice + } else { + // order is correct. just append + fragmented.buffer.Append(packet) + fragmented.fragmentID = fragmentID + } + + // check whether we have disordered slices and try + // to append them if it does fit + if fragmented.fragmentID > 0 && len(fragmented.disorderedSlices) > 0 { + for i := fragmented.fragmentID - 1; i > 0; i-- { + if slice, ok := fragmented.disorderedSlices[i]; ok { + fragmented.buffer.Append(slice) + delete(fragmented.disorderedSlices, i) + fragmented.fragmentID = i + continue + } + break + } + } + + fragmented.lastUpdate = time.Now() + + if fragmented.fragmentID == 1 && len(fragmented.disorderedSlices) == 0 { + // it was the last fragment + delete(dc.fragments, sequenceID) + lib.ReleaseBuffer(fragmented.disordered) + return fragmented.buffer, nil + } + + if dc.checkCleanPending { + return nil, nil + } + + if dc.checkCleanTimer != nil { + dc.checkCleanTimer.Reset(dc.checkCleanTimeout) + return nil, nil + } + + dc.checkCleanTimer = time.AfterFunc(dc.checkCleanTimeout, func() { + dc.fragmentsMutex.Lock() + defer dc.fragmentsMutex.Unlock() + + if len(dc.fragments) == 0 { + dc.checkCleanPending = false + return + } + + valid := time.Now().Add(-dc.checkCleanDeadline) + for sequenceID, fragmented := range dc.fragments { + if fragmented.lastUpdate.Before(valid) { + // dropping due to exceeded deadline + delete(dc.fragments, sequenceID) + } + } + if len(dc.fragments) == 0 { + dc.checkCleanPending = false + return + } + + dc.checkCleanPending = true + dc.checkCleanTimer.Reset(dc.checkCleanTimeout) + }) + + return nil, nil +} + +func (dc *distConnection) decodeDistHeaderAtomCache(packet []byte, proxy *proxySession) ([]etf.Atom, []byte, error) { + var err error + // all the details are here https://erlang.org/doc/apps/erts/erl_ext_dist.html#normal-distribution-header + + // number of atom references are present in package + references := int(packet[0]) + if references == 0 { + return nil, packet[1:], nil + } + + cache := dc.cache.In + if proxy != nil { + cache = proxy.cache.In + } + cached := make([]etf.Atom, references) + flagsLen := references/2 + 1 + if len(packet) < 1+flagsLen { + // malformed + return nil, nil, errMalformed + } + flags := packet[1 : flagsLen+1] + + // The least significant bit in a half byte is flag LongAtoms. + // If it is set, 2 bytes are used for atom lengths instead of 1 byte + // in the distribution header. + headerAtomLength := 1 // if 'LongAtom' is not set + + // extract this bit. just increase headereAtomLength if this flag is set + lastByte := flags[len(flags)-1] + shift := uint((references & 0x01) * 4) + headerAtomLength += int((lastByte >> shift) & 0x01) + + // 1 (number of references) + references/2+1 (length of flags) + packet = packet[1+flagsLen:] + + for i := 0; i < references; i++ { + if len(packet) < 1+headerAtomLength { + // malformed + return nil, nil, errMalformed + } + shift = uint((i & 0x01) * 4) + flag := (flags[i/2] >> shift) & 0x0F + isNewReference := flag&0x08 == 0x08 + idxReference := uint16(flag & 0x07) + idxInternal := uint16(packet[0]) + idx := (idxReference << 8) | idxInternal + + if isNewReference { + atomLen := uint16(packet[1]) + if headerAtomLength == 2 { + atomLen = binary.BigEndian.Uint16(packet[1:3]) + } + // extract atom + packet = packet[1+headerAtomLength:] + if len(packet) < int(atomLen) { + // malformed + return nil, nil, errMalformed + } + atom := etf.Atom(packet[:atomLen]) + // store in temporary cache for decoding + cached[i] = atom + + // store in link' cache + cache.Atoms[idx] = &atom + packet = packet[atomLen:] + continue + } + + c := cache.Atoms[idx] + if c == nil { + packet = packet[1:] + // decode the rest of this cache but set return err = errMissingInCache + err = errMissingInCache + continue + } + cached[i] = *c + packet = packet[1:] + } + + return cached, packet, err +} + +func (dc *distConnection) encodeDistHeaderAtomCache(b *lib.Buffer, + senderAtomCache map[etf.Atom]etf.CacheItem, + encodingAtomCache *etf.EncodingAtomCache) { + + n := encodingAtomCache.Len() + b.AppendByte(byte(n)) // write NumberOfAtomCache + if n == 0 { + return + } + + startPosition := len(b.B) + lenFlags := n/2 + 1 + flags := b.Extend(lenFlags) + flags[lenFlags-1] = 0 // clear last byte to make sure we have valid LongAtom flag + + for i := 0; i < len(encodingAtomCache.L); i++ { + // clean internal name cache + encodingAtomCache.Delete(encodingAtomCache.L[i].Name) + + shift := uint((i & 0x01) * 4) + idxReference := byte(encodingAtomCache.L[i].ID >> 8) // SegmentIndex + idxInternal := byte(encodingAtomCache.L[i].ID & 255) // InternalSegmentIndex + + cachedItem := senderAtomCache[encodingAtomCache.L[i].Name] + if !cachedItem.Encoded { + idxReference |= 8 // set NewCacheEntryFlag + } + + // the 'flags' slice could be changed if b.B was reallocated during the encoding atoms + flags = b.B[startPosition : startPosition+lenFlags] + // clean it up before reuse + if shift == 0 { + flags[i/2] = 0 + } + flags[i/2] |= idxReference << shift + + if cachedItem.Encoded { + b.AppendByte(idxInternal) + continue + } + + if encodingAtomCache.HasLongAtom { + // 1 (InternalSegmentIndex) + 2 (length) + name + allocLen := 1 + 2 + len(encodingAtomCache.L[i].Name) + buf := b.Extend(allocLen) + buf[0] = idxInternal + binary.BigEndian.PutUint16(buf[1:3], uint16(len(encodingAtomCache.L[i].Name))) + copy(buf[3:], encodingAtomCache.L[i].Name) + } else { + // 1 (InternalSegmentIndex) + 1 (length) + name + allocLen := 1 + 1 + len(encodingAtomCache.L[i].Name) + buf := b.Extend(allocLen) + buf[0] = idxInternal + buf[1] = byte(len(encodingAtomCache.L[i].Name)) + copy(buf[2:], encodingAtomCache.L[i].Name) + } + + cachedItem.Encoded = true + senderAtomCache[encodingAtomCache.L[i].Name] = cachedItem + } + + if encodingAtomCache.HasLongAtom { + shift := uint((n & 0x01) * 4) + flags = b.B[startPosition : startPosition+lenFlags] + flags[lenFlags-1] |= 1 << shift // set LongAtom = 1 + } +} + +func (dc *distConnection) sender(sender_id int, send <-chan *sendMessage, options node.ProtoOptions, peerFlags node.Flags) { + var lenMessage, lenAtomCache, lenPacket, startDataPosition int + var atomCacheBuffer, packetBuffer *lib.Buffer + var err error + var compressed bool + var cacheEnabled, fragmentationEnabled, compressionEnabled, encryptionEnabled bool + + // cancel connection context if something went wrong + // it will cause closing connection with stopping all + // goroutines around this connection + defer dc.cancelContext() + + // Header atom cache is encoded right after the control/message encoding process + // but should be stored as a first item in the packet. + // Thats why we do reserve some space for it in order to get rid + // of reallocation packetBuffer data + reserveHeaderAtomCache := 8192 + + // atom cache of this sender + senderAtomCache := make(map[etf.Atom]etf.CacheItem) + // atom cache of this encoding + encodingAtomCache := etf.TakeEncodingAtomCache() + defer etf.ReleaseEncodingAtomCache(encodingAtomCache) + + encrypt := func(data []byte, sessionID string, block cipher.Block) *lib.Buffer { + l := len(data) + padding := aes.BlockSize - l%aes.BlockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) + data = append(data, padtext...) + l = len(data) + + // take another buffer for encrypted message + xBuffer := lib.TakeBuffer() + // 4 (packet len) + 1 (protoProxyX) + 32 (sessionID) + aes.BlockSize + l + xBuffer.Allocate(4 + 1 + 32 + aes.BlockSize + l) + + binary.BigEndian.PutUint32(xBuffer.B, uint32(xBuffer.Len()-4)) + xBuffer.B[4] = protoProxyX + copy(xBuffer.B[5:], sessionID) + iv := xBuffer.B[4+1+32 : 4+1+32+aes.BlockSize] + if _, err := io.ReadFull(crand.Reader, iv); err != nil { + lib.ReleaseBuffer(xBuffer) + return nil + } + cfb := cipher.NewCFBEncrypter(block, iv) + cfb.XORKeyStream(xBuffer.B[4+1+32+aes.BlockSize:], data) + return xBuffer + } + + message := &sendMessage{} + encodingOptions := etf.EncodeOptions{ + EncodingAtomCache: encodingAtomCache, + NodeName: dc.nodename, + PeerName: dc.peername, + } + + for { + // clean up and get back message struct to the pool + message.packet = nil + message.control = nil + message.payload = nil + message.compression = false + message.proxy = nil + sendMessages.Put(message) + + // waiting for the next message + message = <-send + + if message == nil { + // channel was closed + return + } + + if message.packet != nil { + // transit proxy message + if _, err := dc.flusher.Write(message.packet.B); err != nil { + return + } + lib.ReleaseBuffer(message.packet) + continue + } + + packetBuffer = lib.TakeBuffer() + lenMessage, lenAtomCache, lenPacket = 0, 0, 0 + startDataPosition = reserveHeaderAtomCache + + // do reserve for the header 8K, should be enough + packetBuffer.Allocate(reserveHeaderAtomCache) + + // compression feature is always available for the proxy connection + // check whether compress is enabled for the peer and for this message + compressed = false + compressionEnabled = false + if message.compression { + if message.proxy != nil || peerFlags.EnableCompression { + compressionEnabled = true + } + } + + cacheEnabled = false + // atom cache feature is always available for the proxy connection + if message.proxy != nil || peerFlags.EnableHeaderAtomCache { + cacheEnabled = true + encodingAtomCache.Reset() + } + + // fragmentation feature is always available for the proxy connection + fragmentationEnabled = false + if options.FragmentationUnit > 0 { + if message.proxy != nil || peerFlags.EnableFragmentation { + fragmentationEnabled = true + } + } + + // encryption feature is only available for the proxy connection + encryptionEnabled = false + if message.proxy != nil && message.proxy.session.PeerFlags.EnableEncryption { + encryptionEnabled = true + } + + if message.proxy == nil { + // use connection atom cache + encodingOptions.AtomCache = dc.cache.Out + encodingOptions.SenderAtomCache = senderAtomCache + // use connection flags + encodingOptions.FlagBigCreation = peerFlags.EnableBigCreation + encodingOptions.FlagBigPidRef = peerFlags.EnableBigPidRef + + } else { + // use proxy connection atom cache + encodingOptions.AtomCache = message.proxy.cache.Out + if message.proxy.senderCache[sender_id] == nil { + message.proxy.senderCache[sender_id] = make(map[etf.Atom]etf.CacheItem) + } + encodingOptions.SenderAtomCache = message.proxy.senderCache[sender_id] + // these flags are always enabled for the proxy connection + encodingOptions.FlagBigCreation = true + encodingOptions.FlagBigPidRef = true + } + + // We could use gzip writer for the encoder, but we don't know + // the actual size of the control/payload. For small data, gzipping + // is getting extremely inefficient. That's why it is cheaper to + // encode control/payload first and then decide whether to compress it + // according to a threshold value. + + // encode Control + err = etf.Encode(message.control, packetBuffer, encodingOptions) + if err != nil { + lib.Warning("can not encode control message: %s", err) + lib.ReleaseBuffer(packetBuffer) + continue + } + + // encode Message if present + if message.payload != nil { + err = etf.Encode(message.payload, packetBuffer, encodingOptions) + if err != nil { + lib.Warning("can not encode payload message: %s", err) + lib.ReleaseBuffer(packetBuffer) + continue + } + + } + lenMessage = packetBuffer.Len() - reserveHeaderAtomCache + + if compressionEnabled && packetBuffer.Len() > (reserveHeaderAtomCache+message.compressionThreshold) { + var zWriter *gzip.Writer + + //// take another buffer + zBuffer := lib.TakeBuffer() + // allocate extra 4 bytes for the lenMessage (length of unpacked data) + zBuffer.Allocate(reserveHeaderAtomCache + 4) + level := message.compressionLevel + if level == -1 { + level = 0 + } + if w, ok := gzipWriters[level].Get().(*gzip.Writer); ok { + zWriter = w + zWriter.Reset(zBuffer) + } else { + zWriter, _ = gzip.NewWriterLevel(zBuffer, message.compressionLevel) + } + zWriter.Write(packetBuffer.B[reserveHeaderAtomCache:]) + zWriter.Close() + gzipWriters[level].Put(zWriter) + + // swap buffers only if gzipped data less than the original ones + if zBuffer.Len() < packetBuffer.Len() { + binary.BigEndian.PutUint32(zBuffer.B[reserveHeaderAtomCache:], uint32(lenMessage)) + lenMessage = zBuffer.Len() - reserveHeaderAtomCache + packetBuffer, zBuffer = zBuffer, packetBuffer + compressed = true + } + lib.ReleaseBuffer(zBuffer) + } + + // encode Header Atom Cache if its enabled + if cacheEnabled && encodingAtomCache.Len() > 0 { + atomCacheBuffer = lib.TakeBuffer() + atomCacheBuffer.Allocate(1024) + dc.encodeDistHeaderAtomCache(atomCacheBuffer, encodingOptions.SenderAtomCache, encodingAtomCache) + + lenAtomCache = atomCacheBuffer.Len() - 1024 + if lenAtomCache > reserveHeaderAtomCache-1024 { + // we got huge atom cache + atomCacheBuffer.Append(packetBuffer.B[startDataPosition:]) + startDataPosition = 1024 + lib.ReleaseBuffer(packetBuffer) + packetBuffer = atomCacheBuffer + } else { + startDataPosition -= lenAtomCache + copy(packetBuffer.B[startDataPosition:], atomCacheBuffer.B[1024:]) + lib.ReleaseBuffer(atomCacheBuffer) + } + + } else { + lenAtomCache = 1 + startDataPosition -= lenAtomCache + packetBuffer.B[startDataPosition] = byte(0) + } + + for { + // 4 (packet len) + 1 (dist header: 131) + 1 (dist header: protoDistMessage[Z]) + lenAtomCache + lenPacket = 1 + 1 + lenAtomCache + lenMessage + if !fragmentationEnabled || lenMessage < options.FragmentationUnit { + // send as a single packet + startDataPosition -= 1 + if compressed { + packetBuffer.B[startDataPosition] = protoDistMessageZ // 200 + } else { + packetBuffer.B[startDataPosition] = protoDistMessage // 68 + } + + if message.proxy == nil { + // 4 (packet len) + 1 (protoDist) + startDataPosition -= 4 + 1 + + binary.BigEndian.PutUint32(packetBuffer.B[startDataPosition:], uint32(lenPacket)) + packetBuffer.B[startDataPosition+4] = protoDist // 131 + + if _, err := dc.flusher.Write(packetBuffer.B[startDataPosition:]); err != nil { + return + } + break + } + + // proxy message. + if encryptionEnabled == false { + // no encryption + // 4 (packet len) + protoProxy + sessionID + startDataPosition -= 1 + 4 + 32 + l := len(packetBuffer.B[startDataPosition:]) - 4 + binary.BigEndian.PutUint32(packetBuffer.B[startDataPosition:], uint32(l)) + packetBuffer.B[startDataPosition+4] = protoProxy + copy(packetBuffer.B[startDataPosition+5:], message.proxy.session.ID) + if _, err := dc.flusher.Write(packetBuffer.B[startDataPosition:]); err != nil { + return + } + break + } + + // send encrypted proxy message + xBuffer := encrypt(packetBuffer.B[startDataPosition:], + message.proxy.session.ID, message.proxy.session.Block) + if xBuffer == nil { + // can't encrypt message + return + } + if _, err := dc.flusher.Write(xBuffer.B); err != nil { + return + } + lib.ReleaseBuffer(xBuffer) + break + } + + // Message should be fragmented + + // https://erlang.org/doc/apps/erts/erl_ext_dist.html#distribution-header-for-fragmented-messages + // "The entire atom cache and control message has to be part of the starting fragment" + + sequenceID := uint64(atomic.AddInt64(&dc.sequenceID, 1)) + numFragments := lenMessage/options.FragmentationUnit + 1 + + // 1 (dist header: 131) + 1 (dist header: protoDistFragment) + 8 (sequenceID) + 8 (fragmentID) + ... + lenPacket = 1 + 1 + 8 + 8 + lenAtomCache + options.FragmentationUnit + + // 4 (packet len) + 1 (dist header: 131) + 1 (dist header: protoDistFragment[Z]) + 8 (sequenceID) + 8 (fragmentID) + startDataPosition -= 22 + + if compressed { + packetBuffer.B[startDataPosition+5] = protoDistFragment1Z // 201 + } else { + packetBuffer.B[startDataPosition+5] = protoDistFragment1 // 69 + } + + binary.BigEndian.PutUint64(packetBuffer.B[startDataPosition+6:], uint64(sequenceID)) + binary.BigEndian.PutUint64(packetBuffer.B[startDataPosition+14:], uint64(numFragments)) + + if message.proxy == nil { + binary.BigEndian.PutUint32(packetBuffer.B[startDataPosition:], uint32(lenPacket)) + packetBuffer.B[startDataPosition+4] = protoDist // 131 + if _, err := dc.flusher.Write(packetBuffer.B[startDataPosition : startDataPosition+4+lenPacket]); err != nil { + return + } + } else { + // proxy message + if encryptionEnabled == false { + // send proxy message + // shift left on 32 bytes for the session id + binary.BigEndian.PutUint32(packetBuffer.B[startDataPosition-32:], uint32(lenPacket+32)) + packetBuffer.B[startDataPosition-32+4] = protoProxy // 141 + copy(packetBuffer.B[startDataPosition-32+5:], message.proxy.session.ID) + if _, err := dc.flusher.Write(packetBuffer.B[startDataPosition-32 : startDataPosition+4+lenPacket]); err != nil { + return + } + + } else { + // send encrypted proxy message + // encryption makes padding (up to aes.BlockSize = 16 bytes) so we should keep the data + tail16 := [16]byte{} + n := copy(tail16[:], packetBuffer.B[startDataPosition+4+lenPacket:]) + xBuffer := encrypt(packetBuffer.B[startDataPosition+5:startDataPosition+4+lenPacket], + message.proxy.session.ID, message.proxy.session.Block) + if xBuffer == nil { + // can't encrypt message + return + } + if _, err := dc.flusher.Write(xBuffer.B); err != nil { + return + } + // resore tail + copy(packetBuffer.B[startDataPosition+4+lenPacket:], tail16[:n]) + lib.ReleaseBuffer(xBuffer) + } + } + + startDataPosition += 4 + lenPacket + numFragments-- + + nextFragment: + + if len(packetBuffer.B[startDataPosition:]) > options.FragmentationUnit { + lenPacket = 1 + 1 + 8 + 8 + options.FragmentationUnit + // reuse the previous 22 bytes for the next frame header + startDataPosition -= 22 + + } else { + // the last one + lenPacket = 1 + 1 + 8 + 8 + len(packetBuffer.B[startDataPosition:]) + startDataPosition -= 22 + } + + if compressed { + packetBuffer.B[startDataPosition+5] = protoDistFragmentNZ // 202 + } else { + packetBuffer.B[startDataPosition+5] = protoDistFragmentN // 70 + } + + binary.BigEndian.PutUint64(packetBuffer.B[startDataPosition+6:], uint64(sequenceID)) + binary.BigEndian.PutUint64(packetBuffer.B[startDataPosition+14:], uint64(numFragments)) + if message.proxy == nil { + // send fragment + binary.BigEndian.PutUint32(packetBuffer.B[startDataPosition:], uint32(lenPacket)) + packetBuffer.B[startDataPosition+4] = protoDist // 131 + if _, err := dc.flusher.Write(packetBuffer.B[startDataPosition : startDataPosition+4+lenPacket]); err != nil { + return + } + } else { + // wrap it as a proxy message + if encryptionEnabled == false { + binary.BigEndian.PutUint32(packetBuffer.B[startDataPosition-32:], uint32(lenPacket+32)) + packetBuffer.B[startDataPosition-32+4] = protoProxy // 141 + copy(packetBuffer.B[startDataPosition-32+5:], message.proxy.session.ID) + if _, err := dc.flusher.Write(packetBuffer.B[startDataPosition-32 : startDataPosition+4+lenPacket]); err != nil { + return + } + } else { + // send encrypted proxy message + tail16 := [16]byte{} + n := copy(tail16[:], packetBuffer.B[startDataPosition+4+lenPacket:]) + xBuffer := encrypt(packetBuffer.B[startDataPosition+5:startDataPosition+4+lenPacket], + message.proxy.session.ID, message.proxy.session.Block) + if xBuffer == nil { + // can't encrypt message + return + } + if _, err := dc.flusher.Write(xBuffer.B); err != nil { + return + } + // resore tail + copy(packetBuffer.B[startDataPosition+4+lenPacket:], tail16[:n]) + lib.ReleaseBuffer(xBuffer) + } + } + + startDataPosition += 4 + lenPacket + numFragments-- + if numFragments > 0 { + goto nextFragment + } + + // done + break + } + + lib.ReleaseBuffer(packetBuffer) + + if cacheEnabled == false { + continue + } + + // get updates from the connection AtomCache and update the sender's cache (senderAtomCache) + lastAddedAtom, lastAddedID := encodingOptions.AtomCache.LastAdded() + if lastAddedID < 0 { + continue + } + if _, exist := encodingOptions.SenderAtomCache[lastAddedAtom]; exist { + continue + } + + encodingOptions.AtomCache.RLock() + for _, a := range encodingOptions.AtomCache.ListSince(lastAddedID) { + encodingOptions.SenderAtomCache[a] = etf.CacheItem{ID: lastAddedID, Name: a, Encoded: false} + lastAddedID++ + } + encodingOptions.AtomCache.RUnlock() + + } + +} + +func (dc *distConnection) send(to string, creation uint32, msg *sendMessage) error { + i := atomic.AddInt32(&dc.senders.i, 1) + n := i % dc.senders.n + s := dc.senders.sender[n] + if s == nil { + // connection was closed + return node.ErrNoRoute + } + dc.proxySessionsMutex.RLock() + ps, isProxy := dc.proxySessionsByPeerName[to] + dc.proxySessionsMutex.RUnlock() + peer_creation := dc.creation + if isProxy { + msg.proxy = &ps + peer_creation = ps.session.Creation + } else { + // its direct sending, so have to make sure if this peer does support compression + if dc.flags.EnableCompression == false { + msg.compression = false + } + } + + // if this peer is Erlang OTP 22 (and earlier), peer_creation is always 0, so we + // must skip this checking. + if creation > 0 && peer_creation > 0 && peer_creation != creation { + return node.ErrProcessIncarnation + } + + // TODO to decide whether to return error if channel is full + //select { + //case s.sendChannel <- msg: + // return nil + //default: + // return ErrOverloadConnection + //} + + s.Lock() + defer s.Unlock() + + s.sendChannel <- msg + return nil +} + +func proxyFlagsToUint64(pf node.ProxyFlags) uint64 { + var flags uint64 + if pf.EnableLink { + flags |= 1 + } + if pf.EnableMonitor { + flags |= 1 << 1 + } + if pf.EnableRemoteSpawn { + flags |= 1 << 2 + } + if pf.EnableEncryption { + flags |= 1 << 3 + } + return flags +} + +func proxyFlagsFromUint64(f uint64) node.ProxyFlags { + var flags node.ProxyFlags + flags.EnableLink = f&1 > 0 + flags.EnableMonitor = f&(1<<1) > 0 + flags.EnableRemoteSpawn = f&(1<<2) > 0 + flags.EnableEncryption = f&(1<<3) > 0 + return flags +} diff --git a/node/dist/dist_test.go b/proto/dist/proto_test.go similarity index 57% rename from node/dist/dist_test.go rename to proto/dist/proto_test.go index 3474b6e3..f96d8b57 100644 --- a/node/dist/dist_test.go +++ b/proto/dist/proto_test.go @@ -2,9 +2,7 @@ package dist import ( "bytes" - "fmt" "math/rand" - "net" "reflect" "testing" "time" @@ -13,110 +11,13 @@ import ( "github.com/ergo-services/ergo/lib" ) -func TestLinkRead(t *testing.T) { - - server, client := net.Pipe() - defer func() { - server.Close() - client.Close() - }() - - link := Link{ - conn: server, - } - - go client.Write([]byte{0, 0, 0, 0, 0, 0, 0, 1, 0}) - - // read keepalive answer on a client side - go func() { - bb := make([]byte, 10) - for { - _, e := client.Read(bb) - if e != nil { - return - } - } - }() - - c := make(chan bool) - b := lib.TakeBuffer() - go func() { - link.Read(b) - close(c) - }() - select { - case <-c: - fmt.Println("OK", b.B) - case <-time.After(1000 * time.Millisecond): - t.Fatal("incorrect") - } - -} - -func TestComposeName(t *testing.T) { - //link := &Link{ - // Name: "testName", - // Cookie: "testCookie", - // Hidden: false, - - // flags: toNodeFlag(PUBLISHED, UNICODE_IO, DIST_MONITOR, DIST_MONITOR_NAME, - // EXTENDED_PIDS_PORTS, EXTENDED_REFERENCES, - // DIST_HDR_ATOM_CACHE, HIDDEN_ATOM_CACHE, NEW_FUN_TAGS, - // SMALL_ATOM_TAGS, UTF8_ATOMS, MAP_TAG, BIG_CREATION, - // FRAGMENTS, - // ), - - // version: 5, - //} - //b := lib.TakeBuffer() - //defer lib.ReleaseBuffer(b) - //link.composeName(b) - //shouldBe := []byte{} - - //if !bytes.Equal(b.B, shouldBe) { - // t.Fatal("malform value") - //} - -} - -func TestReadName(t *testing.T) { - -} - -func TestComposeStatus(t *testing.T) { - -} - -func TestComposeChallenge(t *testing.T) { - -} - -func TestReadChallenge(t *testing.T) { - -} - -func TestValidateChallengeReply(t *testing.T) { - -} - -func TestComposeChallengeAck(t *testing.T) { - -} - -func TestComposeChalleneReply(t *testing.T) { - -} - -func TestValidateChallengeAck(t *testing.T) { - -} - func TestDecodeDistHeaderAtomCache(t *testing.T) { - link := Link{} + c := &distConnection{} + c.cache = etf.NewAtomCache() a1 := etf.Atom("atom1") a2 := etf.Atom("atom2") - link.cacheIn[1034] = &a1 - link.cacheIn[5] = &a2 + c.cache.In.Atoms[1034] = &a1 + c.cache.In.Atoms[5] = &a2 packet := []byte{ 131, 68, // start dist header 5, 4, 137, 9, // 5 atoms and theirs flags @@ -134,7 +35,7 @@ func TestDecodeDistHeaderAtomCache(t *testing.T) { } cacheExpected := []etf.Atom{"atom1", "atom2", "reg", "call", "set_get_state"} - cacheInExpected := link.cacheIn + cacheInExpected := c.cache.In.Atoms a3 := etf.Atom("reg") a4 := etf.Atom("call") a5 := etf.Atom("set_get_state") @@ -143,13 +44,13 @@ func TestDecodeDistHeaderAtomCache(t *testing.T) { cacheInExpected[494] = &a5 packetExpected := packet[34:] - cache, packet1, _ := link.decodeDistHeaderAtomCache(packet[2:]) + cache, packet1, _ := c.decodeDistHeaderAtomCache(packet[2:], nil) if !bytes.Equal(packet1, packetExpected) { t.Fatal("incorrect packet") } - if !reflect.DeepEqual(link.cacheIn, cacheInExpected) { + if !reflect.DeepEqual(c.cache.In.Atoms, cacheInExpected) { t.Fatal("incorrect cacheIn") } @@ -164,16 +65,16 @@ func TestEncodeDistHeaderAtomCache(t *testing.T) { b := lib.TakeBuffer() defer lib.ReleaseBuffer(b) - writerAtomCache := make(map[etf.Atom]etf.CacheItem) - encodingAtomCache := etf.TakeListAtomCache() - defer etf.ReleaseListAtomCache(encodingAtomCache) + senderAtomCache := make(map[etf.Atom]etf.CacheItem) + encodingAtomCache := etf.TakeEncodingAtomCache() + defer etf.ReleaseEncodingAtomCache(encodingAtomCache) - writerAtomCache["reg"] = etf.CacheItem{ID: 1000, Encoded: false, Name: "reg"} - writerAtomCache["call"] = etf.CacheItem{ID: 499, Encoded: false, Name: "call"} - writerAtomCache["one_more_atom"] = etf.CacheItem{ID: 199, Encoded: true, Name: "one_more_atom"} - writerAtomCache["yet_another_atom"] = etf.CacheItem{ID: 2, Encoded: false, Name: "yet_another_atom"} - writerAtomCache["extra_atom"] = etf.CacheItem{ID: 10, Encoded: true, Name: "extra_atom"} - writerAtomCache["potato"] = etf.CacheItem{ID: 2017, Encoded: true, Name: "potato"} + senderAtomCache["reg"] = etf.CacheItem{ID: 1000, Encoded: false, Name: "reg"} + senderAtomCache["call"] = etf.CacheItem{ID: 499, Encoded: false, Name: "call"} + senderAtomCache["one_more_atom"] = etf.CacheItem{ID: 199, Encoded: true, Name: "one_more_atom"} + senderAtomCache["yet_another_atom"] = etf.CacheItem{ID: 2, Encoded: false, Name: "yet_another_atom"} + senderAtomCache["extra_atom"] = etf.CacheItem{ID: 10, Encoded: true, Name: "extra_atom"} + senderAtomCache["potato"] = etf.CacheItem{ID: 2017, Encoded: true, Name: "potato"} // Encoded field is ignored here encodingAtomCache.Append(etf.CacheItem{ID: 499, Name: "call"}) @@ -190,8 +91,8 @@ func TestEncodeDistHeaderAtomCache(t *testing.T) { } - l := &Link{} - l.encodeDistHeaderAtomCache(b, writerAtomCache, encodingAtomCache) + l := &distConnection{} + l.encodeDistHeaderAtomCache(b, senderAtomCache, encodingAtomCache) if !reflect.DeepEqual(b.B, expected) { t.Fatal("incorrect value") @@ -210,7 +111,7 @@ func TestEncodeDistHeaderAtomCache(t *testing.T) { 97, 110, 111, 116, 104, 101, 114, 95, 97, 116, 111, 109, } - l.encodeDistHeaderAtomCache(b, writerAtomCache, encodingAtomCache) + l.encodeDistHeaderAtomCache(b, senderAtomCache, encodingAtomCache) if !reflect.DeepEqual(b.B, expected) { t.Fatal("incorrect value", b.B) @@ -218,7 +119,7 @@ func TestEncodeDistHeaderAtomCache(t *testing.T) { } func BenchmarkDecodeDistHeaderAtomCache(b *testing.B) { - link := &Link{} + link := &distConnection{} packet := []byte{ 131, 68, // start dist header 5, 4, 137, 9, // 5 atoms and theirs flags @@ -236,25 +137,25 @@ func BenchmarkDecodeDistHeaderAtomCache(b *testing.B) { } b.ResetTimer() for i := 0; i < b.N; i++ { - link.decodeDistHeaderAtomCache(packet[2:]) + link.decodeDistHeaderAtomCache(packet[2:], nil) } } func BenchmarkEncodeDistHeaderAtomCache(b *testing.B) { - link := &Link{} + link := &distConnection{} buf := lib.TakeBuffer() defer lib.ReleaseBuffer(buf) - writerAtomCache := make(map[etf.Atom]etf.CacheItem) - encodingAtomCache := etf.TakeListAtomCache() - defer etf.ReleaseListAtomCache(encodingAtomCache) + senderAtomCache := make(map[etf.Atom]etf.CacheItem) + encodingAtomCache := etf.TakeEncodingAtomCache() + defer etf.ReleaseEncodingAtomCache(encodingAtomCache) - writerAtomCache["reg"] = etf.CacheItem{ID: 1000, Encoded: false, Name: "reg"} - writerAtomCache["call"] = etf.CacheItem{ID: 499, Encoded: false, Name: "call"} - writerAtomCache["one_more_atom"] = etf.CacheItem{ID: 199, Encoded: true, Name: "one_more_atom"} - writerAtomCache["yet_another_atom"] = etf.CacheItem{ID: 2, Encoded: false, Name: "yet_another_atom"} - writerAtomCache["extra_atom"] = etf.CacheItem{ID: 10, Encoded: true, Name: "extra_atom"} - writerAtomCache["potato"] = etf.CacheItem{ID: 2017, Encoded: true, Name: "potato"} + senderAtomCache["reg"] = etf.CacheItem{ID: 1000, Encoded: false, Name: "reg"} + senderAtomCache["call"] = etf.CacheItem{ID: 499, Encoded: false, Name: "call"} + senderAtomCache["one_more_atom"] = etf.CacheItem{ID: 199, Encoded: true, Name: "one_more_atom"} + senderAtomCache["yet_another_atom"] = etf.CacheItem{ID: 2, Encoded: false, Name: "yet_another_atom"} + senderAtomCache["extra_atom"] = etf.CacheItem{ID: 10, Encoded: true, Name: "extra_atom"} + senderAtomCache["potato"] = etf.CacheItem{ID: 2017, Encoded: true, Name: "potato"} // Encoded field is ignored here encodingAtomCache.Append(etf.CacheItem{ID: 499, Name: "call"}) @@ -263,38 +164,39 @@ func BenchmarkEncodeDistHeaderAtomCache(b *testing.B) { encodingAtomCache.Append(etf.CacheItem{ID: 2017, Name: "potato"}) b.ResetTimer() for i := 0; i < b.N; i++ { - link.encodeDistHeaderAtomCache(buf, writerAtomCache, encodingAtomCache) + link.encodeDistHeaderAtomCache(buf, senderAtomCache, encodingAtomCache) } } func TestDecodeFragment(t *testing.T) { - link := &Link{} + link := &distConnection{} link.checkCleanTimeout = 50 * time.Millisecond link.checkCleanDeadline = 150 * time.Millisecond + link.fragments = make(map[uint64]*fragmentedPacket) // decode fragment with fragmentID=0 should return error - fragment0 := []byte{0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3} - if _, e := link.decodeFragment(fragment0, true); e == nil { + fragment0 := []byte{protoDistFragment1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3} + if _, e := link.decodeFragment(fragment0, nil); e == nil { t.Fatal("should be error here") } - fragment1 := []byte{0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3, 1, 2, 3} - fragment2 := []byte{0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2, 4, 5, 6} - fragment3 := []byte{0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 7, 8, 9} + fragment1 := []byte{protoDistFragment1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 2, 3} + fragment2 := []byte{protoDistFragmentN, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2, 4, 5, 6} + fragment3 := []byte{protoDistFragmentN, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 7, 8, 9} - expected := []byte{68, 1, 2, 3, 4, 5, 6, 7, 8, 9} + expected := []byte{68, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9} // add first fragment - if x, e := link.decodeFragment(fragment1, true); x != nil || e != nil { - t.Fatal("should be nil here", e) + if x, e := link.decodeFragment(fragment1, nil); x != nil || e != nil { + t.Fatal("should be nil here", x, e) } // add second one - if x, e := link.decodeFragment(fragment2, false); x != nil || e != nil { + if x, e := link.decodeFragment(fragment2, nil); x != nil || e != nil { t.Fatal("should be nil here", e) } // add the last one. should return *lib.Buffer with assembled packet - if x, e := link.decodeFragment(fragment3, false); x == nil || e != nil { + if x, e := link.decodeFragment(fragment3, nil); x == nil || e != nil { t.Fatal("shouldn't be nil here", e) } else { // x should be *lib.Buffer @@ -313,7 +215,7 @@ func TestDecodeFragment(t *testing.T) { link.checkCleanDeadline = 150 * time.Millisecond // test lost fragment // add the first fragment and wait 160ms - if x, e := link.decodeFragment(fragment1, true); x != nil || e != nil { + if x, e := link.decodeFragment(fragment1, nil); x != nil || e != nil { t.Fatal("should be nil here", e) } if len(link.fragments) == 0 { @@ -330,17 +232,17 @@ func TestDecodeFragment(t *testing.T) { link.checkCleanTimeout = 0 link.checkCleanDeadline = 0 fragments := [][]byte{ - []byte{0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 9, 1, 2, 3}, - []byte{0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8, 4, 5, 6}, - []byte{0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7, 7, 8, 9}, - []byte{0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 10, 11, 12}, - []byte{0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 13, 14, 15}, - []byte{0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4, 16, 17, 18}, - []byte{0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3, 19, 20, 21}, - []byte{0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 22, 23, 24}, - []byte{0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 25, 26, 27}, + {protoDistFragment1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 2, 3}, + {protoDistFragmentN, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8, 4, 5, 6}, + {protoDistFragmentN, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7, 7, 8, 9}, + {protoDistFragmentN, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6, 10, 11, 12}, + {protoDistFragmentN, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 13, 14, 15}, + {protoDistFragmentN, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4, 16, 17, 18}, + {protoDistFragmentN, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3, 19, 20, 21}, + {protoDistFragmentN, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 22, 23, 24}, + {protoDistFragmentN, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 25, 26, 27}, } - expected = []byte{68, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27} + expected = []byte{68, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27} fragmentsReverse := make([][]byte, len(fragments)) l := len(fragments) @@ -350,13 +252,8 @@ func TestDecodeFragment(t *testing.T) { var result *lib.Buffer var e error - var first bool for i := range fragmentsReverse { - first = false - if fragmentsReverse[i][15] == byte(l) { - first = true - } - if result, e = link.decodeFragment(fragmentsReverse[i], first); e != nil { + if result, e = link.decodeFragment(fragmentsReverse[i], nil); e != nil { t.Fatal(e) } @@ -382,11 +279,7 @@ func TestDecodeFragment(t *testing.T) { } for i := range fragmentsShuffle { - first = false - if fragmentsShuffle[i][15] == byte(l) { - first = true - } - if result, e = link.decodeFragment(fragmentsShuffle[i], first); e != nil { + if result, e = link.decodeFragment(fragmentsShuffle[i], nil); e != nil { t.Fatal(e) } diff --git a/proto/dist/resolver.go b/proto/dist/resolver.go new file mode 100644 index 00000000..394b6ae8 --- /dev/null +++ b/proto/dist/resolver.go @@ -0,0 +1,327 @@ +package dist + +import ( + "context" + "encoding/binary" + "fmt" + "io" + "net" + "strconv" + "strings" + "time" + + "github.com/ergo-services/ergo/lib" + "github.com/ergo-services/ergo/node" +) + +const ( + DefaultEPMDPort uint16 = 4369 + + epmdAliveReq = 120 + epmdAliveResp = 121 + epmdAliveRespX = 118 + epmdPortPleaseReq = 122 + epmdPortResp = 119 + epmdNamesReq = 110 + + // wont be implemented + // epmdDumpReq = 100 + // epmdKillReq = 107 + // epmdStopReq = 115 + + // Extra data + ergoExtraMagic = 4411 + ergoExtraVersion1 = 1 +) + +// epmd implements resolver +type epmdResolver struct { + node.Resolver + + // EPMD server + enableEPMD bool + host string + port uint16 + + // Node + name string + nodePort uint16 + nodeName string + nodeHost string + handshakeVersion node.HandshakeVersion + + extra []byte +} + +func CreateResolver() node.Resolver { + resolver := &epmdResolver{ + port: DefaultEPMDPort, + } + return resolver +} + +func CreateResolverWithLocalEPMD(host string, port uint16) node.Resolver { + if port == 0 { + port = DefaultEPMDPort + } + resolver := &epmdResolver{ + enableEPMD: true, + host: host, + port: port, + } + return resolver +} + +func CreateResolverWithRemoteEPMD(host string, port uint16) node.Resolver { + if port == 0 { + port = DefaultEPMDPort + } + resolver := &epmdResolver{ + host: host, + port: port, + } + return resolver +} + +func (e *epmdResolver) Register(ctx context.Context, name string, port uint16, options node.ResolverOptions) error { + n := strings.Split(name, "@") + if len(n) != 2 { + return fmt.Errorf("(EMPD) FQDN for node name is required (example: node@hostname)") + } + + e.name = name + e.nodeName = n[0] + e.nodeHost = n[1] + e.nodePort = port + e.handshakeVersion = options.HandshakeVersion + + e.composeExtra(options) + ready := make(chan error) + + go func() { + buf := make([]byte, 16) + var reconnecting bool + + for { + if ctx.Err() != nil { + // node is stopped + return + } + + // try to start embedded EPMD server + if e.enableEPMD { + startServerEPMD(ctx, e.host, e.port) + } + + // register this node on EPMD server + conn, err := e.registerNode(options) + if err != nil { + lib.Log("EPMD client: can't register node %q (%s). Retry in 3 seconds...", name, err) + time.Sleep(3 * time.Second) + continue + } + + go func() { + <-ctx.Done() + conn.Close() + }() + + if reconnecting == false { + ready <- nil + reconnecting = true + } + + for { + _, err := conn.Read(buf) + if err == nil { + continue + } + break + } + lib.Log("[%s] EPMD client: closing connection", name) + } + }() + + defer close(ready) + return <-ready +} + +func (e *epmdResolver) Resolve(name string) (node.Route, error) { + var route node.Route + + n := strings.Split(name, "@") + if len(n) != 2 { + return node.Route{}, fmt.Errorf("incorrect FQDN node name (example: node@localhost)") + } + conn, err := net.Dial("tcp", net.JoinHostPort(n[1], fmt.Sprintf("%d", e.port))) + if err != nil { + return node.Route{}, err + } + + defer conn.Close() + + if err := e.sendPortPleaseReq(conn, n[0]); err != nil { + return node.Route{}, err + } + + route.Node = n[0] + route.Host = n[1] + + err = e.readPortResp(&route, conn) + if err != nil { + return node.Route{}, err + } + + return route, nil + +} + +func (e *epmdResolver) composeExtra(options node.ResolverOptions) { + buf := make([]byte, 4) + + // 2 bytes: ergoExtraMagic + binary.BigEndian.PutUint16(buf[0:2], uint16(ergoExtraMagic)) + // 1 byte Extra version + buf[2] = ergoExtraVersion1 + // 1 byte flag enabled TLS + if options.EnableTLS { + buf[3] = 1 + } + e.extra = buf + return +} + +func (e *epmdResolver) readExtra(route *node.Route, buf []byte) { + if len(buf) < 4 { + return + } + magic := binary.BigEndian.Uint16(buf[0:2]) + if uint16(ergoExtraMagic) != magic { + return + } + + if buf[2] != ergoExtraVersion1 { + return + } + + if buf[3] == 1 { + route.Options.EnableTLS = true + } + + route.Options.IsErgo = true + return +} + +func (e *epmdResolver) registerNode(options node.ResolverOptions) (net.Conn, error) { + // + resolverHost := e.host + if resolverHost == "" { + resolverHost = e.nodeHost + } + dialer := net.Dialer{ + KeepAlive: 15 * time.Second, + } + dsn := net.JoinHostPort(resolverHost, strconv.Itoa(int(e.port))) + conn, err := dialer.Dial("tcp", dsn) + if err != nil { + return nil, err + } + + if err := e.sendAliveReq(conn); err != nil { + conn.Close() + return nil, err + } + + if err := e.readAliveResp(conn); err != nil { + conn.Close() + return nil, err + } + + lib.Log("[%s] EPMD client: node registered", e.name) + return conn, nil +} + +func (e *epmdResolver) sendAliveReq(conn net.Conn) error { + buf := make([]byte, 2+14+len(e.nodeName)+len(e.extra)) + binary.BigEndian.PutUint16(buf[0:2], uint16(len(buf)-2)) + buf[2] = byte(epmdAliveReq) + binary.BigEndian.PutUint16(buf[3:5], e.nodePort) + // http://erlang.org/doc/reference_manual/distributed.html (section 13.5) + // 77 — regular public node, 72 — hidden + // We use a regular one + buf[5] = 77 + // Protocol TCP + buf[6] = 0 + // HighestVersion + binary.BigEndian.PutUint16(buf[7:9], uint16(HandshakeVersion6)) + // LowestVersion + binary.BigEndian.PutUint16(buf[9:11], uint16(HandshakeVersion5)) + // length Node name + l := len(e.nodeName) + binary.BigEndian.PutUint16(buf[11:13], uint16(l)) + // Node name + offset := (13 + l) + copy(buf[13:offset], e.nodeName) + // Extra data + l = len(e.extra) + binary.BigEndian.PutUint16(buf[offset:offset+2], uint16(l)) + copy(buf[offset+2:offset+2+l], e.extra) + // Send + if _, err := conn.Write(buf); err != nil { + return err + } + return nil +} + +func (e *epmdResolver) readAliveResp(conn net.Conn) error { + buf := make([]byte, 16) + if _, err := conn.Read(buf); err != nil { + return err + } + switch buf[0] { + case epmdAliveResp, epmdAliveRespX: + default: + return fmt.Errorf("Malformed EPMD response %v", buf) + } + if buf[1] != 0 { + return fmt.Errorf("Can't register %q. Code: %v", e.nodeName, buf[1]) + } + return nil +} + +func (e *epmdResolver) sendPortPleaseReq(conn net.Conn, name string) error { + buflen := uint16(2 + len(name) + 1) + buf := make([]byte, buflen) + binary.BigEndian.PutUint16(buf[0:2], uint16(len(buf)-2)) + buf[2] = byte(epmdPortPleaseReq) + copy(buf[3:buflen], name) + _, err := conn.Write(buf) + return err +} + +func (e *epmdResolver) readPortResp(route *node.Route, c net.Conn) error { + + buf := make([]byte, 1024) + n, err := c.Read(buf) + if err != nil && err != io.EOF { + return fmt.Errorf("reading from link - %s", err) + } + buf = buf[:n] + + if buf[0] == epmdPortResp && buf[1] == 0 { + p := binary.BigEndian.Uint16(buf[2:4]) + nameLen := binary.BigEndian.Uint16(buf[10:12]) + route.Port = p + extraStart := 12 + int(nameLen) + // read extra data + buf = buf[extraStart:] + extraLen := binary.BigEndian.Uint16(buf[:2]) + buf = buf[2 : extraLen+2] + e.readExtra(route, buf) + return nil + } else if buf[1] > 0 { + return fmt.Errorf("desired node not found") + } else { + return fmt.Errorf("malformed reply - %#v", buf) + } +} diff --git a/proto/dist/types.go b/proto/dist/types.go new file mode 100644 index 00000000..e526c9ab --- /dev/null +++ b/proto/dist/types.go @@ -0,0 +1,41 @@ +package dist + +// Distributed operations codes (http://www.erlang.org/doc/apps/erts/erl_dist_protocol.html) +const ( + distProtoLINK = 1 + distProtoSEND = 2 + distProtoEXIT = 3 + distProtoUNLINK = 4 + distProtoNODE_LINK = 5 + distProtoREG_SEND = 6 + distProtoGROUP_LEADER = 7 + distProtoEXIT2 = 8 + distProtoSEND_TT = 12 + distProtoEXIT_TT = 13 + distProtoREG_SEND_TT = 16 + distProtoEXIT2_TT = 18 + distProtoMONITOR = 19 + distProtoDEMONITOR = 20 + distProtoMONITOR_EXIT = 21 + distProtoSEND_SENDER = 22 + distProtoSEND_SENDER_TT = 23 + distProtoPAYLOAD_EXIT = 24 + distProtoPAYLOAD_EXIT_TT = 25 + distProtoPAYLOAD_EXIT2 = 26 + distProtoPAYLOAD_EXIT2_TT = 27 + distProtoPAYLOAD_MONITOR_P_EXIT = 28 + distProtoSPAWN_REQUEST = 29 + distProtoSPAWN_REQUEST_TT = 30 + distProtoSPAWN_REPLY = 31 + distProtoSPAWN_REPLY_TT = 32 + distProtoALIAS_SEND = 33 + distProtoALIAS_SEND_TT = 34 + distProtoUNLINK_ID = 35 + distProtoUNLINK_ID_ACK = 36 + + // ergo operations codes + distProtoPROXY_CONNECT_REQUEST = 101 + distProtoPROXY_CONNECT_REPLY = 102 + distProtoPROXY_CONNECT_CANCEL = 103 + distProtoPROXY_DISCONNECT = 104 +) diff --git a/tests/application_test.go b/tests/application_test.go index e2f96584..a3d5d814 100644 --- a/tests/application_test.go +++ b/tests/application_test.go @@ -27,12 +27,12 @@ func (a *testApplication) Load(args ...etf.Term) (gen.ApplicationSpec, error) { Name: name, Description: "My Test Applicatoin", Version: "v.0.1", - Environment: map[string]interface{}{ + Environment: map[gen.EnvKey]interface{}{ "envName1": 123, "envName2": "Hello world", }, Children: []gen.ApplicationChildSpec{ - gen.ApplicationChildSpec{ + { Child: &testAppGenServer{}, Name: nameGS, }, @@ -262,7 +262,7 @@ func TestApplicationBasics(t *testing.T) { tLifeSpan := time.Since(tStart) fmt.Printf("... application should be self stopped in 150ms: ") - if mynode.IsProcessAlive(p) { + if p.IsAlive() { t.Fatal("still alive") } diff --git a/tests/atomcache_test.go b/tests/atomcache_test.go new file mode 100644 index 00000000..8e383f47 --- /dev/null +++ b/tests/atomcache_test.go @@ -0,0 +1,1252 @@ +package tests + +import ( + "fmt" + "math/rand" + "strings" + "testing" + + "github.com/ergo-services/ergo" + "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/gen" + "github.com/ergo-services/ergo/lib" + "github.com/ergo-services/ergo/node" + "github.com/ergo-services/ergo/proto/dist" +) + +func TestAtomCacheLess255Uniq(t *testing.T) { + fmt.Printf("\n=== Test Atom Cache (less 255 uniq) \n") + opts1 := node.Options{} + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 2 + opts1.Proto = dist.CreateProto(protoOptions) + fmt.Printf("Starting node: nodeAtomCache1Less255@localhost with NumHandlers = 2: ") + node1, e := ergo.StartNode("nodeAtomCache1Less255@localhost", "cookie", opts1) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCache2Less255@localhost with NubHandlers = 2: ") + opts2 := node.Options{} + opts2.Proto = dist.CreateProto(protoOptions) + node2, e := ergo.StartNode("nodeAtomCache2Less255@localhost", "cookie", opts2) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + defer node1.Stop() + defer node2.Stop() + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs2 := &testMonitor{ + v: make(chan interface{}, 2), + } + + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + fmt.Printf(" wait for start of gs2 on %#v: ", node2.Name()) + node2gs2, _ := node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) + waitForResultWithValue(t, gs2.v, node2gs2.Self()) + + atoms2K := make(etf.List, 2100) + s := lib.RandomString(240) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + + long := make([]byte, 66*1024) + for i := range long { + long[i] = byte(i % 255) + } + + fmt.Printf("case 1: sending 2.1K atoms: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 2: sending a tuple with 2.1K atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 3: sending 2.1K UTF-8 long atoms: ") + s = strings.Repeat("🚀", 252) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 4: sending a tuple with 2.1K UTF-8 long atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") +} + +func TestAtomCacheMore255Uniq(t *testing.T) { + fmt.Printf("\n=== Test Atom Cache (more 255 uniq) \n") + opts1 := node.Options{} + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 2 + opts1.Proto = dist.CreateProto(protoOptions) + fmt.Printf("Starting node: nodeAtomCache1More255@localhost with NumHandlers = 2: ") + node1, e := ergo.StartNode("nodeAtomCache1More255@localhost", "cookie", opts1) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCache2More255@localhost with NubHandlers = 2: ") + opts2 := node.Options{} + opts2.Proto = dist.CreateProto(protoOptions) + node2, e := ergo.StartNode("nodeAtomCache2More255@localhost", "cookie", opts2) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + defer node1.Stop() + defer node2.Stop() + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs2 := &testMonitor{ + v: make(chan interface{}, 2), + } + + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + fmt.Printf(" wait for start of gs2 on %#v: ", node2.Name()) + node2gs2, _ := node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) + waitForResultWithValue(t, gs2.v, node2gs2.Self()) + + atoms2K := make(etf.List, 2100) + s := lib.RandomString(240) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i)) + } + + long := make([]byte, 66*1024) + for i := range long { + long[i] = byte(i % 255) + } + + fmt.Printf("case 1: sending 2.1K atoms: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 2: sending a tuple with 2.1K atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 3: sending 2.1K UTF-8 long atoms: ") + s = strings.Repeat("🚀", 251) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i)) + } + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 4: sending a tuple with 2.1K UTF-8 long atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") +} + +func TestAtomCacheLess255UniqWithCompression(t *testing.T) { + fmt.Printf("\n=== Test Atom Cache (less 255 uniq) with Compression \n") + opts1 := node.Options{} + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 2 + opts1.Proto = dist.CreateProto(protoOptions) + fmt.Printf("Starting node: nodeAtomCache1Less255Compression@localhost with NumHandlers = 2: ") + node1, e := ergo.StartNode("nodeAtomCache1Less255Compression@localhost", "cookie", opts1) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCache2Less255Compression@localhost with NubHandlers = 2: ") + opts2 := node.Options{} + opts2.Proto = dist.CreateProto(protoOptions) + node2, e := ergo.StartNode("nodeAtomCache2Less255Compression@localhost", "cookie", opts2) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + defer node1.Stop() + defer node2.Stop() + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs2 := &testMonitor{ + v: make(chan interface{}, 2), + } + + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + node1gs1.SetCompression(true) + + fmt.Printf(" wait for start of gs2 on %#v: ", node2.Name()) + node2gs2, _ := node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) + waitForResultWithValue(t, gs2.v, node2gs2.Self()) + + atoms2K := make(etf.List, 2100) + s := lib.RandomString(240) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + + long := make([]byte, 66*1024) + for i := range long { + long[i] = byte(i % 255) + } + + fmt.Printf("case 1: sending 2.1K atoms: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 2: sending a tuple with 2.1K atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 3: sending 2.1K UTF-8 long atoms: ") + s = strings.Repeat("🚀", 252) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 4: sending a tuple with 2.1K UTF-8 long atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") +} +func TestAtomCacheMore255UniqWithCompression(t *testing.T) { + fmt.Printf("\n=== Test Atom Cache (more 255 uniq) with Compression \n") + opts1 := node.Options{} + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 2 + opts1.Proto = dist.CreateProto(protoOptions) + fmt.Printf("Starting node: nodeAtomCache1More255Compression@localhost with NumHandlers = 2: ") + node1, e := ergo.StartNode("nodeAtomCache1More255Compression@localhost", "cookie", opts1) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCache2More255Compression@localhost with NubHandlers = 2: ") + opts2 := node.Options{} + opts2.Proto = dist.CreateProto(protoOptions) + node2, e := ergo.StartNode("nodeAtomCache2More255Compression@localhost", "cookie", opts2) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + defer node1.Stop() + defer node2.Stop() + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs2 := &testMonitor{ + v: make(chan interface{}, 2), + } + + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + node1gs1.SetCompression(true) + + fmt.Printf(" wait for start of gs2 on %#v: ", node2.Name()) + node2gs2, _ := node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) + waitForResultWithValue(t, gs2.v, node2gs2.Self()) + + atoms2K := make(etf.List, 2100) + s := lib.RandomString(240) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i)) + } + + long := make([]byte, 66*1024) + for i := range long { + long[i] = byte(i % 255) + } + + fmt.Printf("case 1: sending 2.1K atoms: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 2: sending a tuple with 2.1K atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 3: sending 2.1K UTF-8 long atoms: ") + s = strings.Repeat("🚀", 251) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i)) + } + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 4: sending a tuple with 2.1K UTF-8 long atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") +} + +func TestAtomCacheLess255UniqViaProxy(t *testing.T) { + fmt.Printf("\n=== Test Atom Cache (less 255 uniq) via Proxy connection \n") + opts1 := node.Options{} + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 2 + opts1.Proto = dist.CreateProto(protoOptions) + fmt.Printf("Starting node: nodeAtomCache1Less255ViaProxy@localhost with NumHandlers = 2: ") + node1, e := ergo.StartNode("nodeAtomCache1Less255ViaProxy@localhost", "cookie", opts1) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCacheTLess255ViaProxy@localhost with NubHandlers = 2: ") + optsT := node.Options{} + optsT.Proxy.Transit = true + optsT.Proto = dist.CreateProto(protoOptions) + nodeT, e := ergo.StartNode("nodeAtomCacheTLess255ViaProxy@localhost", "cookie", optsT) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCache2Less255ViaProxy@localhost with NubHandlers = 2: ") + opts2 := node.Options{} + opts2.Proto = dist.CreateProto(protoOptions) + node2, e := ergo.StartNode("nodeAtomCache2Less255ViaProxy@localhost", "cookie", opts2) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + defer node1.Stop() + defer node2.Stop() + defer nodeT.Stop() + + fmt.Printf(" connect %s with %s via proxy %s: ", node1.Name(), node2.Name(), nodeT.Name()) + route := node.ProxyRoute{ + Proxy: nodeT.Name(), + } + node1.AddProxyRoute(node2.Name(), route) + + if err := node1.Connect(node2.Name()); err != nil { + t.Fatal(err) + } + + indirectNodes := node2.NodesIndirect() + if len(indirectNodes) != 1 || indirectNodes[0] != node1.Name() { + t.Fatal("wrong result:", indirectNodes) + } + fmt.Println("OK") + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs2 := &testMonitor{ + v: make(chan interface{}, 2), + } + + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + fmt.Printf(" wait for start of gs2 on %#v: ", node2.Name()) + node2gs2, _ := node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) + waitForResultWithValue(t, gs2.v, node2gs2.Self()) + + atoms2K := make(etf.List, 2100) + s := lib.RandomString(240) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + + long := make([]byte, 66*1024) + for i := range long { + long[i] = byte(i % 255) + } + + fmt.Printf("case 1: sending 2.1K atoms: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 2: sending a tuple with 2.1K atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 3: sending 2.1K UTF-8 long atoms: ") + s = strings.Repeat("🚀", 252) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 4: sending a tuple with 2.1K UTF-8 long atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") +} + +func TestAtomCacheMore255UniqViaProxy(t *testing.T) { + fmt.Printf("\n=== Test Atom Cache (more 255 uniq) via Proxy connection \n") + opts1 := node.Options{} + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 2 + opts1.Proto = dist.CreateProto(protoOptions) + fmt.Printf("Starting node: nodeAtomCache1More255ViaProxy@localhost with NumHandlers = 2: ") + node1, e := ergo.StartNode("nodeAtomCache1More255ViaProxy@localhost", "cookie", opts1) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCacheTMore255ViaProxy@localhost with NubHandlers = 2: ") + optsT := node.Options{} + optsT.Proxy.Transit = true + optsT.Proto = dist.CreateProto(protoOptions) + nodeT, e := ergo.StartNode("nodeAtomCacheTMore255ViaProxy@localhost", "cookie", optsT) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCache2More255ViaProxy@localhost with NubHandlers = 2: ") + opts2 := node.Options{} + opts2.Proto = dist.CreateProto(protoOptions) + node2, e := ergo.StartNode("nodeAtomCache2More255ViaProxy@localhost", "cookie", opts2) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + defer node1.Stop() + defer node2.Stop() + defer nodeT.Stop() + + fmt.Printf(" connect %s with %s via proxy %s: ", node1.Name(), node2.Name(), nodeT.Name()) + route := node.ProxyRoute{ + Proxy: nodeT.Name(), + } + node1.AddProxyRoute(node2.Name(), route) + + if err := node1.Connect(node2.Name()); err != nil { + t.Fatal(err) + } + + indirectNodes := node2.NodesIndirect() + if len(indirectNodes) != 1 || indirectNodes[0] != node1.Name() { + t.Fatal("wrong result:", indirectNodes) + } + fmt.Println("OK") + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs2 := &testMonitor{ + v: make(chan interface{}, 2), + } + + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + fmt.Printf(" wait for start of gs2 on %#v: ", node2.Name()) + node2gs2, _ := node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) + waitForResultWithValue(t, gs2.v, node2gs2.Self()) + + atoms2K := make(etf.List, 2100) + s := lib.RandomString(240) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i)) + } + + long := make([]byte, 66*1024) + for i := range long { + long[i] = byte(i % 255) + } + + fmt.Printf("case 1: sending 2.1K atoms: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 2: sending a tuple with 2.1K atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 3: sending 2.1K UTF-8 long atoms: ") + s = strings.Repeat("🚀", 252) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 4: sending a tuple with 2.1K UTF-8 long atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") +} + +func TestAtomCacheLess255UniqViaProxyWithEncryption(t *testing.T) { + fmt.Printf("\n=== Test Atom Cache (less 255 uniq) via Proxy connection with Encryption\n") + opts1 := node.Options{} + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 2 + opts1.Proto = dist.CreateProto(protoOptions) + + opts1.Proxy.Flags = node.DefaultProxyFlags() + opts1.Proxy.Flags.EnableEncryption = true + + fmt.Printf("Starting node: nodeAtomCache1Less255ViaProxyEnc@localhost with NumHandlers = 2: ") + node1, e := ergo.StartNode("nodeAtomCache1Less255ViaProxyEnc@localhost", "cookie", opts1) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCacheTLess255ViaProxyEnc@localhost with NubHandlers = 2: ") + optsT := node.Options{} + optsT.Proxy.Transit = true + optsT.Proto = dist.CreateProto(protoOptions) + nodeT, e := ergo.StartNode("nodeAtomCacheTLess255ViaProxyEnc@localhost", "cookie", optsT) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCache2Less255ViaProxyEnc@localhost with NubHandlers = 2: ") + opts2 := node.Options{} + opts2.Proto = dist.CreateProto(protoOptions) + node2, e := ergo.StartNode("nodeAtomCache2Less255ViaProxyEnc@localhost", "cookie", opts2) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + defer node1.Stop() + defer node2.Stop() + defer nodeT.Stop() + + fmt.Printf(" connect %s with %s via proxy %s: ", node1.Name(), node2.Name(), nodeT.Name()) + route := node.ProxyRoute{ + Proxy: nodeT.Name(), + } + node1.AddProxyRoute(node2.Name(), route) + + if err := node1.Connect(node2.Name()); err != nil { + t.Fatal(err) + } + + indirectNodes := node2.NodesIndirect() + if len(indirectNodes) != 1 || indirectNodes[0] != node1.Name() { + t.Fatal("wrong result:", indirectNodes) + } + fmt.Println("OK") + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs2 := &testMonitor{ + v: make(chan interface{}, 2), + } + + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + fmt.Printf(" wait for start of gs2 on %#v: ", node2.Name()) + node2gs2, _ := node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) + waitForResultWithValue(t, gs2.v, node2gs2.Self()) + + atoms2K := make(etf.List, 2100) + s := lib.RandomString(240) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + + long := make([]byte, 66*1024) + for i := range long { + long[i] = byte(i % 255) + } + + fmt.Printf("case 1: sending 2.1K atoms: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 2: sending a tuple with 2.1K atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 3: sending 2.1K UTF-8 long atoms: ") + s = strings.Repeat("🚀", 252) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 4: sending a tuple with 2.1K UTF-8 long atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") +} + +func TestAtomCacheMore255UniqViaProxyWithEncryption(t *testing.T) { + fmt.Printf("\n=== Test Atom Cache (more 255 uniq) via Proxy connection with Encriptin \n") + opts1 := node.Options{} + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 2 + opts1.Proto = dist.CreateProto(protoOptions) + + opts1.Proxy.Flags = node.DefaultProxyFlags() + opts1.Proxy.Flags.EnableEncryption = true + + fmt.Printf("Starting node: nodeAtomCache1More255ViaProxyEnc@localhost with NumHandlers = 2: ") + node1, e := ergo.StartNode("nodeAtomCache1More255ViaProxyEnc@localhost", "cookie", opts1) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCacheTMore255ViaProxyEnc@localhost with NubHandlers = 2: ") + optsT := node.Options{} + optsT.Proxy.Transit = true + optsT.Proto = dist.CreateProto(protoOptions) + nodeT, e := ergo.StartNode("nodeAtomCacheTMore255ViaProxyEnc@localhost", "cookie", optsT) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCache2More255ViaProxyEnc@localhost with NubHandlers = 2: ") + opts2 := node.Options{} + opts2.Proto = dist.CreateProto(protoOptions) + node2, e := ergo.StartNode("nodeAtomCache2More255ViaProxyEnc@localhost", "cookie", opts2) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + defer node1.Stop() + defer node2.Stop() + defer nodeT.Stop() + + fmt.Printf(" connect %s with %s via proxy %s: ", node1.Name(), node2.Name(), nodeT.Name()) + route := node.ProxyRoute{ + Proxy: nodeT.Name(), + } + node1.AddProxyRoute(node2.Name(), route) + + if err := node1.Connect(node2.Name()); err != nil { + t.Fatal(err) + } + + indirectNodes := node2.NodesIndirect() + if len(indirectNodes) != 1 || indirectNodes[0] != node1.Name() { + t.Fatal("wrong result:", indirectNodes) + } + fmt.Println("OK") + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs2 := &testMonitor{ + v: make(chan interface{}, 2), + } + + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + fmt.Printf(" wait for start of gs2 on %#v: ", node2.Name()) + node2gs2, _ := node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) + waitForResultWithValue(t, gs2.v, node2gs2.Self()) + + atoms2K := make(etf.List, 2100) + s := lib.RandomString(240) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i)) + } + + long := make([]byte, 66*1024) + for i := range long { + long[i] = byte(i % 255) + } + + fmt.Printf("case 1: sending 2.1K atoms: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 2: sending a tuple with 2.1K atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 3: sending 2.1K UTF-8 long atoms: ") + s = strings.Repeat("🚀", 252) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 4: sending a tuple with 2.1K UTF-8 long atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") +} + +func TestAtomCacheLess255UniqViaProxyWithEncryptionCompression(t *testing.T) { + fmt.Printf("\n=== Test Atom Cache (less 255 uniq) via Proxy connection with Encryption and Compression\n") + opts1 := node.Options{} + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 2 + opts1.Proto = dist.CreateProto(protoOptions) + + opts1.Proxy.Flags = node.DefaultProxyFlags() + opts1.Proxy.Flags.EnableEncryption = true + + fmt.Printf("Starting node: nodeAtomCache1Less255ViaProxyEncComp@localhost with NumHandlers = 2: ") + node1, e := ergo.StartNode("nodeAtomCache1Less255ViaProxyEncComp@localhost", "cookie", opts1) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCacheTLess255ViaProxyEncComp@localhost with NubHandlers = 2: ") + optsT := node.Options{} + optsT.Proxy.Transit = true + optsT.Proto = dist.CreateProto(protoOptions) + nodeT, e := ergo.StartNode("nodeAtomCacheTLess255ViaProxyEncComp@localhost", "cookie", optsT) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCache2Less255ViaProxyEncComp@localhost with NubHandlers = 2: ") + opts2 := node.Options{} + opts2.Proto = dist.CreateProto(protoOptions) + node2, e := ergo.StartNode("nodeAtomCache2Less255ViaProxyEncComp@localhost", "cookie", opts2) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + defer node1.Stop() + defer node2.Stop() + defer nodeT.Stop() + + fmt.Printf(" connect %s with %s via proxy %s: ", node1.Name(), node2.Name(), nodeT.Name()) + route := node.ProxyRoute{ + Proxy: nodeT.Name(), + } + node1.AddProxyRoute(node2.Name(), route) + + if err := node1.Connect(node2.Name()); err != nil { + t.Fatal(err) + } + + indirectNodes := node2.NodesIndirect() + if len(indirectNodes) != 1 || indirectNodes[0] != node1.Name() { + t.Fatal("wrong result:", indirectNodes) + } + fmt.Println("OK") + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs2 := &testMonitor{ + v: make(chan interface{}, 2), + } + + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + node1gs1.SetCompression(true) + + fmt.Printf(" wait for start of gs2 on %#v: ", node2.Name()) + node2gs2, _ := node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) + waitForResultWithValue(t, gs2.v, node2gs2.Self()) + + atoms2K := make(etf.List, 2100) + s := lib.RandomString(240) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + + long := make([]byte, 66*1024) + for i := range long { + long[i] = byte(i % 255) + } + + fmt.Printf("case 1: sending 2.1K atoms: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 2: sending a tuple with 2.1K atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 3: sending 2.1K UTF-8 long atoms: ") + s = strings.Repeat("🚀", 252) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 4: sending a tuple with 2.1K UTF-8 long atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") +} + +func TestAtomCacheMore255UniqViaProxyWithEncryptionCompression(t *testing.T) { + fmt.Printf("\n=== Test Atom Cache (more 255 uniq) via Proxy connection with Encription and Compression \n") + opts1 := node.Options{} + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 2 + opts1.Proto = dist.CreateProto(protoOptions) + + opts1.Proxy.Flags = node.DefaultProxyFlags() + opts1.Proxy.Flags.EnableEncryption = true + + fmt.Printf("Starting node: nodeAtomCache1More255ViaProxyEncComp@localhost with NumHandlers = 2: ") + node1, e := ergo.StartNode("nodeAtomCache1More255ViaProxyEncComp@localhost", "cookie", opts1) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCacheTMore255ViaProxyEncComp@localhost with NubHandlers = 2: ") + optsT := node.Options{} + optsT.Proxy.Transit = true + optsT.Proto = dist.CreateProto(protoOptions) + nodeT, e := ergo.StartNode("nodeAtomCacheTMore255ViaProxyEncComp@localhost", "cookie", optsT) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("Starting node: nodeAtomCache2More255ViaProxyEncComp@localhost with NubHandlers = 2: ") + opts2 := node.Options{} + opts2.Proto = dist.CreateProto(protoOptions) + node2, e := ergo.StartNode("nodeAtomCache2More255ViaProxyEncComp@localhost", "cookie", opts2) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + defer node1.Stop() + defer node2.Stop() + defer nodeT.Stop() + + fmt.Printf(" connect %s with %s via proxy %s: ", node1.Name(), node2.Name(), nodeT.Name()) + route := node.ProxyRoute{ + Proxy: nodeT.Name(), + } + node1.AddProxyRoute(node2.Name(), route) + + if err := node1.Connect(node2.Name()); err != nil { + t.Fatal(err) + } + + indirectNodes := node2.NodesIndirect() + if len(indirectNodes) != 1 || indirectNodes[0] != node1.Name() { + t.Fatal("wrong result:", indirectNodes) + } + fmt.Println("OK") + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs2 := &testMonitor{ + v: make(chan interface{}, 2), + } + + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + node1gs1.SetCompression(true) + + fmt.Printf(" wait for start of gs2 on %#v: ", node2.Name()) + node2gs2, _ := node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) + waitForResultWithValue(t, gs2.v, node2gs2.Self()) + + atoms2K := make(etf.List, 2100) + s := lib.RandomString(240) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i)) + } + + long := make([]byte, 66*1024) + for i := range long { + long[i] = byte(i % 255) + } + + fmt.Printf("case 1: sending 2.1K atoms: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 2: sending a tuple with 2.1K atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + + fmt.Printf("case 3: sending 2.1K UTF-8 long atoms: ") + s = strings.Repeat("🚀", 252) + for i := range atoms2K { + atoms2K[i] = etf.Atom(fmt.Sprintf("%s%d", s, i/10)) + } + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := atoms2K + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") + fmt.Printf("case 4: sending a tuple with 2.1K UTF-8 long atoms and 66K binary: ") + rand.Shuffle(len(atoms2K), func(i, j int) { + atoms2K[i], atoms2K[j] = atoms2K[j], atoms2K[i] + }) + for i := 0; i < 10; i++ { + result := etf.Tuple{atoms2K, long} + node1gs1.Send(node2gs2.Self(), result) + if err := waitForResultWithValueReturnError(t, gs2.v, result); err != nil { + t.Fatal(err) + } + } + fmt.Println("OK") +} diff --git a/tests/registrar_test.go b/tests/core_test.go similarity index 87% rename from tests/registrar_test.go rename to tests/core_test.go index af96a74d..a22da074 100644 --- a/tests/registrar_test.go +++ b/tests/core_test.go @@ -11,16 +11,16 @@ import ( "github.com/ergo-services/ergo/node" ) -type TestRegistrarGenserver struct { +type TestCoreGenserver struct { gen.Server } -func (trg *TestRegistrarGenserver) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, message etf.Term) (etf.Term, gen.ServerStatus) { - // fmt.Printf("TestRegistrarGenserver ({%s, %s}): HandleCall: %#v, From: %#v\n", trg.process.name, trg.process.Node.Name(), message, from) +func (trg *TestCoreGenserver) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, message etf.Term) (etf.Term, gen.ServerStatus) { + // fmt.Printf("TestCoreGenserver ({%s, %s}): HandleCall: %#v, From: %#v\n", trg.process.name, trg.process.Node.Name(), message, from) return message, gen.ServerStatusOK } -func (trg *TestRegistrarGenserver) HandleDirect(process *gen.ServerProcess, message interface{}) (interface{}, error) { +func (trg *TestCoreGenserver) HandleDirect(process *gen.ServerProcess, message interface{}) (interface{}, error) { switch m := message.(type) { case makeCall: return process.Call(m.to, m.message) @@ -28,7 +28,7 @@ func (trg *TestRegistrarGenserver) HandleDirect(process *gen.ServerProcess, mess return nil, gen.ErrUnsupportedRequest } -func TestRegistrar(t *testing.T) { +func TestCore(t *testing.T) { fmt.Printf("\n=== Test Registrar\n") fmt.Printf("Starting nodes: nodeR1@localhost, nodeR2@localhost: ") node1, _ := ergo.StartNode("nodeR1@localhost", "cookies", node.Options{}) @@ -39,8 +39,8 @@ func TestRegistrar(t *testing.T) { fmt.Println("OK") } - gs := &TestRegistrarGenserver{} - fmt.Printf("Starting TestRegistrarGenserver. registering as 'gs1' on %s and create an alias: ", node1.Name()) + gs := &TestCoreGenserver{} + fmt.Printf("Starting TestCoreGenserver. registering as 'gs1' on %s and create an alias: ", node1.Name()) node1gs1, err := node1.Spawn("gs1", gen.ProcessOptions{}, gs, nil) if err != nil { t.Fatal(err) @@ -100,7 +100,7 @@ func TestRegistrar(t *testing.T) { } fmt.Println("OK") - fmt.Printf("Starting TestRegistrarGenserver and registering as 'gs2' on %s: ", node1.Name()) + fmt.Printf("Starting TestCoreGenserver and registering as 'gs2' on %s: ", node1.Name()) node1gs2, err := node1.Spawn("gs2", gen.ProcessOptions{}, gs, nil) if err != nil { t.Fatal(err) @@ -121,7 +121,7 @@ func TestRegistrar(t *testing.T) { fmt.Println("OK") } -func TestRegistrarAlias(t *testing.T) { +func TestCoreAlias(t *testing.T) { fmt.Printf("\n=== Test Registrar Alias\n") fmt.Printf("Starting node: nodeR1Alias@localhost: ") node1, _ := ergo.StartNode("nodeR1Alias@localhost", "cookies", node.Options{}) @@ -132,7 +132,7 @@ func TestRegistrarAlias(t *testing.T) { fmt.Println("OK") } - gs := &TestRegistrarGenserver{} + gs := &TestCoreGenserver{} fmt.Printf(" Starting gs1 and gs2 GenServers on %s: ", node1.Name()) node1gs1, err := node1.Spawn("gs1", gen.ProcessOptions{}, gs, nil) if err != nil { diff --git a/tests/monitor_test.go b/tests/monitor_test.go index 85061feb..c1d3cec0 100644 --- a/tests/monitor_test.go +++ b/tests/monitor_test.go @@ -2,6 +2,8 @@ package tests import ( "fmt" + "reflect" + "sort" "testing" "github.com/ergo-services/ergo" @@ -44,9 +46,8 @@ func TestMonitorLocalLocal(t *testing.T) { node1, _ := ergo.StartNode("nodeM1LocalLocal@localhost", "cookies", node.Options{}) if node1 == nil { t.Fatal("can't start node") - } else { - fmt.Println("OK") } + fmt.Println("OK") gs1 := &testMonitor{ v: make(chan interface{}, 2), } @@ -64,7 +65,7 @@ func TestMonitorLocalLocal(t *testing.T) { waitForResultWithValue(t, gs2.v, node1gs2.Self()) // by Pid - fmt.Printf("... by Pid Local-Local: gs1 -> gs2. demonitor: ") + fmt.Printf("... by Pid Local-Local: gs1 -> gs2. monitor/demonitor: ") ref := node1gs1.MonitorProcess(node1gs2.Self()) if !node1gs2.IsMonitor(ref) { @@ -76,7 +77,7 @@ func TestMonitorLocalLocal(t *testing.T) { } fmt.Println("OK") - fmt.Printf("... by Pid Local-Local: gs1 -> gs2. terminate: ") + fmt.Printf("... by Pid Local-Local: gs1 -> gs2. monitor/terminate: ") ref = node1gs1.MonitorProcess(node1gs2.Self()) node1gs2.Exit("normal") result := gen.MessageDown{ @@ -90,7 +91,7 @@ func TestMonitorLocalLocal(t *testing.T) { t.Fatal(err) } - fmt.Print("... by Pid Local-Local: gs1 -> unknownPid: ") + fmt.Print("... by Pid Local-Local: gs1 -> monitor unknownPid: ") ref = node1gs1.MonitorProcess(node1gs2.Self()) result = gen.MessageDown{ Ref: ref, @@ -103,7 +104,7 @@ func TestMonitorLocalLocal(t *testing.T) { node1gs2, _ = node1.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) waitForResultWithValue(t, gs2.v, node1gs2.Self()) // by Name - fmt.Printf("... by Name Local-Local: gs1 -> gs2. demonitor: ") + fmt.Printf("... by Name Local-Local: gs1 -> gs2. monitor/demonitor: ") ref = node1gs1.MonitorProcess("gs2") if err := checkCleanProcessRef(node1gs1, ref); err == nil { t.Fatal("monitor reference has been lost") @@ -114,7 +115,7 @@ func TestMonitorLocalLocal(t *testing.T) { } fmt.Println("OK") - fmt.Printf("... by Name Local-Local: gs1 -> gs2. terminate: ") + fmt.Printf("... by Name Local-Local: gs1 -> gs2. monitor/terminate: ") ref = node1gs1.MonitorProcess("gs2") node1gs2.Exit("normal") result = gen.MessageDown{ @@ -126,7 +127,7 @@ func TestMonitorLocalLocal(t *testing.T) { if err := checkCleanProcessRef(node1gs1, ref); err != nil { t.Fatal(err) } - fmt.Print("... by Name Local-Local: gs1 -> unknownPid: ") + fmt.Print("... by Name Local-Local: gs1 -> monitor unknown name: ") ref = node1gs1.MonitorProcess("asdfasdf") result = gen.MessageDown{ Ref: ref, @@ -216,14 +217,14 @@ func TestMonitorLocalRemoteByPid(t *testing.T) { waitForResultWithValue(t, gs2.v, node2gs2.Self()) // by Pid - fmt.Printf("... by Pid Local-Remote: gs1 -> gs2. demonitor: ") + fmt.Printf("... by Pid Local-Remote: gs1 -> gs2. monitor/demonitor: ") ref := node1gs1.MonitorProcess(node2gs2.Self()) // wait a bit for the MessageDown if something went wrong waitForTimeout(t, gs1.v) - if err := checkCleanProcessRef(node1gs1, ref); err == nil { + if node1gs1.IsMonitor(ref) == false { t.Fatal("monitor reference has been lost on node 1") } - if err := checkCleanProcessRef(node2gs2, ref); err == nil { + if node2gs2.IsMonitor(ref) == false { t.Fatal("monitor reference has been lost on node 2") } if found := node1gs1.DemonitorProcess(ref); found == false { @@ -240,7 +241,7 @@ func TestMonitorLocalRemoteByPid(t *testing.T) { } fmt.Println("OK") - fmt.Printf("... by Pid Local-Remote: gs1 -> gs2. terminate: ") + fmt.Printf("... by Pid Local-Remote: gs1 -> gs2. monitor/terminate: ") ref = node1gs1.MonitorProcess(node2gs2.Self()) // wait a bit for the MessageDown if something went wrong waitForTimeout(t, gs1.v) @@ -259,7 +260,7 @@ func TestMonitorLocalRemoteByPid(t *testing.T) { t.Fatal(err) } - fmt.Printf("... by Pid Local-Remote: gs1 -> unknownPid: ") + fmt.Printf("... by Pid Local-Remote: gs1 -> monitor unknownPid: ") ref = node1gs1.MonitorProcess(node2gs2.Self()) result = gen.MessageDown{ Ref: ref, @@ -275,10 +276,11 @@ func TestMonitorLocalRemoteByPid(t *testing.T) { node2gs2, _ = node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) waitForResultWithValue(t, gs2.v, node2gs2.Self()) - fmt.Printf("... by Pid Local-Remote: gs1 -> gs2. onNodeDown: ") + fmt.Printf("... by Pid Local-Remote: gs1 -> gs2. monitor/NodeDown: ") ref = node1gs1.MonitorProcess(node2gs2.Self()) // wait a bit for the MessageDown if something went wrong waitForTimeout(t, gs1.v) + node1.Disconnect(node2.Name()) node2.Stop() result = gen.MessageDown{ Ref: ref, @@ -290,7 +292,7 @@ func TestMonitorLocalRemoteByPid(t *testing.T) { t.Fatal(err) } - fmt.Printf("... by Pid Local-Remote: gs1 -> gs2. UnknownNode: ") + fmt.Printf("... by Pid Local-Remote: gs1 -> gs2. monitor unknown node: ") ref = node1gs1.MonitorProcess(node2gs2.Self()) result.Ref = ref waitForResultWithValue(t, gs1.v, result) @@ -329,7 +331,7 @@ func TestMonitorLocalRemoteByName(t *testing.T) { processID := gen.ProcessID{Name: "gs2", Node: node2.Name()} - fmt.Printf("... by gen.ProcessID{Name, Node} Local-Remote: gs1 -> gs2. demonitor: ") + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Remote: gs1 -> gs2. monitor/demonitor: ") ref := node1gs1.MonitorProcess(processID) // wait a bit for the MessageDown if something went wrong waitForTimeout(t, gs1.v) @@ -353,7 +355,7 @@ func TestMonitorLocalRemoteByName(t *testing.T) { } fmt.Println("OK") - fmt.Printf("... by gen.ProcessID{Name, Node} Local-Remote: gs1 -> gs2. terminate: ") + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Remote: gs1 -> gs2. monitor/terminate: ") ref = node1gs1.MonitorProcess(processID) // wait a bit for the MessageDown if something went wrong waitForTimeout(t, gs1.v) @@ -372,7 +374,7 @@ func TestMonitorLocalRemoteByName(t *testing.T) { t.Fatal("monitor ref is still alive") } - fmt.Printf("... by gen.ProcessID{Name, Node} Local-Remote: gs1 -> unknownPid: ") + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Remote: gs1 -> monitor unknown remote name: ") ref = node1gs1.MonitorProcess(processID) result = gen.MessageDown{ Ref: ref, @@ -388,22 +390,343 @@ func TestMonitorLocalRemoteByName(t *testing.T) { node2gs2, _ = node2.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) waitForResultWithValue(t, gs2.v, node2gs2.Self()) - fmt.Printf("... by gen.ProcessID{Name, Node} Local-Remote: gs1 -> gs2. onNodeDown: ") + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Remote: gs1 -> gs2. monitor/onNodeDown: ") ref = node1gs1.MonitorProcess(processID) + node1.Disconnect(node2.Name()) + node2.Stop() result = gen.MessageDown{ Ref: ref, ProcessID: processID, Reason: "noconnection", } + waitForResultWithValue(t, gs1.v, result) + if node1gs1.IsMonitor(ref) { + t.Fatal("monitor ref is still alive") + } + + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Remote: gs1 -> gs2. monitor unknown node: ") + ref = node1gs1.MonitorProcess(processID) + result.Ref = ref + waitForResultWithValue(t, gs1.v, result) + if node1gs1.IsMonitor(ref) { + t.Fatal("monitor ref is still alive") + } + node1.Stop() +} + +func TestMonitorLocalProxyRemoteByPid(t *testing.T) { + fmt.Printf("\n=== Test Monitor Remote via Proxy by Pid\n") + fmt.Printf("Starting nodes: nodeM1ProxyRemoteByPid@localhost, nodeM2ProxyRemoteByPid@localhost, nodeM3ProxyRemoteByPid@localhost : ") + opts1 := node.Options{} + opts1.Proxy.Flags = node.DefaultProxyFlags() + opts1.Proxy.Flags.EnableMonitor = false + node1, err := ergo.StartNode("nodeM1ProxyRemoteByPid@localhost", "cookies", opts1) + if err != nil { + t.Fatal("can't start node:", err) + } + opts2 := node.Options{} + opts2.Proxy.Transit = true + node2, err := ergo.StartNode("nodeM2ProxyRemoteByPid@localhost", "cookies", opts2) + if err != nil { + t.Fatal("can't start node:", err, node2.Name()) + } + node3, err := ergo.StartNode("nodeM3ProxyRemoteByPid@localhost", "cookies", node.Options{}) + if err != nil { + t.Fatal("can't start node:", err) + } + + route := node.ProxyRoute{ + Proxy: node2.Name(), + } + node1.AddProxyRoute(node3.Name(), route) + node1.Connect(node3.Name()) + fmt.Println("OK") + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs3 := &testMonitor{ + v: make(chan interface{}, 2), + } + + // starting gen servers + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + fmt.Printf(" wait for start of gs3 on %#v: ", node3.Name()) + node3gs3, _ := node3.Spawn("gs3", gen.ProcessOptions{}, gs3, nil) + waitForResultWithValue(t, gs3.v, node3gs3.Self()) + + // by Pid + fmt.Printf("... by Pid Local-Proxy-Remote: gs1 -> gs3. monitor/demonitor: ") + ref := node1gs1.MonitorProcess(node3gs3.Self()) + // wait a bit for the MessageDown if something went wrong + waitForTimeout(t, gs1.v) + if node1gs1.IsMonitor(ref) == false { + t.Fatal("monitor reference has been lost on node 1") + } + if node3gs3.IsMonitor(ref) == false { + t.Fatal("monitor reference has been lost on node 3") + } + if found := node1gs1.DemonitorProcess(ref); found == false { + t.Fatal("lost monitoring reference on node1") + } + // Demonitoring is the async message with nothing as a feedback. + // use waitForTimeout just as a short timer + waitForTimeout(t, gs1.v) + if err := checkCleanProcessRef(node1gs1, ref); err != nil { + t.Fatal(err) + } + if err := checkCleanProcessRef(node3gs3, ref); err != nil { + t.Fatal(err) + } + fmt.Println("OK") + + fmt.Printf("... by Pid Local-Proxy-Remote: gs1 -> gs3. monitor/terminate: ") + ref = node1gs1.MonitorProcess(node3gs3.Self()) // wait a bit for the MessageDown if something went wrong waitForTimeout(t, gs1.v) + node3gs3.Exit("normal") + result := gen.MessageDown{ + Ref: ref, + Pid: node3gs3.Self(), + Reason: "normal", + } + waitForResultWithValue(t, gs1.v, result) + + if err := checkCleanProcessRef(node1gs1, ref); err != nil { + t.Fatal(err) + } + if err := checkCleanProcessRef(node3gs3, ref); err != nil { + t.Fatal(err) + } + + fmt.Printf("... by Pid Local-Proxy-Remote: gs1 -> monitor unknownPid: ") + ref = node1gs1.MonitorProcess(node3gs3.Self()) + result = gen.MessageDown{ + Ref: ref, + Pid: node3gs3.Self(), + Reason: "noproc", + } + waitForResultWithValue(t, gs1.v, result) + if err := checkCleanProcessRef(node1gs1, ref); err != nil { + t.Fatal(err) + } + + fmt.Printf(" wait for start of gs3 on %#v: ", node3.Name()) + node3gs3, _ = node3.Spawn("gs3", gen.ProcessOptions{}, gs3, nil) + waitForResultWithValue(t, gs3.v, node3gs3.Self()) + + fmt.Printf("... by Pid Local-Proxy-Remote: gs3 -> gs1. monitor/(node1: ProxyFlags.EnableMonitor = false): ") + + ref = node3gs3.MonitorProcess(node1gs1.Self()) + result = gen.MessageDown{ + Ref: ref, + Pid: node1gs1.Self(), + Reason: "unsupported", + } + waitForResultWithValue(t, gs3.v, result) + + fmt.Printf("... by Pid Local-Proxy-Remote: gs1 -> gs3. monitor/ProxyDown: ") + ref = node1gs1.MonitorProcess(node3gs3.Self()) + waitForTimeout(t, gs1.v) node2.Stop() + result = gen.MessageDown{ + Ref: ref, + Pid: node3gs3.Self(), + Reason: "noproxy", + } + waitForResultWithValue(t, gs1.v, result) + if err := checkCleanProcessRef(node1gs1, ref); err != nil { + t.Fatal(err) + } + + node2, err = ergo.StartNode("nodeM2ProxyRemoteByPid@localhost", "cookies", opts2) + if err != nil { + t.Fatal("can't start node:", err, node2.Name()) + } + + fmt.Printf("... by Pid Local-Proxy-Remote: gs1 -> gs3. monitor/NodeDown: ") + ref = node1gs1.MonitorProcess(node3gs3.Self()) + // wait a bit for the MessageDown if something went wrong + waitForTimeout(t, gs1.v) + node3.Stop() + result = gen.MessageDown{ + Ref: ref, + Pid: node3gs3.Self(), + Reason: "noconnection", + } + waitForResultWithValue(t, gs1.v, result) + if err := checkCleanProcessRef(node1gs1, ref); err != nil { + t.Fatal(err) + } + + fmt.Printf("... by Pid Local-Proxy-Remote: gs1 -> gs3. monitor unknown node: ") + ref = node1gs1.MonitorProcess(node3gs3.Self()) + result.Ref = ref + waitForResultWithValue(t, gs1.v, result) + if err := checkCleanProcessRef(node1gs1, ref); err != nil { + t.Fatal(err) + } + node1.Stop() +} + +func TestMonitorLocalProxyRemoteByName(t *testing.T) { + fmt.Printf("\n=== Test Monitor Local-Proxy-Remote by Name\n") + fmt.Printf("Starting nodes: nodeM1ProxyRemoteByName@localhost, nodeM2RemoteByName@localhost, nodeM3RemoteByName@localhost: ") + opts1 := node.Options{} + opts1.Proxy.Flags = node.DefaultProxyFlags() + opts1.Proxy.Flags.EnableMonitor = false + node1, err := ergo.StartNode("nodeM1RemoteByName@localhost", "cookies", opts1) + if err != nil { + t.Fatal("can't start node:", err) + } + opts2 := node.Options{} + opts2.Proxy.Transit = true + node2, err := ergo.StartNode("nodeM2RemoteByName@localhost", "cookies", opts2) + if err != nil { + t.Fatal("can't start node:", err) + } + node3, err := ergo.StartNode("nodeM3RemoteByName@localhost", "cookies", node.Options{}) + if err != nil { + t.Fatal("can't start node:", err) + } + route := node.ProxyRoute{ + Proxy: node2.Name(), + } + node1.AddProxyRoute(node3.Name(), route) + node1.Connect(node3.Name()) + fmt.Println("OK") + + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs3 := &testMonitor{ + v: make(chan interface{}, 2), + } + + // starting gen servers + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + fmt.Printf(" wait for start of gs3 on %#v: ", node3.Name()) + node3gs3, _ := node3.Spawn("gs3", gen.ProcessOptions{}, gs3, nil) + waitForResultWithValue(t, gs3.v, node3gs3.Self()) + + processID := gen.ProcessID{Name: "gs3", Node: node3.Name()} + + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Proxy-Remote: gs1 -> gs3. monitor/demonitor: ") + ref := node1gs1.MonitorProcess(processID) + // wait a bit for the MessageDown if something went wrong + waitForTimeout(t, gs1.v) + if err := checkCleanProcessRef(node1gs1, ref); err == nil { + t.Fatal("monitor reference has been lost on node 1") + } + if err := checkCleanProcessRef(node3gs3, ref); err == nil { + t.Fatal("monitor reference has been lost on node 3") + } + if found := node1gs1.DemonitorProcess(ref); found == false { + t.Fatal("lost monitoring reference on node1") + } + // Demonitoring is the async message with nothing as a feedback. + // use waitForTimeout just as a short timer + waitForTimeout(t, gs1.v) + if err := checkCleanProcessRef(node1gs1, ref); err != nil { + t.Fatal(err) + } + if err := checkCleanProcessRef(node3gs3, ref); err != nil { + t.Fatal(err) + } + fmt.Println("OK") + + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Proxy-Remote: gs1 -> gs3. monitor/terminate: ") + ref = node1gs1.MonitorProcess(processID) + // wait a bit for the MessageDown if something went wrong + waitForTimeout(t, gs1.v) + if node1gs1.IsMonitor(ref) == false { + t.Fatal("monitor reference has been lost on node 1") + } + if node3gs3.IsMonitor(ref) == false { + t.Fatal("monitor reference has been lost on node 3") + } + node3gs3.Exit("normal") + result := gen.MessageDown{ + Ref: ref, + ProcessID: processID, + Reason: "normal", + } + waitForResultWithValue(t, gs1.v, result) + + if node1gs1.IsMonitor(ref) { + t.Fatal("monitor ref is still alive") + } + if node3gs3.IsMonitor(ref) { + t.Fatal("monitor ref is still alive") + } + + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Proxy-Remote: gs1 -> monitor unknown remote name: ") + ref = node1gs1.MonitorProcess(processID) + result = gen.MessageDown{ + Ref: ref, + ProcessID: processID, + Reason: "noproc", + } + waitForResultWithValue(t, gs1.v, result) + if node1gs1.IsMonitor(ref) { + t.Fatal("monitor ref is still alive") + } + + fmt.Printf(" wait for start of gs3 on %#v: ", node3.Name()) + node3gs3, _ = node3.Spawn("gs3", gen.ProcessOptions{}, gs3, nil) + waitForResultWithValue(t, gs3.v, node3gs3.Self()) + + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Proxy-Remote: gs3 -> gs1. monitor/(node1: ProxyFlags.EnableMonitor = false): ") + + processID1 := gen.ProcessID{Name: node1gs1.Name(), Node: node1.Name()} + ref = node3gs3.MonitorProcess(processID1) + result = gen.MessageDown{ + Ref: ref, + ProcessID: processID1, + Reason: "unsupported", + } + waitForResultWithValue(t, gs3.v, result) + + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Proxy-Remote: gs1 -> gs3. monitor/ProxyDown: ") + ref = node1gs1.MonitorProcess(processID) + waitForTimeout(t, gs1.v) + node2.Stop() + result = gen.MessageDown{ + Ref: ref, + ProcessID: processID, + Reason: "noproxy", + } + waitForResultWithValue(t, gs1.v, result) + if err := checkCleanProcessRef(node1gs1, ref); err != nil { + t.Fatal(err) + } + + node2, err = ergo.StartNode("nodeM2RemoteByName@localhost", "cookies", opts2) + if err != nil { + t.Fatal("can't start node:", err, node2.Name()) + } + + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Proxy-Remote: gs1 -> gs3. monitor/NodeDown: ") + ref = node1gs1.MonitorProcess(processID) + waitForTimeout(t, gs1.v) + node3.Stop() + result = gen.MessageDown{ + Ref: ref, + ProcessID: processID, + Reason: "noconnection", + } waitForResultWithValue(t, gs1.v, result) if node1gs1.IsMonitor(ref) { t.Fatal("monitor ref is still alive") } - fmt.Printf("... by gen.ProcessID{Name, Node} Local-Remote: gs1 -> gs2. UnknownNode: ") + fmt.Printf("... by gen.ProcessID{Name, Node} Local-Proxy-Remote: gs1 -> gs3. monitor unknown node: ") ref = node1gs1.MonitorProcess(processID) result.Ref = ref waitForResultWithValue(t, gs1.v, result) @@ -419,8 +742,8 @@ func TestMonitorLocalRemoteByName(t *testing.T) { by Pid - equal_pids, already_linked, doesnt_exist, terminate, unlink */ -func TestLinkLocalLocal(t *testing.T) { - fmt.Printf("\n=== Test Link Local-Local\n") +func TestLinkLocal(t *testing.T) { + fmt.Printf("\n=== Test Link Local\n") fmt.Printf("Starting node: nodeL1LocalLocal@localhost: ") node1, _ := ergo.StartNode("nodeL1LocalLocal@localhost", "cookies", node.Options{}) if node1 == nil { @@ -562,8 +885,8 @@ func TestLinkLocalLocal(t *testing.T) { Link by Pid - already_linked, doesnt_exist, terminate, unlink, node_down, node_unknown */ -func TestLinkLocalRemote(t *testing.T) { - fmt.Printf("\n=== Test Link Local-Remote by Pid\n") +func TestLinkRemote(t *testing.T) { + fmt.Printf("\n=== Test Link Remote by Pid\n") fmt.Printf("Starting nodes: nodeL1LocalRemoteByPid@localhost, nodeL2LocalRemoteByPid@localhost: ") node1, _ := ergo.StartNode("nodeL1LocalRemoteByPid@localhost", "cookies", node.Options{}) node2, _ := ergo.StartNode("nodeL2LocalRemoteByPid@localhost", "cookies", node.Options{}) @@ -747,10 +1070,419 @@ func TestLinkLocalRemote(t *testing.T) { node1.Stop() } +func TestLinkRemoteProxy(t *testing.T) { + fmt.Printf("\n=== Test Link Remote Via Proxy\n") + fmt.Printf("Starting nodes: nodeL1RemoteViaProxy@localhost, nodeL2RemoteViaProxy@localhost, nodeL3RemoteViaProxy@localhost: ") + node1, err := ergo.StartNode("nodeL1RemoteViaProxy@localhost", "cookies", node.Options{}) + if err != nil { + t.Fatal(err) + } + node2opts := node.Options{} + node2opts.Proxy.Transit = true + node2, err := ergo.StartNode("nodeL2RemoteViaProxy@localhost", "cookies", node2opts) + if err != nil { + t.Fatal(err) + } + node3, err := ergo.StartNode("nodeL3RemoteViaProxy@localhost", "cookies", node.Options{}) + if err != nil { + t.Fatal(err) + } + fmt.Println("OK") + + route := node.ProxyRoute{ + Proxy: node2.Name(), + } + route.Flags = node.DefaultProxyFlags() + route.Flags.EnableLink = false + node1.AddProxyRoute(node3.Name(), route) + + fmt.Printf(" check connectivity of %s with %s via proxy %s: ", node1.Name(), node3.Name(), node2.Name()) + if err := node1.Connect(node3.Name()); err != nil { + t.Fatal(err) + } + node1indirect := node1.NodesIndirect() + node3indirect := node3.NodesIndirect() + if len(node1indirect) != 1 || len(node3indirect) != 1 { + t.Fatal("wrong indirect nodes (node1:", node1indirect, "; node3:", node3indirect, ")") + } + if node1indirect[0] != node3.Name() || node3indirect[0] != node1.Name() { + t.Fatal("wrong indirect nodes (node1:", node1indirect, "; node3:", node3indirect, ")") + } + fmt.Println("OK") + gs1 := &testMonitor{ + v: make(chan interface{}, 2), + } + gs3 := &testMonitor{ + v: make(chan interface{}, 2), + } + + // starting gen servers + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + + fmt.Printf(" wait for start of gs3 on %#v: ", node3.Name()) + node3gs3, _ := node3.Spawn("gs3", gen.ProcessOptions{}, gs3, nil) + waitForResultWithValue(t, gs3.v, node3gs3.Self()) + + fmt.Printf("Testing Link process Local-Proxy-Remote: gs1 -> gs3. unlink: ") + node1gs1.Link(node3gs3.Self()) + // wait a bit since linking process is async + waitForTimeout(t, gs1.v) + + if checkLinkPid(node1gs1, node3gs3.Self()) != nil { + t.Fatal("link missing on node1gs1") + } + if checkLinkPid(node3gs3, node1gs1.Self()) != nil { + t.Fatal("link missing on node3gs3 ") + } + + node1gs1.Unlink(node3gs3.Self()) + // wait a bit since unlinking process is async + waitForTimeout(t, gs1.v) + if err := checkCleanLinkPid(node1gs1, node3gs3.Self()); err != nil { + t.Fatal(err) + } + if err := checkCleanLinkPid(node3gs3, node1gs1.Self()); err != nil { + t.Fatal(err) + } + fmt.Println("OK") + + fmt.Printf("Testing Link process Local-Proxy-Remote: gs1 -> gs3. already_linked: ") + node1gs1.Link(node3gs3.Self()) + if checkLinkPid(node1gs1, node3gs3.Self()) != nil { + t.Fatal("link missing on node1gs1") + } + // wait a bit since linking process is async + waitForTimeout(t, gs1.v) + if checkLinkPid(node3gs3, node1gs1.Self()) != nil { + t.Fatal("link missing on node3gs3") + } + ll1 := len(node1gs1.Links()) + ll3 := len(node3gs3.Links()) + + node3gs3.Link(node1gs1.Self()) + // wait a bit since linking process is async + waitForTimeout(t, gs3.v) + + if ll1 != len(node1gs1.Links()) || ll3 != len(node3gs3.Links()) { + t.Fatal("number of links has changed on the second Link call") + } + fmt.Println("OK") + + fmt.Printf("Testing Link process Local-Proxy-Remote: gs1 -> gs3. terminate (trap_exit = true): ") + // do not link these process since they are already linked after the previous test + + node1gs1.SetTrapExit(true) + + node3gs3.Exit("normal") + result := gen.MessageExit{Pid: node3gs3.Self(), Reason: "normal"} + waitForResultWithValue(t, gs1.v, result) + + if err := checkCleanLinkPid(node1gs1, node3gs3.Self()); err != nil { + t.Fatal(err) + } + if err := checkCleanLinkPid(node3gs3, node1gs1.Self()); err != nil { + t.Fatal(err) + } + if !node1gs1.IsAlive() { + t.Fatal("gs1 should be alive after gs3 exit due to enabled trap exit on gs1") + } + + fmt.Printf("Testing Link process Local-Proxy-Remote: gs1 -> gs3. doesnt_exist: ") + ll1 = len(node1gs1.Links()) + node1gs1.Link(node3gs3.Self()) + result = gen.MessageExit{Pid: node3gs3.Self(), Reason: "noproc"} + waitForResultWithValue(t, gs1.v, result) + if ll1 != len(node1gs1.Links()) { + t.Fatal("number of links has changed on the second Link call") + } + + fmt.Printf(" wait for start of gs3 on %#v: ", node1.Name()) + node3gs3, _ = node3.Spawn("gs3", gen.ProcessOptions{}, gs3, nil) + waitForResultWithValue(t, gs3.v, node3gs3.Self()) + + node1gs1.SetTrapExit(false) + fmt.Printf("Testing Link process Local-Proxy-Remote: gs1 -> gs3. terminate (trap_exit = false): ") + node1gs1.Link(node3gs3.Self()) + waitForTimeout(t, gs3.v) + + if checkLinkPid(node1gs1, node3gs3.Self()) != nil { + t.Fatal("link missing on node1gs1") + } + if checkLinkPid(node3gs3, node1gs1.Self()) != nil { + t.Fatal("link missing on node3gs3") + } + + node3gs3.Exit("normal") + + // wait a bit to make sure if we receive anything (shouldnt receive) + waitForTimeout(t, gs1.v) + + if err := checkCleanLinkPid(node1gs1, node3gs3.Self()); err != nil { + t.Fatal(err) + } + if err := checkCleanLinkPid(node3gs3, node1gs1.Self()); err != nil { + t.Fatal(err) + } + if node1gs1.IsAlive() { + t.Fatal("gs1 shouldnt be alive after gs3 exit due to disable trap exit on gs1") + } + if node3gs3.IsAlive() { + t.Fatal("gs3 must be terminated") + } + fmt.Println("OK") + + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) + node1gs1, _ = node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) + waitForResultWithValue(t, gs1.v, node1gs1.Self()) + fmt.Printf(" wait for start of gs3 on %#v: ", node3.Name()) + node3gs3, _ = node3.Spawn("gs3", gen.ProcessOptions{}, gs3, nil) + waitForResultWithValue(t, gs3.v, node3gs3.Self()) + + node1gs1.SetTrapExit(true) + fmt.Printf("Testing Link process Local-Proxy-Remote: gs1 -> gs3. node_down: ") + node1gs1.Link(node3gs3.Self()) + waitForTimeout(t, gs1.v) + + if checkLinkPid(node1gs1, node3gs3.Self()) != nil { + t.Fatal("link missing for node1gs1") + } + if checkCleanLinkPid(node3gs3, node1gs1.Self()) == nil { + t.Fatal("link missing for node3gs3") + } + + // race conditioned case. + // processing of the process termination (on the remote peer) can be done faster than + // the link termination there, so MessageExit with "kill" reason will be arrived + // earlier. + node3.Stop() + result1 := gen.MessageExit{Pid: node3gs3.Self(), Reason: "noconnection"} + result2 := gen.MessageExit{Pid: node3gs3.Self(), Reason: "kill"} + + waitForResultWithValueOrValue(t, gs1.v, result1, result2) + + if err := checkCleanLinkPid(node1gs1, node3gs3.Self()); err != nil { + t.Fatal(err) + } + // must wait a bit + waitForTimeout(t, gs1.v) + if err := checkCleanLinkPid(node3gs3, node1gs1.Self()); err != nil { + t.Fatal(err) + } + + ll1 = len(node1gs1.Links()) + fmt.Printf("Testing Link process Local-Proxy-Remote: gs1 -> gs3. node_unknown: ") + node1gs1.Link(node3gs3.Self()) + result = gen.MessageExit{Pid: node3gs3.Self(), Reason: "noconnection"} + waitForResultWithValue(t, gs1.v, result) + + if ll1 != len(node1gs1.Links()) { + t.Fatal("number of links has changed on the second Link call") + } + + node3, err = ergo.StartNode("nodeL3RemoteViaProxy@localhost", "cookies", node.Options{}) + fmt.Printf(" starting node: %s", node3.Name()) + if err != nil { + t.Fatal(err) + } + fmt.Println("OK") + fmt.Printf(" wait for start of gs3 on %#v: ", node3.Name()) + node3gs3, _ = node3.Spawn("gs3", gen.ProcessOptions{}, gs3, nil) + waitForResultWithValue(t, gs3.v, node3gs3.Self()) + + if err := node1.Connect(node3.Name()); err != nil { + t.Fatal(err) + } + + node3gs3.SetTrapExit(true) + fmt.Printf("Testing Proxy Local-Proxy-Remote for link gs3 -> gs1 (Node1 ProxyFlags.EnableLink = false): ") + node3gs3.Link(node1gs1.Self()) + result = gen.MessageExit{Pid: node1gs1.Self(), Reason: node.ErrPeerUnsupported.Error()} + waitForResultWithValue(t, gs3.v, result) + + node1gs1.Link(node3gs3.Self()) + waitForTimeout(t, gs1.v) + + if checkLinkPid(node1gs1, node3gs3.Self()) != nil { + t.Fatal("link missing for node1gs1") + } + if checkCleanLinkPid(node3gs3, node1gs1.Self()) == nil { + t.Fatal("link missing for node3gs3") + } + fmt.Println("Testing Proxy Down Local-Proxy-Remote for linked gs1 -> gs3 (trap_exit = true): ") + node2.Stop() + + fmt.Printf(" wait for MessageExit with reason 'noproxy' on gs1: ") + result = gen.MessageExit{Pid: node3gs3.Self(), Reason: "noproxy"} + waitForResultWithValue(t, gs1.v, result) + + fmt.Printf(" wait for MessageExit with reason 'noproxy' on gs3: ") + result = gen.MessageExit{Pid: node1gs1.Self(), Reason: "noproxy"} + waitForResultWithValue(t, gs3.v, result) + + node1.Stop() +} + +func TestMonitorNode(t *testing.T) { + fmt.Printf("\n=== Test Monitor Node \n") + fmt.Printf("... start nodes A, B, C, D: ") + optsA := node.Options{} + nodeA, e := ergo.StartNode("monitornodeAproxy@localhost", "secret", optsA) + if e != nil { + t.Fatal(e) + } + optsB := node.Options{} + optsB.Proxy.Transit = true + nodeB, e := ergo.StartNode("monitornodeBproxy@localhost", "secret", optsB) + if e != nil { + t.Fatal(e) + } + optsC := node.Options{} + optsC.Proxy.Transit = true + nodeC, e := ergo.StartNode("monitornodeCproxy@localhost", "secret", optsC) + if e != nil { + t.Fatal(e) + } + + optsD := node.Options{} + nodeD, e := ergo.StartNode("monitornodeDproxy@localhost", "secret", optsD) + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + gsA := &testMonitor{ + v: make(chan interface{}, 2), + } + gsB := &testMonitor{ + v: make(chan interface{}, 2), + } + gsD := &testMonitor{ + v: make(chan interface{}, 2), + } + fmt.Printf("... start processA on node A: ") + pA, err := nodeA.Spawn("", gen.ProcessOptions{}, gsA) + if err != nil { + t.Fatal(err) + } + waitForResultWithValue(t, gsA.v, pA.Self()) + fmt.Printf("... start processB on node B: ") + pB, err := nodeB.Spawn("", gen.ProcessOptions{}, gsB) + if err != nil { + t.Fatal(err) + } + waitForResultWithValue(t, gsB.v, pB.Self()) + fmt.Printf("... start processD on node D: ") + pD, err := nodeD.Spawn("", gen.ProcessOptions{}, gsD) + if err != nil { + t.Fatal(err) + } + waitForResultWithValue(t, gsD.v, pD.Self()) + fmt.Printf("... add proxy route on A to the node D via B: ") + routeAtoDviaB := node.ProxyRoute{ + Proxy: nodeB.Name(), + } + if err := nodeA.AddProxyRoute(nodeD.Name(), routeAtoDviaB); err != nil { + t.Fatal(err) + } + fmt.Println("OK") + + fmt.Printf("... add proxy transit route on B to the node D via C: ") + route := node.ProxyRoute{ + Proxy: nodeC.Name(), + } + if err := nodeB.AddProxyRoute(nodeD.Name(), route); err != nil { + t.Fatal(err) + } + fmt.Println("OK") + + fmt.Printf("... monitor D by processA (via proxy connection): ") + refA := pA.MonitorNode(nodeD.Name()) + fmt.Println("OK") + fmt.Printf("... monitor A by processD (via proxy connection): ") + refD := pD.MonitorNode(nodeA.Name()) + fmt.Println("OK") + fmt.Printf("... monitor C by processB (via direct connection): ") + refB := pB.MonitorNode(nodeC.Name()) + fmt.Println("OK") + fmt.Printf("... check connectivity (A -> D via B and C, D -> A via C and B): ") + nodelist := []string{nodeB.Name(), nodeD.Name()} + nodesA := nodeA.Nodes() + sort.Strings(nodesA) + if reflect.DeepEqual(nodesA, nodelist) == false { + t.Fatal("node A has wrong peers", nodeA.Nodes()) + } + if reflect.DeepEqual(nodeA.NodesIndirect(), []string{nodeD.Name()}) == false { + t.Fatal("node A has wrong proxy peer", nodeA.NodesIndirect()) + } + nodelist = []string{nodeA.Name(), nodeC.Name()} + nodesB := nodeB.Nodes() + sort.Strings(nodesB) + if reflect.DeepEqual(nodesB, nodelist) == false { + t.Fatal("node B has wrong peers", nodeB.Nodes()) + } + if reflect.DeepEqual(nodeB.NodesIndirect(), []string{}) == false { + t.Fatal("node B has wrong proxy peer", nodeB.NodesIndirect()) + } + nodelist = []string{nodeB.Name(), nodeD.Name()} + nodesC := nodeC.Nodes() + sort.Strings(nodesC) + if reflect.DeepEqual(nodesC, nodelist) == false { + t.Fatal("node C has wrong peers", nodeC.Nodes()) + } + if reflect.DeepEqual(nodeC.NodesIndirect(), []string{}) == false { + t.Fatal("node C has wrong proxy peer", nodeC.NodesIndirect()) + } + nodelist = []string{nodeA.Name(), nodeC.Name()} + nodesD := nodeD.Nodes() + sort.Strings(nodesD) + if reflect.DeepEqual(nodesD, nodelist) == false { + t.Fatal("node D has wrong peers", nodeD.Nodes()) + } + if reflect.DeepEqual(nodeD.NodesIndirect(), []string{nodeA.Name()}) == false { + t.Fatal("node D has wrong proxy peer", nodeD.NodesIndirect()) + } + fmt.Println("OK") + fmt.Printf("... stop node C : ") + nodeC.Stop() + fmt.Println("OK") + resultMessageProxyDown := gen.MessageProxyDown{Ref: refD, Node: nodeC.Name(), Proxy: nodeD.Name(), Reason: "noconnection"} + fmt.Printf("... processD must receive gen.MessageProxyDown{Node: C, Proxy: D,...}: ") + waitForResultWithValue(t, gsD.v, resultMessageProxyDown) + resultMessageProxyDown = gen.MessageProxyDown{Ref: refA, Node: nodeC.Name(), Proxy: nodeB.Name(), Reason: "noconnection"} + fmt.Printf("... processA must receive gen.MessageProxyDown{Node: C, Proxy: B,...}: ") + waitForResultWithValue(t, gsA.v, resultMessageProxyDown) + resultMessageDown := gen.MessageNodeDown{Ref: refB, Name: nodeC.Name()} + fmt.Printf("... processB must receive gen.MessageDown: ") + waitForResultWithValue(t, gsB.v, resultMessageDown) + + fmt.Printf("... check connectivity (A <-> B, C is down, D has no peers): ") + if reflect.DeepEqual(nodeA.Nodes(), []string{nodeB.Name()}) == false { + t.Fatal("node A has wrong peer", nodeA.Nodes()) + } + if reflect.DeepEqual(nodeB.Nodes(), []string{nodeA.Name()}) == false { + t.Fatal("node B has wrong peer", nodeB.Nodes()) + } + if nodeC.IsAlive() == true { + t.Fatal("node C is still alive") + } + if reflect.DeepEqual(nodeC.Nodes(), []string{}) == false { + t.Fatal("node C has peers", nodeC.Nodes()) + } + if reflect.DeepEqual(nodeD.Nodes(), []string{}) == false { + t.Fatal("node D has peers", nodeD.Nodes()) + } + fmt.Println("OK") + nodeD.Stop() + nodeA.Stop() + nodeB.Stop() +} + // helpers func checkCleanProcessRef(p gen.Process, ref etf.Ref) error { if p.IsMonitor(ref) { - return fmt.Errorf("monitor process reference hasnt clean correctly") + return fmt.Errorf("monitor process reference hasn't been cleaned correctly") } return nil @@ -759,7 +1491,7 @@ func checkCleanProcessRef(p gen.Process, ref etf.Ref) error { func checkCleanLinkPid(p gen.Process, pid etf.Pid) error { for _, l := range p.Links() { if l == pid { - return fmt.Errorf("process link reference hasnt cleaned correctly") + return fmt.Errorf("process link reference hasn't been cleaned correctly") } } return nil diff --git a/tests/node_test.go b/tests/node_test.go index 3c368fda..fb066f9b 100644 --- a/tests/node_test.go +++ b/tests/node_test.go @@ -1,20 +1,22 @@ package tests import ( + "context" "crypto/md5" "fmt" "math/rand" "net" "reflect" - "runtime" "sync" "testing" + "time" "github.com/ergo-services/ergo" "github.com/ergo-services/ergo/etf" "github.com/ergo-services/ergo/gen" + "github.com/ergo-services/ergo/lib" "github.com/ergo-services/ergo/node" - "github.com/ergo-services/ergo/node/dist" + "github.com/ergo-services/ergo/proto/dist" ) type benchCase struct { @@ -23,13 +25,13 @@ type benchCase struct { } func TestNode(t *testing.T) { + ctx := context.Background() opts := node.Options{ - ListenRangeBegin: 25001, - ListenRangeEnd: 25001, - EPMDPort: 24999, + Listen: 25001, + Resolver: dist.CreateResolverWithLocalEPMD("", 24999), } - node1, _ := ergo.StartNode("node@localhost", "cookies", opts) + node1, _ := ergo.StartNodeWithContext(ctx, "node@localhost", "cookies", opts) if conn, err := net.Dial("tcp", ":25001"); err != nil { fmt.Println("Connect to the node' listening port FAILED") @@ -53,8 +55,8 @@ func TestNode(t *testing.T) { t.Fatal(e) } - if !node1.IsProcessAlive(p) { - t.Fatal("IsProcessAlive: expect 'true', but got 'false'") + if !p.IsAlive() { + t.Fatal("IsAlive: expect 'true', but got 'false'") } _, ee := node1.ProcessInfo(p.Self()) @@ -159,63 +161,42 @@ func TestNodeFragmentation(t *testing.T) { wg.Wait() } -func TestNodeAtomCache(t *testing.T) { - - node1, _ := ergo.StartNode("nodeT1AtomCache@localhost", "secret", node.Options{}) - node2, _ := ergo.StartNode("nodeT2AtomCache@localhost", "secret", node.Options{}) - - tgs := &benchGS{} - p1, e1 := node1.Spawn("", gen.ProcessOptions{}, tgs) - p2, e2 := node2.Spawn("", gen.ProcessOptions{}, tgs) +func TestNodeStaticRoute(t *testing.T) { + nodeName1 := "nodeT1StaticRoute@localhost" + nodeName2 := "nodeT2StaticRoute@localhost" + nodeStaticPort := uint16(9876) + node1, e1 := ergo.StartNode(nodeName1, "secret", node.Options{}) if e1 != nil { t.Fatal(e1) } + defer node1.Stop() + + node2, e2 := ergo.StartNode(nodeName2, "secret", node.Options{}) if e2 != nil { t.Fatal(e2) } + defer node2.Stop() - message := etf.Tuple{ - etf.Atom("a1"), - etf.Atom("a2"), - etf.Atom("a3"), - etf.Atom("a4"), - etf.Atom("a5"), - } - for i := 0; i < 2*runtime.GOMAXPROCS(-1); i++ { - call := makeCall{ - to: p2.Self(), - message: message, - } - if _, e := p1.Direct(call); e != nil { - t.Fatal(e) - } - } -} - -func TestNodeStaticRoute(t *testing.T) { - nodeName := "nodeT1StaticRoute@localhost" - nodeStaticPort := 9876 - - node1, _ := ergo.StartNode(nodeName, "secret", node.Options{}) - nr, err := node1.Resolve(nodeName) + nr, err := node1.Resolve(nodeName2) if err != nil { - t.Fatal("Can't resolve port number for ", nodeName) + t.Fatal("Can't resolve port number for ", nodeName2, err) } - e := node1.AddStaticRoute(nodeName, uint16(nodeStaticPort)) + // override route for nodeName2 with static port + e := node1.AddStaticRoutePort(nodeName2, nodeStaticPort, node.RouteOptions{}) if e != nil { t.Fatal(e) } // should be overrided by the new value of nodeStaticPort - if nr, err := node1.Resolve(nodeName); err != nil || nr.Port != nodeStaticPort { - t.Fatal("Wrong port number after adding static route. Got", nr.Port, "Expected", nodeStaticPort) + if r, err := node1.Resolve(nodeName2); err != nil || r.Port != nodeStaticPort { + t.Fatal("Wrong port number after adding static route. Got", r.Port, "Expected", nodeStaticPort) } - node1.RemoveStaticRoute(nodeName) + node1.RemoveStaticRoute(nodeName2) // should be resolved into the original port number - if nr2, err := node1.Resolve(nodeName); err != nil || nr.Port != nr2.Port { + if nr2, err := node1.Resolve(nodeName2); err != nil || nr.Port != nr2.Port { t.Fatal("Wrong port number after removing static route") } } @@ -242,20 +223,16 @@ func (h *handshakeGenServer) HandleDirect(process *gen.ServerProcess, message in func TestNodeDistHandshake(t *testing.T) { fmt.Printf("\n=== Test Node Handshake versions\n") - nodeOptions5 := node.Options{ - HandshakeVersion: dist.ProtoHandshake5, + // handshake version 5 + handshake5options := dist.HandshakeOptions{ + Version: dist.HandshakeVersion5, } - nodeOptions6 := node.Options{ - HandshakeVersion: dist.ProtoHandshake6, - } - nodeOptions5WithTLS := node.Options{ - HandshakeVersion: dist.ProtoHandshake5, - TLSMode: node.TLSModeAuto, - } - nodeOptions6WithTLS := node.Options{ - HandshakeVersion: dist.ProtoHandshake6, - TLSMode: node.TLSModeAuto, + + // handshake version 6 + handshake6options := dist.HandshakeOptions{ + Version: dist.HandshakeVersion6, } + hgs := &handshakeGenServer{} type Pair struct { @@ -263,68 +240,110 @@ func TestNodeDistHandshake(t *testing.T) { nodeA node.Node nodeB node.Node } - node1, e1 := ergo.StartNode("node1Handshake5@localhost", "secret", nodeOptions5) + node1Options5 := node.Options{ + Handshake: dist.CreateHandshake(handshake5options), + } + node1, e1 := ergo.StartNode("node1Handshake5@localhost", "secret", node1Options5) if e1 != nil { t.Fatal(e1) } - node2, e2 := ergo.StartNode("node2Handshake5@localhost", "secret", nodeOptions5) + node2Options5 := node.Options{ + Handshake: dist.CreateHandshake(handshake5options), + } + node2, e2 := ergo.StartNode("node2Handshake5@localhost", "secret", node2Options5) if e2 != nil { t.Fatal(e2) } - node3, e3 := ergo.StartNode("node3Handshake5@localhost", "secret", nodeOptions5) + node3Options5 := node.Options{ + Handshake: dist.CreateHandshake(handshake5options), + } + node3, e3 := ergo.StartNode("node3Handshake5@localhost", "secret", node3Options5) if e3 != nil { t.Fatal(e3) } - node4, e4 := ergo.StartNode("node4Handshake6@localhost", "secret", nodeOptions6) + node4Options6 := node.Options{ + Handshake: dist.CreateHandshake(handshake6options), + } + node4, e4 := ergo.StartNode("node4Handshake6@localhost", "secret", node4Options6) if e4 != nil { t.Fatal(e4) } // node5, _ := ergo.StartNode("node5Handshake6@localhost", "secret", nodeOptions6) // node6, _ := ergo.StartNode("node6Handshake5@localhost", "secret", nodeOptions5) - node7, e7 := ergo.StartNode("node7Handshake6@localhost", "secret", nodeOptions6) + node7Options6 := node.Options{ + Handshake: dist.CreateHandshake(handshake6options), + } + node7, e7 := ergo.StartNode("node7Handshake6@localhost", "secret", node7Options6) if e7 != nil { t.Fatal(e7) } - node8, e8 := ergo.StartNode("node8Handshake6@localhost", "secret", nodeOptions6) + node8Options6 := node.Options{ + Handshake: dist.CreateHandshake(handshake6options), + } + node8, e8 := ergo.StartNode("node8Handshake6@localhost", "secret", node8Options6) if e8 != nil { t.Fatal(e8) } - node9, e9 := ergo.StartNode("node9Handshake5@localhost", "secret", nodeOptions5WithTLS) + node9Options5WithTLS := node.Options{ + Handshake: dist.CreateHandshake(handshake5options), + TLS: node.TLS{Enable: true}, + } + node9, e9 := ergo.StartNode("node9Handshake5@localhost", "secret", node9Options5WithTLS) if e9 != nil { t.Fatal(e9) } - node10, e10 := ergo.StartNode("node10Handshake5@localhost", "secret", nodeOptions5WithTLS) + node10Options5WithTLS := node.Options{ + Handshake: dist.CreateHandshake(handshake5options), + TLS: node.TLS{Enable: true}, + } + node10, e10 := ergo.StartNode("node10Handshake5@localhost", "secret", node10Options5WithTLS) if e10 != nil { t.Fatal(e10) } - node11, e11 := ergo.StartNode("node11Handshake5@localhost", "secret", nodeOptions5WithTLS) + node11Options5WithTLS := node.Options{ + Handshake: dist.CreateHandshake(handshake5options), + TLS: node.TLS{Enable: true}, + } + node11, e11 := ergo.StartNode("node11Handshake5@localhost", "secret", node11Options5WithTLS) if e11 != nil { t.Fatal(e11) } - node12, e12 := ergo.StartNode("node12Handshake6@localhost", "secret", nodeOptions6WithTLS) + node12Options6WithTLS := node.Options{ + Handshake: dist.CreateHandshake(handshake6options), + TLS: node.TLS{Enable: true}, + } + node12, e12 := ergo.StartNode("node12Handshake6@localhost", "secret", node12Options6WithTLS) if e12 != nil { t.Fatal(e12) } // node13, _ := ergo.StartNode("node13Handshake6@localhost", "secret", nodeOptions6WithTLS) // node14, _ := ergo.StartNode("node14Handshake5@localhost", "secret", nodeOptions5WithTLS) - node15, e15 := ergo.StartNode("node15Handshake6@localhost", "secret", nodeOptions6WithTLS) + node15Options6WithTLS := node.Options{ + Handshake: dist.CreateHandshake(handshake6options), + TLS: node.TLS{Enable: true}, + } + node15, e15 := ergo.StartNode("node15Handshake6@localhost", "secret", node15Options6WithTLS) if e15 != nil { t.Fatal(e15) } - node16, e16 := ergo.StartNode("node16Handshake6@localhost", "secret", nodeOptions6WithTLS) + node16Options6WithTLS := node.Options{ + Handshake: dist.CreateHandshake(handshake6options), + TLS: node.TLS{Enable: true}, + } + node16, e16 := ergo.StartNode("node16Handshake6@localhost", "secret", node16Options6WithTLS) if e16 != nil { t.Fatal(e16) } nodes := []Pair{ - Pair{"No TLS. version 5 -> version 5", node1, node2}, - Pair{"No TLS. version 5 -> version 6", node3, node4}, + {"No TLS. version 5 -> version 5", node1, node2}, + {"No TLS. version 5 -> version 6", node3, node4}, //Pair{ "No TLS. version 6 -> version 5", node5, node6 }, - Pair{"No TLS. version 6 -> version 6", node7, node8}, - Pair{"With TLS. version 5 -> version 5", node9, node10}, - Pair{"With TLS. version 5 -> version 6", node11, node12}, + {"No TLS. version 6 -> version 6", node7, node8}, + {"With TLS. version 5 -> version 5", node9, node10}, + {"With TLS. version 5 -> version 6", node11, node12}, //Pair{ "With TLS. version 6 -> version 5", node13, node14 }, - Pair{"With TLS. version 6 -> version 6", node15, node16}, + {"With TLS. version 6 -> version 6", node15, node16}, } defer func(nodes []Pair) { @@ -339,7 +358,7 @@ func TestNodeDistHandshake(t *testing.T) { var result etf.Term for i := range nodes { pair := nodes[i] - fmt.Printf(" %s: ", pair.name) + fmt.Printf(" %s %s -> %s: ", pair.name, pair.nodeA.Name(), pair.nodeB.Name()) pA, e = pair.nodeA.Spawn("", gen.ProcessOptions{}, hgs) if e != nil { t.Fatal(e) @@ -364,35 +383,823 @@ func TestNodeDistHandshake(t *testing.T) { } } -func TestNodeRemoteSpawn(t *testing.T) { - fmt.Printf("\n=== Test Node Remote Spawn\n") - node1, _ := ergo.StartNode("node1remoteSpawn@localhost", "secret", node.Options{}) - node2, _ := ergo.StartNode("node2remoteSpawn@localhost", "secret", node.Options{}) +func TestNodeRemoteSpawn(t *testing.T) { + fmt.Printf("\n=== Test Node Remote Spawn\n") + node1opts := node.Options{} + node1opts.Proxy.Flags = node.DefaultProxyFlags() + node1opts.Proxy.Flags.EnableRemoteSpawn = false + + node1, _ := ergo.StartNode("node1remoteSpawn@localhost", "secret", node1opts) + node2opts := node.Options{} + node2opts.Proxy.Transit = true + node2, _ := ergo.StartNode("node2remoteSpawn@localhost", "secret", node2opts) + node3, _ := ergo.StartNode("node3remoteSpawn@localhost", "secret", node.Options{}) + route := node.ProxyRoute{ + Proxy: node2.Name(), + } + node1.AddProxyRoute(node3.Name(), route) + defer node1.Stop() + defer node2.Stop() + defer node3.Stop() + + if err := node1.Connect(node3.Name()); err != nil { + t.Fatal(err) + } + + node2.ProvideRemoteSpawn("remote", &handshakeGenServer{}) + process, err := node1.Spawn("gs1", gen.ProcessOptions{}, &handshakeGenServer{}) + if err != nil { + t.Fatal(err) + } + + opts := gen.RemoteSpawnOptions{ + Name: "remote", + } + fmt.Printf(" process gs1@node1 request to spawn new process on node2 and register this process with name 'remote': ") + gotPid, err := process.RemoteSpawn(node2.Name(), "remote", opts, 1, 2, 3) + if err != nil { + t.Fatal(err) + } + p := node2.ProcessByName("remote") + if p == nil { + t.Fatal("can't find process 'remote' on node2") + } + if gotPid != p.Self() { + t.Fatal("process pid mismatch") + } + fmt.Println("OK") + + fmt.Printf(" process gs1@node1 request to spawn new process on node2 with the same name (must be failed): ") + _, err = process.RemoteSpawn(node2.Name(), "remote", opts, 1, 2, 3) + if err != node.ErrTaken { + t.Fatal(err) + } + fmt.Println("OK") + fmt.Printf(" process gs1@node1 request to spawn new process on node2 with unregistered behavior name (must be failed): ") + _, err = process.RemoteSpawn(node2.Name(), "randomname", opts, 1, 2, 3) + if err != node.ErrBehaviorUnknown { + t.Fatal(err) + } + fmt.Println("OK") + + fmt.Printf(" process gs1@node1 request to spawn new process on node3 via proxy node2 and register this process with name 'remote': ") + node3.ProvideRemoteSpawn("remote", &handshakeGenServer{}) + gotPid, err = process.RemoteSpawn(node3.Name(), "remote", opts, 1, 2, 3) + if err != nil { + t.Fatal(err) + } + p = node3.ProcessByName("remote") + if p == nil { + t.Fatal("can't find process 'remote' on node2") + } + if gotPid != p.Self() { + t.Fatal("process pid mismatch") + } + fmt.Println("OK") + fmt.Printf(" process gs3@node3 request to spawn new process on node1 via proxy node2 (node1 ProxyFlags.RemoteSpawn: false): ") + process3, err := node3.Spawn("gs3", gen.ProcessOptions{}, &handshakeGenServer{}) + if err != nil { + t.Fatal(err) + } + gotPid, err = process3.RemoteSpawn(node1.Name(), "remote", opts, 1, 2, 3) + if err != node.ErrPeerUnsupported { + t.Fatal(err) + } + fmt.Println("OK") +} + +func TestNodeResolveExtra(t *testing.T) { + fmt.Printf("\n=== Test Node Resolve Extra \n") + fmt.Printf("... starting node1 with disabled TLS: ") + node1, err := ergo.StartNode("node1resolveExtra@localhost", "secret", node.Options{}) + if err != nil { + t.Fatal(err) + } + defer node1.Stop() + fmt.Println("OK") + opts := node.Options{} + opts.TLS.Enable = true + fmt.Printf("... starting node2 with enabled TLS: ") + node2, err := ergo.StartNode("node2resolveExtra@localhost", "secret", opts) + if err != nil { + t.Fatal(err) + } + defer node2.Stop() + fmt.Println("OK") + + fmt.Printf("... node1 resolves node2 with enabled TLS: ") + route1, err := node1.Resolve("node2resolveExtra@localhost") + if err != nil { + t.Fatal(err) + } + if route1.Options.EnableTLS == false { + t.Fatal("expected true value") + } + fmt.Println("OK") + + fmt.Printf("... node2 resolves node1 with disabled TLS: ") + route2, err := node2.Resolve("node1resolveExtra@localhost") + if err != nil { + t.Fatal(err) + } + if route2.Options.EnableTLS == true { + t.Fatal("expected true value") + } + fmt.Println("OK") + + fmt.Printf("... node1 connect to node2: ") + if err := node1.Connect(node2.Name()); err != nil { + t.Fatal(err) + } + if len(node1.Nodes()) != 1 { + t.Fatal("no peers") + } + if node1.Nodes()[0] != node2.Name() { + t.Fatal("wrong peer") + } + fmt.Println("OK") + + fmt.Printf("... disconnecting nodes: ") + time.Sleep(300 * time.Millisecond) + if err := node1.Disconnect(node2.Name()); err != nil { + t.Fatal(err) + } + if len(node1.Nodes()) > 0 { + t.Fatal("still connected") + } + fmt.Println("OK") + + fmt.Printf("... node2 connect to node1: ") + if err := node2.Connect(node1.Name()); err != nil { + t.Fatal(err) + } + if len(node2.Nodes()) != 1 { + t.Fatal("no peers") + } + if node2.Nodes()[0] != node1.Name() { + t.Fatal("wrong peer") + } + fmt.Println("OK") +} + +type failoverServer struct { + gen.Server + v chan interface{} +} + +func (f *failoverServer) Init(process *gen.ServerProcess, args ...etf.Term) error { + return nil +} +func (f *failoverServer) HandleInfo(process *gen.ServerProcess, message etf.Term) gen.ServerStatus { + if _, yes := gen.IsMessageFallback(message); yes { + f.v <- message + return gen.ServerStatusOK + } + time.Sleep(300 * time.Millisecond) + return gen.ServerStatusOK +} +func TestNodeProcessFallback(t *testing.T) { + fmt.Printf("\n=== Test Node Process Fallback\n") + fmt.Printf("... start node1: ") + node1, e := ergo.StartNode("node1processfallback@localhost", "secret", node.Options{}) + if e != nil { + t.Fatal(e) + } + defer node1.Stop() + fmt.Println("OK") + popts1 := gen.ProcessOptions{ + MailboxSize: 2, + Fallback: gen.ProcessFallback{ + Name: "fp", + Tag: "test_tag", + }, + } + gsf := &failoverServer{ + v: make(chan interface{}, 2), + } + + fmt.Printf("... start process p1 (with mailbox size = 2 and fallback process = \"fp\"): ") + p1, err := node1.Spawn("", popts1, &failoverServer{}) + if err != nil { + t.Fatal(e) + } + fmt.Println("OK") + fmt.Printf("... start failover process p2 (with name = \"fp\"): ") + _, err = node1.Spawn("fp", gen.ProcessOptions{}, gsf) + if err != nil { + t.Fatal(e) + } + fmt.Println("OK") + fmt.Printf("... sending 4 messages to p1 (4th must wrapped into gen.MessageFallback and forwarded to \"fp\" ): ") + p1.Send(p1.Self(), "m1") + p1.Send(p1.Self(), "m2") + p1.Send(p1.Self(), "m3") + // bellow message must be forwarded + p1.Send(p1.Self(), "m4") + + result := gen.MessageFallback{Process: p1.Self(), Tag: "test_tag", Message: "m4"} + waitForResultWithValue(t, gsf.v, result) +} + +type compressionServer struct { + gen.Server +} + +func (c *compressionServer) Init(process *gen.ServerProcess, args ...etf.Term) error { + return nil +} + +func (c *compressionServer) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, message etf.Term) (etf.Term, gen.ServerStatus) { + blob := message.(etf.Tuple)[1].([]byte) + md5original := message.(etf.Tuple)[0].(string) + md5sum := fmt.Sprint(md5.Sum(blob)) + result := etf.Atom("ok") + if !reflect.DeepEqual(md5original, md5sum) { + result = etf.Atom("mismatch") + } + return result, gen.ServerStatusOK +} +func (c *compressionServer) HandleDirect(process *gen.ServerProcess, message interface{}) (interface{}, error) { + switch m := message.(type) { + case makeCall: + return process.Call(m.to, m.message) + } + return nil, gen.ErrUnsupportedRequest +} +func TestNodeCompression(t *testing.T) { + fmt.Printf("\n=== Test Node Compression \n") + opts1 := node.Options{} + opts1.Compression.Enable = true + // need 1 handler to make Atom cache work + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 1 + opts1.Proto = dist.CreateProto(protoOptions) + node1, e := ergo.StartNode("node1compression@localhost", "secret", opts1) + if e != nil { + t.Fatal(e) + } + defer node1.Stop() + node2, e := ergo.StartNode("node2compression@localhost", "secret", node.Options{}) + if e != nil { + t.Fatal(e) + } + defer node2.Stop() + + n1p1, err := node1.Spawn("", gen.ProcessOptions{}, &compressionServer{}) + if err != nil { + t.Fatal(err) + } + n2p1, err := node2.Spawn("", gen.ProcessOptions{}, &compressionServer{}) + if err != nil { + t.Fatal(err) + } + + fmt.Printf("... send 1MB compressed. no fragmentation: ") + // empty data (no fragmentation) + blob := make([]byte, 1024*1024) + md5sum := fmt.Sprint(md5.Sum(blob)) + message := etf.Tuple{md5sum, blob} + + // send 3 times. that is how atom cache is working - + // atoms are encoding from cache on 2nd or 3rd sending + call := makeCall{ + to: n2p1.Self(), + message: message, + } + for i := 0; i < 3; i++ { + result, e := n1p1.Direct(call) + if e != nil { + t.Fatal(e) + } + if result != etf.Atom("ok") { + t.Fatal(result) + } + } + fmt.Println("OK") + + fmt.Printf("... send 1MB compressed. with fragmentation: ") + // will be fragmented + rnd := lib.RandomString(1024 * 1024) + blob = []byte(rnd) // compression rate for random string around 50% + //rand.Read(blob[:66000]) // compression rate for 1MB of random data - 0 % (entropy too big) + md5sum = fmt.Sprint(md5.Sum(blob)) + message = etf.Tuple{md5sum, blob} + + call = makeCall{ + to: n2p1.Self(), + message: message, + } + for i := 0; i < 3; i++ { + result, e := n1p1.Direct(call) + if e != nil { + t.Fatal(e) + } + if result != etf.Atom("ok") { + t.Fatal(result) + } + } + fmt.Println("OK") +} + +func TestNodeProxyConnect(t *testing.T) { + fmt.Printf("\n=== Test Node Proxy\n") + fmt.Printf("... connect NodeA to NodeC via NodeB: ") + optsA := node.Options{} + nodeA, e := ergo.StartNode("nodeAproxy@localhost", "secret", optsA) + if e != nil { + t.Fatal(e) + } + route := node.ProxyRoute{ + Proxy: "nodeBproxy@localhost", + } + nodeA.AddProxyRoute("nodeCproxy@localhost", route) + + optsB := node.Options{} + optsB.Proxy.Transit = true + nodeB, e := ergo.StartNode("nodeBproxy@localhost", "secret", optsB) + if e != nil { + t.Fatal(e) + } + optsC := node.Options{} + nodeC, e := ergo.StartNode("nodeCproxy@localhost", "secret", optsC) + if e != nil { + t.Fatal(e) + } + + if err := nodeA.Connect("nodeCproxy@localhost"); err != nil { + t.Fatal(err) + } + + indirectNodes := nodeA.NodesIndirect() + if len(indirectNodes) != 1 { + t.Fatal("wrong result:", indirectNodes) + } + if indirectNodes[0] != "nodeCproxy@localhost" { + t.Fatal("wrong result:", indirectNodes) + } + indirectNodes = nodeC.NodesIndirect() + if len(indirectNodes) != 1 { + t.Fatal("wrong result:", indirectNodes) + } + if indirectNodes[0] != "nodeAproxy@localhost" { + t.Fatal("wrong result:", indirectNodes) + } + if len(nodeB.NodesIndirect()) > 0 { + t.Fatal("wrong result:", nodeB.NodesIndirect()) + } + fmt.Println("OK") + + fmt.Printf("... disconnect NodeC from NodeA: ") + nodeC.Disconnect("nodeAproxy@localhost") + if len(nodeC.NodesIndirect()) > 0 { + t.Fatal("wrong result:", nodeC.NodesIndirect()) + } + + time.Sleep(100 * time.Millisecond) + if len(nodeA.NodesIndirect()) > 0 { + t.Fatal("wrong result:", nodeA.NodesIndirect()) + } + fmt.Println("OK") + nodeB.Stop() + optsB.Proxy.Transit = false + nodeB, e = ergo.StartNode("nodeBproxy@localhost", "secret", optsB) + if e != nil { + t.Fatal(e) + } + fmt.Printf("... connect NodeA to NodeC via NodeB(transit proxy disabled): ") + e = nodeA.Connect("nodeCproxy@localhost") + if e == nil { + t.Fatal("must be error here") + } + errMessage := "[nodeBproxy@localhost] proxy feature disabled" + if e.Error() != errMessage { + t.Fatal(e) + } + fmt.Println("OK") + nodeB.Stop() + nodeC.Stop() + + nodeB.Stop() + optsB.Proxy.Transit = true + nodeB, e = ergo.StartNode("nodeBproxy@localhost", "secret", optsB) + if e != nil { + t.Fatal(e) + } + + optsC.Flags = node.DefaultFlags() + optsC.Flags.EnableProxy = false + nodeC, e = ergo.StartNode("nodeCproxy@localhost", "secret", optsC) + if e != nil { + t.Fatal(e) + } + fmt.Printf("... connect NodeA to NodeC (proxy feature support disabled) via NodeB: ") + e = nodeA.Connect("nodeCproxy@localhost") + if e == nil { + t.Fatal("must be error here") + } + errMessage = "[nodeBproxy@localhost] peer does not support this feature" + if e.Error() != errMessage { + t.Fatal(e) + } + fmt.Println("OK") + nodeC.Stop() + + optsC = node.Options{} + optsC.Proxy.Cookie = "123" + nodeC, e = ergo.StartNode("nodeCproxy@localhost", "secret", optsC) + if e != nil { + t.Fatal(e) + } + fmt.Printf("... connect NodeA to NodeC (with wrong cookie) via NodeB: ") + e = nodeA.Connect("nodeCproxy@localhost") + if e == nil { + t.Fatal("must be error here") + } + errMessage = "[nodeCproxy@localhost] can't establish proxy connection" + if e.Error() != errMessage { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("... connect NodeA to NodeC (with correct cookie) via NodeB: ") + if nodeA.RemoveProxyRoute("nodeCproxy@localhost") == false { + t.Fatal("proxy route not found") + } + route = node.ProxyRoute{ + Proxy: "nodeBproxy@localhost", + Cookie: "123", + } + nodeA.AddProxyRoute("nodeCproxy@localhost", route) + + e = nodeA.Connect("nodeCproxy@localhost") + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + fmt.Printf("... connect NodeA to NodeD (with enabled encryption) via NodeB: ") + optsD := node.Options{} + optsD.Proxy.Cookie = "123" + optsD.Proxy.Flags = node.DefaultProxyFlags() + optsD.Proxy.Flags.EnableEncryption = true + nodeD, e := ergo.StartNode("nodeDproxy@localhost", "secret", optsD) + if e != nil { + t.Fatal(e) + } + + route = node.ProxyRoute{ + Proxy: "nodeBproxy@localhost", + Cookie: "123", + } + nodeA.AddProxyRoute("nodeDproxy@localhost", route) + e = nodeA.Connect("nodeDproxy@localhost") + if e != nil { + t.Fatal(e) + } + fmt.Println("OK") + + // use gen serv from test_monitor + gsA := &testMonitor{ + v: make(chan interface{}, 2), + } + gsC := &testMonitor{ + v: make(chan interface{}, 2), + } + gsD := &testMonitor{ + v: make(chan interface{}, 2), + } + fmt.Printf("... start processA on NodeA: ") + pA, err := nodeA.Spawn("", gen.ProcessOptions{}, gsA) + if err != nil { + t.Fatal(err) + } + waitForResultWithValue(t, gsA.v, pA.Self()) + fmt.Printf("... start processC on NodeC: ") + pC, err := nodeC.Spawn("", gen.ProcessOptions{}, gsC) + if err != nil { + t.Fatal(err) + } + waitForResultWithValue(t, gsC.v, pC.Self()) + fmt.Printf("... start processD on NodeD: ") + pD, err := nodeD.Spawn("", gen.ProcessOptions{}, gsD) + if err != nil { + t.Fatal(err) + } + waitForResultWithValue(t, gsD.v, pD.Self()) + + fmt.Printf("... processA send short message to processC: ") + if e := pA.Send(pC.Self(), "test"); e != nil { + t.Fatal(e) + } + waitForResultWithValue(t, gsC.v, "test") + + fmt.Printf("... processA send short message to processD (encrypted): ") + pA.Send(pD.Self(), "test") + waitForResultWithValue(t, gsD.v, "test") + + randomString := []byte(lib.RandomString(1024 * 10)) + pA.SetCompression(true) + fmt.Printf("... processA send 10K message to processC (compressed): ") + pA.Send(pC.Self(), randomString) + waitForResultWithValue(t, gsC.v, randomString) + + fmt.Printf("... processA send 10K message to processD (compressed, encrypted): ") + pA.Send(pD.Self(), randomString) + waitForResultWithValue(t, gsD.v, randomString) + + pA.SetCompression(false) + randomString = []byte(lib.RandomString(1024 * 100)) + fmt.Printf("... processA send 100K message to processC (fragmented): ") + pA.Send(pC.Self(), randomString) + waitForResultWithValue(t, gsC.v, randomString) + + fmt.Printf("... processA send 100K message to processD (fragmented, encrypted): ") + pA.Send(pD.Self(), randomString) + waitForResultWithValue(t, gsD.v, randomString) + + pA.SetCompression(true) + randomString = []byte(lib.RandomString(1024 * 1024)) + fmt.Printf("... processA send 1M message to processC (fragmented, compressed): ") + pA.Send(pC.Self(), randomString) + waitForResultWithValue(t, gsC.v, randomString) + + fmt.Printf("... processA send 1M message to processD (fragmented, compressed, encrypted): ") + pA.Send(pD.Self(), randomString) + waitForResultWithValue(t, gsD.v, randomString) + + nodeA.Stop() + nodeB.Stop() + nodeC.Stop() + nodeD.Stop() + +} + +func TestNodeIncarnation(t *testing.T) { + fmt.Printf("\n=== Test Node Incarnation\n") + fmt.Printf("... start nodes: ") + optsA := node.Options{} + nodeA, e := ergo.StartNode("nodeAincarnation@localhost", "secret", optsA) + if e != nil { + t.Fatal(e) + } + route := node.ProxyRoute{ + Proxy: "nodeBincarnation@localhost", + } + nodeA.AddProxyRoute("nodeCincarnation@localhost", route) + // add sleep to get Creation different value for the next node + optsB := node.Options{} + optsB.Proxy.Transit = true + nodeB, e := ergo.StartNode("nodeBincarnation@localhost", "secret", optsB) + if e != nil { + t.Fatal(e) + } + optsC := node.Options{ + Creation: 1234, + } + nodeC, e := ergo.StartNode("nodeCincarnation@localhost", "secret", optsC) + if e != nil { + t.Fatal(e) + } + + if err := nodeA.Connect("nodeCincarnation@localhost"); err != nil { + t.Fatal(err) + } + + indirectNodes := nodeA.NodesIndirect() + if len(indirectNodes) != 1 { + t.Fatal("wrong result:", indirectNodes) + } + if indirectNodes[0] != "nodeCincarnation@localhost" { + t.Fatal("wrong result:", indirectNodes) + } + indirectNodes = nodeC.NodesIndirect() + if len(indirectNodes) != 1 { + t.Fatal("wrong result:", indirectNodes) + } + if indirectNodes[0] != "nodeAincarnation@localhost" { + t.Fatal("wrong result:", indirectNodes) + } + if len(nodeB.NodesIndirect()) > 0 { + t.Fatal("wrong result:", nodeB.NodesIndirect()) + } + fmt.Println("OK") + + // use gen serv from test_monitor + gsA := &testMonitor{ + v: make(chan interface{}, 2), + } + gsB := &testMonitor{ + v: make(chan interface{}, 2), + } + gsC := &testMonitor{ + v: make(chan interface{}, 2), + } + fmt.Printf("... start processA on NodeA: ") + pA, err := nodeA.Spawn("", gen.ProcessOptions{}, gsA) + if err != nil { + t.Fatal(err) + } + waitForResultWithValue(t, gsA.v, pA.Self()) + + fmt.Printf("... start processB on NodeB: ") + pB, err := nodeB.Spawn("", gen.ProcessOptions{}, gsB) + if err != nil { + t.Fatal(err) + } + waitForResultWithValue(t, gsB.v, pB.Self()) + + fmt.Printf("... start processC on NodeC: ") + pC, err := nodeC.Spawn("", gen.ProcessOptions{}, gsC) + if err != nil { + t.Fatal(err) + } + waitForResultWithValue(t, gsC.v, pC.Self()) + + pidC := pC.Self() + + fmt.Printf("... processA send a message to processC (via proxy): ") + if e := pA.Send(pidC, "test"); e != nil { + t.Fatal(e) + } + waitForResultWithValue(t, gsC.v, "test") + fmt.Printf("... processB send short message to processC: ") + if e := pB.Send(pidC, "test"); e != nil { + t.Fatal(e) + } + waitForResultWithValue(t, gsC.v, "test") + fmt.Printf("... restart nodeC and processC: ") + nodeC.Stop() + nodeC.Wait() + + optsC.Creation = 12345 + nodeC, e = ergo.StartNode("nodeCincarnation@localhost", "secret", optsC) + if e != nil { + t.Fatal(e) + } + + if err := nodeA.Connect("nodeCincarnation@localhost"); err != nil { + t.Fatal(err) + } + pC, err = nodeC.Spawn("", gen.ProcessOptions{}, gsC) + if err != nil { + t.Fatal(err) + } + waitForResultWithValue(t, gsC.v, pC.Self()) + + fmt.Printf("... processA send a message to previous incarnation of processC (via proxy): ") + if e := pA.Send(pidC, "test"); e != node.ErrProcessIncarnation { + t.Fatal("must be ErrProcessIncarnation here", e) + } + fmt.Println("OK") + fmt.Printf("... processB send short message to previous incarnation of processC: ") + if e := pB.Send(pidC, "test"); e != node.ErrProcessIncarnation { + t.Fatal(e) + } + fmt.Println("OK") + + indirectNodes = nodeA.NodesIndirect() + if len(indirectNodes) != 1 { + t.Fatal("wrong result:", indirectNodes) + } + if indirectNodes[0] != "nodeCincarnation@localhost" { + t.Fatal("wrong result:", indirectNodes) + } + indirectNodes = nodeC.NodesIndirect() + if len(indirectNodes) != 1 { + t.Fatal("wrong result:", indirectNodes) + } + if indirectNodes[0] != "nodeAincarnation@localhost" { + t.Fatal("wrong result:", indirectNodes) + } + if len(nodeB.NodesIndirect()) > 0 { + t.Fatal("wrong result:", nodeB.NodesIndirect()) + } +} + +func BenchmarkNodeCompressionDisabled1MBempty(b *testing.B) { + node1name := fmt.Sprintf("nodeB1compressionDis_%d@localhost", b.N) + node2name := fmt.Sprintf("nodeB2compressionDis_%d@localhost", b.N) + node1, _ := ergo.StartNode(node1name, "bench", node.Options{}) + node2, _ := ergo.StartNode(node2name, "bench", node.Options{}) + defer node1.Stop() + defer node2.Stop() + if err := node1.Connect(node2.Name()); err != nil { + b.Fatal(err) + } + + bgs := &benchGS{} + + var empty [1024 * 1024]byte + b.SetParallelism(15) + b.RunParallel(func(pb *testing.PB) { + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p2, e2 := node2.Spawn("", gen.ProcessOptions{}, bgs) + if e2 != nil { + b.Fatal(e2) + } + b.ResetTimer() + for pb.Next() { + call := makeCall{ + to: p2.Self(), + message: empty, + } + _, e := p1.DirectWithTimeout(call, 30) + if e != nil { + b.Fatal(e) + } + } + + }) +} +func BenchmarkNodeCompressionEnabled1MBempty(b *testing.B) { + node1name := fmt.Sprintf("nodeB1compressionEn_%d@localhost", b.N) + node2name := fmt.Sprintf("nodeB2compressionEn_%d@localhost", b.N) + node1, _ := ergo.StartNode(node1name, "bench", node.Options{}) + node2, _ := ergo.StartNode(node2name, "bench", node.Options{}) + defer node1.Stop() + defer node2.Stop() + if err := node1.Connect(node2.Name()); err != nil { + b.Fatal(err) + } + + bgs := &benchGS{} + + var empty [1024 * 1024]byte + //b.SetParallelism(15) + b.RunParallel(func(pb *testing.PB) { + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p1.SetCompression(true) + p1.SetCompressionLevel(5) + p2, e2 := node2.Spawn("", gen.ProcessOptions{}, bgs) + if e2 != nil { + b.Fatal(e2) + } + b.ResetTimer() + for pb.Next() { + call := makeCall{ + to: p2.Self(), + message: empty, + } + _, e := p1.DirectWithTimeout(call, 30) + if e != nil { + b.Fatal(e) + } + } + + }) +} + +func BenchmarkNodeCompressionEnabled1MBstring(b *testing.B) { + node1name := fmt.Sprintf("nodeB1compressionEnStr_%d@localhost", b.N) + node2name := fmt.Sprintf("nodeB2compressionEnStr_%d@localhost", b.N) + node1, e := ergo.StartNode(node1name, "bench", node.Options{}) + if e != nil { + b.Fatal(e) + } + node2, e := ergo.StartNode(node2name, "bench", node.Options{}) + if e != nil { + b.Fatal(e) + } defer node1.Stop() defer node2.Stop() - - node2.ProvideRemoteSpawn("remote", &handshakeGenServer{}) - process, err := node1.Spawn("gs1", gen.ProcessOptions{}, &handshakeGenServer{}) - if err != nil { - t.Fatal(err) + if err := node1.Connect(node2.Name()); err != nil { + b.Fatal(err) } - opts := gen.RemoteSpawnOptions{ - RegisterName: "remote", - } - fmt.Printf(" process gs1@node1 requests spawn new process on node2 and register this process with name 'remote': ") - _, err = process.RemoteSpawn(node2.Name(), "remote", opts, 1, 2, 3) - if err != nil { - t.Fatal(err) - } - fmt.Println("OK") - fmt.Printf(" process gs1@node1 requests spawn new process on node2 with the same name (must be failed): ") + bgs := &benchGS{} - _, err = process.RemoteSpawn(node2.Name(), "remote", opts, 1, 2, 3) - if err != node.ErrTaken { - t.Fatal(err) - } - fmt.Println("OK") + randomString := []byte(lib.RandomString(1024 * 1024)) + b.SetParallelism(15) + b.RunParallel(func(pb *testing.PB) { + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p1.SetCompression(true) + p1.SetCompressionLevel(5) + p2, e2 := node2.Spawn("", gen.ProcessOptions{}, bgs) + if e2 != nil { + b.Fatal(e2) + } + b.ResetTimer() + for pb.Next() { + call := makeCall{ + to: p2.Self(), + message: randomString, + } + _, e := p1.DirectWithTimeout(call, 30) + if e != nil { + b.Fatal(e) + } + } + + }) } type benchGS struct { @@ -405,16 +1212,16 @@ func (b *benchGS) HandleCall(process *gen.ServerProcess, from gen.ServerFrom, me func (b *benchGS) HandleDirect(process *gen.ServerProcess, message interface{}) (interface{}, error) { switch m := message.(type) { case makeCall: - return process.Call(m.to, m.message) + return process.CallWithTimeout(m.to, m.message, 30) } return nil, gen.ErrUnsupportedRequest } -func BenchmarkNodeSequential(b *testing.B) { +func BenchmarkNodeSequentialNetwork(b *testing.B) { node1name := fmt.Sprintf("nodeB1_%d@localhost", b.N) node2name := fmt.Sprintf("nodeB2_%d@localhost", b.N) - node1, _ := ergo.StartNode(node1name, "bench", node.Options{DisableHeaderAtomCache: false}) + node1, _ := ergo.StartNode(node1name, "bench", node.Options{}) node2, _ := ergo.StartNode(node2name, "bench", node.Options{}) bgs := &benchGS{} @@ -454,10 +1261,10 @@ func BenchmarkNodeSequential(b *testing.B) { } } -func BenchmarkNodeSequentialSingleNode(b *testing.B) { +func BenchmarkNodeSequentialLocal(b *testing.B) { node1name := fmt.Sprintf("nodeB1Local_%d@localhost", b.N) - node1, _ := ergo.StartNode(node1name, "bench", node.Options{DisableHeaderAtomCache: true}) + node1, _ := ergo.StartNode(node1name, "bench", node.Options{}) bgs := &benchGS{} @@ -500,7 +1307,7 @@ func BenchmarkNodeParallel(b *testing.B) { node1name := fmt.Sprintf("nodeB1Parallel_%d@localhost", b.N) node2name := fmt.Sprintf("nodeB2Parallel_%d@localhost", b.N) - node1, _ := ergo.StartNode(node1name, "bench", node.Options{DisableHeaderAtomCache: false}) + node1, _ := ergo.StartNode(node1name, "bench", node.Options{}) node2, _ := ergo.StartNode(node2name, "bench", node.Options{}) bgs := &benchGS{} @@ -550,7 +1357,7 @@ func BenchmarkNodeParallel(b *testing.B) { func BenchmarkNodeParallelSingleNode(b *testing.B) { node1name := fmt.Sprintf("nodeB1ParallelLocal_%d@localhost", b.N) - node1, _ := ergo.StartNode(node1name, "bench", node.Options{DisableHeaderAtomCache: false}) + node1, _ := ergo.StartNode(node1name, "bench", node.Options{}) bgs := &benchGS{} @@ -594,17 +1401,328 @@ func BenchmarkNodeParallelSingleNode(b *testing.B) { }) } + +func BenchmarkNodeProxy_NodeA_to_NodeC_direct_Message_1K(b *testing.B) { + node1name := fmt.Sprintf("nodeB1ProxyDisabled%d@localhost", b.N) + node2name := fmt.Sprintf("nodeB2ProxyDisabled%d@localhost", b.N) + node1, _ := ergo.StartNode(node1name, "bench", node.Options{}) + node2, _ := ergo.StartNode(node2name, "bench", node.Options{}) + defer node1.Stop() + defer node2.Stop() + + bgs := &benchGS{} + + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p2, e2 := node2.Spawn("", gen.ProcessOptions{}, bgs) + if e2 != nil { + b.Fatal(e2) + } + + call := makeCall{ + to: p2.Self(), + message: "hi", + } + if _, e := p1.Direct(call); e != nil { + b.Fatal("single ping", e) + } + + randomString := []byte(lib.RandomString(1024)) + b.SetParallelism(15) + b.RunParallel(func(pb *testing.PB) { + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p2, e2 := node2.Spawn("", gen.ProcessOptions{}, bgs) + if e2 != nil { + b.Fatal(e2) + } + b.ResetTimer() + for pb.Next() { + call := makeCall{ + to: p2.Self(), + message: randomString, + } + _, e := p1.Direct(call) + if e != nil { + b.Fatal(e) + } + } + + }) +} +func BenchmarkNodeProxy_NodeA_to_NodeC_via_NodeB_Message_1K(b *testing.B) { + node1name := fmt.Sprintf("nodeB1ProxyEnabled1K%d@localhost", b.N) + node2name := fmt.Sprintf("nodeB2ProxyEnabled1K%d@localhost", b.N) + node3name := fmt.Sprintf("nodeB3ProxyEnabled1K%d@localhost", b.N) + node1, _ := ergo.StartNode(node1name, "bench", node.Options{}) + opts2 := node.Options{} + opts2.Proxy.Transit = true + node2, _ := ergo.StartNode(node2name, "bench", opts2) + node3, _ := ergo.StartNode(node3name, "bench", node.Options{}) + defer node1.Stop() + defer node2.Stop() + defer node3.Stop() + route := node.ProxyRoute{ + Proxy: node2.Name(), + } + node1.AddProxyRoute(node3.Name(), route) + + bgs := &benchGS{} + + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p3, e3 := node3.Spawn("", gen.ProcessOptions{}, bgs) + if e3 != nil { + b.Fatal(e3) + } + + call := makeCall{ + to: p3.Self(), + message: "hi", + } + if _, e := p1.Direct(call); e != nil { + b.Fatal("single ping", e) + } + + randomString := []byte(lib.RandomString(1024)) + b.SetParallelism(15) + b.RunParallel(func(pb *testing.PB) { + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p3, e3 := node3.Spawn("", gen.ProcessOptions{}, bgs) + if e3 != nil { + b.Fatal(e3) + } + b.ResetTimer() + for pb.Next() { + call := makeCall{ + to: p3.Self(), + message: randomString, + } + _, e := p1.Direct(call) + if e != nil { + b.Fatal(e) + } + } + + }) +} + +func BenchmarkNodeProxy_NodeA_to_NodeC_via_NodeB_Message_1K_Encrypted(b *testing.B) { + node1name := fmt.Sprintf("nodeB1ProxyEnabled1K%d@localhost", b.N) + node2name := fmt.Sprintf("nodeB2ProxyEnabled1K%d@localhost", b.N) + node3name := fmt.Sprintf("nodeB3ProxyEnabled1K%d@localhost", b.N) + opts1 := node.Options{} + opts1.Proxy.Flags = node.DefaultProxyFlags() + opts1.Proxy.Flags.EnableEncryption = true + node1, _ := ergo.StartNode(node1name, "bench", opts1) + opts2 := node.Options{} + opts2.Proxy.Transit = true + node2, _ := ergo.StartNode(node2name, "bench", opts2) + node3, _ := ergo.StartNode(node3name, "bench", node.Options{}) + defer node1.Stop() + defer node2.Stop() + defer node3.Stop() + route := node.ProxyRoute{ + Proxy: node2.Name(), + } + node1.AddProxyRoute(node3.Name(), route) + + bgs := &benchGS{} + + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p3, e3 := node3.Spawn("", gen.ProcessOptions{}, bgs) + if e3 != nil { + b.Fatal(e3) + } + + call := makeCall{ + to: p3.Self(), + message: "hi", + } + if _, e := p1.Direct(call); e != nil { + b.Fatal("single ping", e) + } + + randomString := []byte(lib.RandomString(1024)) + b.SetParallelism(15) + b.RunParallel(func(pb *testing.PB) { + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p3, e3 := node3.Spawn("", gen.ProcessOptions{}, bgs) + if e3 != nil { + b.Fatal(e3) + } + b.ResetTimer() + for pb.Next() { + call := makeCall{ + to: p3.Self(), + message: randomString, + } + _, e := p1.Direct(call) + if e != nil { + b.Fatal(e) + } + } + + }) +} + +func BenchmarkNodeProxy_NodeA_to_NodeC_via_NodeB_Message_1M_Compressed(b *testing.B) { + node1name := fmt.Sprintf("nodeB1ProxyEnabled1K%d@localhost", b.N) + node2name := fmt.Sprintf("nodeB2ProxyEnabled1K%d@localhost", b.N) + node3name := fmt.Sprintf("nodeB3ProxyEnabled1K%d@localhost", b.N) + opts1 := node.Options{} + opts1.Proxy.Flags = node.DefaultProxyFlags() + opts1.Proxy.Flags.EnableEncryption = false + node1, _ := ergo.StartNode(node1name, "bench", opts1) + opts2 := node.Options{} + opts2.Proxy.Transit = true + node2, _ := ergo.StartNode(node2name, "bench", opts2) + node3, _ := ergo.StartNode(node3name, "bench", node.Options{}) + defer node1.Stop() + defer node2.Stop() + defer node3.Stop() + route := node.ProxyRoute{ + Proxy: node2.Name(), + } + node1.AddProxyRoute(node3.Name(), route) + + bgs := &benchGS{} + + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p3, e3 := node3.Spawn("", gen.ProcessOptions{}, bgs) + if e3 != nil { + b.Fatal(e3) + } + + call := makeCall{ + to: p3.Self(), + message: "hi", + } + if _, e := p1.Direct(call); e != nil { + b.Fatal("single ping", e) + } + + randomString := []byte(lib.RandomString(1024 * 1024)) + b.SetParallelism(15) + b.RunParallel(func(pb *testing.PB) { + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p3, e3 := node3.Spawn("", gen.ProcessOptions{}, bgs) + if e3 != nil { + b.Fatal(e3) + } + b.ResetTimer() + for pb.Next() { + call := makeCall{ + to: p3.Self(), + message: randomString, + } + _, e := p1.Direct(call) + if e != nil { + b.Fatal(e) + } + } + + }) +} + +func BenchmarkNodeProxy_NodeA_to_NodeC_via_NodeB_Message_1M_CompressedEncrypted(b *testing.B) { + node1name := fmt.Sprintf("nodeB1ProxyEnabled1K%d@localhost", b.N) + node2name := fmt.Sprintf("nodeB2ProxyEnabled1K%d@localhost", b.N) + node3name := fmt.Sprintf("nodeB3ProxyEnabled1K%d@localhost", b.N) + opts1 := node.Options{} + opts1.Proxy.Flags = node.DefaultProxyFlags() + opts1.Proxy.Flags.EnableEncryption = true + node1, _ := ergo.StartNode(node1name, "bench", opts1) + opts2 := node.Options{} + opts2.Proxy.Transit = true + node2, _ := ergo.StartNode(node2name, "bench", opts2) + node3, _ := ergo.StartNode(node3name, "bench", node.Options{}) + defer node1.Stop() + defer node2.Stop() + defer node3.Stop() + route := node.ProxyRoute{ + Proxy: node2.Name(), + } + node1.AddProxyRoute(node3.Name(), route) + + bgs := &benchGS{} + + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p3, e3 := node3.Spawn("", gen.ProcessOptions{}, bgs) + if e3 != nil { + b.Fatal(e3) + } + + call := makeCall{ + to: p3.Self(), + message: "hi", + } + if _, e := p1.Direct(call); e != nil { + b.Fatal("single ping", e) + } + + randomString := []byte(lib.RandomString(1024 * 1024)) + b.SetParallelism(15) + b.RunParallel(func(pb *testing.PB) { + p1, e1 := node1.Spawn("", gen.ProcessOptions{}, bgs) + if e1 != nil { + b.Fatal(e1) + } + p1.SetCompression(true) + p3, e3 := node3.Spawn("", gen.ProcessOptions{}, bgs) + if e3 != nil { + b.Fatal(e3) + } + b.ResetTimer() + for pb.Next() { + call := makeCall{ + to: p3.Self(), + message: randomString, + } + _, e := p1.Direct(call) + if e != nil { + b.Fatal(e) + } + } + + }) +} + func benchCases() []benchCase { return []benchCase{ - benchCase{"number", 12345}, - benchCase{"string", "hello world"}, - benchCase{"tuple (PID)", + {"number", 12345}, + {"string", "hello world"}, + {"tuple (PID)", etf.Pid{ Node: "node@localhost", ID: 1000, Creation: 1, }, }, - benchCase{"binary 1MB", make([]byte, 1024*1024)}, + {"binary 1MB", make([]byte, 1024*1024)}, } } diff --git a/tests/raft_data_test.go b/tests/raft_data_test.go new file mode 100644 index 00000000..1f252f18 --- /dev/null +++ b/tests/raft_data_test.go @@ -0,0 +1,209 @@ +//go:build !manual + +package tests + +import ( + "fmt" + "testing" + "time" + + "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/gen" +) + +// F - follower +// M - quorum member +// L - leader +// Q - quorum (all members) +// +// append cases: +// 1. F -> M -> L -> Q ... broadcast +// 2. F -> L -> Q ... broadcast +// 3. M -> L - Q ... broadcast +// 4. L -> Q ... broadcast +// + +func TestRaftAppend(t *testing.T) { + fmt.Printf("\n=== Test GenRaft - append data\n") + server := &testRaft{ + n: 6, + qstate: gen.RaftQuorumState5, + } + nodes, rafts, leaderSerial := startRaftCluster("append", server) + + fmt.Printf(" append on a follower (send to the quorum member and forward to the leader: ") + for _, raft := range rafts { + q := raft.Quorum() + // find the follower + if q.Member == true { + continue + } + + // cases 1 and 2 - send to the quorum member. + // the follower isn't able to send it to the leader (case 2) + // since it has no info about the quorum leader (quorum member forwards it + // to the leader under the hood) + ref, err := raft.Append("asdfkey", "asdfvalue") + if err != nil { + t.Fatal(err) + } + leaderSerial = checkAppend(t, server, ref, rafts, leaderSerial) + break + } + fmt.Println("OK") + fmt.Printf(" append on a quorum member (send to the leader): ") + for _, raft := range rafts { + q := raft.Quorum() + // find the quorum member + if q.Member == false { + continue + } + + // case 3 - quorum member sends append to the leader + ref, err := raft.Append("asdfkey", "asdfvalue") + if err != nil { + t.Fatal(err) + } + leaderSerial = checkAppend(t, server, ref, rafts, leaderSerial) + break + } + fmt.Println("OK") + fmt.Printf(" append on a leader: ") + for _, raft := range rafts { + l := raft.Leader() + // finde the quorum leader + if l == nil || l.Leader != raft.Self() { + continue + } + + // case 4 - leader makes append + ref, err := raft.Append("asdfkey", "asdfvalue") + if err != nil { + t.Fatal(err) + } + leaderSerial = checkAppend(t, server, ref, rafts, leaderSerial) + break + } + fmt.Println("OK") + + for _, node := range nodes { + node.Stop() + } +} + +// get serial case: run through the raft process and get all the data to achieve +// the same serial +func TestRaftGet(t *testing.T) { + fmt.Printf("\n=== Test GenRaft - get data\n") + server := &testRaft{ + n: 6, + qstate: gen.RaftQuorumState5, + } + nodes, rafts, leaderSerial := startRaftCluster("get", server) + for _, raft := range rafts { + fmt.Println(" started raft process", raft.Self(), "with serial", raft.Serial()) + } + + for _, raft := range rafts { + _, err := raft.Get(12341234) + if err != gen.ErrRaftNoSerial { + t.Fatal("must be ErrRaftNoSerial here") + } + + if raft.Serial() == leaderSerial { + fmt.Println(" raft process", raft.Self(), "has already latest serial. skip it") + continue + } + + fmt.Printf(" get serials (%d...%d) on %s to reach the leader's serial: ", raft.Serial()+1, leaderSerial, raft.Self()) + serial := raft.Serial() + gotFrom := []etf.Pid{} + for i := serial; i < leaderSerial; i++ { + ref, err := raft.Get(i + 1) + if err != nil { + t.Fatal(err) + } + result := waitGetRef(t, server, ref) + + // compare with the original + original := data[result.key] + if original.serial != result.serial { + t.Fatal("wrong serial") + } + if original.value != result.value { + t.Fatal("wrong value") + } + // check internal data + internal := raft.State.(*raftState) + if internal.serials[result.serial] != result.key { + t.Fatal("serial mismatch the result key") + } + d, exist := internal.data[result.key] + if exist == false { + t.Fatal("internal data hasn't been updated") + } + if d.serial != result.serial { + t.Fatal("intenal data serial mismatch") + } + if d.value != result.value { + t.Fatal("intarnal data value mismatch") + } + gotFrom = append(gotFrom, result.process.Self()) + } + fmt.Println("OK") + fmt.Println(" got from:", gotFrom, ")") + } + + //fmt.Println("-----------") + //for _, raft := range rafts { + // s := raft.State.(*raftState) + // fmt.Println(raft.Self(), "INTERNAL", s) + //} + + for _, node := range nodes { + node.Stop() + } +} + +func waitGetRef(t *testing.T, server *testRaft, ref etf.Ref) raftResult { + var result raftResult + select { + case result = <-server.s: + if result.ref != ref { + t.Fatal("wrong ref") + } + return result + case <-time.After(30 * time.Second): + t.Fatal("get timeout") + } + return result +} + +func checkAppend(t *testing.T, server *testRaft, ref etf.Ref, rafts []*gen.RaftProcess, serial uint64) uint64 { + appends := 0 + for { + select { + case result := <-server.a: + if result.serial != serial+1 { + t.Fatalf("wrong serial %d (must be %d)", result.serial, serial+1) + } + appends++ + //fmt.Println("got append on ", result.process.Self(), "total appends", appends) + if appends != len(rafts) { + continue + } + // check serials + for _, r := range rafts { + s := r.Serial() + if s != serial+1 { + t.Fatalf("wrong serial %d on %s", s, r.Self()) + } + } + return serial + 1 + case <-time.After(30 * time.Second): + t.Fatal("append timeout") + + } + } + +} diff --git a/tests/raft_manual_test.go b/tests/raft_manual_test.go new file mode 100644 index 00000000..41567501 --- /dev/null +++ b/tests/raft_manual_test.go @@ -0,0 +1,138 @@ +//go:build manual +// +build manual + +// to run this test: +// go test -run TestRaft -ergo.norecover -tags manual +// +// enable debug printing in the gen/raft.go +// quorum building debuging: %s/\/\/ QUODBG // +// leader election debuging: %s/\/\/ LDRDBG // +// heartbeat debuging: %s/\/\/ HRTDBG // + +package tests + +import ( + "fmt" + "math/rand" + "testing" + "time" + + "github.com/ergo-services/ergo" + "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/gen" + "github.com/ergo-services/ergo/node" +) + +type testRaft struct { + gen.Raft + res chan interface{} +} + +func (tr *testRaft) InitRaft(process *gen.RaftProcess, args ...etf.Term) (gen.RaftOptions, error) { + var options gen.RaftOptions + if len(args) > 0 { + options.Peers = args[0].([]gen.ProcessID) + options.Serial = uint64(rand.Intn(10)) + } + + fmt.Println(process.Self(), process.Name(), " ----------", options.Serial) + return options, gen.RaftStatusOK +} + +func (tr *testRaft) HandleQuorum(process *gen.RaftProcess, q *gen.RaftQuorum) gen.RaftStatus { + if q == nil { + fmt.Println("QQQ quorum", process.Name(), "state: NONE") + return gen.RaftStatusOK + } else { + fmt.Println("QQQ quorum", process.Name(), "state:", q.State, q.Member, q.Peers) + } + if sent, _ := process.State.(int); sent != 1 { + process.SendAfter(process.Self(), "ok", 7*time.Second) + process.State = 1 + } + //tr.res <- qs + return gen.RaftStatusOK +} + +func (tr *testRaft) HandleLeader(process *gen.RaftProcess, leader *gen.RaftLeader) gen.RaftStatus { + fmt.Println("LLL leader", process.Name(), leader) + return gen.RaftStatusOK +} + +func (tr *testRaft) HandleAppend(process *gen.RaftProcess, ref etf.Ref, serial uint64, key string, value etf.Term) gen.RaftStatus { + fmt.Println("AAA append", ref, serial, value) + return gen.RaftStatusOK +} + +func (tr *testRaft) HandleGet(process *gen.RaftProcess, serial uint64) (string, etf.Term, gen.RaftStatus) { + fmt.Println("GGG get", process.Name(), serial) + return "", nil, gen.RaftStatusOK +} + +func (tr *testRaft) HandleRaftInfo(process *gen.RaftProcess, message etf.Term) gen.ServerStatus { + q := process.Quorum() + if q == nil { + fmt.Println("III info", process.Name(), "state: NONE", "message:", message) + } else { + fmt.Println("III info", process.Name(), "Q:", q.State, q.Member, "", process.Leader(), "message:", message) + } + if l := process.Leader(); l != nil && l.Leader == process.Self() { + fmt.Println("III i'm leader. freeze", process.Self()) + time.Sleep(35 * time.Second) + } + process.State = 0 + return gen.ServerStatusOK +} + +func TestRaftLeader(t *testing.T) { + fmt.Printf("\n=== Test GenRaft\n") + var N int = 4 + + fmt.Printf("Starting %d nodes: nodeGenRaftXX@localhost...", N) + + nodes := make([]node.Node, N) + for i := range nodes { + name := fmt.Sprintf("nodeGenRaft%02d@localhost", i) + node, err := ergo.StartNode(name, "cookies", node.Options{}) + if err != nil { + t.Fatal(err) + } + nodes[i] = node + } + + defer func() { + for i := range nodes { + nodes[i].Stop() + } + }() + fmt.Println("OK") + + rafts := make([]gen.Process, N) + results := make([]chan interface{}, N) + var args []etf.Term + var peer gen.ProcessID + for i := range rafts { + name := fmt.Sprintf("raft%02d", i+1) + if i == 0 { + args = nil + } else { + peer.Node = nodes[i-1].Name() + peer.Name = rafts[i-1].Name() + peers := []gen.ProcessID{peer} + args = []etf.Term{peers} + } + tr := &testRaft{ + res: make(chan interface{}, 2), + } + results[i] = tr.res + raft, err := nodes[i].Spawn(name, gen.ProcessOptions{}, tr, args...) + if err != nil { + t.Fatal(err) + } + rafts[i] = raft + //time.Sleep(300 * time.Millisecond) + } + + time.Sleep(50 * time.Second) + +} diff --git a/tests/raft_test.go b/tests/raft_test.go new file mode 100644 index 00000000..7dc6fcc7 --- /dev/null +++ b/tests/raft_test.go @@ -0,0 +1,361 @@ +//go:build !manual + +package tests + +import ( + "fmt" + "math/rand" + "testing" + "time" + + "github.com/ergo-services/ergo" + "github.com/ergo-services/ergo/etf" + "github.com/ergo-services/ergo/gen" + "github.com/ergo-services/ergo/node" +) + +type testCaseRaft struct { + n int + state gen.RaftQuorumState + name string +} + +var ( + ql string = "quorum of %2d members with 1 leader: " + qlf string = "quorum of %2d members with 1 leader + %d follower(s): " + cases = []testCaseRaft{ + testCaseRaft{n: 2, name: "no quorum, no leader: "}, + testCaseRaft{n: 3, name: ql, state: gen.RaftQuorumState3}, + testCaseRaft{n: 4, name: qlf, state: gen.RaftQuorumState3}, + testCaseRaft{n: 5, name: ql, state: gen.RaftQuorumState5}, + testCaseRaft{n: 6, name: qlf, state: gen.RaftQuorumState5}, + testCaseRaft{n: 7, name: ql, state: gen.RaftQuorumState7}, + + // + // cases below are work well, but quorum building takes too long some time. + //testCaseRaft{n: 8, name: qlf, state: gen.RaftQuorumState7}, + //testCaseRaft{n: 9, name: ql, state: gen.RaftQuorumState9}, + //testCaseRaft{n: 10, name: qlf, state: gen.RaftQuorumState9}, + //testCaseRaft{n: 11, name: ql, state: gen.RaftQuorumState11}, + //testCaseRaft{n: 12, name: qlf, state: gen.RaftQuorumState11}, + //testCaseRaft{n: 15, name: qlf, state: gen.RaftQuorumState11}, + //testCaseRaft{n: 25, name: qlf, state: gen.RaftQuorumState11}, + } + + data = map[string]dataValueSerial{ + "key0": dataValueSerial{"value0", 0}, + "key1": dataValueSerial{"value1", 1}, + "key2": dataValueSerial{"value2", 2}, + "key3": dataValueSerial{"value3", 3}, + "key4": dataValueSerial{"value4", 4}, + "key5": dataValueSerial{"value5", 5}, + "key6": dataValueSerial{"value6", 6}, + "key7": dataValueSerial{"value7", 7}, + "key8": dataValueSerial{"value8", 8}, + "key9": dataValueSerial{"value9", 9}, + } + keySerials = []string{ + "key0", + "key1", + "key2", + "key3", + "key4", + "key5", + "key6", + "key7", + "key8", + "key9", + } +) + +type dataValueSerial struct { + value string + serial uint64 +} + +type testRaft struct { + gen.Raft + n int + qstate gen.RaftQuorumState + p chan *gen.RaftProcess // Init + q chan *gen.RaftQuorum // HandleQuorum + l chan *gen.RaftLeader // HandleLeader + a chan raftResult // HandleAppend + s chan raftResult // HandleSerial +} + +type raftResult struct { + process *gen.RaftProcess + ref etf.Ref + serial uint64 + key string + value etf.Term +} + +type raftArgs struct { + peers []gen.ProcessID + serial uint64 +} +type raftState struct { + data map[string]dataValueSerial + serials []string +} + +func (tr *testRaft) InitRaft(process *gen.RaftProcess, args ...etf.Term) (gen.RaftOptions, error) { + var options gen.RaftOptions + ra := args[0].(raftArgs) + options.Peers = ra.peers + options.Serial = ra.serial + + state := &raftState{ + data: make(map[string]dataValueSerial), + } + for i := 0; i < int(ra.serial)+1; i++ { + key := keySerials[i] + state.data[key] = data[key] + state.serials = append(state.serials, key) + } + process.State = state + tr.p <- process + + return options, gen.RaftStatusOK +} + +func (tr *testRaft) HandleQuorum(process *gen.RaftProcess, quorum *gen.RaftQuorum) gen.RaftStatus { + //fmt.Println(process.Self(), "QQQ", quorum) + if quorum != nil { + tr.q <- quorum + } + return gen.RaftStatusOK +} + +func (tr *testRaft) HandleLeader(process *gen.RaftProcess, leader *gen.RaftLeader) gen.RaftStatus { + //fmt.Println(process.Self(), "LLL", leader) + // leader elected within a quorum + q := process.Quorum() + if q == nil { + return gen.RaftStatusOK + } + if leader != nil && q.State == tr.qstate { + tr.l <- leader + } + + return gen.RaftStatusOK +} + +func (tr *testRaft) HandleAppend(process *gen.RaftProcess, ref etf.Ref, serial uint64, key string, value etf.Term) gen.RaftStatus { + //fmt.Println(process.Self(), "HANDLE APPEND member:", process.Quorum().Member, "append", ref, serial, key, value) + + result := raftResult{ + process: process, + ref: ref, + serial: serial, + key: key, + value: value, + } + tr.a <- result + return gen.RaftStatusOK +} + +func (tr *testRaft) HandleGet(process *gen.RaftProcess, serial uint64) (string, etf.Term, gen.RaftStatus) { + var key string + //fmt.Println(process.Self(), "HANDLE GET member:", process.Quorum().Member, "get", serial) + + state := process.State.(*raftState) + if len(state.serials) < int(serial) { + // fmt.Println(process.Self(), "NO DATA for", serial) + return key, nil, gen.RaftStatusOK + } + key = state.serials[int(serial)] + data := state.data[key] + return key, data.value, gen.RaftStatusOK +} + +func (tr *testRaft) HandleSerial(process *gen.RaftProcess, ref etf.Ref, serial uint64, key string, value etf.Term) gen.RaftStatus { + //fmt.Println(process.Self(), "HANDLE SERIAL member:", process.Quorum().Member, "append", ref, serial, key, value) + result := raftResult{ + process: process, + ref: ref, + serial: serial, + key: key, + value: value, + } + s := process.Serial() + if s != serial { + fmt.Println(process.Self(), "ERROR: disordered serial request") + tr.s <- raftResult{} + return gen.RaftStatusOK + } + state := process.State.(*raftState) + state.serials = append(state.serials, key) + state.data[key] = dataValueSerial{ + value: value.(string), + serial: serial, + } + tr.s <- result + return gen.RaftStatusOK +} +func (tr *testRaft) HandleCancel(process *gen.RaftProcess, ref etf.Ref, reason string) gen.RaftStatus { + fmt.Println("CCC cancel", ref, reason) + return gen.RaftStatusOK +} + +func (tr *testRaft) HandleRaftInfo(process *gen.RaftProcess, message etf.Term) gen.ServerStatus { + return gen.ServerStatusOK +} + +func TestRaftLeader(t *testing.T) { + fmt.Printf("\n=== Test GenRaft - build quorum, leader election\n") + for _, c := range cases { + fmt.Printf(" cluster with %2d distributed raft processes. ", c.n) + if c.n == 2 { + fmt.Printf(c.name) + } else { + f := c.n - int(c.state) + if f == 0 { + fmt.Printf(c.name, c.state) + } else { + fmt.Printf(c.name, c.state, c.n-int(c.state)) + } + } + + server := &testRaft{ + n: c.n, + qstate: c.state, + } + // start distributed raft processes and wait until + // they build a quorum and elect their leader + nodes, rafts, leaderSerial := startRaftCluster("append", server) + ok := true + if c.n > 2 { + ok = false + for _, raft := range rafts { + q := raft.Quorum() + if q == nil { + continue + } + if q.Member == false { + continue + } + + l := raft.Leader() + if l == nil { + continue + } + if l.Serial != leaderSerial { + t.Fatal("wrong leader serial") + } + ok = true + break + } + } + if ok == false { + t.Fatal("no quorum or leader found") + } + fmt.Println("OK") + // stop cluster + for _, node := range nodes { + node.Stop() + } + } + +} + +func startRaftCluster(name string, server *testRaft) ([]node.Node, []*gen.RaftProcess, uint64) { + nodes := make([]node.Node, server.n) + for i := range nodes { + name := fmt.Sprintf("nodeGenRaft-%s-cluster-%02dNode%02d@localhost", name, server.n, i) + node, err := ergo.StartNode(name, "cookies", node.Options{}) + if err != nil { + panic(err) + } + nodes[i] = node + } + + processes := make([]gen.Process, server.n) + server.p = make(chan *gen.RaftProcess, 1000) + server.q = make(chan *gen.RaftQuorum, 1000) + server.l = make(chan *gen.RaftLeader, 1000) + server.a = make(chan raftResult, 1000) + server.s = make(chan raftResult, 1000) + leaderSerial := uint64(0) + var peer gen.ProcessID + for i := range processes { + name := fmt.Sprintf("raft%02d", i+1) + args := raftArgs{ + serial: uint64(rand.Intn(9)), + } + if args.serial > leaderSerial { + leaderSerial = args.serial + } + if i > 0 { + peer.Node = nodes[i-1].Name() + peer.Name = processes[i-1].Name() + args.peers = []gen.ProcessID{peer} + } + p, err := nodes[i].Spawn(name, gen.ProcessOptions{}, server, args) + if err != nil { + panic(err) + } + processes[i] = p + } + + rafts := []*gen.RaftProcess{} + // how many results with 'leader' should be awaiting + resultsL := 0 + // how many results with 'quorum' should be awaiting + resultsQ := 0 + for { + select { + case p := <-server.p: + rafts = append(rafts, p) + + if len(rafts) < server.n { + continue + } + + if server.n == 2 { + // no leader, no quorum + return nodes, rafts, leaderSerial + } + continue + + case q := <-server.q: + + if q.State != server.qstate { + continue + } + + resultsQ++ + if resultsQ < int(server.qstate) { + continue + } + + if resultsL < int(server.qstate) { + continue + } + // all quorum members are received leader election result + return nodes, rafts, leaderSerial + + case l := <-server.l: + if l.State != server.qstate { + continue + } + + resultsL++ + if resultsL < int(server.qstate) { + continue + } + + if resultsQ < server.n { + continue + } + + // all quorum members are received leader election result + return nodes, rafts, leaderSerial + + case <-time.After(30 * time.Second): + panic("can't start raft cluster") + } + } +} diff --git a/tests/rpc_test.go b/tests/rpc_test.go index 143ef9da..12aa709b 100644 --- a/tests/rpc_test.go +++ b/tests/rpc_test.go @@ -54,7 +54,7 @@ func TestRPC(t *testing.T) { return a[len(a)-1] } - fmt.Printf("Registering RPC method 'testMod.testFun' on %s: ", node1.NodeName()) + fmt.Printf("Registering RPC method 'testMod.testFun' on %s: ", node1.Name()) time.Sleep(100 * time.Millisecond) // waiting for start 'rex' gen_server if e := node1.ProvideRPC("testMod", "testFun", testFun1); e != nil { t.Fatal(e) @@ -64,7 +64,7 @@ func TestRPC(t *testing.T) { node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) - fmt.Printf("Call RPC method 'testMod.testFun' with 1 arg on %s: ", node1.NodeName()) + fmt.Printf("Call RPC method 'testMod.testFun' with 1 arg on %s: ", node1.Name()) case1 := testRPCCase1{ node: "nodeRPC@localhost", mod: "testMod", @@ -76,7 +76,7 @@ func TestRPC(t *testing.T) { } waitForResultWithValue(t, gs1.res, 12345) - fmt.Printf("Call RPC method 'testMod.testFun' with 3 arg on %s: ", node1.NodeName()) + fmt.Printf("Call RPC method 'testMod.testFun' with 3 arg on %s: ", node1.Name()) case1 = testRPCCase1{ node: "nodeRPC@localhost", mod: "testMod", @@ -88,14 +88,14 @@ func TestRPC(t *testing.T) { } waitForResultWithValue(t, gs1.res, node1gs1.Self()) - fmt.Printf("Revoking RPC method 'testMod.testFun' on %s: ", node1.NodeName()) + fmt.Printf("Revoking RPC method 'testMod.testFun' on %s: ", node1.Name()) if e := node1.RevokeRPC("testMod", "testFun"); e != nil { t.Fatal(e) } else { fmt.Println("OK") } - fmt.Printf("Call revoked RPC method 'testMod.testFun' with 1 arg on %s: ", node1.NodeName()) + fmt.Printf("Call revoked RPC method 'testMod.testFun' with 1 arg on %s: ", node1.Name()) expected1 := etf.Tuple{etf.Atom("badrpc"), etf.Tuple{etf.Atom("EXIT"), etf.Tuple{etf.Atom("undef"), @@ -115,7 +115,7 @@ func TestRPC(t *testing.T) { } waitForResultWithValue(t, gs1.res, expected1) - fmt.Printf("Call RPC unknown method 'xxx.xxx' on %s: ", node1.NodeName()) + fmt.Printf("Call RPC unknown method 'xxx.xxx' on %s: ", node1.Name()) expected2 := etf.Tuple{etf.Atom("badrpc"), etf.Tuple{etf.Atom("EXIT"), etf.Tuple{etf.Atom("undef"), diff --git a/tests/saga_dist_test.go b/tests/saga_dist_test.go index 1868ef51..f6c6e802 100644 --- a/tests/saga_dist_test.go +++ b/tests/saga_dist_test.go @@ -10,6 +10,7 @@ import ( "github.com/ergo-services/ergo/etf" "github.com/ergo-services/ergo/gen" "github.com/ergo-services/ergo/node" + "github.com/ergo-services/ergo/proto/dist" ) // this test implemets distributed case of computing sum for the given @@ -123,7 +124,7 @@ func (gs *testSaga1) HandleSagaDirect(process *gen.SagaProcess, message interfac switch m := message.(type) { case task: values := splitSlice(m.value, m.split) - fmt.Printf(" process %v txs with %v value(s) each distributing them on Saga2 and Saga3: ", len(values), m.split) + fmt.Printf(" process %d txs with %v value(s) each distributing them on Saga2 and Saga3: ", len(values), m.split) for i := range values { txValue := taskTX{ value: values[i], @@ -238,21 +239,30 @@ func TestSagaDist(t *testing.T) { fmt.Printf("\n=== Test GenSagaDist\n") fmt.Printf("Starting node: nodeGenSagaDist01@localhost...") - node1, _ := ergo.StartNode("nodeGenSagaDist01@localhost", "cookies", node.Options{}) + opts1 := node.Options{} + protoOptions := node.DefaultProtoOptions() + protoOptions.NumHandlers = 2 + opts1.Proto = dist.CreateProto(protoOptions) + node1, _ := ergo.StartNode("nodeGenSagaDist01@localhost", "cookies", opts1) if node1 == nil { t.Fatal("can't start node") return } fmt.Println("OK") fmt.Printf("Starting node: nodeGenSagaDist02@localhost...") - node2, _ := ergo.StartNode("nodeGenSagaDist02@localhost", "cookies", node.Options{}) + opts2 := node.Options{} + opts2.Proto = dist.CreateProto(protoOptions) + node2, _ := ergo.StartNode("nodeGenSagaDist02@localhost", "cookies", opts2) if node2 == nil { t.Fatal("can't start node") return } fmt.Println("OK") fmt.Printf("Starting node: nodeGenSagaDist03@localhost...") - node3, _ := ergo.StartNode("nodeGenSagaDist03@localhost", "cookies", node.Options{}) + opts3 := node.Options{} + opts3.Proto = dist.CreateProto(protoOptions) + fmt.Printf("Starting node: nodeGenSagaDist02@localhost...") + node3, _ := ergo.StartNode("nodeGenSagaDist03@localhost", "cookies", opts3) if node3 == nil { t.Fatal("can't start node") return @@ -317,9 +327,6 @@ func TestSagaDist(t *testing.T) { // stop all nodes node3.Stop() - node3.Wait() node2.Stop() - node2.Wait() node1.Stop() - node1.Wait() } diff --git a/tests/server_test.go b/tests/server_test.go index f1f62fe4..faa97379 100644 --- a/tests/server_test.go +++ b/tests/server_test.go @@ -84,19 +84,19 @@ func TestServer(t *testing.T) { err: make(chan error, 2), } - fmt.Printf(" wait for start of gs1 on %#v: ", node1.NodeName()) + fmt.Printf(" wait for start of gs1 on %#v: ", node1.Name()) node1gs1, _ := node1.Spawn("gs1", gen.ProcessOptions{}, gs1, nil) waitForResultWithValue(t, gs1.res, nil) - fmt.Printf(" wait for start of gs2 on %#v: ", node1.NodeName()) + fmt.Printf(" wait for start of gs2 on %#v: ", node1.Name()) node1gs2, _ := node1.Spawn("gs2", gen.ProcessOptions{}, gs2, nil) waitForResultWithValue(t, gs2.res, nil) - fmt.Printf(" wait for start of gs3 on %#v: ", node2.NodeName()) + fmt.Printf(" wait for start of gs3 on %#v: ", node2.Name()) node2gs3, _ := node2.Spawn("gs3", gen.ProcessOptions{}, gs3, nil) waitForResultWithValue(t, gs3.res, nil) - fmt.Printf(" wait for start of gsDirect on %#v: ", node2.NodeName()) + fmt.Printf(" wait for start of gsDirect on %#v: ", node2.Name()) node2gsDirect, _ := node2.Spawn("gsDirect", gen.ProcessOptions{}, gsDirect, nil) waitForResult(t, gsDirect.err) @@ -219,7 +219,7 @@ func TestServer(t *testing.T) { } fmt.Printf(" process.Send (by Name) local (gs1) -> remote (gs3) : ") - processName := gen.ProcessID{Name: "gs3", Node: node2.NodeName()} + processName := gen.ProcessID{Name: "gs3", Node: node2.Name()} node1gs1.Send(processName, etf.Atom("hi")) waitForResultWithValue(t, gs3.res, etf.Atom("hi")) @@ -319,7 +319,7 @@ func TestServer(t *testing.T) { } fmt.Println("OK") - fmt.Printf("Stopping nodes: %v, %v\n", node1.NodeName(), node2.NodeName()) + fmt.Printf("Stopping nodes: %v, %v\n", node1.Name(), node2.Name()) node1.Stop() node2.Stop() } @@ -432,10 +432,13 @@ func (gs *GSCallPanic) HandleDirect(process *gen.ServerProcess, message interfac if !ok { return nil, fmt.Errorf("not a pid") } + fmt.Printf(" making a call p1node1 -> p1node2 (panic): ") if _, err := process.CallWithTimeout(pids[0], "panic", 1); err == nil { return nil, fmt.Errorf("must be error here") + } else { + fmt.Println("OK") } - + fmt.Printf(" making a call p1node1 -> p2node2: ") v, err := process.Call(pids[1], "test") if err != nil { return nil, err @@ -443,6 +446,7 @@ func (gs *GSCallPanic) HandleDirect(process *gen.ServerProcess, message interfac if v.(string) != "ok" { return nil, fmt.Errorf("wrong result %#v", v) } + fmt.Println("OK") return nil, nil } @@ -464,15 +468,15 @@ func TestServerCallServerWithPanic(t *testing.T) { fmt.Println("OK") } - p1n1, err := node1.Spawn("", gen.ProcessOptions{}, &GSCallPanic{}) + p1n1, err := node1.Spawn("p1node1", gen.ProcessOptions{}, &GSCallPanic{}) if err != nil { t.Fatal(err) } - p1n2, err := node2.Spawn("", gen.ProcessOptions{}, &GSCallPanic{}) + p1n2, err := node2.Spawn("p1node2", gen.ProcessOptions{}, &GSCallPanic{}) if err != nil { t.Fatal(err) } - p2n2, err := node2.Spawn("", gen.ProcessOptions{}, &GSCallPanic{}) + p2n2, err := node2.Spawn("2node2", gen.ProcessOptions{}, &GSCallPanic{}) if err != nil { t.Fatal(err) } @@ -482,7 +486,6 @@ func TestServerCallServerWithPanic(t *testing.T) { if _, err := p1n1.Direct(pids); err != nil { t.Fatal(err) } - fmt.Println("OK") } func TestServerMessageOrder(t *testing.T) { @@ -507,21 +510,21 @@ func TestServerMessageOrder(t *testing.T) { res: make(chan interface{}, 2), } - fmt.Printf(" wait for start of gs1order on %#v: ", node1.NodeName()) + fmt.Printf(" wait for start of gs1order on %#v: ", node1.Name()) node1gs1, err1 := node1.Spawn("gs1order", gen.ProcessOptions{}, gs1, nil) if err1 != nil { panic(err1) } waitForResultWithValue(t, gs1.res, nil) - fmt.Printf(" wait for start of gs2order on %#v: ", node1.NodeName()) + fmt.Printf(" wait for start of gs2order on %#v: ", node1.Name()) node1gs2, err2 := node1.Spawn("gs2order", gen.ProcessOptions{}, gs2, nil) if err2 != nil { panic(err2) } waitForResultWithValue(t, gs2.res, nil) - fmt.Printf(" wait for start of gs3order on %#v: ", node1.NodeName()) + fmt.Printf(" wait for start of gs3order on %#v: ", node1.Name()) node1gs3, err3 := node1.Spawn("gs3order", gen.ProcessOptions{}, gs3, nil) if err3 != nil { panic(err3) @@ -536,7 +539,6 @@ func TestServerMessageOrder(t *testing.T) { } } waitForResultWithValue(t, gs2.res, 1000) - fmt.Println("OK") fmt.Printf(" making Direct call with making a call from gs2 to gs3 1 time: ") _, err := node1gs2.Direct(testCase3{n: 1}) @@ -561,7 +563,6 @@ func TestServerMessageOrder(t *testing.T) { } } waitForResultWithValue(t, gs2.res, 123) - fmt.Println("OK") node1gs3.Exit("normal") node1.Stop() node1.Wait() @@ -719,27 +720,27 @@ func TestServerMessageFlood(t *testing.T) { gsdest := &messageFloodDestGS{ res: make(chan interface{}, 2), } - fmt.Printf(" wait for start of gs1source on %#v: ", node1.NodeName()) + fmt.Printf(" wait for start of gs1source on %#v: ", node1.Name()) gs1sourceProcess, _ := node1.Spawn("gs1source", gen.ProcessOptions{}, gs1source, nil) waitForResultWithValue(t, gs1source.res, nil) - fmt.Printf(" wait for start of gs2source on %#v: ", node1.NodeName()) + fmt.Printf(" wait for start of gs2source on %#v: ", node1.Name()) gs2sourceProcess, _ := node1.Spawn("gs2source", gen.ProcessOptions{}, gs2source, nil) waitForResultWithValue(t, gs2source.res, nil) - fmt.Printf(" wait for start of gs3source on %#v: ", node1.NodeName()) + fmt.Printf(" wait for start of gs3source on %#v: ", node1.Name()) gs3sourceProcess, _ := node1.Spawn("gs3source", gen.ProcessOptions{}, gs3source, nil) waitForResultWithValue(t, gs3source.res, nil) - fmt.Printf(" wait for start of gs4source on %#v: ", node1.NodeName()) + fmt.Printf(" wait for start of gs4source on %#v: ", node1.Name()) gs4sourceProcess, _ := node1.Spawn("gs4source", gen.ProcessOptions{}, gs4source, nil) waitForResultWithValue(t, gs4source.res, nil) - fmt.Printf(" wait for start of gs5source on %#v: ", node1.NodeName()) + fmt.Printf(" wait for start of gs5source on %#v: ", node1.Name()) gs5sourceProcess, _ := node1.Spawn("gs5source", gen.ProcessOptions{}, gs5source, nil) waitForResultWithValue(t, gs5source.res, nil) - fmt.Printf(" wait for start of gsdest on %#v: ", node1.NodeName()) + fmt.Printf(" wait for start of gsdest on %#v: ", node1.Name()) node1.Spawn("gsdest", gen.ProcessOptions{}, gsdest, nil) waitForResultWithValue(t, gsdest.res, nil) @@ -820,6 +821,20 @@ func waitForResultWithValue(t *testing.T, w chan interface{}, value interface{}) } } +func waitForResultWithValueReturnError(t *testing.T, w chan interface{}, value interface{}) error { + select { + case v := <-w: + if reflect.DeepEqual(v, value) { + return nil + } else { + return fmt.Errorf("expected: %#v , got: %#v", value, v) + } + + case <-time.After(time.Second * time.Duration(2)): + return fmt.Errorf("result timeout") + } +} + func waitForResultWithValueOrValue(t *testing.T, w chan interface{}, value1, value2 interface{}) { select { case v := <-w: diff --git a/tests/stage_test.go b/tests/stage_test.go index 3291e67a..4249b9d2 100644 --- a/tests/stage_test.go +++ b/tests/stage_test.go @@ -571,6 +571,7 @@ func TestStageDistributed(t *testing.T) { fmt.Printf("... Consumer@node2 handled subscription confirmation from Producer@node1 (StageCancelTemporary): ") waitForResultWithValue(t, consumer.value, sub3) + node2.Disconnect(node1.Name()) node1.Stop() fmt.Printf("... Stopping node1: ") if err := node1.WaitWithTimeout(1000 * time.Millisecond); err != nil { diff --git a/tests/supervisor_ofa_test.go b/tests/supervisor_ofa_test.go index 78e0e250..7dee39cb 100644 --- a/tests/supervisor_ofa_test.go +++ b/tests/supervisor_ofa_test.go @@ -91,17 +91,17 @@ func TestSupervisorOneForAll(t *testing.T) { // testing permanent testCases := []ChildrenTestCase{ - ChildrenTestCase{ + { reason: "normal", statuses: []string{"new", "new", "new"}, events: 6, // waiting for 3 terminating and 3 starting }, - ChildrenTestCase{ + { reason: "abnormal", statuses: []string{"new", "new", "new"}, events: 6, }, - ChildrenTestCase{ + { reason: "shutdown", statuses: []string{"new", "new", "new"}, events: 6, @@ -155,17 +155,17 @@ func TestSupervisorOneForAll(t *testing.T) { // testing transient testCases = []ChildrenTestCase{ - ChildrenTestCase{ + { reason: "normal", statuses: []string{"empty", "new", "new"}, events: 5, // waiting for 3 terminates and 2 starts }, - ChildrenTestCase{ + { reason: "abnormal", statuses: []string{"empty", "new", "new"}, events: 4, // waiting for 2 terminates and 2 starts }, - ChildrenTestCase{ + { reason: "shutdown", statuses: []string{"empty", "new", "empty"}, events: 3, // waiting for 2 terminates and 1 start @@ -211,17 +211,17 @@ func TestSupervisorOneForAll(t *testing.T) { // restart strategy is rest_for_one or one_for_all and a sibling's death // causes the temporary process to be terminated). testCases = []ChildrenTestCase{ - ChildrenTestCase{ + { reason: "normal", statuses: []string{"empty", "empty", "empty"}, events: 3, // waiting for 3 terminates }, - ChildrenTestCase{ + { reason: "abnormal", statuses: []string{"empty", "empty", "empty"}, events: 3, // waiting for 3 terminates }, - ChildrenTestCase{ + { reason: "shutdown", statuses: []string{"empty", "empty", "empty"}, events: 3, // waiting for 3 terminate @@ -279,17 +279,17 @@ func (ts *testSupervisorOneForAll) Init(args ...etf.Term) (gen.SupervisorSpec, e ch := args[1].(chan interface{}) return gen.SupervisorSpec{ Children: []gen.SupervisorChildSpec{ - gen.SupervisorChildSpec{ + { Name: "testGS1", Child: &testSupervisorGenServer{}, Args: []etf.Term{ch, 0}, }, - gen.SupervisorChildSpec{ + { Name: "testGS2", Child: &testSupervisorGenServer{}, Args: []etf.Term{ch, 1}, }, - gen.SupervisorChildSpec{ + { Name: "testGS3", Child: &testSupervisorGenServer{}, Args: []etf.Term{ch, 2}, diff --git a/tests/supervisor_ofo_test.go b/tests/supervisor_ofo_test.go index 4f2d8d71..32dace72 100644 --- a/tests/supervisor_ofo_test.go +++ b/tests/supervisor_ofo_test.go @@ -264,17 +264,17 @@ func (ts *testSupervisorOneForOne) Init(args ...etf.Term) (gen.SupervisorSpec, e ch := args[1].(chan interface{}) return gen.SupervisorSpec{ Children: []gen.SupervisorChildSpec{ - gen.SupervisorChildSpec{ + { Name: "testGS1", Child: &testSupervisorGenServer{}, Args: []etf.Term{ch, 0}, }, - gen.SupervisorChildSpec{ + { Name: "testGS2", Child: &testSupervisorGenServer{}, Args: []etf.Term{ch, 1}, }, - gen.SupervisorChildSpec{ + { Name: "testGS3", Child: &testSupervisorGenServer{}, Args: []etf.Term{ch, 2}, diff --git a/tests/supervisor_rfo_test.go b/tests/supervisor_rfo_test.go index f5603871..2a7bde95 100644 --- a/tests/supervisor_rfo_test.go +++ b/tests/supervisor_rfo_test.go @@ -89,17 +89,17 @@ func TestSupervisorRestForOne(t *testing.T) { // testing permanent testCases := []ChildrenTestCase{ - ChildrenTestCase{ + { reason: "normal", statuses: []string{"new", "new", "new"}, events: 6, // waiting for 3 terminates and 3 starts }, - ChildrenTestCase{ + { reason: "abnormal", statuses: []string{"old", "new", "new"}, events: 4, // waiting for 2 terminates and 2 starts }, - ChildrenTestCase{ + { reason: "shutdown", statuses: []string{"old", "old", "new"}, events: 2, // waiting for 1 terminates and 1 starts @@ -158,17 +158,17 @@ func TestSupervisorRestForOne(t *testing.T) { // testing transient testCases = []ChildrenTestCase{ - ChildrenTestCase{ + { reason: "abnormal", statuses: []string{"new", "new", "new"}, events: 6, // waiting for 3 terminates and 3 starts }, - ChildrenTestCase{ + { reason: "normal", statuses: []string{"old", "empty", "new"}, events: 3, // waiting for 2 terminates and 1 starts }, - ChildrenTestCase{ + { reason: "shutdown", statuses: []string{"old", "empty", "empty"}, events: 1, // waiting for 1 terminates @@ -214,17 +214,17 @@ func TestSupervisorRestForOne(t *testing.T) { // restart strategy is rest_for_one or one_for_all and a sibling's death // causes the temporary process to be terminated). testCases = []ChildrenTestCase{ - ChildrenTestCase{ + { reason: "normal", statuses: []string{"empty", "empty", "empty"}, events: 3, // waiting for 3 terminates }, - ChildrenTestCase{ + { reason: "abnormal", statuses: []string{"old", "empty", "empty"}, events: 2, // waiting for 2 terminates }, - ChildrenTestCase{ + { reason: "shutdown", statuses: []string{"old", "old", "empty"}, events: 1, // waiting for 1 terminate @@ -284,17 +284,17 @@ func (ts *testSupervisorRestForOne) Init(args ...etf.Term) (gen.SupervisorSpec, ch := args[1].(chan interface{}) return gen.SupervisorSpec{ Children: []gen.SupervisorChildSpec{ - gen.SupervisorChildSpec{ + { Name: "testGS1", Child: &testSupervisorGenServer{}, Args: []etf.Term{ch, 0}, }, - gen.SupervisorChildSpec{ + { Name: "testGS2", Child: &testSupervisorGenServer{}, Args: []etf.Term{ch, 1}, }, - gen.SupervisorChildSpec{ + { Name: "testGS3", Child: &testSupervisorGenServer{}, Args: []etf.Term{ch, 2}, diff --git a/tests/supervisor_sofo_test.go b/tests/supervisor_sofo_test.go index 14514414..3ffafea8 100644 --- a/tests/supervisor_sofo_test.go +++ b/tests/supervisor_sofo_test.go @@ -38,17 +38,17 @@ func TestSupervisorSimpleOneForOne(t *testing.T) { // =================================================================================================== // test SupervisorStrategyRestartPermanent testCases := []ChildrenTestCase{ - ChildrenTestCase{ + { reason: "abnormal", statuses: []string{"new", "new", "new", "new", "new", "new"}, events: 12, // waiting for 6 terminates and 6 restarts }, - ChildrenTestCase{ + { reason: "normal", statuses: []string{"new", "new", "new", "new", "new", "new"}, events: 12, // waiting for 6 terminates and 6 restarts }, - ChildrenTestCase{ + { reason: "shutdown", statuses: []string{"new", "new", "new", "new", "new", "new"}, events: 12, // waiting for 6 terminates and 6 restarts @@ -146,17 +146,17 @@ func TestSupervisorSimpleOneForOne(t *testing.T) { // =================================================================================================== // test SupervisorStrategyRestartTransient testCases = []ChildrenTestCase{ - ChildrenTestCase{ + { reason: "abnormal", statuses: []string{"new", "new", "new", "new", "new", "new"}, events: 12, // waiting for 6 terminates and 6 restarts }, - ChildrenTestCase{ + { reason: "normal", statuses: []string{"empty", "empty", "empty", "empty", "empty", "empty"}, events: 6, // waiting for 6 terminates }, - ChildrenTestCase{ + { reason: "shutdown", statuses: []string{"empty", "empty", "empty", "empty", "empty", "empty"}, events: 6, // waiting for 6 terminates @@ -254,17 +254,17 @@ func TestSupervisorSimpleOneForOne(t *testing.T) { // =================================================================================================== // test SupervisorStrategyRestartTemporary testCases = []ChildrenTestCase{ - ChildrenTestCase{ + { reason: "abnormal", statuses: []string{"empty", "empty", "empty", "empty", "empty", "empty"}, events: 6, // waiting for 6 terminates }, - ChildrenTestCase{ + { reason: "normal", statuses: []string{"empty", "empty", "empty", "empty", "empty", "empty"}, events: 6, // waiting for 6 terminates }, - ChildrenTestCase{ + { reason: "shutdown", statuses: []string{"empty", "empty", "empty", "empty", "empty", "empty"}, events: 6, // waiting for 6 terminates @@ -364,15 +364,15 @@ func (ts *testSupervisorSimpleOneForOne) Init(args ...etf.Term) (gen.SupervisorS restart := args[0].(string) return gen.SupervisorSpec{ Children: []gen.SupervisorChildSpec{ - gen.SupervisorChildSpec{ + { Name: "testGS1", Child: &testSupervisorGenServer{}, }, - gen.SupervisorChildSpec{ + { Name: "testGS2", Child: &testSupervisorGenServer{}, }, - gen.SupervisorChildSpec{ + { Name: "testGS3", Child: &testSupervisorGenServer{}, }, diff --git a/version.go b/version.go index bafd4690..ef2acdc7 100644 --- a/version.go +++ b/version.go @@ -1,7 +1,7 @@ package ergo const ( - Version = "2.0.0" - VersionPrefix = "ergo" - VersionOTP int = 24 + Version = "2.1.0" // Ergo Framework version + VersionPrefix = "ergo" // Prefix using for the full version name + VersionOTP int = 24 // Erlang version support )