diff --git a/config/dev.exs b/config/dev.exs index 960b54bcca..8e495fae2e 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -16,6 +16,30 @@ config :teiserver, Teiserver.SpringTcpServer, heartbeat_interval: nil, heartbeat_timeout: nil +raw_http_port = System.get_env("TEISERVER_HTTP_PORT", "4002") |> String.to_integer() + +# Secondary dev nodes should disable the asset watchers, otherwise both nodes +# race to write the same asset output files (TEISERVER_WATCHERS=off) +watchers = + if System.get_env("TEISERVER_WATCHERS") == "off" do + [] + else + [ + esbuild: {Esbuild, :install_and_run, [:teiserver, ~w(--sourcemap=inline --watch)]}, + tailwind: {Tailwind, :install_and_run, [:teiserver, ~w(--watch)]}, + dark_sass: { + DartSass, + :install_and_run, + [:dark, ~w(--embed-source-map --source-map-urls=absolute --watch)] + }, + light_sass: { + DartSass, + :install_and_run, + [:light, ~w(--embed-source-map --source-map-urls=absolute --watch)] + } + ] + end + # For development, we disable any cache and enable # debugging and code reloading. # @@ -24,23 +48,10 @@ config :teiserver, Teiserver.SpringTcpServer, # with webpack to recompile .js and .css sources. config :teiserver, TeiserverWeb.Endpoint, url: [host: "localhost"], - http: [ip: {0, 0, 0, 0}, port: 4000], - watchers: [ - esbuild: {Esbuild, :install_and_run, [:teiserver, ~w(--sourcemap=inline --watch)]}, - tailwind: {Tailwind, :install_and_run, [:teiserver, ~w(--watch)]}, - dark_sass: { - DartSass, - :install_and_run, - [:dark, ~w(--embed-source-map --source-map-urls=absolute --watch)] - }, - light_sass: { - DartSass, - :install_and_run, - [:light, ~w(--embed-source-map --source-map-urls=absolute --watch)] - } - ] + http: [ip: {0, 0, 0, 0}, port: raw_http_port], + watchers: watchers -config :teiserver, Teiserver.OAuth, issuer: "http://localhost:4000" +config :teiserver, Teiserver.OAuth, issuer: "http://localhost:#{raw_http_port}" config :teiserver, Teiserver, certs: [ diff --git a/config/runtime.exs b/config/runtime.exs index 3d1879d878..2051887281 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -254,3 +254,37 @@ end config :teiserver, TeiserverWeb.Monitoring, port: Teiserver.ConfigHelpers.get_env("TEI_METRICS_SERVER_PORT", 4001, :int) + +# Each test partition gets its own database and ports so that +# `mix test --partitions N` instances can run concurrently +if config_env() == :test do + partition = System.get_env("MIX_TEST_PARTITION", "0") + partition_port = String.to_integer(partition) * 100 + + config :teiserver, Teiserver.Repo, database: "teiserver_test#{partition}" + + http_port = 4002 + partition_port + + config :teiserver, TeiserverWeb.Endpoint, http: [port: http_port] + config :teiserver, Teiserver.OAuth, issuer: "http://localhost:#{http_port}" + + config :teiserver, TeiserverWeb.Monitoring, + port: Teiserver.ConfigHelpers.get_env("TEI_METRICS_SERVER_PORT", 4001, :int) + partition_port +end + +# Cluster formation, requires the node to run in distributed mode +# (--name/--sname). Topologies are ignored when not distributed. +is_local? = config_env() == :dev +is_prod? = config_env() == :prod + +if is_local? do + # Connects to every distributed node registered with the local epmd, + # useful for local development (see the justfile) + config :libcluster, + topologies: [teiserver: [strategy: Cluster.Strategy.LocalEpmd]] +end + +if is_prod? do + # TODO: set libcluster topology for prod, e.g. Cluster.Strategy.Epmd with a + # static host list or Cluster.Strategy.DNSPoll for discovery +end diff --git a/justfile b/justfile new file mode 100644 index 0000000000..bb74a20d40 --- /dev/null +++ b/justfile @@ -0,0 +1,14 @@ + +# List available recipes +default: + @just --list + +# Run a dev cluster node, e.g. `just iex` or `just iex 2` +# Instance 1: http 4002, spring tcp 8200, metrics 4001; each further +# instance is offset by 100. Asset watchers only run on instance 1. +node instance='1': + TEISERVER_HTTP_PORT=$((4002 + ({{instance}} - 1) * 100)) \ + TEI_SPRING_TCP_PORT=$((8200 + ({{instance}} - 1) * 100)) \ + TEI_METRICS_SERVER_PORT=$((4001 + ({{instance}} - 1) * 100)) \ + {{ if instance == '1' { '' } else { 'TEISERVER_WATCHERS=off' } }} \ + iex --name node{{instance}}@127.0.0.1 --cookie teiserver-dev -S mix phx.server diff --git a/lib/teiserver/application.ex b/lib/teiserver/application.ex index f19995dec6..8991ee4e41 100644 --- a/lib/teiserver/application.ex +++ b/lib/teiserver/application.ex @@ -25,9 +25,19 @@ defmodule Teiserver.Application do LoggerBackends.add({LoggerFileBackend, :info_log}) Logger.add_handlers(:teiserver) + # Topologies are only usable when running in distributed mode, otherwise + # libcluster logs a warning on every reconnect attempt + cluster_topologies = + if Node.alive?() do + Application.get_env(:libcluster, :topologies, []) + else + [] + end + children = [ Teiserver.PromEx, + {Cluster.Supervisor, [cluster_topologies, [name: Teiserver.ClusterSupervisor]]}, # Migrations {Ecto.Migrator, repos: Application.fetch_env!(:teiserver, :ecto_repos), @@ -78,8 +88,12 @@ defmodule Teiserver.Application do {Horde.Registry, [keys: :unique, members: :auto, name: Teiserver.LobbyRegistry]}, {Horde.Registry, [keys: :unique, members: :auto, name: Teiserver.ClientRegistry]}, {Horde.Registry, [keys: :unique, members: :auto, name: Teiserver.PartyRegistry]}, - {Horde.Registry, [keys: :unique, members: :auto, name: Teiserver.QueueWaitRegistry]}, - {Horde.Registry, [keys: :unique, members: :auto, name: Teiserver.QueueMatchRegistry]}, + + # Cluster-wide singletons (CoordinatorServer, MatchMonitorServer, + # AutomodServer, LobbyIdServer); children of a dead node are restarted + # on a surviving node + {Horde.DynamicSupervisor, + [strategy: :one_for_one, members: :auto, name: Teiserver.SingletonSupervisor]}, # These are for tracking the number of servers on the local node {Registry, keys: :duplicate, name: Teiserver.LocalPoolRegistry}, @@ -221,6 +235,7 @@ defmodule Teiserver.Application do @impl Application @spec prep_stop(map()) :: map() def prep_stop(state) do + # TODO - PubSub doesn't work here because the endpoint is already stopped PubSub.broadcast( Teiserver.PubSub, "application", diff --git a/lib/teiserver/battle/servers/match_monitor_server.ex b/lib/teiserver/battle/servers/match_monitor_server.ex index e68073d759..8449850266 100644 --- a/lib/teiserver/battle/servers/match_monitor_server.ex +++ b/lib/teiserver/battle/servers/match_monitor_server.ex @@ -3,6 +3,7 @@ defmodule Teiserver.Battle.MatchMonitorServer do The server used to monitor the autohosts and get data from them """ + alias Horde.DynamicSupervisor, as: HordeSupervisor alias Phoenix.PubSub alias Teiserver.Account alias Teiserver.Account.Auth @@ -22,7 +23,7 @@ defmodule Teiserver.Battle.MatchMonitorServer do alias Teiserver.Telemetry use Plugins - use GenServer + use GenServer, restart: :transient require Logger @@ -30,19 +31,26 @@ defmodule Teiserver.Battle.MatchMonitorServer do @spec do_start() :: :ok def do_start do - # Start the supervisor server - {:ok, _monitor_pid} = - DynamicSupervisor.start_child(Teiserver.Coordinator.DynamicSupervisor, { + # Start the singleton server, another node may have already done so + result = + HordeSupervisor.start_child(Teiserver.SingletonSupervisor, { Teiserver.Battle.MatchMonitorServer, name: Teiserver.Battle.MatchMonitorServer, data: %{} }) - :ok + case result do + {:ok, _monitor_pid} -> :ok + {:error, {:already_started, _pid}} -> :ok + end end @spec start_link(list()) :: :ignore | {:error, any} | {:ok, pid} def start_link(opts) do - GenServer.start_link(__MODULE__, opts[:data], []) + GenServer.start_link(__MODULE__, opts[:data], name: via_tuple()) + end + + defp via_tuple do + {:via, Horde.Registry, {Teiserver.ServerRegistry, "MatchMonitorServer", :match_monitor}} end @spec get_match_monitor_userid() :: User.id() | nil @@ -375,6 +383,12 @@ defmodule Teiserver.Battle.MatchMonitorServer do {:noreply, state} end + # Another node won the singleton registration during a Horde registry + # merge; step down cleanly so the :transient restart does not respawn us + def handle_info({:EXIT, _from, {:name_conflict, _key, _registry, _winning_pid}}, state) do + {:stop, :normal, state} + end + # Catchall handle_info def handle_info(msg, state) do Logger.warning( @@ -542,12 +556,6 @@ defmodule Teiserver.Battle.MatchMonitorServer do send(self(), :begin) Logger.metadata(request_id: "MatchMonitorServer") - Horde.Registry.register( - Teiserver.ServerRegistry, - "MatchMonitorServer", - :match_monitor - ) - {:ok, %{}} end diff --git a/lib/teiserver/coordinator.ex b/lib/teiserver/coordinator.ex index 579ac5e28f..45b499b894 100644 --- a/lib/teiserver/coordinator.ex +++ b/lib/teiserver/coordinator.ex @@ -10,8 +10,9 @@ defmodule Teiserver.Coordinator do def start_coordinator do case get_coordinator_pid() do nil -> - # Start the supervisor server - DynamicSupervisor.start_child(Teiserver.Coordinator.DynamicSupervisor, { + # Start the singleton server, another node may win the race and + # start it first + Horde.DynamicSupervisor.start_child(Teiserver.SingletonSupervisor, { Teiserver.Coordinator.CoordinatorServer, name: Teiserver.Coordinator.CoordinatorServer, data: %{} }) diff --git a/lib/teiserver/coordinator/automod_server.ex b/lib/teiserver/coordinator/automod_server.ex index a276c66e98..0839c22d04 100644 --- a/lib/teiserver/coordinator/automod_server.ex +++ b/lib/teiserver/coordinator/automod_server.ex @@ -1,6 +1,7 @@ defmodule Teiserver.Coordinator.AutomodServer do @moduledoc false + alias Horde.DynamicSupervisor, as: HordeSupervisor alias Phoenix.PubSub alias Teiserver.Account alias Teiserver.Account.Auth @@ -9,7 +10,7 @@ defmodule Teiserver.Coordinator.AutomodServer do alias Teiserver.Config alias Teiserver.Coordinator alias Teiserver.Moderation - use GenServer + use GenServer, restart: :transient require Logger import Teiserver.Logging.Helpers, only: [add_audit_log: 4] @@ -33,18 +34,28 @@ defmodule Teiserver.Coordinator.AutomodServer do @spec do_start() :: :ok defp do_start do - {:ok, _automod_pid} = - DynamicSupervisor.start_child(Teiserver.Coordinator.DynamicSupervisor, { + # Another node may have already started the singleton + result = + HordeSupervisor.start_child(Teiserver.SingletonSupervisor, { Teiserver.Coordinator.AutomodServer, name: Teiserver.Coordinator.AutomodServer, data: %{} }) + case result do + {:ok, _automod_pid} -> :ok + {:error, {:already_started, _pid}} -> :ok + end + :ok end @spec start_link(list()) :: :ignore | {:error, any} | {:ok, pid} def start_link(opts) do - GenServer.start_link(__MODULE__, opts[:data], []) + GenServer.start_link(__MODULE__, opts[:data], name: via_tuple()) + end + + defp via_tuple do + {:via, Horde.Registry, {Teiserver.ServerRegistry, "AutomodServer", :automod}} end def handle_info(:begin, state) do @@ -110,6 +121,12 @@ defmodule Teiserver.Coordinator.AutomodServer do def handle_info(%{channel: "telemetry_user_properties"}, state), do: {:noreply, state} + # Another node won the singleton registration during a Horde registry + # merge; step down cleanly so the :transient restart does not respawn us + def handle_info({:EXIT, _from, {:name_conflict, _key, _registry, _winning_pid}}, state) do + {:stop, :normal, state} + end + # Catchall handle_info def handle_info(msg, state) do Logger.error("AutoMod handle_info error. No handler for msg of #{Kernel.inspect(msg)}") @@ -123,11 +140,7 @@ defmodule Teiserver.Coordinator.AutomodServer do @spec init(map()) :: {:ok, map()} def init(_opts) do - Horde.Registry.register( - Teiserver.ServerRegistry, - "AutomodServer", - :automod - ) + Process.flag(:trap_exit, true) :timer.send_after(500, :begin) {:ok, %{}} diff --git a/lib/teiserver/coordinator/coordinator_server.ex b/lib/teiserver/coordinator/coordinator_server.ex index 43e91a00eb..145e929045 100644 --- a/lib/teiserver/coordinator/coordinator_server.ex +++ b/lib/teiserver/coordinator/coordinator_server.ex @@ -19,7 +19,7 @@ defmodule Teiserver.Coordinator.CoordinatorServer do alias Teiserver.Room alias Teiserver.Telemetry - use GenServer + use GenServer, restart: :transient require Logger @@ -32,7 +32,11 @@ defmodule Teiserver.Coordinator.CoordinatorServer do @spec start_link(list()) :: :ignore | {:error, any} | {:ok, pid} def start_link(opts) do - GenServer.start_link(__MODULE__, opts[:data], []) + GenServer.start_link(__MODULE__, opts[:data], name: via_tuple()) + end + + defp via_tuple do + {:via, Horde.Registry, {Teiserver.ServerRegistry, "CoordinatorServer", :coordinator}} end @impl GenServer @@ -310,6 +314,12 @@ defmodule Teiserver.Coordinator.CoordinatorServer do {:noreply, state} end + # Another node won the singleton registration during a Horde registry + # merge; step down cleanly so the :transient restart does not respawn us + def handle_info({:EXIT, _from, {:name_conflict, _key, _registry, _winning_pid}}, state) do + {:stop, :normal, state} + end + # Catchall handle_info def handle_info(msg, state) do Logger.error( @@ -369,12 +379,6 @@ defmodule Teiserver.Coordinator.CoordinatorServer do def init(_opts) do Process.flag(:trap_exit, true) - Horde.Registry.register( - Teiserver.ServerRegistry, - "CoordinatorServer", - :coordinator - ) - send(self(), :begin) {:ok, %{client: %{}}} end diff --git a/lib/teiserver/data/lobby_id_server.ex b/lib/teiserver/data/lobby_id_server.ex index f6fd3295db..ce80c0e3b5 100644 --- a/lib/teiserver/data/lobby_id_server.ex +++ b/lib/teiserver/data/lobby_id_server.ex @@ -3,19 +3,25 @@ defmodule Teiserver.LobbyIdServer do Used as a singleton to create lobby_ids and ensure we increment the counter each time across the cluster. """ - use GenServer + alias Horde.DynamicSupervisor, as: HordeSupervisor + + use GenServer, restart: :transient @spec start_lobby_id_server() :: :ok | {:failure, String.t()} def start_lobby_id_server do case get_server_pid() do nil -> - {:ok, _coordinator_pid} = - DynamicSupervisor.start_child(Teiserver.Coordinator.DynamicSupervisor, { + # Another node may win the race and start the singleton first + result = + HordeSupervisor.start_child(Teiserver.SingletonSupervisor, { __MODULE__, name: __MODULE__, data: %{} }) - :ok + case result do + {:ok, _pid} -> :ok + {:error, {:already_started, _pid}} -> {:failure, "Already started"} + end _pid -> {:failure, "Already started"} @@ -51,7 +57,11 @@ defmodule Teiserver.LobbyIdServer do @spec start_link(list()) :: :ignore | {:error, any} | {:ok, pid} def start_link(_opts) do - GenServer.start_link(__MODULE__, [], []) + GenServer.start_link(__MODULE__, [], name: via_tuple()) + end + + defp via_tuple do + {:via, Horde.Registry, {Teiserver.ServerRegistry, "LobbyIdServer", :lobby_id_server}} end def handle_call(:next_id, _from, state) do @@ -62,13 +72,15 @@ defmodule Teiserver.LobbyIdServer do {:noreply, %{state | next_id: next_id}} end + # Another node won the singleton registration during a Horde registry + # merge; step down cleanly so the :transient restart does not respawn us + def handle_info({:EXIT, _from, {:name_conflict, _key, _registry, _winning_pid}}, state) do + {:stop, :normal, state} + end + @spec init(map()) :: {:ok, map()} def init(_opts) do - Horde.Registry.register( - Teiserver.ServerRegistry, - "LobbyIdServer", - :lobby_id_server - ) + Process.flag(:trap_exit, true) {:ok, %{ diff --git a/lib/teiserver/matchmaking/queue_supervisor.ex b/lib/teiserver/matchmaking/queue_supervisor.ex index 4721f62098..e5df4c3bd7 100644 --- a/lib/teiserver/matchmaking/queue_supervisor.ex +++ b/lib/teiserver/matchmaking/queue_supervisor.ex @@ -50,8 +50,10 @@ defmodule Teiserver.Matchmaking.QueueSupervisor do def start_queue!(state) do case HordeSupervisor.start_child(__MODULE__, {QueueServer, state}) do - {:error, err} -> raise "Cannot start queue: #{inspect(err)}" {:ok, pid} -> {:ok, pid} + # another node in the cluster already runs this queue + {:error, {:already_started, pid}} -> {:ok, pid} + {:error, err} -> raise "Cannot start queue: #{inspect(err)}" end end diff --git a/lib/teiserver/servers/telemetry_server.ex b/lib/teiserver/servers/telemetry_server.ex index 223f56d3f0..f665681de4 100644 --- a/lib/teiserver/servers/telemetry_server.ex +++ b/lib/teiserver/servers/telemetry_server.ex @@ -239,9 +239,7 @@ defmodule Teiserver.Telemetry.TelemetryServer do balancer_servers: Horde.Registry.count(Teiserver.BalancerRegistry), lobby_servers: Horde.Registry.count(Teiserver.LobbyRegistry), client_servers: Horde.Registry.count(Teiserver.ClientRegistry), - party_servers: Horde.Registry.count(Teiserver.PartyRegistry), - queue_wait_servers: Horde.Registry.count(Teiserver.QueueWaitRegistry), - queue_match_servers: Horde.Registry.count(Teiserver.QueueMatchRegistry) + party_servers: Horde.Registry.count(Teiserver.PartyRegistry) } process_counts = diff --git a/mix.exs b/mix.exs index 3841bd62dc..efdcb2e9ea 100644 --- a/mix.exs +++ b/mix.exs @@ -131,6 +131,7 @@ defmodule Teiserver.MixProject do {:mdex, "~> 0.2"}, {:ranch, "~> 1.8"}, {:horde, "~> 0.10"}, + {:libcluster, "~> 3.4"}, {:etop, "~> 0.7.0"}, {:cowlib, "~> 2.11", hex: :remedy_cowlib, override: true}, {:json_xema, "~> 0.3"}, diff --git a/mix.lock b/mix.lock index 8f2036005a..722572afd7 100644 --- a/mix.lock +++ b/mix.lock @@ -54,6 +54,7 @@ "json_xema": {:hex, :json_xema, "0.6.5", "060459c9c9152650edb4427b1acbc61fa43a23bcea0301d200cafa76e0880f37", [:mix], [{:conv_case, "~> 0.2", [hex: :conv_case, repo: "hexpm", optional: false]}, {:xema, "~> 0.16", [hex: :xema, repo: "hexpm", optional: false]}], "hexpm", "b8ffdbc2f67aa8b91b44e1ba0ab77eb5c0b0142116f8fbb804977fb939d470ef"}, "jump_credo_checks": {:hex, :jump_credo_checks, "0.1.0", "8ff038eb868d36bfce6b47916619c68df99ea802adae2a95702b59463120ebb1", [:mix], [{:credo, "~> 1.7", [hex: :credo, repo: "hexpm", optional: false]}], "hexpm", "bb76a8bff31a1f42289a9ba03f4f5666e6ae061168f6f13280550ad072851a1f"}, "lazy_html": {:hex, :lazy_html, "0.1.11", "136c8e9cd616b4f4e9c1562daa683880891120b759606dc4c3b6b18058ba5d79", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9.0", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "3b1be592929c31eca1a21673d25696e5c14cddfe922d9d1a3e3b48be4163883b"}, + "libcluster": {:hex, :libcluster, "3.5.0", "5ee4cfde4bdf32b2fef271e33ce3241e89509f4344f6c6a8d4069937484866ba", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebf6561fcedd765a4cd43b4b8c04b1c87f4177b5fb3cbdfe40a780499d72f743"}, "libring": {:hex, :libring, "1.7.0", "4f245d2f1476cd7ed8f03740f6431acba815401e40299208c7f5c640e1883bda", [:mix], [], "hexpm", "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"}, "logger_backends": {:hex, :logger_backends, "1.0.0", "09c4fad6202e08cb0fbd37f328282f16539aca380f512523ce9472b28edc6bdf", [:mix], [], "hexpm", "1faceb3e7ec3ef66a8f5746c5afd020e63996df6fd4eb8cdb789e5665ae6c9ce"}, "logger_file_backend": {:hex, :logger_file_backend, "0.0.14", "774bb661f1c3fed51b624d2859180c01e386eb1273dc22de4f4a155ef749a602", [:mix], [], "hexpm", "071354a18196468f3904ef09413af20971d55164267427f6257b52cfba03f9e6"}, diff --git a/test/support/teiserver_test_lib.ex b/test/support/teiserver_test_lib.ex index 7f83adc3af..3a1f10ad99 100644 --- a/test/support/teiserver_test_lib.ex +++ b/test/support/teiserver_test_lib.ex @@ -2,6 +2,7 @@ defmodule Teiserver.TeiserverTestLib do @moduledoc false alias ExUnit.Callbacks + alias Horde.DynamicSupervisor, as: HordeSupervisor alias Teiserver.Account alias Teiserver.Account.AccoladeLib alias Teiserver.CacheUser @@ -519,7 +520,7 @@ defmodule Teiserver.TeiserverTestLib do on_exit(fn -> assert :ok = - DynamicSupervisor.terminate_child(Coordinator.DynamicSupervisor, pid) + HordeSupervisor.terminate_child(Teiserver.SingletonSupervisor, pid) end) end