apply yaml from stdin

cli/config/config.go

@@ -48,10 +48,22 @@ func processEnvVars(yamlContent []byte) ([]byte, error) {
 
 // Interpolate reads a YAML file, processes environment variables, and outputs the result
 func Interpolate(inputPath, outputPath string) error {
-	// Read YAML file
-	yamlContent, err := os.ReadFile(inputPath)
-	if err != nil {
-		return fmt.Errorf("failed to read YAML file: %w", err)
+	var yamlContent []byte
+	var err error
+
+	// Read YAML content from stdin or file
+	if inputPath == "-" {
+		// Read from stdin
+		yamlContent, err = io.ReadAll(os.Stdin)
+		if err != nil {
+			return fmt.Errorf("failed to read from stdin: %w", err)
+		}
+	} else {
+		// Read from file
+		yamlContent, err = os.ReadFile(inputPath)
+		if err != nil {
+			return fmt.Errorf("failed to read YAML file: %w", err)
+		}
 	}
 
 	// Process environment variables

config/runtime.exs

@@ -227,8 +227,6 @@ if config_env() == :prod and self_hosted do
 
   config :sequin,
     api_base_url: "http://#{server_host}:#{server_port}",
-    config_file_path: System.get_env("CONFIG_FILE_PATH"),
-    config_file_yaml: System.get_env("CONFIG_FILE_YAML"),
     release_version: System.get_env("RELEASE_VERSION"),
     backfill_max_pending_messages: backfill_max_pending_messages,
     max_memory_bytes: ConfigParser.max_memory_bytes(env_vars),

lib/sequin/yaml_loader.ex

@@ -25,35 +25,22 @@ defmodule Sequin.YamlLoader do
     defstruct [:description, :action]
   end
 
-  def config_file_path do
-    Application.get_env(@app, :config_file_path)
-  end
-
-  def apply! do
-    Logger.info("Applying config")
+  def apply_from_stdin! do
     load_app()
     ensure_repo_started!()
 
-    cond do
-      not self_hosted?() ->
-        Logger.info("Not self-hosted, skipping config loading")
-
-      not is_nil(config_file_yaml()) ->
-        Logger.info("Loading from config file YAML")
+    Logger.info("Reading config from stdin")
 
-        config_file_yaml()
-        |> Base.decode64!()
-        |> apply_from_yml!()
+    case IO.read(:stdio, :eof) do
+      :eof ->
+        Logger.info("No config data received from stdin")
 
-      not is_nil(config_file_path()) ->
-        Logger.info("Loading from config file path")
+      {:error, reason} ->
+        raise "Failed to read config from stdin: #{inspect(reason)}"
 
-        config_file_path()
-        |> File.read!()
-        |> apply_from_yml!()
-
-      true ->
-        Logger.info("No config file YAML or path, skipping config loading")
+      yml when is_binary(yml) ->
+        Logger.info("Received config data, applying...")
+        apply_from_yml!(yml)
     end
 
     :ok
@@ -976,10 +963,6 @@ defmodule Sequin.YamlLoader do
   ## Utilities ##
   ###############
 
-  defp config_file_yaml do
-    Application.get_env(@app, :config_file_yaml)
-  end
-
   defp load_app do
     Application.load(@app)
   end

scripts/start_commands.sh

@@ -47,45 +47,25 @@ migrate() {
 apply_config() {
   echo "Applying config"
 
-  # Get the config file path from the application
-  CONFIG_FILE_PATH=$(./prod/rel/sequin/bin/sequin eval "IO.puts Sequin.YamlLoader.config_file_path()")
-
   # Check if CONFIG_FILE_YAML is provided (base64 encoded YAML)
   if [ -n "${CONFIG_FILE_YAML:-}" ]; then
-    echo "CONFIG_FILE_YAML environment variable found, decoding and using it"
-    YAML_FILENAME="config_from_env.yml"
-    RAW_CONFIG_PATH="${APP_HOME_DIR}/${YAML_FILENAME}"
-
-    # Decode base64 content and write to file
-    echo "${CONFIG_FILE_YAML}" | base64 -d > "${RAW_CONFIG_PATH}"
-    echo "Decoded YAML config to ${RAW_CONFIG_PATH}"
-
-    # Update CONFIG_FILE_PATH to use this file
-    CONFIG_FILE_PATH="${RAW_CONFIG_PATH}"
-
-    # Unset CONFIG_FILE_YAML to prevent the app from attempting to use it directly
+    echo "CONFIG_FILE_YAML environment variable found, decoding and piping to application"
+    echo "${CONFIG_FILE_YAML}" \
+        | base64 -d \
+        | sequin config interpolate - \
+        | ./prod/rel/sequin/bin/sequin eval "Sequin.YamlLoader.apply_from_stdin!"
+    echo "Config applied from environment variable"
     unset CONFIG_FILE_YAML
-    echo "Unset CONFIG_FILE_YAML after processing"
-  fi
-
-  if [ -n "${CONFIG_FILE_PATH}" ] && [ -f "${CONFIG_FILE_PATH}" ]; then
-    echo "Substituting environment variables in ${CONFIG_FILE_PATH}"
-    # Copy to app home directory where the app user has write permissions
-    CONFIG_FILENAME=$(basename "${CONFIG_FILE_PATH}")
-    INTERPOLATED_CONFIG_PATH="${APP_HOME_DIR}/${CONFIG_FILENAME}.interpolated.yml"
-
-    # Use sequin to interpolate environment variables
-    sequin config interpolate "${CONFIG_FILE_PATH}" --output "${INTERPOLATED_CONFIG_PATH}"
-    echo "Environment variable substitution complete in ${INTERPOLATED_CONFIG_PATH}"
-
-    # Update CONFIG_FILE_PATH to use the interpolated file
-    export CONFIG_FILE_PATH="${INTERPOLATED_CONFIG_PATH}"
   else
-    echo "No config file found or path is empty, skipping environment variable substitution"
+    if [ -n "${CONFIG_FILE_PATH}" ] && [ -f "${CONFIG_FILE_PATH}" ]; then
+      echo "Interpolating and applying config from ${CONFIG_FILE_PATH}"
+      sequin config interpolate "${CONFIG_FILE_PATH}" \
+          | ./prod/rel/sequin/bin/sequin eval "Sequin.YamlLoader.apply_from_stdin!"
+      echo "Config applied from file"
+    else
+      echo "No config file found or path is empty, skipping config loading"
+    fi
   fi
-
-  ./prod/rel/sequin/bin/sequin eval "Sequin.YamlLoader.apply!"
-  echo "Config applied"
 }
 
 start_application() {