diff --git a/cli/cmd/local/dev.go b/cli/cmd/local/dev.go new file mode 100644 index 0000000..1f2aec0 --- /dev/null +++ b/cli/cmd/local/dev.go @@ -0,0 +1,109 @@ +package local + +import ( + _ "embed" + "regexp" + "strings" + + "github.com/charmbracelet/log" + "github.com/spf13/cobra" + "github.com/typestreamio/typestream/cli/pkg/compose" +) + +var devCreating = regexp.MustCompile(`Container typestream-dev-(.*)-1 Creating`) +var devStarted = regexp.MustCompile(`Container typestream-dev-(.*)-1 Started`) +var devHealthy = regexp.MustCompile(`Container typestream-dev-(.*)-1 Healthy`) + +var devStopCmd = &cobra.Command{ + Use: "stop", + Short: "Stop development mode services", + Run: func(cmd *cobra.Command, args []string) { + log.Info("✋ Stopping development services") + runner := compose.NewDevRunner() + go func() { + for m := range runner.StdOut { + log.Info(m) + } + }() + err := runner.RunCommand("down") + if err != nil { + log.Fatalf("💥 failed to stop dev services: %v", err) + } + log.Info("👋 Development services stopped") + }, +} + +var devCmd = &cobra.Command{ + Use: "dev", + Short: "Start TypeStream in development mode (dependencies only)", + Long: `Starts only the infrastructure services (Redpanda, Envoy, UI) in Docker. +The TypeStream server should be run separately on the host for fast iteration. + +After running this command: +1. In another terminal, run: ./gradlew server:run --continuous +2. Edit Kotlin files and watch them reload automatically (~5s) + +Or use the helper script: ./scripts/dev/server.sh + +To stop dev services: ./typestream local dev stop`, + Run: func(cmd *cobra.Command, args []string) { + log.Info("🚀 starting TypeStream in development mode") + log.Info("📦 starting infrastructure services (Redpanda, Envoy, UI)") + + runner := compose.NewDevRunner() + go func() { + log.Info("🐳 starting docker compose") + for m := range runner.StdOut { + if strings.Contains(m, "Error response from daemon") { + log.Error("💥 " + m) + } + if strings.Contains(m, "redpanda Pulling") { + log.Info("📦 downloading redpanda") + log.Info("⏳ this may take a while...") + } + if strings.Contains(m, "redpanda Pulled") { + log.Info("✅ redpanda downloaded") + } + + if devCreating.MatchString(m) { + capture := devCreating.FindStringSubmatch(m) + log.Info("🛫 starting " + capture[1]) + } + + if devStarted.MatchString(m) { + capture := devStarted.FindStringSubmatch(m) + log.Info("✨ " + capture[1] + " started") + } + + if devHealthy.MatchString(m) { + capture := devHealthy.FindStringSubmatch(m) + log.Info("✅ " + capture[1] + " healthy") + } + } + }() + + err := runner.RunCommand("up", "--detach", "--wait", "--force-recreate", "--remove-orphans") + if err != nil { + log.Fatalf("💥 failed to run docker compose: %v", err) + } + + log.Info("✅ infrastructure ready") + log.Info("") + log.Info("📝 Next steps:") + log.Info(" 1. Run the server: ./gradlew server:run --continuous") + log.Info(" 2. Or use the script: ./scripts/dev/server.sh") + log.Info(" 3. Edit Kotlin files and watch auto-reload!") + log.Info("") + log.Info("🌐 Services:") + log.Info(" • React UI: http://localhost:5173") + log.Info(" • Envoy Proxy: http://localhost:8080") + log.Info(" • Kafbat UI: http://localhost:8088") + log.Info(" • Kafka: localhost:19092") + log.Info(" • Schema Reg: http://localhost:18081") + }, +} + +func init() { + devCmd.AddCommand(devStopCmd) + localCmd.AddCommand(devCmd) +} diff --git a/cli/pkg/compose/compose-deps.yml.tmpl b/cli/pkg/compose/compose-deps.yml.tmpl new file mode 100644 index 0000000..5295096 --- /dev/null +++ b/cli/pkg/compose/compose-deps.yml.tmpl @@ -0,0 +1,74 @@ +networks: + typestream_network: + driver: bridge +volumes: + redpanda: null + uiv2_node_modules: null +services: + redpanda: + image: docker.redpanda.com/redpandadata/redpanda:v24.2.9 + command: + - redpanda start + - --smp 1 + - --overprovisioned + - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092 + - --advertise-kafka-addr internal://redpanda:9092,external://localhost:19092 + - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082 + - --advertise-pandaproxy-addr internal://redpanda:8082,external://localhost:18082 + - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081 + - --rpc-addr redpanda:33145 + - --advertise-rpc-addr redpanda:33145 + ports: + - 18081:18081 + - 18082:18082 + - 19092:19092 + - 19644:9644 + volumes: + - redpanda:/var/lib/redpanda/data + networks: + - typestream_network + healthcheck: + test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 5s + # NOTE: server service is NOT included - runs on host for development + kafbat-ui: + container_name: kafbat-ui + image: ghcr.io/kafbat/kafka-ui:latest + ports: + - 8088:8080 + networks: + - typestream_network + environment: + KAFKA_CLUSTERS_0_NAME: typestream_redpanda + KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: redpanda:9092 + KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://redpanda:8081 + envoy: + image: envoyproxy/envoy:v1.28-latest + volumes: + - {{ .ProjectRoot }}/cli/pkg/compose/envoy.yaml:/etc/envoy/envoy.yaml + ports: + - "8080:8080" + networks: + - typestream_network + extra_hosts: + - "host.docker.internal:host-gateway" + uiv2: + image: node:20-alpine + working_dir: /app + volumes: + # Mount uiv2 directory for live reloading + - {{ .ProjectRoot }}/uiv2:/app + # Use named volume for node_modules to avoid platform issues + - uiv2_node_modules:/app/node_modules + ports: + - "5173:5173" + networks: + - typestream_network + command: sh -c "corepack enable && pnpm install --frozen-lockfile --force && pnpm dev --host 0.0.0.0 --port 5173" + depends_on: + - envoy + environment: + - VITE_SERVER_URL=http://localhost:8080 diff --git a/cli/pkg/compose/dev_runner.go b/cli/pkg/compose/dev_runner.go new file mode 100644 index 0000000..c90b8e5 --- /dev/null +++ b/cli/pkg/compose/dev_runner.go @@ -0,0 +1,141 @@ +package compose + +import ( + "bufio" + "bytes" + _ "embed" + "html/template" + "os" + "os/exec" + "path/filepath" + "sync" + + "github.com/charmbracelet/log" + "github.com/typestreamio/typestream/cli/pkg/version" +) + +// devProjectName is the Docker Compose project name for dev mode +// Using a separate name avoids conflicts with production "typestream" project +const devProjectName = "typestream-dev" + +// getProjectRoot returns the absolute path to the typestream project root +func getProjectRoot() string { + // Get the executable path and navigate to project root + exe, err := os.Executable() + if err != nil { + // Fallback to current working directory + cwd, _ := os.Getwd() + return cwd + } + // The CLI binary is at /cli/typestream, so go up one level + return filepath.Dir(filepath.Dir(exe)) +} + +// getDevComposeFilePath returns a fixed path for the dev compose file +// Using a fixed path instead of temp files ensures Docker Compose can track state properly +func getDevComposeFilePath() string { + homeDir, err := os.UserHomeDir() + if err != nil { + homeDir = "/tmp" + } + typestreamDir := filepath.Join(homeDir, ".typestream") + if err := os.MkdirAll(typestreamDir, 0755); err != nil { + log.Warn("Could not create ~/.typestream, using /tmp", "error", err) + return "/tmp/typestream-compose-deps.yml" + } + return filepath.Join(typestreamDir, "compose-deps.yml") +} + +//go:embed compose-deps.yml.tmpl +var composeDepsFile string + +type DevRunner struct { + StdOut chan string +} + +func NewDevRunner() *DevRunner { + return &DevRunner{ + StdOut: make(chan string), + } +} + +func (runner *DevRunner) Show() string { + buf := bytes.Buffer{} + tmpl, err := template.New("compose-deps-template").Parse(composeDepsFile) + if err != nil { + log.Fatal("💥 failed to parse compose-deps template: %v", err) + } + + err = tmpl.Execute(&buf, struct { + Image string + ProjectRoot string + }{ + Image: version.DockerImage("typestream/server"), + ProjectRoot: getProjectRoot(), + }) + if err != nil { + log.Fatal("💥 failed to execute compose-deps template: %v", err) + } + return buf.String() +} + +func (runner *DevRunner) RunCommand(arg ...string) error { + composeFilePath := getDevComposeFilePath() + + // Write the compose file to the fixed location + composeFile, err := os.Create(composeFilePath) + if err != nil { + log.Fatalf("💥 failed to create compose file at %s: %v", composeFilePath, err) + } + + tmpl, err := template.New("compose-deps-template").Parse(composeDepsFile) + if err != nil { + log.Fatal("💥 failed to parse compose-deps template: %v", err) + } + + err = tmpl.Execute(composeFile, struct { + Image string + ProjectRoot string + }{ + Image: version.DockerImage("typestream/server"), + ProjectRoot: getProjectRoot(), + }) + + if err != nil { + log.Fatal("💥 failed to execute compose-deps template: %v", err) + } + + err = composeFile.Close() + if err != nil { + log.Fatalf("💥 failed to close compose file: %v", err) + } + + // Use fixed project name and file path to ensure Docker Compose can track state + args := append([]string{"-p", devProjectName, "-f", composeFilePath}, arg...) + cmd := exec.Command("docker-compose", args...) + + stdErr, err := cmd.StderrPipe() + if err != nil { + log.Fatalf("Failed to get stderr pipe: %v", err) + } + + err = cmd.Start() + if err != nil { + log.Fatalf("Failed to start docker compose: %v", err) + } + + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + scanner := bufio.NewScanner(stdErr) + for scanner.Scan() { + m := scanner.Text() + runner.StdOut <- m + } + }() + + wg.Wait() + + return cmd.Wait() +} diff --git a/cli/pkg/compose/envoy.yaml b/cli/pkg/compose/envoy.yaml index 4160128..2097e3f 100644 --- a/cli/pkg/compose/envoy.yaml +++ b/cli/pkg/compose/envoy.yaml @@ -41,8 +41,9 @@ static_resources: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router clusters: - name: typestream_service - connect_timeout: 0.25s - type: logical_dns + connect_timeout: 5s + type: strict_dns + dns_lookup_family: V4_ONLY lb_policy: round_robin http2_protocol_options: {} load_assignment: @@ -52,5 +53,5 @@ static_resources: - endpoint: address: socket_address: - address: server + address: host.docker.internal port_value: 4242 diff --git a/config/src/main/kotlin/io/typestream/config/Config.kt b/config/src/main/kotlin/io/typestream/config/Config.kt index fdabf80..ec25651 100644 --- a/config/src/main/kotlin/io/typestream/config/Config.kt +++ b/config/src/main/kotlin/io/typestream/config/Config.kt @@ -62,14 +62,23 @@ data class Config( val configFilePath: String = if (envFile != null) { logger.info { "loading configuration from TYPESTREAM_CONFIG" } - val defaultPath = Paths.get(systemConfigPath, "typestream.toml") - Paths.get(systemConfigPath).toFile().mkdir() - val configFile = defaultPath.toFile() - configFile.createNewFile() + val parentDir = Paths.get(systemConfigPath).toFile() + + // Try to create the system config directory, fall back to temp if it fails + val actualConfigPath = if (!parentDir.exists() && !parentDir.mkdirs()) { + logger.warn { "cannot write to $systemConfigPath, using temporary directory" } + val tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "typestream").toFile() + tempDir.mkdirs() + tempDir.absolutePath + } else { + systemConfigPath + } + val configFile = Paths.get(actualConfigPath, "typestream.toml").toFile() + configFile.createNewFile() configFile.writeText(envFile) - systemConfigPath + actualConfigPath } else { val paths = SystemEnv["TYPESTREAM_CONFIG_PATH"] ?: ".:$systemConfigPath" @@ -87,13 +96,23 @@ data class Config( require(fileStream != null) { "default configuration not found" } val defaultPath = Paths.get(systemConfigPath, "typestream.toml") - Paths.get(systemConfigPath).toFile().mkdir() - val configFile = defaultPath.toFile() - configFile.createNewFile() - - configFile.writeText(fileStream.readAllBytes().decodeToString().trimIndent()) - - systemConfigPath + val parentDir = Paths.get(systemConfigPath).toFile() + + // Try to create the system config directory, fall back to temp if it fails + val actualConfigPath = if (!parentDir.exists() && !parentDir.mkdirs()) { + logger.warn { "cannot write to $systemConfigPath, using temporary directory" } + val tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "typestream").toFile() + tempDir.mkdirs() + tempDir.absolutePath + } else { + systemConfigPath + } + + val actualConfigFile = Paths.get(actualConfigPath, "typestream.toml").toFile() + actualConfigFile.createNewFile() + actualConfigFile.writeText(fileStream.readAllBytes().decodeToString().trimIndent()) + + actualConfigPath } } diff --git a/scripts/dev/server.sh b/scripts/dev/server.sh index 072a6a6..71a3f73 100755 --- a/scripts/dev/server.sh +++ b/scripts/dev/server.sh @@ -1,8 +1,17 @@ #!/usr/bin/env bash +# Run TypeStream server locally with hot reload -set -euo pipefail -IFS=$'\n\t' +set -e -script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -TYPESTREAM_CONFIG_PATH=$script_dir ./gradlew server:run -q --console=plain +export TYPESTREAM_CONFIG_PATH="$SCRIPT_DIR" + +cd "$PROJECT_ROOT" + +echo "🚀 Starting TypeStream server with hot reload..." +echo "📝 Config path: $TYPESTREAM_CONFIG_PATH" +echo "" + +./gradlew server:run --continuous diff --git a/scripts/dev/typestream.toml b/scripts/dev/typestream.toml index 21f0d43..7934403 100644 --- a/scripts/dev/typestream.toml +++ b/scripts/dev/typestream.toml @@ -1,6 +1,6 @@ [grpc] port=4242 [sources.kafka.local] -bootstrapServers="localhost:9092" -schemaRegistry.url="http://localhost:8081" +bootstrapServers="localhost:19092" +schemaRegistry.url="http://localhost:18081" fsRefreshRate=10 diff --git a/server/build.gradle.kts b/server/build.gradle.kts index dc8a62c..6c41d58 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -16,6 +16,11 @@ application { mainClass.set("io.typestream.MainKt") } +tasks.named("run") { + // Inherit all environment variables from the shell + environment(System.getenv()) +} + dependencies { implementation(project(":config")) implementation(project(":libs:k8s-client")) @@ -35,6 +40,7 @@ dependencies { testImplementation(libs.bundles.testcontainers) testImplementation(libs.grpc.testing) + testImplementation(libs.mockk) testImplementation(libs.okhttp.mockwebserver) testImplementation(libs.test.containers.redpanda) }