🔧 [fix] Fix Port Mapping Issue (#62)

* 🐛 Fix: Add port initializiation for Docker Compose

This commit adds significant logging for debugging as well as port initialization if a specified port is not found in the existing config file.

* 🚑️ Add port validation before manuscript deployment

This commit validates and assigns ports before deploying manuscripts, ensuring that ports for graphQL , the database, and the manuscript itself are all properly assigned.

* ♻️ [chore] Move Port Verification to Pkg

This commit moves the port verification out of deploy manuscript and into the common_command.go file in the pkg package. The idea would be to gradually migrate common methods to a single source so that the script is easier to follow and debug.

* 🐛 [fix] Fix overlapping ports upon manuscript init

This commit modifies GetListeningPorts() in common_command.go so as to make it more robust. Instead of just looking at  system ports. It evaluates docker container ports and system ports comprehensively to ensure that port conflicts don't occur. It also continues the migration of system functionality to the common_command.go inside pkg package for better encapsulation.

* 🎨 DRY out createDockerComposeFile function

Applied principle of D.R.Y. - don't repeat yourself - to init_manuscripts.go. Now, instead of reusing similar logic to assign ports, it uses the common pkg.InitialzePorts function on the manuscript  to assign new ports. This is a significantly cleaner implementation as it means that we'll only need to change the logic  in one location going forward.

* 🎨 Clean up duplicate FindAvailablePort definition

Due to previous commmits, the FindAvailablePort() is now defined in pkg/common_command.go and no longer needs to be put here in init_manuscript.go. It is now being removed for clarity.

commands/deploy_manuscript.go

@@ -64,19 +64,20 @@ func DeployManuscript(args []string) {
 			}
 			return nil
 		}},
-		{"Step 2: Checking manuscript is already deployed", func() error {
+		{"Step 2: Verifying Port Initialization", func() error { return pkg.InitializePorts(&ms) }},
+		{"Step 3: Checking manuscript is already deployed", func() error {
 			err := CheckManuscriptExist(ms)
 			if err != nil {
 				return err
 			}
 			return nil
 		}},
-		{"Step 3: Create Directory", func() error { return createDirectory(manuscriptDir) }},
-		{"Step 4: Create ManuscriptFile", func() error { return copyManuscriptFile(manuscriptDir, manuscriptPath) }},
-		{"Step 5: Create DockerComposeFile", func() error { return createDockerComposeFile(manuscriptDir, &ms) }},
-		{"Step 6: Check Docker Installed", func() error { return checkDockerInstalled() }},
-		{"Step 7: Start Docker Containers", func() error { return startDockerContainers(manuscriptDir) }},
-		{"Step 8: Check Container Status", func() error { return checkContainerStatus(&ms) }},
+		{"Step 4: Create Directory", func() error { return createDirectory(manuscriptDir) }},
+		{"Step 5: Create ManuscriptFile", func() error { return copyManuscriptFile(manuscriptDir, manuscriptPath) }},
+		{"Step 6: Create DockerComposeFile", func() error { return createDockerComposeFile(manuscriptDir, &ms) }},
+		{"Step 7: Check Docker Installed", func() error { return checkDockerInstalled() }},
+		{"Step 8: Start Docker Containers", func() error { return startDockerContainers(manuscriptDir) }},
+		{"Step 9: Check Container Status", func() error { return checkContainerStatus(&ms) }},
 	}
 
 	for _, step := range steps {

commands/init_manuscript.go

@@ -147,49 +147,50 @@ func createManuscriptFile(dir string, ms pkg.Manuscript) error {
 }
 
 func createDockerComposeFile(dir string, ms *pkg.Manuscript) error {
+	fmt.Printf("Debug: Creating docker-compose file in directory: %s\n", dir)
 	composeFilePath := filepath.Join(dir, "docker-compose.yml")
 	dockComposeTemplate := static.DockerComposeTemplate
 	switch ms.Sink {
 	case defaultSink:
 		m, err := pkg.LoadConfig(manuscriptConfig)
 		if err != nil {
-			return fmt.Errorf("failed to load manuscript config: %w", err)
+			fmt.Printf("Debug: Creating new config as none exists or error loading: %v\n", err)
+			m = &pkg.Config{
+				BaseDir:     getHomeDir(),
+				Manuscripts: []pkg.Manuscript{},
+			}
 		}
 
 		ms.GraphQLImage = graphQLImage
 		if runtime.GOARCH == "arm64" || runtime.GOARCH == "arm" {
 			ms.GraphQLImage = graphQLARMImage
 		}
 
-		var excludePorts []int
-		for _, m := range m.Manuscripts {
-			if m.Name == ms.Name {
-				ms.Port = m.Port
-				ms.DbPort = m.DbPort
-				ms.DbUser = m.DbUser
-				ms.DbPassword = m.DbPassword
-				ms.GraphQLPort = m.GraphQLPort
-				dockComposeTemplate = static.DockerComposeWithPostgresqlContent
-				return createTemplateFile(composeFilePath, dockComposeTemplate, ms)
+		// Check if manuscript already exists in config
+		var existingMs *pkg.Manuscript
+		for _, manuscript := range m.Manuscripts {
+			if manuscript.Name == ms.Name {
+				existingMs = &manuscript
+				break
 			}
-			excludePorts = append(excludePorts, m.Port, m.GraphQLPort)
-		}
-		port, err := FindAvailablePort(8081, 8181, excludePorts)
-		if err != nil {
-			return fmt.Errorf("failed to find available port: %w", err)
-		}
-		ms.Port = port
-		excludePorts = append(excludePorts, port)
-		graphQLPort, err := FindAvailablePort(8081, 8181, excludePorts)
-		if err != nil {
-			return fmt.Errorf("failed to find available port: %w", err)
 		}
-		dbPort, err := FindAvailablePort(15432, 15532, excludePorts)
-		if err != nil {
-			return fmt.Errorf("failed to find available port: %w", err)
+
+		if existingMs != nil {
+			// Use existing ports if manuscript is already configured
+			ms.Port = existingMs.Port
+			ms.DbPort = existingMs.DbPort
+			ms.DbUser = existingMs.DbUser
+			ms.DbPassword = existingMs.DbPassword
+			ms.GraphQLPort = existingMs.GraphQLPort
+		} else {
+			// Initialize new ports using the common function
+			if err := pkg.InitializePorts(ms); err != nil {
+				return fmt.Errorf("failed to initialize ports: %w", err)
+			}
 		}
-		ms.DbPort = dbPort
-		ms.GraphQLPort = graphQLPort
+
+		fmt.Printf("Debug: Using ports - Flink: %d, GraphQL: %d, DB: %d\n",
+			ms.Port, ms.GraphQLPort, ms.DbPort)
 		dockComposeTemplate = static.DockerComposeWithPostgresqlContent
 	default:
 	}
@@ -295,29 +296,6 @@ func createTemplateFile(filePath, tmplContent string, data interface{}) error {
 	return nil
 }
 
-func FindAvailablePort(startPort, endPort int, exclude []int) (int, error) {
-	listeningPorts, err := pkg.GetListeningPorts()
-	if err != nil {
-		return 0, err
-	}
-
-	portMap := make(map[int]bool)
-	for _, port := range listeningPorts {
-		portMap[port] = true
-	}
-	for _, port := range exclude {
-		portMap[port] = true
-	}
-
-	for port := startPort; port <= endPort; port++ {
-		if !portMap[port] {
-			return port, nil
-		}
-	}
-
-	return 0, fmt.Errorf("no available ports in the range %d-%d", startPort, endPort)
-}
-
 func promptInput(prompt, defaultVal string) string {
 	fmt.Printf("\r\033[33m%s\u001B[0m", prompt)
 	reader := bufio.NewReader(os.Stdin)

pkg/common_command.go

@@ -3,39 +3,123 @@ package pkg
 import (
 	"bufio"
 	"bytes"
+	"fmt"
 	"os/exec"
 	"regexp"
 	"strconv"
 )
 
 func GetListeningPorts() ([]int, error) {
+	ports := make(map[int]bool)
+
+	// Check system ports using lsof
 	cmd := exec.Command("lsof", "-nP", "-iTCP", "-sTCP:LISTEN")
 	var out bytes.Buffer
 	cmd.Stdout = &out
 
 	if err := cmd.Run(); err != nil {
-		return nil, err
+		// Don't return error here, continue to check Docker ports
+		fmt.Printf("Warning: Unable to check system ports: %v\n", err)
+	} else {
+		re := regexp.MustCompile(`:(\d+)\s+\(LISTEN\)`)
+		scanner := bufio.NewScanner(&out)
+		for scanner.Scan() {
+			line := scanner.Text()
+			matches := re.FindStringSubmatch(line)
+			if len(matches) > 1 {
+				port, err := strconv.Atoi(matches[1])
+				if err != nil {
+					continue
+				}
+				ports[port] = true
+			}
+		}
 	}
 
-	re := regexp.MustCompile(`:(\d+)\s+\(LISTEN\)`)
+	// Check Docker container ports
+	dockerCmd := exec.Command("docker", "ps", "--format", "{{.Ports}}")
+	var dockerOut bytes.Buffer
+	dockerCmd.Stdout = &dockerOut
 
-	var ports []int
-	scanner := bufio.NewScanner(&out)
+	if err := dockerCmd.Run(); err != nil {
+		return nil, fmt.Errorf("failed to check Docker ports: %w", err)
+	}
+
+	// Parse Docker port mappings
+	scanner := bufio.NewScanner(&dockerOut)
+	portRegex := regexp.MustCompile(`0\.0\.0\.0:(\d+)`)
 	for scanner.Scan() {
 		line := scanner.Text()
-		matches := re.FindStringSubmatch(line)
-		if len(matches) > 1 {
-			port, err := strconv.Atoi(matches[1])
-			if err != nil {
-				continue
+		matches := portRegex.FindAllStringSubmatch(line, -1)
+		for _, match := range matches {
+			if len(match) > 1 {
+				port, err := strconv.Atoi(match[1])
+				if err != nil {
+					continue
+				}
+				ports[port] = true
 			}
-			ports = append(ports, port)
 		}
 	}
 
-	if err := scanner.Err(); err != nil {
-		return nil, err
+	// Convert map to slice
+	var result []int
+	for port := range ports {
+		result = append(result, port)
+	}
+	return result, nil
+}
+
+func InitializePorts(ms *Manuscript) error {
+	// Initialize Flink port if not set
+	if ms.Port == 0 {
+		port, err := FindAvailablePort(8081, 8181, nil)
+		if err != nil {
+			return fmt.Errorf("failed to find available port for Flink: %w", err)
+		}
+		ms.Port = port
+	}
+
+	// Initialize GraphQL port if not set
+	if ms.GraphQLPort == 0 {
+		graphQLPort, err := FindAvailablePort(8082, 8182, []int{ms.Port})
+		if err != nil {
+			return fmt.Errorf("failed to find available port for GraphQL: %w", err)
+		}
+		ms.GraphQLPort = graphQLPort
+	}
+
+	// Initialize DB port if not set
+	if ms.DbPort == 0 {
+		dbPort, err := FindAvailablePort(15432, 15532, []int{ms.Port, ms.GraphQLPort})
+		if err != nil {
+			return fmt.Errorf("failed to find available port for DB: %w", err)
+		}
+		ms.DbPort = dbPort
+	}
+
+	return nil
+}
+
+func FindAvailablePort(startPort, endPort int, exclude []int) (int, error) {
+	listeningPorts, err := GetListeningPorts()
+	if err != nil {
+		return 0, err
+	}
+
+	portMap := make(map[int]bool)
+	for _, port := range listeningPorts {
+		portMap[port] = true
+	}
+	for _, port := range exclude {
+		portMap[port] = true
+	}
+
+	for port := startPort; port <= endPort; port++ {
+		if !portMap[port] {
+			return port, nil
+		}
 	}
 
-	return ports, nil
+	return 0, fmt.Errorf("no available ports in the range %d-%d", startPort, endPort)
 }