feat: [DEV-53] Total analysis, additional parse +@

build.sh

@@ -1,4 +1,4 @@
-tag=v0.1.1
+tag=v0.1.3
 
 docker build -t ghcr.io/beaverhouse/ba-data-process:$tag .
 docker push ghcr.io/beaverhouse/ba-data-process:$tag

cmd/process_raid/main.go

@@ -17,6 +17,14 @@ import (
 	"github.com/joho/godotenv"
 )
 
+// RaidListItem represents an item in raids.json
+type RaidListItem struct {
+	ID           string `json:"id"`
+	Name         string `json:"name"`
+	TopLevel     string `json:"top_level"`
+	PartyUpdated bool   `json:"party_updated"`
+}
+
 func main() {
 	if logic.IsLocalEnv() {
 		if err := godotenv.Load(); err != nil {
@@ -33,12 +41,17 @@ func main() {
 
 	queries := postgres.New(pool)
 
-	contentIDs, err := queries.ListContentIDs(context.Background())
+	// Get all contents for raid list
+	contents, err := queries.ListContentsForRaidList(context.Background())
 	if err != nil {
-		log.Fatal(fmt.Errorf("failed to list content IDs: %w", err))
+		log.Fatal(fmt.Errorf("failed to list contents: %w", err))
 	}
 
-	for _, contentID := range contentIDs {
+	// Track party_updated status for each content
+	partyUpdated := make(map[string]bool)
+
+	for _, content := range contents {
+		contentID := content.ContentID
 		log.Printf("\n=== Processing content: %s ===", contentID)
 
 		contentInfo, err := queries.GetContentByID(context.Background(), contentID)
@@ -51,8 +64,10 @@ func main() {
 		partyData, filterResult, err := logic_duckdb.ParseDuckDB(contentID, contentInfo.StartDate.Time)
 		if err != nil {
 			log.Printf("Skipping content %s: %v", contentID, err)
+			partyUpdated[contentID] = false
 			continue
 		}
+		partyUpdated[contentID] = true
 
 		fileName := fmt.Sprintf("%s.json", contentID)
 
@@ -175,5 +190,21 @@ func main() {
 		log.Printf("✓ Successfully processed content: %s\n", contentID)
 	}
 
+	// Generate raids.json
+	log.Println("\n=== Generating raids.json ===")
+	var raidList []RaidListItem
+	for _, content := range contents {
+		raidList = append(raidList, RaidListItem{
+			ID:           content.ContentID,
+			Name:         content.Title,
+			TopLevel:     string(content.TopLevel),
+			PartyUpdated: partyUpdated[content.ContentID],
+		})
+	}
+
+	if err := logic_upload.MarshalAndUpload(raidList, "batorment/v3", "raids.json", *dryRun, "Raids list uploaded"); err != nil {
+		log.Printf("Failed to upload raids.json: %v", err)
+	}
+
 	fmt.Println("\n=== Successfully processed all raids ===")
 }

cmd/total_analysis/main.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+
+	"ba-torment-data-process/internal/db/postgres"
+	"ba-torment-data-process/internal/logic"
+	"ba-torment-data-process/internal/logic/analysis"
+	logic_upload "ba-torment-data-process/internal/logic/upload"
+
+	"github.com/joho/godotenv"
+)
+
+func main() {
+	if logic.IsLocalEnv() {
+		if err := godotenv.Load(); err != nil {
+			log.Fatalf("Failed to load .env file: %v", err)
+		}
+	}
+
+	dryRun := flag.Bool("dry-run", false, "Run in dry-run mode (no actual uploads)")
+	flag.Parse()
+
+	// Initialize database connection
+	pool := postgres.InitFromEnv()
+	defer pool.Close()
+
+	queries := postgres.New(pool)
+
+	// Get all content IDs sorted by start_date
+	contents, err := queries.ListContentIDsWithStartDate(context.Background())
+	if err != nil {
+		log.Fatal(fmt.Errorf("failed to list content IDs: %w", err))
+	}
+
+	// Extract content IDs in order
+	contentIDs := make([]string, len(contents))
+	for i, c := range contents {
+		contentIDs[i] = c.ContentID
+	}
+
+	log.Printf("Found %d content IDs", len(contentIDs))
+
+	// Download all party data
+	log.Println("Downloading party data from S3...")
+	partyDataMap := analysis.DownloadAllPartyData(contentIDs)
+	log.Printf("Successfully downloaded %d/%d party data", len(partyDataMap), len(contentIDs))
+
+	if len(partyDataMap) == 0 {
+		log.Fatal("No party data available for analysis")
+	}
+
+	// Run analysis
+	log.Println("Running total analysis...")
+	result := analysis.RunTotalAnalysis(partyDataMap, contentIDs)
+
+	// Upload result
+	err = logic_upload.MarshalAndUpload(
+		result,
+		"batorment/v3/total-analysis",
+		"analysis.json",
+		*dryRun,
+		"Total analysis completed",
+	)
+	if err != nil {
+		log.Fatalf("Failed to upload analysis result: %v", err)
+	}
+
+	log.Println("Total analysis completed successfully!")
+}

docker-entrypoint.sh

@@ -7,7 +7,7 @@ echo "============================================"
 echo ""
 
 # Step 1: Update SchaleDB data
-echo "[1/2] Updating student data from SchaleDB..."
+echo "[1/3] Updating student data from SchaleDB..."
 echo "--------------------------------------------"
 /app/bin/update_from_schaledb
 if [ $? -eq 0 ]; then
@@ -18,7 +18,7 @@ else
 fi
 
 echo ""
-echo "[2/2] Processing raid data..."
+echo "[2/3] Processing raid data..."
 echo "--------------------------------------------"
 /app/bin/process_raid
 if [ $? -eq 0 ]; then
@@ -28,6 +28,17 @@ else
     exit 1
 fi
 
+echo ""
+echo "[3/3] Running total analysis..."
+echo "--------------------------------------------"
+/app/bin/total_analysis
+if [ $? -eq 0 ]; then
+    echo "✓ Total analysis completed successfully"
+else
+    echo "✗ Total analysis failed with exit code $?"
+    exit 1
+fi
+
 echo ""
 echo "============================================"
 echo "All tasks completed successfully!"

internal/db/postgres/sql/query/contents.sql

@@ -3,3 +3,9 @@ SELECT content_id, start_date FROM batorment_v3.contents WHERE content_id = $1;
 
 -- name: ListContentIDs :many
 SELECT content_id FROM batorment_v3.contents WHERE deleted_at IS NULL;
+
+-- name: ListContentIDsWithStartDate :many
+SELECT content_id, start_date FROM batorment_v3.contents WHERE deleted_at IS NULL ORDER BY start_date ASC;
+
+-- name: ListContentsForRaidList :many
+SELECT content_id, title, top_level FROM batorment_v3.contents WHERE deleted_at IS NULL ORDER BY start_date ASC;

internal/logic/analysis/analysis.go

@@ -0,0 +1,91 @@
+package analysis
+
+import (
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+	"time"
+
+	"ba-torment-data-process/internal/types"
+)
+
+const PartyDataBaseURL = "https://twauaebyyujvvvusbrwe.supabase.co/storage/v1/object/public/pb7h4uvn2b6m0lyu7i6r3j8ac/batorment/v3/party"
+
+// DownloadAllPartyData downloads party data for all content IDs
+func DownloadAllPartyData(contentIDs []string) map[string]*types.BATormentPartyData {
+	result := make(map[string]*types.BATormentPartyData)
+
+	for _, contentID := range contentIDs {
+		url := PartyDataBaseURL + "/" + contentID + ".json"
+		data, err := fetchPartyData(url)
+		if err != nil {
+			log.Printf("Failed to fetch party data for %s: %v", contentID, err)
+			continue
+		}
+
+		var partyData types.BATormentPartyData
+		if err := json.Unmarshal(data, &partyData); err != nil {
+			log.Printf("Failed to parse party data for %s: %v", contentID, err)
+			continue
+		}
+
+		result[contentID] = &partyData
+		log.Printf("Downloaded party data for %s: %d parties", contentID, len(partyData.PartyDetail))
+	}
+
+	return result
+}
+
+// fetchPartyData fetches party data from URL (non-fatal on error)
+func fetchPartyData(url string) ([]byte, error) {
+	start := time.Now()
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return nil, &httpError{StatusCode: resp.StatusCode, URL: url}
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Printf("Fetched: url=%s, duration=%s", url, time.Since(start))
+	return body, nil
+}
+
+type httpError struct {
+	StatusCode int
+	URL        string
+}
+
+func (e *httpError) Error() string {
+	return "HTTP " + string(rune(e.StatusCode)) + " for " + e.URL
+}
+
+// RunTotalAnalysis runs the complete analysis
+// sortedContentIDs provides the order for raidAnalyses (sorted by start_date)
+func RunTotalAnalysis(partyDataMap map[string]*types.BATormentPartyData, sortedContentIDs []string) *types.TotalAnalysisOutput {
+	log.Printf("Starting total analysis with %d raids", len(partyDataMap))
+
+	// Raid analysis
+	log.Println("Running raid analyses...")
+	raidAnalyses := RunRaidAnalyses(partyDataMap, sortedContentIDs)
+	log.Printf("Completed raid analyses: %d raids", len(raidAnalyses))
+
+	// Character analysis
+	log.Println("Running character analyses...")
+	characterAnalyses := RunCharacterAnalyses(partyDataMap, sortedContentIDs)
+	log.Printf("Completed character analyses: %d characters", len(characterAnalyses))
+
+	return &types.TotalAnalysisOutput{
+		GeneratedAt:       time.Now().Format(time.RFC3339),
+		RaidAnalyses:      raidAnalyses,
+		CharacterAnalyses: characterAnalyses,
+	}
+}