diff --git a/docs/user-guide/cli-reference.md b/docs/user-guide/cli-reference.md index b397ea17..49dd549b 100644 --- a/docs/user-guide/cli-reference.md +++ b/docs/user-guide/cli-reference.md @@ -16,6 +16,10 @@ Available for all commands: |---------|-------------| | `codanna init` | Set up .codanna directory with default configuration | | `codanna index` | Build searchable index from codebase | +| `codanna add-folder` | Add a folder to be indexed | +| `codanna remove-folder` | Remove a folder from indexed paths | +| `codanna list-folders` | List all folders that are being indexed | +| `codanna clean` | Remove symbols from folders no longer in indexed paths | | `codanna retrieve` | Query symbols, relationships, and dependencies | | `codanna serve` | Start MCP server | | `codanna config` | Display active settings | @@ -33,11 +37,12 @@ Set up .codanna directory with default configuration **Options:** - `-f, --force` - Force overwrite existing configuration -`codanna index ` +`codanna index [PATHS...]` Build searchable index from codebase **Arguments:** -- `` - Path to file or directory to index +- `[PATHS...]` - Paths to files or directories to index (multiple paths allowed) +- If no paths provided, uses `indexed_paths` from configuration (must be configured via `add-folder`) **Options:** - `-t, --threads ` - Number of threads to use (overrides config) @@ -46,6 +51,118 @@ Build searchable index from codebase - `--dry-run` - Dry run - show what would be indexed without indexing - `--max-files ` - Maximum number of files to index +**Examples:** +```bash +# Index a single directory +codanna index src --progress + +# Index multiple directories at once +codanna index src lib tests --progress + +# Use configured indexed paths +codanna index --progress +``` + +**Behavior:** +- Accepts multiple paths for indexing in a single operation +- When run without arguments, uses folders from `indexed_paths` configuration +- Automatically cleans up symbols from removed folders when using configuration +- Backward compatible with single-path usage + +`codanna add-folder ` +Add a folder to the indexed paths list + +**Arguments:** +- `` - Path to folder to add + +**Examples:** +```bash +# Add a folder to be indexed +codanna add-folder /path/to/project + +# Add multiple folders +codanna add-folder src +codanna add-folder lib +codanna add-folder tests +``` + +**Behavior:** +- Adds folder to `indexed_paths` in configuration +- Saves configuration to `.codanna/settings.toml` +- Paths are canonicalized to absolute paths +- Prevents duplicate entries +- Does not automatically index the folder (run `codanna index` after) + +`codanna remove-folder ` +Remove a folder from the indexed paths list + +**Arguments:** +- `` - Path to folder to remove + +**Examples:** +```bash +# Remove a folder from indexed paths +codanna remove-folder /path/to/old-project + +# Remove by relative path (will be canonicalized) +codanna remove-folder tests +``` + +**Behavior:** +- Removes folder from `indexed_paths` in configuration +- Saves configuration to `.codanna/settings.toml` +- Does not automatically clean symbols (run `codanna clean` or `codanna index` after) +- Path must exist in configuration or error is returned + +`codanna list-folders` +List all folders that are being indexed + +**Examples:** +```bash +# List all indexed folders +codanna list-folders +``` + +**Output:** +``` +Indexed folders: + - /path/to/project1 + - /path/to/project2 + - /path/to/project3 +``` + +Or if none configured: +``` +Indexed folders: + (none configured) + +To add folders: codanna add-folder +``` + +**Behavior:** +- Displays all folders in `indexed_paths` configuration +- Shows helpful message if empty +- Useful for verifying configuration state + +`codanna clean` +Remove symbols from folders no longer in indexed paths + +**Examples:** +```bash +# Clean up symbols from removed folders +codanna clean +``` + +**Behavior:** +- Compares current `indexed_paths` with files in index +- Removes symbols from files not under any configured folder +- Reports number of files cleaned +- Saves updated index +- Safe to run multiple times (idempotent) +- Requires `indexed_paths` to be configured + +**Note:** Running `codanna index` automatically performs cleanup, so this command is optional in most workflows. + `codanna retrieve ` Query indexed symbols, relationships, and dependencies diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 33fea4e7..bc0cfdad 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -112,6 +112,109 @@ threads = 8 # Number of threads for parallel indexing max_file_size_mb = 10 # Skip files larger than this ``` +## Multi-Folder Indexing + +Index multiple directories simultaneously with persistent configuration. + +### Configuration + +```toml +[indexing] +indexed_paths = [ + "/absolute/path/to/project1", + "/absolute/path/to/project2", + "/absolute/path/to/project3" +] +``` + +### Managing Indexed Folders + +**Add folders:** +```bash +# Add individual folders +codanna add-folder /path/to/project1 +codanna add-folder /path/to/project2 + +# Or use relative paths (will be converted to absolute) +codanna add-folder src +codanna add-folder lib +``` + +**List configured folders:** +```bash +codanna list-folders +``` + +**Remove a folder:** +```bash +codanna remove-folder /path/to/project1 +``` + +**Clean up symbols from removed folders:** +```bash +codanna clean +``` + +### Usage Examples + +**Multi-project workspace:** +```bash +# Configure folders +codanna add-folder ~/workspace/project-api +codanna add-folder ~/workspace/project-web +codanna add-folder ~/workspace/shared-lib + +# Index all configured folders +codanna index --progress +``` + +**Monorepo support:** +```bash +# Index specific packages +codanna add-folder packages/backend +codanna add-folder packages/frontend +codanna add-folder packages/shared +codanna index --progress +``` + +**Selective indexing:** +```bash +# Index only specific directories +codanna index src lib tests --progress + +# Or configure for repeated indexing +codanna add-folder src +codanna add-folder lib +codanna index --progress # Uses configured paths +``` + +### Behavior + +**Default behavior:** +- If no `indexed_paths` configured, `codanna index` requires explicit path arguments (backward compatible) +- Paths are stored as canonical absolute paths +- Duplicate paths are automatically prevented + +**Automatic cleanup:** +- Running `codanna index` without arguments uses configured paths +- Automatically removes symbols from folders no longer in configuration +- Manual cleanup available via `codanna clean` command + +**Path canonicalization:** +- Relative paths are converted to absolute +- Symlinks are resolved to actual paths +- Prevents duplicate entries for the same folder + +### Use Cases + +**Multi-project workspaces** - Index multiple related projects together for cross-project symbol resolution + +**Monorepo support** - Index different components separately while maintaining cross-references + +**Selective indexing** - Only index specific directories within large codebases + +**Dynamic workflows** - Add and remove folders as your project structure changes + ## Ignore Patterns Codanna respects `.gitignore` and adds its own `.codannaignore`: diff --git a/src/config.rs b/src/config.rs index 8cc7022f..7821dfa5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -20,7 +20,7 @@ use figment::{ }; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::OnceLock; #[derive(Debug, Deserialize, Serialize, Clone)] @@ -94,6 +94,11 @@ pub struct IndexingConfig { /// Patterns to ignore during indexing #[serde(default)] pub ignore_patterns: Vec, + + /// List of directories to index + /// This list is managed by the add-folder and remove-folder commands + #[serde(default)] + pub indexed_paths: Vec, } #[derive(Debug, Deserialize, Serialize, Clone)] @@ -289,6 +294,7 @@ impl Default for IndexingConfig { ".git/**".to_string(), "*.generated.*".to_string(), ], + indexed_paths: Vec::new(), } } } @@ -539,7 +545,7 @@ impl Settings { /// Find the workspace root by looking for .codanna directory /// Searches from current directory up to root - fn find_workspace_config() -> Option { + pub fn find_workspace_config() -> Option { let current = std::env::current_dir().ok()?; let local_dir = crate::init::local_dir_name(); @@ -752,6 +758,11 @@ impl Settings { result.push_str("# Exponential backoff: 100ms, 200ms, 400ms delays\n"); } else if line.starts_with("ignore_patterns = ") { result.push_str("\n# Additional patterns to ignore during indexing\n"); + } else if line.starts_with("indexed_paths = ") { + result.push_str("\n# List of directories to index\n"); + result.push_str("# Add folders using: codanna add-folder \n"); + result.push_str("# Remove folders using: codanna remove-folder \n"); + result.push_str("# List all folders using: codanna list-folders\n"); } else if line == "[mcp]" { result.push_str("\n[mcp]\n"); prev_line_was_section = true; @@ -923,6 +934,58 @@ __pycache__/ Ok(()) } + + /// Add a folder to the list of indexed paths + pub fn add_indexed_path(&mut self, path: PathBuf) -> Result<(), String> { + // Canonicalize the path to avoid duplicates + let canonical_path = path + .canonicalize() + .map_err(|e| format!("Invalid path: {e}"))?; + + // Check if path already exists + for existing in &self.indexing.indexed_paths { + if let Ok(existing_canonical) = existing.canonicalize() { + if existing_canonical == canonical_path { + return Err(format!("Path already indexed: {}", path.display())); + } + } + } + + // Add the path + self.indexing.indexed_paths.push(canonical_path); + Ok(()) + } + + /// Remove a folder from the list of indexed paths + pub fn remove_indexed_path(&mut self, path: &Path) -> Result<(), String> { + let canonical_path = path + .canonicalize() + .map_err(|e| format!("Invalid path: {e}"))?; + + let original_len = self.indexing.indexed_paths.len(); + self.indexing.indexed_paths.retain(|p| { + if let Ok(existing_canonical) = p.canonicalize() { + existing_canonical != canonical_path + } else { + true + } + }); + + if self.indexing.indexed_paths.len() == original_len { + return Err(format!( + "Path not found in indexed paths: {}", + path.display() + )); + } + + Ok(()) + } + + /// Get all indexed paths + /// Returns empty vector if none are configured (maintains backward compatibility) + pub fn get_indexed_paths(&self) -> Vec { + self.indexing.indexed_paths.clone() + } } /// Global check for whether debug logging is enabled. @@ -1151,4 +1214,165 @@ enabled = true ); println!("=== TEST PASSED ==="); } + + #[test] + fn test_add_indexed_path() { + let temp_dir = TempDir::new().unwrap(); + let test_folder = temp_dir.path().join("test_folder"); + fs::create_dir(&test_folder).unwrap(); + + let mut settings = Settings::default(); + + // Add a path + assert!(settings.add_indexed_path(test_folder.clone()).is_ok()); + assert_eq!(settings.indexing.indexed_paths.len(), 1); + + // Try to add the same path again - should fail + let result = settings.add_indexed_path(test_folder.clone()); + assert!(result.is_err()); + assert_eq!(settings.indexing.indexed_paths.len(), 1); + } + + #[test] + fn test_remove_indexed_path() { + let temp_dir = TempDir::new().unwrap(); + let test_folder = temp_dir.path().join("test_folder"); + fs::create_dir(&test_folder).unwrap(); + + let mut settings = Settings::default(); + + // Add a path + settings.add_indexed_path(test_folder.clone()).unwrap(); + assert_eq!(settings.indexing.indexed_paths.len(), 1); + + // Remove the path + assert!(settings.remove_indexed_path(&test_folder).is_ok()); + assert_eq!(settings.indexing.indexed_paths.len(), 0); + + // Try to remove it again - should fail + let result = settings.remove_indexed_path(&test_folder); + assert!(result.is_err()); + } + + #[test] + fn test_multiple_indexed_paths() { + let temp_dir = TempDir::new().unwrap(); + let folder1 = temp_dir.path().join("folder1"); + let folder2 = temp_dir.path().join("folder2"); + let folder3 = temp_dir.path().join("folder3"); + + fs::create_dir(&folder1).unwrap(); + fs::create_dir(&folder2).unwrap(); + fs::create_dir(&folder3).unwrap(); + + let mut settings = Settings::default(); + + // Add multiple paths + settings.add_indexed_path(folder1.clone()).unwrap(); + settings.add_indexed_path(folder2.clone()).unwrap(); + settings.add_indexed_path(folder3.clone()).unwrap(); + + assert_eq!(settings.indexing.indexed_paths.len(), 3); + + // Remove one path + settings.remove_indexed_path(&folder2).unwrap(); + assert_eq!(settings.indexing.indexed_paths.len(), 2); + + // Verify the right paths remain + let canonical_folder1 = folder1.canonicalize().unwrap(); + let canonical_folder3 = folder3.canonicalize().unwrap(); + + let remaining_paths: Vec<_> = settings + .indexing + .indexed_paths + .iter() + .filter_map(|p| p.canonicalize().ok()) + .collect(); + + assert!(remaining_paths.contains(&canonical_folder1)); + assert!(remaining_paths.contains(&canonical_folder3)); + } + + #[test] + fn test_get_indexed_paths_with_default() { + let settings = Settings::default(); + + // Should return empty vector when no paths configured (backward compatible) + let paths = settings.get_indexed_paths(); + assert_eq!(paths.len(), 0); + } + + #[test] + fn test_get_indexed_paths_with_configured_paths() { + let temp_dir = TempDir::new().unwrap(); + let test_folder = temp_dir.path().join("test_folder"); + fs::create_dir(&test_folder).unwrap(); + + let mut settings = Settings::default(); + settings.add_indexed_path(test_folder.clone()).unwrap(); + + // Should return the configured paths + let paths = settings.get_indexed_paths(); + assert_eq!(paths.len(), 1); + + let canonical_test = test_folder.canonicalize().unwrap(); + let canonical_returned = paths[0].canonicalize().unwrap(); + assert_eq!(canonical_returned, canonical_test); + } + + #[test] + fn test_indexed_paths_from_toml() { + let temp_dir = TempDir::new().unwrap(); + let config_path = temp_dir.path().join("settings.toml"); + let test_folder1 = temp_dir.path().join("src"); + let test_folder2 = temp_dir.path().join("lib"); + + fs::create_dir(&test_folder1).unwrap(); + fs::create_dir(&test_folder2).unwrap(); + + // Convert paths to strings with forward slashes for TOML compatibility + let path1_str = test_folder1.display().to_string().replace('\\', "/"); + let path2_str = test_folder2.display().to_string().replace('\\', "/"); + + let toml_content = format!( + r#" +version = 1 + +[indexing] +indexed_paths = ["{path1_str}", "{path2_str}"] +"# + ); + + fs::write(&config_path, toml_content).unwrap(); + + let settings = Settings::load_from(&config_path).unwrap(); + assert_eq!(settings.indexing.indexed_paths.len(), 2); + assert_eq!(settings.indexing.indexed_paths[0], test_folder1); + assert_eq!(settings.indexing.indexed_paths[1], test_folder2); + } + + #[test] + fn test_save_indexed_paths_to_toml() { + let temp_dir = TempDir::new().unwrap(); + let config_path = temp_dir.path().join("settings.toml"); + let test_folder = temp_dir.path().join("test_folder"); + + fs::create_dir(&test_folder).unwrap(); + + let mut settings = Settings::default(); + settings.add_indexed_path(test_folder.clone()).unwrap(); + + // Save to file + settings.save(&config_path).unwrap(); + + // Load from file and verify + let loaded_settings = Settings::load_from(&config_path).unwrap(); + assert_eq!(loaded_settings.indexing.indexed_paths.len(), 1); + + let canonical_test = test_folder.canonicalize().unwrap(); + let canonical_loaded = loaded_settings.indexing.indexed_paths[0] + .canonicalize() + .unwrap(); + assert_eq!(canonical_loaded, canonical_test); + } } diff --git a/src/indexing/simple.rs b/src/indexing/simple.rs index 60c3a38d..a2d57cf2 100644 --- a/src/indexing/simple.rs +++ b/src/indexing/simple.rs @@ -93,6 +93,8 @@ pub struct SimpleIndexer { file_languages: std::collections::HashMap, /// Language behaviors with persistent state (imports, etc.) file_behaviors: std::collections::HashMap>, + /// Indexed folder paths (canonicalized) to track which folders are currently indexed + indexed_folders: std::collections::HashSet, } impl Default for SimpleIndexer { @@ -144,6 +146,7 @@ impl SimpleIndexer { semantic_search: None, file_languages: std::collections::HashMap::new(), file_behaviors: std::collections::HashMap::new(), + indexed_folders: std::collections::HashSet::new(), }; // Try to load symbol cache for fast lookups @@ -187,6 +190,7 @@ impl SimpleIndexer { semantic_search: None, file_languages: std::collections::HashMap::new(), file_behaviors: std::collections::HashMap::new(), + indexed_folders: std::collections::HashSet::new(), }; // Resolution system now handled through LanguageBehavior: @@ -1937,6 +1941,87 @@ impl SimpleIndexer { }) } + /// Track a folder as indexed (stores canonicalized path) + pub fn add_indexed_folder(&mut self, folder_path: &Path) -> IndexResult<()> { + let canonical = folder_path + .canonicalize() + .map_err(|e| IndexError::FileRead { + path: folder_path.to_path_buf(), + source: e, + })?; + self.indexed_folders.insert(canonical); + Ok(()) + } + + /// Get all currently tracked indexed folders + pub fn get_indexed_folders(&self) -> &std::collections::HashSet { + &self.indexed_folders + } + + /// Remove symbols from folders that are no longer in the indexed folders list + pub fn clean_removed_folders(&mut self, current_folders: &[PathBuf]) -> IndexResult { + // Canonicalize current folders + let canonical_folders: std::collections::HashSet = current_folders + .iter() + .filter_map(|p| p.canonicalize().ok()) + .collect(); + + // Get all indexed file paths + let all_files = self.get_all_indexed_paths(); + + let mut removed_count = 0; + + // Find files that are no longer under any of the current folders + for file_path in all_files { + let file_canonical = match file_path.canonicalize() { + Ok(p) => p, + Err(_) => continue, // File may have been deleted + }; + + // Check if this file is under any of the current folders + let is_under_current_folder = canonical_folders + .iter() + .any(|folder| file_canonical.starts_with(folder)); + + if !is_under_current_folder { + // File is from a removed folder, delete all its documents (file, symbols, relationships, imports) + if let Err(e) = self + .document_index + .remove_file_documents(&file_path.to_string_lossy()) + { + debug_print!( + self, + "Failed to remove documents for {}: {}", + file_path.display(), + e + ); + } else { + debug_print!(self, "Removed all documents from {}", file_path.display()); + removed_count += 1; + } + } + } + + // Commit the deletions to Tantivy + if removed_count > 0 { + if let Err(e) = self.document_index.commit_batch() { + debug_print!(self, "Failed to commit deletions: {}", e); + return Err(IndexError::PersistenceError { + path: self.settings.index_path.clone(), + source: Box::new(e), + }); + } + + // Don't start a new batch here - let the next indexing operation do that. + // This ensures the reader has fully reloaded before any new operations. + } + + // Update tracked folders + self.indexed_folders = canonical_folders; + + Ok(removed_count) + } + /// Search documentation using natural language query /// Returns symbols with their similarity scores, sorted by relevance pub fn semantic_search_docs( diff --git a/src/main.rs b/src/main.rs index 0ffd0044..34e8a7f1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -70,10 +70,12 @@ fn create_custom_help() -> String { } else { help.push_str(&format!("{}\n", style("Quick Start:").cyan().bold())); } - help.push_str(" $ codanna init # Initialize in current directory\n"); - help.push_str(" $ codanna index src # Index your source code\n"); - help.push_str(" $ codanna serve --http --watch # HTTP server with OAuth\n"); - help.push_str(" $ codanna serve --https --watch # HTTPS server with TLS\n\n"); + help.push_str(" $ codanna init # Initialize in current directory\n"); + help.push_str(" $ codanna index src lib # Index multiple directories\n"); + help.push_str(" $ codanna add-folder tests # Add tests folder to indexed paths\n"); + help.push_str(" $ codanna list-folders # List all indexed folders\n"); + help.push_str(" $ codanna serve --http --watch # HTTP server with OAuth\n"); + help.push_str(" $ codanna serve --https --watch # HTTPS server with TLS\n\n"); // About section help.push_str("Index code and query relationships, symbols, and dependencies.\n\n"); @@ -92,17 +94,21 @@ fn create_custom_help() -> String { } else { help.push_str(&format!("{}\n", style("Commands:").cyan().bold())); } - help.push_str(" init Set up .codanna directory\n"); - help.push_str(" index Build searchable index from codebase\n"); - help.push_str(" retrieve Query symbols, relationships, and dependencies\n"); - help.push_str(" serve Start MCP server\n"); - help.push_str(" config Display active settings\n"); - help.push_str(" mcp-test Test MCP connection\n"); - help.push_str(" mcp Execute MCP tools directly\n"); - help.push_str(" benchmark Benchmark parser performance\n"); - help.push_str(" parse Output AST nodes in JSONL format\n"); - help.push_str(" plugin Manage Claude Code plugins\n"); - help.push_str(" help Print this message or the help of the given subcommand(s)\n\n"); + help.push_str(" init Set up .codanna directory\n"); + help.push_str(" index Build searchable index from codebase\n"); + help.push_str(" add-folder Add a folder to be indexed\n"); + help.push_str(" remove-folder Remove a folder from indexed paths\n"); + help.push_str(" list-folders List all folders that are being indexed\n"); + help.push_str(" clean Clean up symbols from removed folders\n"); + help.push_str(" retrieve Query symbols, relationships, and dependencies\n"); + help.push_str(" serve Start MCP server\n"); + help.push_str(" config Display active settings\n"); + help.push_str(" mcp-test Test MCP connection\n"); + help.push_str(" mcp Execute MCP tools directly\n"); + help.push_str(" benchmark Benchmark parser performance\n"); + help.push_str(" parse Output AST nodes in JSONL format\n"); + help.push_str(" plugin Manage Claude Code plugins\n"); + help.push_str(" help Print this message or the help of the given subcommand(s)\n\n"); help.push_str("See 'codanna help ' for more information on a specific command.\n\n"); @@ -166,8 +172,9 @@ enum Commands { /// Index source files or directories #[command(about = "Build searchable index from codebase")] Index { - /// Path to file or directory to index - path: PathBuf, + /// Paths to files or directories to index (multiple paths allowed) + #[arg(value_name = "PATH")] + paths: Vec, /// Number of threads to use (overrides config) #[arg(short, long)] @@ -190,6 +197,28 @@ enum Commands { max_files: Option, }, + /// Add a folder to the indexed paths list + #[command(about = "Add a folder to be indexed")] + AddFolder { + /// Path to folder to add + path: PathBuf, + }, + + /// Remove a folder from the indexed paths list + #[command(about = "Remove a folder from indexed paths")] + RemoveFolder { + /// Path to folder to remove + path: PathBuf, + }, + + /// List all indexed folders + #[command(about = "List all folders that are being indexed")] + ListFolders, + + /// Clean up symbols from removed folders + #[command(about = "Remove symbols from folders no longer in indexed paths")] + Clean, + /// Query code relationships and dependencies #[command( about = "Search symbols, find callers/callees, analyze impact", @@ -829,8 +858,7 @@ async fn main() { let mut indexer = if skip_index_load { SimpleIndexer::with_settings(settings.clone()) // Empty indexer, won't be used } else { - let force_recreate_index = - matches!(cli.command, Commands::Index { force: true, ref path, .. } if path.is_dir()); + let force_recreate_index = matches!(cli.command, Commands::Index { force: true, ref paths, .. } if !paths.is_empty() && paths.iter().all(|p| p.is_dir())); if persistence.exists() && !force_recreate_index { if config.debug { eprintln!( @@ -1104,205 +1132,332 @@ async fn main() { } Commands::Index { - path, + paths, force, progress, dry_run, max_files, .. } => { - // Determine if path is a file or directory - if path.is_file() { - // Initialize project resolution providers even for single file - // (needed for proper alias resolution) - let provider_registry = create_provider_registry(); - if let Err(e) = initialize_providers(&provider_registry, &config) { - eprintln!("\n{e}"); - - // Display recovery suggestions - let suggestions = e.recovery_suggestions(); - if !suggestions.is_empty() { - eprintln!("\nRecovery steps:"); - for suggestion in suggestions { - eprintln!(" • {suggestion}"); - } + // If no paths provided, use indexed_paths from config + let paths_to_index = if paths.is_empty() { + config.get_indexed_paths() + } else { + paths.clone() + }; + + if paths_to_index.is_empty() { + eprintln!("Error: No paths specified and no indexed paths configured"); + eprintln!("Usage: codanna index [...]"); + eprintln!(" or: codanna add-folder to configure indexed paths"); + std::process::exit(1); + } + + // Initialize project resolution providers before indexing + let provider_registry = create_provider_registry(); + if let Err(e) = initialize_providers(&provider_registry, &config) { + eprintln!("\n{e}"); + + // Display recovery suggestions + let suggestions = e.recovery_suggestions(); + if !suggestions.is_empty() { + eprintln!("\nRecovery steps:"); + for suggestion in suggestions { + eprintln!(" • {suggestion}"); } + } - // Exit with ConfigError code - use codanna::io::ExitCode; - let exit_code = ExitCode::from_error(&e); - std::process::exit(exit_code as i32); + // Exit with ConfigError code + use codanna::io::ExitCode; + let exit_code = ExitCode::from_error(&e); + std::process::exit(exit_code as i32); + } + + // Clean up symbols from removed folders if using configured paths + if paths.is_empty() && !config.indexing.indexed_paths.is_empty() { + match indexer.clean_removed_folders(&config.indexing.indexed_paths) { + Ok(count) if count > 0 => { + println!("Cleaned {count} files from removed folders"); + } + Ok(_) => {} // No files removed + Err(e) => eprintln!("Warning: Failed to clean removed folders: {e}"), } + } - // Single file indexing - match indexer.index_file_with_force(&path, force) { - Ok(result) => { - let language_name = path - .extension() - .and_then(|ext| ext.to_str()) - .and_then(|ext| { - let registry = codanna::parsing::get_registry(); - registry.lock().ok().and_then(|r| { - r.get_by_extension(ext).map(|def| def.name().to_string()) + // Process each path + for path in &paths_to_index { + if path.is_file() { + // Single file indexing + match indexer.index_file_with_force(path, force) { + Ok(result) => { + let language_name = path + .extension() + .and_then(|ext| ext.to_str()) + .and_then(|ext| { + let registry = codanna::parsing::get_registry(); + registry.lock().ok().and_then(|r| { + r.get_by_extension(ext).map(|def| def.name().to_string()) + }) }) - }) - .unwrap_or_else(|| "unknown".to_string()); - - if result.is_cached() { - println!( - "Successfully loaded from cache: {} [{}]", - path.display(), - language_name - ); - } else { - println!( - "Successfully indexed: {} [{}]", - path.display(), - language_name - ); - } - println!("File ID: {}", result.file_id().value()); - - // Get symbols for just this file - let file_symbols = indexer.get_symbols_by_file(result.file_id()); - println!("Found {} symbols in this file", file_symbols.len()); - println!("Total symbols in index: {}", indexer.symbol_count()); - - // Show summary of what was found in this file - let functions = file_symbols - .iter() - .filter(|s| s.kind == SymbolKind::Function) - .count(); - let methods = file_symbols - .iter() - .filter(|s| s.kind == SymbolKind::Method) - .count(); - let structs = file_symbols - .iter() - .filter(|s| s.kind == SymbolKind::Struct) - .count(); - let traits = file_symbols - .iter() - .filter(|s| s.kind == SymbolKind::Trait) - .count(); - - println!(" Functions: {functions}"); - println!(" Methods: {methods}"); - println!(" Structs: {structs}"); - println!(" Traits: {traits}"); - - // Save the index - if config.debug { - eprintln!( - "DEBUG: Saving index with {} symbols", - indexer.symbol_count() - ); + .unwrap_or_else(|| "unknown".to_string()); + + if result.is_cached() { + println!( + "Successfully loaded from cache: {} [{}]", + path.display(), + language_name + ); + } else { + println!( + "Successfully indexed: {} [{}]", + path.display(), + language_name + ); + } + println!("File ID: {}", result.file_id().value()); + + // Get symbols for just this file + let file_symbols = indexer.get_symbols_by_file(result.file_id()); + println!("Found {} symbols in this file", file_symbols.len()); + println!("Total symbols in index: {}", indexer.symbol_count()); + + // Show summary of what was found in this file + let functions = file_symbols + .iter() + .filter(|s| s.kind == SymbolKind::Function) + .count(); + let methods = file_symbols + .iter() + .filter(|s| s.kind == SymbolKind::Method) + .count(); + let structs = file_symbols + .iter() + .filter(|s| s.kind == SymbolKind::Struct) + .count(); + let traits = file_symbols + .iter() + .filter(|s| s.kind == SymbolKind::Trait) + .count(); + + println!(" Functions: {functions}"); + println!(" Methods: {methods}"); + println!(" Structs: {structs}"); + println!(" Traits: {traits}"); } - match persistence.save(&indexer) { - Ok(_) => { - println!("\nIndex saved to: {}", config.index_path.display()); - if config.debug { - eprintln!("DEBUG: Index saved successfully"); + Err(e) => { + eprintln!("Error indexing file {}: {e}", path.display()); + + // Display recovery suggestions + let suggestions = e.recovery_suggestions(); + if !suggestions.is_empty() { + eprintln!("\nSuggestions:"); + for suggestion in suggestions { + eprintln!(" • {suggestion}"); } } - Err(e) => eprintln!("\nWarning: Could not save index: {e}"), + + std::process::exit(1); } } - Err(e) => { - eprintln!("Error indexing file: {e}"); - - // Display recovery suggestions - let suggestions = e.recovery_suggestions(); - if !suggestions.is_empty() { - eprintln!("\nSuggestions:"); - for suggestion in suggestions { - eprintln!(" • {suggestion}"); + } else if path.is_dir() { + // Directory indexing + if let Some(max) = max_files { + println!( + "Indexing directory: {} (limited to {} files)", + path.display(), + max + ); + } else { + println!("Indexing directory: {}", path.display()); + } + + // Track this folder as indexed + if let Err(e) = indexer.add_indexed_folder(path) { + eprintln!("Warning: Failed to track indexed folder: {e}"); + } + + match indexer + .index_directory_with_options(path, progress, dry_run, force, max_files) + { + Ok(stats) => { + stats.display(); + } + Err(e) => { + eprintln!("Error indexing directory {}: {e}", path.display()); + + // Display recovery suggestions + let suggestions = e.recovery_suggestions(); + if !suggestions.is_empty() { + eprintln!("\nSuggestions:"); + for suggestion in suggestions { + eprintln!(" • {suggestion}"); + } } + + std::process::exit(1); } + } + } else { + eprintln!("Error: Path does not exist: {}", path.display()); + std::process::exit(1); + } + } + // After processing all paths, save the index if not in dry-run mode + if !dry_run { + // Build symbol cache before saving + if let Err(e) = indexer.build_symbol_cache() { + eprintln!("Warning: Failed to build symbol cache: {e}"); + } + + // Save the index + eprintln!( + "\nSaving index with {} total symbols, {} total relationships...", + indexer.symbol_count(), + indexer.relationship_count() + ); + match persistence.save(&indexer) { + Ok(_) => { + println!("Index saved to: {}", config.index_path.display()); + } + Err(e) => { + eprintln!("Error: Could not save index: {e}"); std::process::exit(1); } } - } else if path.is_dir() { - // Initialize project resolution providers before indexing - let provider_registry = create_provider_registry(); - if let Err(e) = initialize_providers(&provider_registry, &config) { - eprintln!("\n{e}"); - - // Display recovery suggestions - let suggestions = e.recovery_suggestions(); - if !suggestions.is_empty() { - eprintln!("\nRecovery steps:"); - for suggestion in suggestions { - eprintln!(" • {suggestion}"); - } + } + } + + Commands::AddFolder { path } => { + // Load the config file + let config_path = if let Some(custom_path) = &cli.config { + custom_path.clone() + } else { + Settings::find_workspace_config().unwrap_or_else(|| { + eprintln!("Error: No configuration file found. Run 'codanna init' first."); + std::process::exit(1); + }) + }; + + // Load settings from file + let mut settings = Settings::load_from(&config_path).unwrap_or_else(|e| { + eprintln!("Error loading configuration: {e}"); + std::process::exit(1); + }); + + // Add the folder + match settings.add_indexed_path(path.clone()) { + Ok(_) => { + println!("Added folder to indexed paths: {}", path.display()); + + // Save the updated configuration + if let Err(e) = settings.save(&config_path) { + eprintln!("Error saving configuration: {e}"); + std::process::exit(1); } - // Exit with ConfigError code - use codanna::io::ExitCode; - let exit_code = ExitCode::from_error(&e); - std::process::exit(exit_code as i32); + println!("Configuration saved to: {}", config_path.display()); + println!("\nCurrent indexed paths:"); + for indexed_path in &settings.indexing.indexed_paths { + println!(" - {}", indexed_path.display()); + } } - - // Directory indexing - if let Some(max) = max_files { - println!( - "Indexing directory: {} (limited to {} files)", - path.display(), - max - ); - } else { - println!("Indexing directory: {}", path.display()); + Err(e) => { + eprintln!("Error: {e}"); + std::process::exit(1); } + } + } - match indexer - .index_directory_with_options(&path, progress, dry_run, force, max_files) - { - Ok(stats) => { - stats.display(); + Commands::RemoveFolder { path } => { + // Load the config file + let config_path = if let Some(custom_path) = &cli.config { + custom_path.clone() + } else { + Settings::find_workspace_config().unwrap_or_else(|| { + eprintln!("Error: No configuration file found. Run 'codanna init' first."); + std::process::exit(1); + }) + }; - if !dry_run && stats.files_indexed > 0 { - // Build symbol cache before saving - if let Err(e) = indexer.build_symbol_cache() { - eprintln!("Warning: Failed to build symbol cache: {e}"); - } + // Load settings from file + let mut settings = Settings::load_from(&config_path).unwrap_or_else(|e| { + eprintln!("Error loading configuration: {e}"); + std::process::exit(1); + }); - // Save the index - eprintln!( - "\nSaving index with {} total symbols, {} total relationships...", - indexer.symbol_count(), - indexer.relationship_count() - ); - match persistence.save(&indexer) { - Ok(_) => { - println!("Index saved to: {}", config.index_path.display()); - } - Err(e) => { - eprintln!("Error: Could not save index: {e}"); - std::process::exit(1); - } - } - } - } - Err(e) => { - eprintln!("Error indexing directory: {e}"); - - // Display recovery suggestions - let suggestions = e.recovery_suggestions(); - if !suggestions.is_empty() { - eprintln!("\nSuggestions:"); - for suggestion in suggestions { - eprintln!(" • {suggestion}"); - } - } + // Remove the folder + match settings.remove_indexed_path(&path) { + Ok(_) => { + println!("Removed folder from indexed paths: {}", path.display()); + // Save the updated configuration + if let Err(e) = settings.save(&config_path) { + eprintln!("Error saving configuration: {e}"); std::process::exit(1); } + + println!("Configuration saved to: {}", config_path.display()); + + if settings.indexing.indexed_paths.is_empty() { + println!("\nNo indexed paths configured."); + } else { + println!("\nRemaining indexed paths:"); + for indexed_path in &settings.indexing.indexed_paths { + println!(" - {}", indexed_path.display()); + } + } + } + Err(e) => { + eprintln!("Error: {e}"); + std::process::exit(1); } + } + } + + Commands::ListFolders => { + println!("Indexed folders:"); + if config.indexing.indexed_paths.is_empty() { + println!(" (none configured)"); + println!("\nTo add folders: codanna add-folder "); } else { - eprintln!("Error: Path does not exist: {}", path.display()); + for path in &config.indexing.indexed_paths { + println!(" - {}", path.display()); + } + } + } + + Commands::Clean => { + if config.indexing.indexed_paths.is_empty() { + eprintln!("No indexed paths configured. Use 'codanna add-folder ' first."); std::process::exit(1); } + + println!("Cleaning symbols from removed folders..."); + match indexer.clean_removed_folders(&config.indexing.indexed_paths) { + Ok(count) => { + if count > 0 { + println!("Successfully removed symbols from {count} files"); + + // Save the updated index + match persistence.save(&indexer) { + Ok(_) => { + println!("Index saved to: {}", config.index_path.display()); + } + Err(e) => { + eprintln!("Error saving index: {e}"); + std::process::exit(1); + } + } + } else { + println!("No files to clean - all indexed files are under current folders"); + } + } + Err(e) => { + eprintln!("Error cleaning removed folders: {e}"); + std::process::exit(1); + } + } } Commands::Retrieve { query } => { diff --git a/tests/integration/test_multi_folder_indexing.rs b/tests/integration/test_multi_folder_indexing.rs new file mode 100644 index 00000000..b318b06f --- /dev/null +++ b/tests/integration/test_multi_folder_indexing.rs @@ -0,0 +1,887 @@ +//! Integration tests for multi-folder indexing functionality + +use codanna::{IndexPersistence, Settings, SimpleIndexer}; +use std::fs; +use std::sync::Arc; +use tempfile::TempDir; + +#[test] +fn test_index_multiple_folders() { + // Create a temporary directory structure + let temp_dir = TempDir::new().unwrap(); + let workspace = temp_dir.path(); + + // Create multiple source folders with Rust files + let src_dir = workspace.join("src"); + let lib_dir = workspace.join("lib"); + let tests_dir = workspace.join("tests"); + + fs::create_dir_all(&src_dir).unwrap(); + fs::create_dir_all(&lib_dir).unwrap(); + fs::create_dir_all(&tests_dir).unwrap(); + + // Create sample files in each directory + fs::write( + src_dir.join("main.rs"), + r#" +fn main() { + println!("Hello from main!"); +} + +fn helper() -> i32 { + 42 +} +"#, + ) + .unwrap(); + + fs::write( + lib_dir.join("utils.rs"), + r#" +pub fn add(a: i32, b: i32) -> i32 { + a + b +} + +pub fn multiply(a: i32, b: i32) -> i32 { + a * b +} +"#, + ) + .unwrap(); + + fs::write( + tests_dir.join("test_utils.rs"), + r#" +#[test] +fn test_add() { + assert_eq!(2 + 2, 4); +} + +#[test] +fn test_multiply() { + assert_eq!(2 * 3, 6); +} +"#, + ) + .unwrap(); + + // Create a config with indexed paths + let mut settings = Settings::default(); + settings.add_indexed_path(src_dir.clone()).unwrap(); + settings.add_indexed_path(lib_dir.clone()).unwrap(); + settings.add_indexed_path(tests_dir.clone()).unwrap(); + + // Setup index persistence + let index_path = workspace.join(".codanna/index"); + fs::create_dir_all(&index_path).unwrap(); + settings.index_path = index_path.clone(); + + let settings = Arc::new(settings); + let mut indexer = SimpleIndexer::with_settings(settings.clone()); + + // Index all configured folders + let paths_to_index = settings.get_indexed_paths(); + assert_eq!(paths_to_index.len(), 3); + + for path in &paths_to_index { + indexer + .index_directory_with_options(path, false, false, false, None) + .unwrap(); + } + + // Verify symbols from all folders were indexed + let total_symbols = indexer.symbol_count(); + assert!(total_symbols > 0, "Should have indexed some symbols"); + + // Check that we have symbols from different files + let files = indexer.get_all_indexed_paths(); + assert_eq!(files.len(), 3, "Should have indexed 3 files"); + + // Verify we can find symbols from each directory + let main_symbol = indexer.find_symbols_by_name("main", None); + assert!( + !main_symbol.is_empty(), + "Should find main function from src/" + ); + + let add_symbol = indexer.find_symbols_by_name("add", None); + assert!(!add_symbol.is_empty(), "Should find add function from lib/"); + + let test_add_symbol = indexer.find_symbols_by_name("test_add", None); + assert!( + !test_add_symbol.is_empty(), + "Should find test_add function from tests/" + ); + + // Test persistence + let persistence = IndexPersistence::new(index_path.clone()); + persistence.save(&indexer).unwrap(); + + // Load the index back and verify + let loaded_indexer = persistence + .load_with_settings(settings.clone(), false) + .unwrap(); + assert_eq!( + loaded_indexer.symbol_count(), + total_symbols, + "Loaded index should have same number of symbols" + ); +} + +#[test] +fn test_add_and_remove_folders_from_config() { + let temp_dir = TempDir::new().unwrap(); + let workspace = temp_dir.path(); + + // Create test folders + let folder1 = workspace.join("folder1"); + let folder2 = workspace.join("folder2"); + let folder3 = workspace.join("folder3"); + + fs::create_dir_all(&folder1).unwrap(); + fs::create_dir_all(&folder2).unwrap(); + fs::create_dir_all(&folder3).unwrap(); + + // Create config file + let config_dir = workspace.join(".codanna"); + fs::create_dir_all(&config_dir).unwrap(); + let config_path = config_dir.join("settings.toml"); + + let mut settings = Settings::default(); + + // Add folders + settings.add_indexed_path(folder1.clone()).unwrap(); + settings.add_indexed_path(folder2.clone()).unwrap(); + settings.add_indexed_path(folder3.clone()).unwrap(); + + assert_eq!(settings.indexing.indexed_paths.len(), 3); + + // Save config + settings.save(&config_path).unwrap(); + + // Load config and verify + let loaded_settings = Settings::load_from(&config_path).unwrap(); + assert_eq!(loaded_settings.indexing.indexed_paths.len(), 3); + + // Remove a folder + let mut modified_settings = loaded_settings; + modified_settings.remove_indexed_path(&folder2).unwrap(); + assert_eq!(modified_settings.indexing.indexed_paths.len(), 2); + + // Save and reload + modified_settings.save(&config_path).unwrap(); + let final_settings = Settings::load_from(&config_path).unwrap(); + assert_eq!(final_settings.indexing.indexed_paths.len(), 2); + + // Verify correct folders remain + let canonical_folder1 = folder1.canonicalize().unwrap(); + let canonical_folder3 = folder3.canonicalize().unwrap(); + + let remaining_paths: Vec<_> = final_settings + .indexing + .indexed_paths + .iter() + .filter_map(|p| p.canonicalize().ok()) + .collect(); + + assert!(remaining_paths.contains(&canonical_folder1)); + assert!(remaining_paths.contains(&canonical_folder3)); +} + +#[test] +fn test_index_with_no_configured_paths_uses_default() { + let temp_dir = TempDir::new().unwrap(); + let workspace = temp_dir.path(); + + // Create a Rust file in the workspace + fs::write( + workspace.join("test.rs"), + r#" +fn example() { + println!("test"); +} +"#, + ) + .unwrap(); + + // Create settings with no configured paths + let settings = Settings::default(); + let settings = Arc::new(settings); + + // get_indexed_paths should return empty vector when not configured (backward compatible) + let paths = settings.get_indexed_paths(); + assert_eq!(paths.len(), 0); +} + +#[test] +fn test_index_prevents_duplicate_paths() { + let temp_dir = TempDir::new().unwrap(); + let test_folder = temp_dir.path().join("test_folder"); + fs::create_dir(&test_folder).unwrap(); + + let mut settings = Settings::default(); + + // Add the folder once + assert!(settings.add_indexed_path(test_folder.clone()).is_ok()); + + // Try to add the same folder again - should fail + let result = settings.add_indexed_path(test_folder.clone()); + assert!(result.is_err()); + + // Should still only have one path + assert_eq!(settings.indexing.indexed_paths.len(), 1); +} + +#[test] +fn test_add_folder_indexes_new_symbols() { + // This test verifies that adding a new folder and reindexing adds its symbols to the index + let temp_dir = TempDir::new().unwrap(); + let workspace = temp_dir.path(); + + // Create initial folder + let src_dir = workspace.join("src"); + fs::create_dir_all(&src_dir).unwrap(); + fs::write( + src_dir.join("main.rs"), + r#" +fn main() { + println!("Hello"); +} + +fn helper() -> i32 { + 42 +} +"#, + ) + .unwrap(); + + // Setup index + let index_path = workspace.join(".codanna/index"); + fs::create_dir_all(&index_path).unwrap(); + + let mut settings = Settings { + index_path: index_path.clone(), + ..Settings::default() + }; + settings.add_indexed_path(src_dir.clone()).unwrap(); + + let settings = Arc::new(settings); + let mut indexer = SimpleIndexer::with_settings(settings.clone()); + + // Index initial folder + indexer + .index_directory_with_options(&src_dir, false, false, false, None) + .unwrap(); + + let initial_symbol_count = indexer.symbol_count(); + assert!( + initial_symbol_count > 0, + "Should have indexed initial symbols" + ); + + // Verify main function exists + let main_symbols = indexer.find_symbols_by_name("main", None); + assert!(!main_symbols.is_empty(), "Should find main function"); + + // Verify utility function does NOT exist yet + let add_symbols = indexer.find_symbols_by_name("add_numbers", None); + assert!( + add_symbols.is_empty(), + "Should NOT find add_numbers function yet" + ); + + // Save the index + let persistence = IndexPersistence::new(index_path.clone()); + persistence.save(&indexer).unwrap(); + + // Now add a new folder with new code + let lib_dir = workspace.join("lib"); + fs::create_dir_all(&lib_dir).unwrap(); + fs::write( + lib_dir.join("utils.rs"), + r#" +pub fn add_numbers(a: i32, b: i32) -> i32 { + a + b +} + +pub fn multiply(a: i32, b: i32) -> i32 { + a * b +} +"#, + ) + .unwrap(); + + // Update settings to include new folder + let mut updated_settings = Settings { + index_path: index_path.clone(), + ..Settings::default() + }; + updated_settings.add_indexed_path(src_dir.clone()).unwrap(); + updated_settings.add_indexed_path(lib_dir.clone()).unwrap(); + + // Load the existing index + let updated_settings = Arc::new(updated_settings); + let mut updated_indexer = persistence + .load_with_settings(updated_settings.clone(), false) + .unwrap(); + + // Index the new folder (incremental) + updated_indexer + .index_directory_with_options(&lib_dir, false, false, false, None) + .unwrap(); + + // Verify symbol count increased + let final_symbol_count = updated_indexer.symbol_count(); + assert!( + final_symbol_count > initial_symbol_count, + "Symbol count should increase after adding new folder: {initial_symbol_count} -> {final_symbol_count}" + ); + + // Verify we can now find symbols from BOTH folders + let main_symbols = updated_indexer.find_symbols_by_name("main", None); + assert!(!main_symbols.is_empty(), "Should still find main function"); + + let add_symbols = updated_indexer.find_symbols_by_name("add_numbers", None); + assert!( + !add_symbols.is_empty(), + "Should NOW find add_numbers function from new folder" + ); + + let multiply_symbols = updated_indexer.find_symbols_by_name("multiply", None); + assert!( + !multiply_symbols.is_empty(), + "Should find multiply function from new folder" + ); +} + +#[test] +fn test_remove_folder_cleans_symbols() { + // This test verifies that removing a folder and cleaning removes its symbols from the index + let temp_dir = TempDir::new().unwrap(); + let workspace = temp_dir.path(); + + // Create two folders with different code + let src_dir = workspace.join("src"); + let lib_dir = workspace.join("lib"); + + fs::create_dir_all(&src_dir).unwrap(); + fs::create_dir_all(&lib_dir).unwrap(); + + fs::write( + src_dir.join("main.rs"), + r#" +fn main() { + println!("Hello"); +} + +fn src_helper() -> i32 { + 42 +} +"#, + ) + .unwrap(); + + fs::write( + lib_dir.join("utils.rs"), + r#" +pub fn lib_function() -> String { + "from lib".to_string() +} + +pub fn lib_helper() -> i32 { + 100 +} +"#, + ) + .unwrap(); + + // Setup index with both folders + let index_path = workspace.join(".codanna/index"); + fs::create_dir_all(&index_path).unwrap(); + + let mut settings = Settings { + index_path: index_path.clone(), + ..Settings::default() + }; + settings.add_indexed_path(src_dir.clone()).unwrap(); + settings.add_indexed_path(lib_dir.clone()).unwrap(); + + let settings = Arc::new(settings); + let mut indexer = SimpleIndexer::with_settings(settings.clone()); + + // Index both folders + indexer + .index_directory_with_options(&src_dir, false, false, false, None) + .unwrap(); + indexer + .index_directory_with_options(&lib_dir, false, false, false, None) + .unwrap(); + + let initial_symbol_count = indexer.symbol_count(); + assert!( + initial_symbol_count > 0, + "Should have indexed symbols from both folders" + ); + + // Verify symbols from both folders exist + let main_symbols = indexer.find_symbols_by_name("main", None); + assert!(!main_symbols.is_empty(), "Should find main from src/"); + + let src_helper_symbols = indexer.find_symbols_by_name("src_helper", None); + assert!( + !src_helper_symbols.is_empty(), + "Should find src_helper from src/" + ); + + let lib_function_symbols = indexer.find_symbols_by_name("lib_function", None); + assert!( + !lib_function_symbols.is_empty(), + "Should find lib_function from lib/" + ); + + let lib_helper_symbols = indexer.find_symbols_by_name("lib_helper", None); + assert!( + !lib_helper_symbols.is_empty(), + "Should find lib_helper from lib/" + ); + + // Save the index + let persistence = IndexPersistence::new(index_path.clone()); + persistence.save(&indexer).unwrap(); + + // Now remove lib folder from configuration + let mut updated_settings = Settings { + index_path: index_path.clone(), + ..Settings::default() + }; + updated_settings.add_indexed_path(src_dir.clone()).unwrap(); + // Note: NOT adding lib_dir + + // Load the index + let updated_settings = Arc::new(updated_settings); + let mut updated_indexer = persistence + .load_with_settings(updated_settings.clone(), false) + .unwrap(); + + // Debug: print all indexed paths before cleanup + eprintln!("Files before cleanup:"); + for path in updated_indexer.get_all_indexed_paths() { + eprintln!(" - {}", path.display()); + } + + eprintln!("Configured folders:"); + for folder in &updated_settings.indexing.indexed_paths { + eprintln!(" - {}", folder.display()); + } + + // Clean removed folders + let removed_count = updated_indexer + .clean_removed_folders(&updated_settings.indexing.indexed_paths) + .unwrap(); + + eprintln!("Removed {removed_count} files"); + + assert!( + removed_count > 0, + "Should have removed files from lib/ folder" + ); + + // Note: Due to Tantivy's soft-delete mechanism, the symbol count might not decrease immediately + // deleted documents are only physically removed during segment merges. + // Instead, we verify that the symbols can't be found anymore. + + // Verify symbols from src/ still exist + let main_symbols = updated_indexer.find_symbols_by_name("main", None); + assert!( + !main_symbols.is_empty(), + "Should STILL find main from src/ (not removed)" + ); + + let src_helper_symbols = updated_indexer.find_symbols_by_name("src_helper", None); + assert!( + !src_helper_symbols.is_empty(), + "Should STILL find src_helper from src/ (not removed)" + ); + + // Verify symbols from lib/ are gone + let lib_function_symbols = updated_indexer.find_symbols_by_name("lib_function", None); + eprintln!("lib_function symbols found: {}", lib_function_symbols.len()); + for sym in &lib_function_symbols { + eprintln!(" - {} at {}", sym.name, sym.file_path); + } + assert!( + lib_function_symbols.is_empty(), + "Should NOT find lib_function anymore (should be removed)" + ); + + let lib_helper_symbols = updated_indexer.find_symbols_by_name("lib_helper", None); + assert!( + lib_helper_symbols.is_empty(), + "Should NOT find lib_helper anymore (should be removed)" + ); + + // Save and reload to verify persistence + persistence.save(&updated_indexer).unwrap(); + let reloaded_indexer = persistence + .load_with_settings(updated_settings.clone(), false) + .unwrap(); + + // Verify the cleanup persisted - symbols should still not be findable + let lib_function_after_reload = reloaded_indexer.find_symbols_by_name("lib_function", None); + assert!( + lib_function_after_reload.is_empty(), + "lib_function should STILL be gone after reload" + ); + + let src_helper_after_reload = reloaded_indexer.find_symbols_by_name("src_helper", None); + assert!( + !src_helper_after_reload.is_empty(), + "src_helper should STILL exist after reload" + ); +} + +#[test] +fn test_nested_folders_no_duplicate_symbols() { + // Test that indexing both a parent and child folder doesn't create duplicate symbols + let temp_dir = TempDir::new().unwrap(); + let workspace = temp_dir.path(); + + // Create nested structure: src/ and src/utils/ + let src_dir = workspace.join("src"); + let utils_dir = src_dir.join("utils"); + + fs::create_dir_all(&src_dir).unwrap(); + fs::create_dir_all(&utils_dir).unwrap(); + + // File in parent folder + fs::write( + src_dir.join("main.rs"), + r#" +fn main() { + println!("Hello"); +} +"#, + ) + .unwrap(); + + // File in nested folder + fs::write( + utils_dir.join("helper.rs"), + r#" +pub fn helper() -> i32 { + 42 +} +"#, + ) + .unwrap(); + + let index_path = workspace.join(".codanna/index"); + fs::create_dir_all(&index_path).unwrap(); + + // Index only the parent folder first + let mut settings = Settings { + index_path: index_path.clone(), + ..Settings::default() + }; + settings.add_indexed_path(src_dir.clone()).unwrap(); + + let settings = Arc::new(settings); + let mut indexer = SimpleIndexer::with_settings(settings.clone()); + + indexer + .index_directory_with_options(&src_dir, false, false, false, None) + .unwrap(); + + let symbol_count_parent_only = indexer.symbol_count(); + assert!(symbol_count_parent_only > 0, "Should have indexed symbols"); + + // Should find symbols from both files since parent includes child + let main_symbols = indexer.find_symbols_by_name("main", None); + assert!(!main_symbols.is_empty(), "Should find main from parent"); + + let helper_symbols = indexer.find_symbols_by_name("helper", None); + assert!( + !helper_symbols.is_empty(), + "Should find helper from nested folder" + ); + + // Now explicitly add the nested folder too + let persistence = IndexPersistence::new(index_path.clone()); + persistence.save(&indexer).unwrap(); + + let mut settings_with_nested = Settings { + index_path: index_path.clone(), + ..Settings::default() + }; + settings_with_nested + .add_indexed_path(src_dir.clone()) + .unwrap(); + settings_with_nested + .add_indexed_path(utils_dir.clone()) + .unwrap(); + + let settings_with_nested = Arc::new(settings_with_nested); + let mut indexer_with_nested = persistence + .load_with_settings(settings_with_nested.clone(), false) + .unwrap(); + + // Index the nested folder explicitly + indexer_with_nested + .index_directory_with_options(&utils_dir, false, false, false, None) + .unwrap(); + + let symbol_count_with_nested = indexer_with_nested.symbol_count(); + + // Symbol count should be similar (might have some duplicates from re-indexing) + // The important thing is it doesn't explode with duplicates + assert!( + symbol_count_with_nested <= symbol_count_parent_only + 5, + "Should not have massive duplication: {symbol_count_parent_only} vs {symbol_count_with_nested}" + ); + + // Should still find both symbols + let main_symbols = indexer_with_nested.find_symbols_by_name("main", None); + assert!(!main_symbols.is_empty(), "Should still find main"); + + let helper_symbols = indexer_with_nested.find_symbols_by_name("helper", None); + assert!(!helper_symbols.is_empty(), "Should still find helper"); +} + +#[test] +fn test_overlapping_paths_cleanup_protection() { + // Test that files under overlapping paths are protected from cleanup + let temp_dir = TempDir::new().unwrap(); + let workspace = temp_dir.path(); + + let root_dir = workspace.join("project"); + let sub_dir = root_dir.join("submodule"); + + fs::create_dir_all(&root_dir).unwrap(); + fs::create_dir_all(&sub_dir).unwrap(); + + // File in root + fs::write( + root_dir.join("root.rs"), + r#" +fn root_function() { + println!("root"); +} +"#, + ) + .unwrap(); + + // File in subdirectory + fs::write( + sub_dir.join("sub.rs"), + r#" +fn sub_function() { + println!("sub"); +} +"#, + ) + .unwrap(); + + let index_path = workspace.join(".codanna/index"); + fs::create_dir_all(&index_path).unwrap(); + + // Index both overlapping paths + let mut settings = Settings { + index_path: index_path.clone(), + ..Settings::default() + }; + settings.add_indexed_path(root_dir.clone()).unwrap(); + settings.add_indexed_path(sub_dir.clone()).unwrap(); + + let settings = Arc::new(settings); + let mut indexer = SimpleIndexer::with_settings(settings.clone()); + + indexer + .index_directory_with_options(&root_dir, false, false, false, None) + .unwrap(); + indexer + .index_directory_with_options(&sub_dir, false, false, false, None) + .unwrap(); + + let initial_symbol_count = indexer.symbol_count(); + assert!(initial_symbol_count > 0); + + // Both functions should be findable + let root_symbols = indexer.find_symbols_by_name("root_function", None); + assert!(!root_symbols.is_empty(), "Should find root_function"); + + let sub_symbols = indexer.find_symbols_by_name("sub_function", None); + assert!(!sub_symbols.is_empty(), "Should find sub_function"); + + // Save and remove only the root path (keep sub) + let persistence = IndexPersistence::new(index_path.clone()); + persistence.save(&indexer).unwrap(); + + let mut updated_settings = Settings { + index_path: index_path.clone(), + ..Settings::default() + }; + updated_settings.add_indexed_path(sub_dir.clone()).unwrap(); + // NOT adding root_dir + + let updated_settings = Arc::new(updated_settings); + let mut updated_indexer = persistence + .load_with_settings(updated_settings.clone(), false) + .unwrap(); + + // Clean removed folders + let removed_count = updated_indexer + .clean_removed_folders(&updated_settings.indexing.indexed_paths) + .unwrap(); + + // Should only remove the root.rs file, NOT sub.rs (because sub_dir is still indexed) + assert_eq!(removed_count, 1, "Should only remove root.rs"); + + // sub_function should STILL exist (protected by sub_dir being indexed) + let sub_symbols = updated_indexer.find_symbols_by_name("sub_function", None); + assert!( + !sub_symbols.is_empty(), + "sub_function should STILL exist (protected by submodule path)" + ); + + // root_function should be gone + let root_symbols = updated_indexer.find_symbols_by_name("root_function", None); + assert!(root_symbols.is_empty(), "root_function should be removed"); +} + +#[test] +#[cfg(unix)] // Symlinks work differently on Windows +fn test_symlinks_are_canonicalized() { + // Test that symlinks are properly canonicalized to avoid duplicates + let temp_dir = TempDir::new().unwrap(); + let workspace = temp_dir.path(); + + // Create real directory + let real_dir = workspace.join("real"); + fs::create_dir_all(&real_dir).unwrap(); + + fs::write( + real_dir.join("code.rs"), + r#" +fn real_function() { + println!("real"); +} +"#, + ) + .unwrap(); + + // Create symlink to real directory + let symlink_dir = workspace.join("symlink"); + #[cfg(unix)] + std::os::unix::fs::symlink(&real_dir, &symlink_dir).unwrap(); + + let index_path = workspace.join(".codanna/index"); + fs::create_dir_all(&index_path).unwrap(); + + // Try to add both the real path and the symlink + let mut settings = Settings { + index_path: index_path.clone(), + ..Settings::default() + }; + + // Add real directory + settings.add_indexed_path(real_dir.clone()).unwrap(); + + // Try to add symlink - should resolve to same canonical path + let result = settings.add_indexed_path(symlink_dir.clone()); + + // Should fail because they canonicalize to the same path + assert!( + result.is_err(), + "Should not allow adding symlink if real path already added" + ); + + // Should still only have one path + assert_eq!( + settings.indexing.indexed_paths.len(), + 1, + "Should only have one path (real path)" + ); + + // Verify the path is the canonical one + let canonical_real = real_dir.canonicalize().unwrap(); + let stored_path = settings.indexing.indexed_paths[0].canonicalize().unwrap(); + assert_eq!(stored_path, canonical_real, "Should store canonical path"); +} + +#[test] +#[cfg(unix)] +fn test_symlink_removal_works_correctly() { + // Test that removing a folder works correctly even when accessed via symlink + let temp_dir = TempDir::new().unwrap(); + let workspace = temp_dir.path(); + + let dir1 = workspace.join("dir1"); + let dir2 = workspace.join("dir2"); + fs::create_dir_all(&dir1).unwrap(); + fs::create_dir_all(&dir2).unwrap(); + + fs::write(dir1.join("file1.rs"), r#"fn func1() { println!("1"); }"#).unwrap(); + + fs::write(dir2.join("file2.rs"), r#"fn func2() { println!("2"); }"#).unwrap(); + + // Create symlink to dir2 + let symlink_to_dir2 = workspace.join("link2"); + #[cfg(unix)] + std::os::unix::fs::symlink(&dir2, &symlink_to_dir2).unwrap(); + + let index_path = workspace.join(".codanna/index"); + fs::create_dir_all(&index_path).unwrap(); + + // Index both directories (dir1 by real path, dir2 by symlink) + let mut settings = Settings { + index_path: index_path.clone(), + ..Settings::default() + }; + settings.add_indexed_path(dir1.clone()).unwrap(); + settings.add_indexed_path(symlink_to_dir2.clone()).unwrap(); + + let settings = Arc::new(settings); + let mut indexer = SimpleIndexer::with_settings(settings.clone()); + + indexer + .index_directory_with_options(&dir1, false, false, false, None) + .unwrap(); + indexer + .index_directory_with_options(&symlink_to_dir2, false, false, false, None) + .unwrap(); + + // Both functions should exist + let func1 = indexer.find_symbols_by_name("func1", None); + let func2 = indexer.find_symbols_by_name("func2", None); + assert!(!func1.is_empty() && !func2.is_empty()); + + // Save and remove dir2 (using real path, not symlink) + let persistence = IndexPersistence::new(index_path.clone()); + persistence.save(&indexer).unwrap(); + + let mut updated_settings = Settings { + index_path: index_path.clone(), + ..Settings::default() + }; + updated_settings.add_indexed_path(dir1.clone()).unwrap(); + // NOT adding dir2 or symlink_to_dir2 + + let updated_settings = Arc::new(updated_settings); + let mut updated_indexer = persistence + .load_with_settings(updated_settings.clone(), false) + .unwrap(); + + // Clean should work correctly because paths are canonicalized + let removed = updated_indexer + .clean_removed_folders(&updated_settings.indexing.indexed_paths) + .unwrap(); + + assert!(removed > 0, "Should remove dir2 files"); + + // func2 should be gone, func1 should remain + let func1 = updated_indexer.find_symbols_by_name("func1", None); + let func2 = updated_indexer.find_symbols_by_name("func2", None); + + assert!(!func1.is_empty(), "func1 should still exist"); + assert!(func2.is_empty(), "func2 should be removed"); +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index cfe83333..d4ca4e64 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -45,3 +45,6 @@ mod test_typescript_object_property_call; #[path = "integration/test_external_import_resolution.rs"] mod test_external_import_resolution; + +#[path = "integration/test_multi_folder_indexing.rs"] +mod test_multi_folder_indexing;