diff --git a/.gitignore b/.gitignore index d101654b..56ea81a7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ .env .vscode .DS_Store + +# Example binaries +examples/*/semantic-graph +examples/*/*.exe diff --git a/docs/SEMANTIC_GRAPH_SUMMARY.md b/docs/SEMANTIC_GRAPH_SUMMARY.md new file mode 100644 index 00000000..948c75d3 --- /dev/null +++ b/docs/SEMANTIC_GRAPH_SUMMARY.md @@ -0,0 +1,316 @@ +# Semantic Graph with Qdrant - Implementation Guide + +## Answer to the Question + +**Question**: *"Please investigate how I could apply data extraction to the qdrant vector db to model a semantic graph (where entities have relations to each other) while keeping the data at the leafs in vector form. would this be possible?"* + +## Short Answer + +**Yes, it is absolutely possible!** + +You can model a semantic graph in Qdrant by: +1. Storing entities as **points** with vector embeddings (keeping leaf data in vector form) +2. Encoding relationships in the **payload metadata** as structured fields +3. Using Qdrant's **hybrid search** (vector similarity + metadata filtering) for graph queries +4. Implementing graph traversal by following relationship chains through metadata filters + +This approach gives you the best of both worlds: semantic search through vectors and graph navigation through metadata. + +## Complete Implementation + +This repository now includes a complete implementation for semantic graph modeling with Qdrant: + +### πŸ“š Documentation + +1. **[Semantic Graph with Qdrant](./semantic-graph-qdrant.md)** - Comprehensive guide + - Concept and architecture overview + - Implementation strategy + - Practical examples with code + - Query patterns and best practices + - Integration with Wingman platform + +2. **[Architecture Diagrams](./semantic-graph-architecture.md)** - Visual representations + - Component architecture + - Data flow diagrams + - Query type illustrations + - Entity-relationship models + - Storage layout in Qdrant + +3. **[Quick Reference](./semantic-graph-quickref.md)** - Cheat sheet + - Common operations + - Query patterns + - Configuration examples + - Performance tips + - Common pitfalls to avoid + +### πŸ’» Code Implementation + +1. **Graph Interface Extensions** (`pkg/index/graph.go`) + - `GraphDocument` - Entity with relationships + - `GraphProvider` - Interface for graph operations + - `Relation` - Typed edges between entities + - Methods: `IndexEntity`, `QueryRelated`, `QueryGraph`, `TraverseGraph` + +2. **Qdrant Implementation** (`pkg/index/qdrant/client.go`) + - Reference implementation of `GraphProvider` + - Vector search + metadata filtering + - Relationship encoding and traversal + - Multi-hop graph navigation + +3. **Working Example** (`examples/semantic-graph/`) + - Complete end-to-end demonstration + - Entity extraction and indexing + - Multiple query patterns + - Graph traversal examples + - Configuration file included + +## Key Features + +### 1. Vector Embeddings at Leaf Nodes βœ… +```go +entity := index.GraphDocument{ + Document: index.Document{ + Content: "John Doe is a software engineer...", + Embedding: [0.1, 0.2, ..., 0.n], // Vector form! + }, + // ... entity and relation metadata +} +``` + +### 2. Relationship Metadata βœ… +```go +Relations: []index.Relation{ + {Type: "works_at", TargetID: "org-acme-corp"}, + {Type: "knows", TargetID: "person-jane-smith"}, + {Type: "works_on", TargetID: "project-ml-platform"}, +} +``` + +### 3. Hybrid Queries βœ… +```go +// Semantic search + graph constraints +relationFilter := map[string]string{ + "works_at": "org-acme-corp", +} +results := graphProvider.QueryGraph( + ctx, + "machine learning expert", // Semantic query + relationFilter, // Graph constraint + opts, +) +``` + +### 4. Graph Traversal βœ… +```go +// Multi-hop navigation +path := []string{"works_at", "employs", "works_on"} +projects := graphProvider.TraverseGraph( + ctx, + "person-john-doe", + path, + opts, +) +``` + +## Architecture Overview + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Qdrant Collection β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Point: person-john-doe β”‚ +β”‚ β”œβ”€> Vector: [embeddings...] ← Leaf data! β”‚ +β”‚ └─> Payload: β”‚ +β”‚ β”œβ”€> entity_type: "person" β”‚ +β”‚ β”œβ”€> content: "John Doe is..." β”‚ +β”‚ └─> relations: [ ← Graph! β”‚ +β”‚ "works_at:org-acme", β”‚ +β”‚ "knows:person-jane" β”‚ +β”‚ ] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## How It Works + +### Data Storage +1. **Entities** are stored as Qdrant points +2. **Content** is embedded into vectors (semantic representation) +3. **Relationships** are stored in payload metadata (graph structure) +4. **Attributes** are stored in metadata (filterable properties) + +### Query Types + +| Query Type | Vector | Metadata | Use Case | +|------------|--------|----------|----------| +| Semantic Search | βœ… | ❌ | Find similar entities | +| Relationship Query | ❌ | βœ… | Navigate graph edges | +| Hybrid Query | βœ… | βœ… | Semantic + structural | +| Graph Traversal | ❌ | βœ… | Multi-hop navigation | + +### Example Queries + +```go +// 1. Semantic: Find ML experts (vector-based) +Query(ctx, "machine learning expert", opts) + +// 2. Graph: Find John's coworkers (metadata-based) +QueryRelated(ctx, "person-john", "works_at", opts) + .then(QueryRelated(ctx, company, "employs", opts)) + +// 3. Hybrid: Find ML experts at Google (vector + metadata) +QueryGraph(ctx, "ML expert", {"works_at": "org-google"}, opts) + +// 4. Traversal: Find projects of John's colleagues (multi-hop) +TraverseGraph(ctx, "person-john", ["works_at", "employs", "works_on"], opts) +``` + +## Advantages + +βœ… **Semantic Understanding** - Vector embeddings capture meaning +βœ… **Graph Structure** - Explicit relationships between entities +βœ… **Hybrid Queries** - Combine semantic and structural constraints +βœ… **Scalability** - Qdrant handles millions of entities efficiently +βœ… **Flexibility** - Add new relation types without schema changes +βœ… **Single Database** - No need for separate graph database +βœ… **Fast Queries** - HNSW for vectors + indexed payloads for metadata + +## Use Cases + +1. **Knowledge Graphs** - Model organizational knowledge with semantic search +2. **Recommendation Systems** - Find similar entities with specific relationships +3. **RAG Enhancement** - Enrich context with related entities from graph +4. **Entity Resolution** - Link related entities across documents +5. **Relationship Discovery** - Infer connections through graph traversal +6. **Question Answering** - Navigate knowledge graph to find answers +7. **Expert Finding** - Locate people with specific skills and connections + +## Getting Started + +### 1. Set Up Qdrant +```bash +docker run -p 6333:6333 qdrant/qdrant +``` + +### 2. Configure Collection +```yaml +indexes: + knowledge-graph: + type: qdrant + url: http://localhost:6333 + collection: knowledge_graph + vector_size: 1536 + distance: cosine + graph_mode: true +``` + +### 3. Index Entities +```go +entity := index.GraphDocument{ + Document: index.Document{ + Content: entityContent, + Embedding: generateEmbedding(entityContent), + }, + EntityID: "person-john-doe", + EntityType: "person", + Relations: []index.Relation{ + {Type: "works_at", TargetID: "org-acme"}, + }, +} +graphProvider.IndexEntity(ctx, entity) +``` + +### 4. Query the Graph +```go +// Semantic search +results := graphProvider.Query(ctx, "AI expert", opts) + +// Relationship query +related := graphProvider.QueryRelated(ctx, entityID, "works_at", opts) + +// Hybrid query +hybrid := graphProvider.QueryGraph(ctx, query, relationFilter, opts) +``` + +## Integration with Data Extraction + +The implementation works seamlessly with existing Wingman extractors: + +```go +// 1. Extract document content +doc := extractor.Extract(ctx, input, opts) + +// 2. Extract entities and relationships (using LLM or NLP) +entities := extractEntitiesFromText(doc.Content) + +// 3. Generate embeddings +for _, entity := range entities { + embedding := embedder.Embed(ctx, []string{entity.Content}) + entity.Embedding = embedding.Embeddings[0] +} + +// 4. Index in graph +for _, entity := range entities { + graphProvider.IndexEntity(ctx, entity) +} + +// 5. Query the graph +results := graphProvider.QueryGraph(ctx, query, filters, opts) +``` + +## Performance Considerations + +- **Vector Search**: O(log n) with HNSW - scales to millions +- **Metadata Filtering**: O(1) with indexed payloads - very fast +- **Graph Traversal**: O(k * r) where k=hops, r=relations - limit depth +- **Storage**: ~1KB per entity (embedding + metadata) + +## Next Steps + +1. **Try the Example**: Run the code in `examples/semantic-graph/` +2. **Read the Docs**: Review detailed documentation +3. **Experiment**: Build your own semantic graph with your data +4. **Extend**: Add custom entity types and relationship types +5. **Integrate**: Connect with Wingman extractors and LLM tools + +## Files Added + +``` +docs/ +β”œβ”€β”€ semantic-graph-qdrant.md # Comprehensive guide +β”œβ”€β”€ semantic-graph-architecture.md # Visual diagrams +└── semantic-graph-quickref.md # Quick reference + +pkg/index/ +β”œβ”€β”€ graph.go # Graph interface extensions +└── qdrant/ + └── client.go # Qdrant implementation + +examples/semantic-graph/ +β”œβ”€β”€ main.go # Working example +β”œβ”€β”€ config.yaml # Configuration +└── README.md # Example documentation +``` + +## Conclusion + +**Yes, it is absolutely possible to apply data extraction to Qdrant vector DB to model a semantic graph while keeping data at the leafs in vector form.** + +The implementation provided demonstrates: +- βœ… Vector embeddings for semantic content (leaf data in vector form) +- βœ… Metadata-based relationship tracking (graph structure) +- βœ… Hybrid queries (semantic + structural) +- βœ… Graph traversal (multi-hop navigation) +- βœ… Complete working example +- βœ… Full integration with Wingman platform + +This approach leverages Qdrant's strengths in both vector search and metadata filtering to create a powerful semantic graph that maintains the benefits of vector representations while adding explicit relationship modeling. + +## Questions? + +If you have questions about implementation details, performance optimization, or specific use cases, feel free to: +- Review the detailed documentation +- Check the example code +- Experiment with the implementation +- Reach out for clarification + +The semantic graph approach opens up powerful possibilities for knowledge representation and retrieval! diff --git a/docs/automatic-entity-extraction.md b/docs/automatic-entity-extraction.md new file mode 100644 index 00000000..396d2fe7 --- /dev/null +++ b/docs/automatic-entity-extraction.md @@ -0,0 +1,641 @@ +# Automatic Entity and Relationship Extraction + +This guide demonstrates how to automatically detect and infer entities and their relationships from data, eliminating the need to manually specify EntityID, EntityType, and Relations. + +## Overview + +There are several approaches to automatic entity and relationship extraction: + +1. **LLM-based extraction** (Recommended) - Use GPT-4, Claude, or other LLMs +2. **NLP-based extraction** - Use spaCy, Stanford NER, or similar tools +3. **Hybrid approach** - Combine LLM reasoning with NLP pipelines +4. **Custom models** - Fine-tuned models for domain-specific extraction + +## Approach 1: LLM-based Extraction (Recommended) + +### Why LLMs? +- βœ… Understand context and semantics +- βœ… Can infer implicit relationships +- βœ… Handle diverse text formats +- βœ… No training data required +- βœ… Easily customizable with prompts + +### Implementation + +```go +package extraction + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/adrianliechti/wingman/pkg/index" + "github.com/adrianliechti/wingman/pkg/provider" +) + +// EntityExtractor automatically extracts entities and relationships from text +type EntityExtractor struct { + completer provider.Completer + embedder provider.Embedder +} + +// NewEntityExtractor creates a new automatic entity extractor +func NewEntityExtractor(completer provider.Completer, embedder provider.Embedder) *EntityExtractor { + return &EntityExtractor{ + completer: completer, + embedder: embedder, + } +} + +// ExtractedEntity represents an entity with its relationships +type ExtractedEntity struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Content string `json:"content"` + Attributes map[string]string `json:"attributes"` + Relations []ExtractedRelation `json:"relations"` +} + +// ExtractedRelation represents a relationship between entities +type ExtractedRelation struct { + Type string `json:"type"` + TargetID string `json:"target_id"` + TargetName string `json:"target_name"` +} + +// ExtractEntitiesFromText uses an LLM to extract entities and relationships +func (e *EntityExtractor) ExtractEntitiesFromText(ctx context.Context, text string) ([]index.GraphDocument, error) { + // Step 1: Use LLM to extract structured entities and relationships + extractedEntities, err := e.extractWithLLM(ctx, text) + if err != nil { + return nil, fmt.Errorf("failed to extract entities: %w", err) + } + + // Step 2: Convert to GraphDocuments with embeddings + var graphDocs []index.GraphDocument + for _, entity := range extractedEntities { + // Generate embedding for the entity content + embedding, err := e.embedder.Embed(ctx, []string{entity.Content}) + if err != nil { + return nil, fmt.Errorf("failed to generate embedding for %s: %w", entity.ID, err) + } + + // Convert to GraphDocument + relations := make([]index.Relation, len(entity.Relations)) + for i, rel := range entity.Relations { + relations[i] = index.Relation{ + Type: rel.Type, + TargetID: rel.TargetID, + Metadata: map[string]string{ + "target_name": rel.TargetName, + }, + } + } + + graphDoc := index.GraphDocument{ + Document: index.Document{ + Content: entity.Content, + Embedding: embedding.Embeddings[0], + Metadata: entity.Attributes, + }, + EntityID: entity.ID, + EntityType: entity.Type, + Relations: relations, + } + + graphDocs = append(graphDocs, graphDoc) + } + + return graphDocs, nil +} + +// extractWithLLM uses the LLM to extract entities and relationships +func (e *EntityExtractor) extractWithLLM(ctx context.Context, text string) ([]ExtractedEntity, error) { + // Construct extraction prompt + prompt := buildExtractionPrompt(text) + + // Call LLM + completion, err := e.completer.Complete(ctx, []provider.Message{ + { + Role: provider.MessageRoleUser, + Content: prompt, + }, + }, &provider.CompleteOptions{ + Temperature: floatPtr(0.1), // Low temperature for consistent extraction + }) + if err != nil { + return nil, err + } + + // Parse JSON response + var entities []ExtractedEntity + if err := json.Unmarshal([]byte(completion.Message.Content), &entities); err != nil { + return nil, fmt.Errorf("failed to parse LLM response: %w", err) + } + + return entities, nil +} + +// buildExtractionPrompt creates a prompt for entity extraction +func buildExtractionPrompt(text string) string { + return fmt.Sprintf(`You are an expert at extracting entities and relationships from text. + +Given the following text, extract all entities (people, organizations, projects, locations, etc.) and their relationships. + +For each entity, provide: +- id: A unique identifier in format "{type}-{normalized-name}" (e.g., "person-john-doe", "org-acme-corp") +- type: The entity type (person, organization, project, location, product, event, etc.) +- name: The entity's name as it appears in the text +- content: A brief description of the entity (1-2 sentences) +- attributes: Key attributes as key-value pairs (e.g., title, department, industry, etc.) +- relations: Relationships to other entities with type and target_id + +Common relationship types: +- works_at / employs (person to organization) +- manages / managed_by (management relationships) +- knows / knows (personal connections) +- works_on / involves (project participation) +- located_in / contains (location relationships) +- part_of / contains (organizational hierarchy) +- founded / founded_by (founding relationships) +- sponsors / sponsored_by (sponsorship) +- partners_with (partnerships) + +Text to analyze: +""" +%s +""" + +Return ONLY a JSON array of entities. Example format: +[ + { + "id": "person-john-doe", + "type": "person", + "name": "John Doe", + "content": "John Doe is a software engineer specializing in machine learning.", + "attributes": { + "title": "Software Engineer", + "expertise": "Machine Learning" + }, + "relations": [ + { + "type": "works_at", + "target_id": "org-acme-corp", + "target_name": "Acme Corp" + } + ] + } +]`, text) +} + +func floatPtr(f float64) *float64 { + return &f +} +``` + +### Usage Example + +```go +package main + +import ( + "context" + "log" + + "github.com/adrianliechti/wingman/pkg/index" + "github.com/adrianliechti/wingman/pkg/provider/openai" + "example.com/extraction" +) + +func main() { + ctx := context.Background() + + // Initialize LLM and embedder + completer := openai.NewCompleter("gpt-4", "your-api-key") + embedder := openai.NewEmbedder("text-embedding-3-small", "your-api-key") + + // Create extractor + extractor := extraction.NewEntityExtractor(completer, embedder) + + // Your document text + text := ` + John Doe is a Senior Software Engineer at Acme Corp, where he leads + the machine learning team. He works closely with Jane Smith, the + Product Manager, on the AI Platform project. Acme Corp, founded in + 2015, is a technology company based in San Francisco specializing in + artificial intelligence solutions. + ` + + // Extract entities automatically + entities, err := extractor.ExtractEntitiesFromText(ctx, text) + if err != nil { + log.Fatal(err) + } + + // Index in graph + for _, entity := range entities { + log.Printf("Extracted entity: %s (%s) with %d relations", + entity.EntityID, entity.EntityType, len(entity.Relations)) + + err := graphProvider.IndexEntity(ctx, entity) + if err != nil { + log.Printf("Failed to index %s: %v", entity.EntityID, err) + } + } +} +``` + +## Approach 2: NLP-based Extraction with spaCy + +For cases where you prefer open-source NLP tools or need offline processing: + +```python +# nlp_extractor.py +import spacy +from typing import List, Dict, Any +import json + +class NLPEntityExtractor: + def __init__(self): + # Load spaCy model with named entity recognition + self.nlp = spacy.load("en_core_web_trf") # Transformer-based model + + def extract_entities(self, text: str) -> List[Dict[str, Any]]: + doc = self.nlp(text) + + entities = [] + entity_map = {} + + # Extract named entities + for ent in doc.ents: + entity_id = self._generate_id(ent.label_, ent.text) + entity_type = self._normalize_type(ent.label_) + + entity = { + "id": entity_id, + "type": entity_type, + "name": ent.text, + "content": self._extract_context(ent, doc), + "attributes": {}, + "relations": [] + } + + entities.append(entity) + entity_map[ent.text] = entity_id + + # Extract relationships using dependency parsing + for sent in doc.sents: + relations = self._extract_relations(sent, entity_map) + for rel in relations: + # Add relation to source entity + for entity in entities: + if entity["id"] == rel["source_id"]: + entity["relations"].append({ + "type": rel["type"], + "target_id": rel["target_id"], + "target_name": rel["target_name"] + }) + + return entities + + def _normalize_type(self, spacy_label: str) -> str: + """Convert spaCy entity types to our types""" + mapping = { + "PERSON": "person", + "ORG": "organization", + "GPE": "location", + "PRODUCT": "product", + "EVENT": "event", + "WORK_OF_ART": "document", + } + return mapping.get(spacy_label, "entity") + + def _generate_id(self, entity_type: str, name: str) -> str: + """Generate consistent entity ID""" + normalized = name.lower().replace(" ", "-") + return f"{entity_type.lower()}-{normalized}" + + def _extract_context(self, ent, doc) -> str: + """Extract sentence containing the entity""" + for sent in doc.sents: + if ent.start >= sent.start and ent.end <= sent.end: + return sent.text + return ent.text + + def _extract_relations(self, sent, entity_map) -> List[Dict]: + """Extract relationships using dependency parsing""" + relations = [] + + # Pattern: PERSON works at/for ORG + for token in sent: + if token.dep_ in ["prep", "pobj"]: + if token.text.lower() in ["at", "for", "with"]: + # Find subject and object + subj = self._find_subject(token) + obj = self._find_object(token) + + if subj and obj: + if subj.text in entity_map and obj.text in entity_map: + relations.append({ + "source_id": entity_map[subj.text], + "type": "works_at" if token.text == "at" else "affiliated_with", + "target_id": entity_map[obj.text], + "target_name": obj.text + }) + + return relations + + def _find_subject(self, token): + """Find subject in dependency tree""" + for ancestor in token.ancestors: + if ancestor.dep_ in ["nsubj", "nsubjpass"]: + return ancestor + return None + + def _find_object(self, token): + """Find object in dependency tree""" + for child in token.children: + if child.dep_ in ["pobj", "dobj"]: + return child + return None + +# Usage +if __name__ == "__main__": + extractor = NLPEntityExtractor() + text = "John Doe works at Acme Corp in San Francisco." + entities = extractor.extract_entities(text) + print(json.dumps(entities, indent=2)) +``` + +## Approach 3: Hybrid LLM + NLP + +Combine the precision of NLP with the reasoning of LLMs: + +```go +// HybridExtractor combines NLP and LLM approaches +type HybridExtractor struct { + llmExtractor *EntityExtractor + nlpEndpoint string // Python NLP service endpoint +} + +func (h *HybridExtractor) Extract(ctx context.Context, text string) ([]index.GraphDocument, error) { + // Step 1: Use NLP to get candidate entities + nlpEntities, err := h.extractWithNLP(ctx, text) + if err != nil { + return nil, err + } + + // Step 2: Use LLM to enrich and validate + enrichedEntities, err := h.enrichWithLLM(ctx, text, nlpEntities) + if err != nil { + return nil, err + } + + // Step 3: Convert to GraphDocuments + return h.toGraphDocuments(ctx, enrichedEntities) +} +``` + +## Integration with Wingman + +### Option 1: Custom Extractor Tool + +```yaml +# config.yaml +tools: + entity-extractor: + type: custom + url: http://localhost:8080/extract + description: "Automatically extract entities and relationships from text" + +extractors: + auto-entity: + type: custom + url: http://localhost:8081/entities + llm_model: gpt-4 +``` + +### Option 2: Built-in Pipeline + +```go +// In your application +func ProcessDocument(ctx context.Context, filePath string) error { + // 1. Extract document content + doc, err := extractor.Extract(ctx, extractor.Input{ + File: &provider.File{Name: filePath}, + }, nil) + if err != nil { + return err + } + + // 2. Automatically extract entities and relationships + entities, err := entityExtractor.ExtractEntitiesFromText(ctx, string(doc.Content)) + if err != nil { + return err + } + + // 3. Index in semantic graph + for _, entity := range entities { + if err := graphProvider.IndexEntity(ctx, entity); err != nil { + log.Printf("Failed to index entity %s: %v", entity.EntityID, err) + } + } + + return nil +} +``` + +## Best Practices + +### 1. Entity ID Generation + +```go +func GenerateEntityID(entityType, name string) string { + // Normalize name: lowercase, replace spaces with hyphens + normalized := strings.ToLower(name) + normalized = strings.ReplaceAll(normalized, " ", "-") + + // Remove special characters + normalized = regexp.MustCompile(`[^a-z0-9-]`).ReplaceAllString(normalized, "") + + // Limit length + if len(normalized) > 50 { + normalized = normalized[:50] + } + + return fmt.Sprintf("%s-%s", entityType, normalized) +} +``` + +### 2. Deduplication + +```go +func DeduplicateEntities(entities []ExtractedEntity) []ExtractedEntity { + seen := make(map[string]*ExtractedEntity) + + for i := range entities { + entity := &entities[i] + + if existing, ok := seen[entity.ID]; ok { + // Merge relations + existing.Relations = append(existing.Relations, entity.Relations...) + // Merge attributes + for k, v := range entity.Attributes { + if _, exists := existing.Attributes[k]; !exists { + existing.Attributes[k] = v + } + } + } else { + seen[entity.ID] = entity + } + } + + result := make([]ExtractedEntity, 0, len(seen)) + for _, entity := range seen { + result = append(result, *entity) + } + + return result +} +``` + +### 3. Confidence Scoring + +```go +type ExtractedEntityWithConfidence struct { + ExtractedEntity + Confidence float64 `json:"confidence"` +} + +// Filter low-confidence extractions +func FilterByConfidence(entities []ExtractedEntityWithConfidence, minConfidence float64) []ExtractedEntity { + var result []ExtractedEntity + for _, entity := range entities { + if entity.Confidence >= minConfidence { + result = append(result, entity.ExtractedEntity) + } + } + return result +} +``` + +## Advanced: Fine-tuned Models + +For domain-specific extraction, consider fine-tuning: + +### 1. Prepare Training Data + +```json +{ + "text": "Dr. Smith works at Stanford Hospital.", + "entities": [ + {"text": "Dr. Smith", "label": "PERSON", "start": 0, "end": 9}, + {"text": "Stanford Hospital", "label": "ORG", "start": 19, "end": 36} + ], + "relations": [ + {"head": 0, "tail": 1, "label": "works_at"} + ] +} +``` + +### 2. Fine-tune with spaCy + +```python +import spacy +from spacy.training import Example + +# Load base model +nlp = spacy.load("en_core_web_sm") + +# Add custom NER labels +ner = nlp.get_pipe("ner") +ner.add_label("CUSTOM_ENTITY_TYPE") + +# Train +for epoch in range(10): + for text, annotations in training_data: + example = Example.from_dict(nlp.make_doc(text), annotations) + nlp.update([example]) + +# Save +nlp.to_disk("./custom_model") +``` + +## Performance Considerations + +| Approach | Speed | Accuracy | Cost | Offline | +|----------|-------|----------|------|---------| +| LLM-based | Slow | High | High | No | +| NLP-based | Fast | Medium | Free | Yes | +| Hybrid | Medium | High | Medium | Partial | +| Fine-tuned | Fast | High* | Training cost | Yes | + +*High accuracy for domain-specific data + +## Example: Complete Pipeline + +```go +package main + +import ( + "context" + "log" +) + +func main() { + ctx := context.Background() + + // Initialize components + completer := openai.NewCompleter("gpt-4", apiKey) + embedder := openai.NewEmbedder("text-embedding-3-small", apiKey) + extractor := extraction.NewEntityExtractor(completer, embedder) + graphProvider := qdrant.NewClient(config, embedder) + + // Process document + text := ` + TechCorp announced today that Sarah Johnson has been appointed as + the new Chief Technology Officer. She will lead the engineering + team and work closely with CEO Michael Chen on the company's AI + strategy. TechCorp, based in Seattle, has been developing machine + learning platforms since 2018. + ` + + // Automatic extraction - no manual specification needed! + entities, err := extractor.ExtractEntitiesFromText(ctx, text) + if err != nil { + log.Fatal(err) + } + + // Results (automatically extracted): + // - person-sarah-johnson (CTO at TechCorp) + // - person-michael-chen (CEO at TechCorp) + // - org-techcorp (company in Seattle) + // Relations: + // - sarah-johnson works_at org-techcorp + // - sarah-johnson manages engineering-team + // - michael-chen works_at org-techcorp + // - org-techcorp located_in seattle + + // Index in graph + for _, entity := range entities { + log.Printf("Auto-extracted: %s (%s)", entity.EntityID, entity.EntityType) + graphProvider.IndexEntity(ctx, entity) + } +} +``` + +## Summary + +To automatically extract entities and relationships: + +1. **Use LLM-based extraction** (GPT-4, Claude) for best accuracy and flexibility +2. **Use NLP tools** (spaCy) for fast, offline processing +3. **Implement deduplication** to handle entity mentions across documents +4. **Generate consistent IDs** using type and normalized name +5. **Add confidence scoring** to filter low-quality extractions +6. **Consider fine-tuning** for domain-specific use cases + +The LLM approach is recommended as it requires no manual specification - just provide the text and get fully structured entities with relationships automatically extracted! diff --git a/docs/semantic-graph-architecture.md b/docs/semantic-graph-architecture.md new file mode 100644 index 00000000..7de9f927 --- /dev/null +++ b/docs/semantic-graph-architecture.md @@ -0,0 +1,301 @@ +# Semantic Graph Architecture Diagram + +## Overview + +This document provides visual representations of the semantic graph architecture using Qdrant vector database. + +## Component Architecture + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Application Layer β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Semantic β”‚ β”‚ Relation β”‚ β”‚ Multi-hop β”‚ β”‚ +β”‚ β”‚ Search β”‚ β”‚ Query β”‚ β”‚ Traversal β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ GraphProvider Interface β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ β€’ Query(query, filters) β”‚ β”‚ +β”‚ β”‚ β€’ QueryRelated(entityID, relationType) β”‚ β”‚ +β”‚ β”‚ β€’ QueryGraph(query, relationFilters) β”‚ β”‚ +β”‚ β”‚ β€’ TraverseGraph(startEntityID, path) β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Qdrant Client Implementation β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Vector Search β”‚ β”‚ Payload Filtering β”‚ β”‚ +β”‚ β”‚ - HNSW Index β”‚ β”‚ - Keyword Match β”‚ β”‚ +β”‚ β”‚ - Cosine Distance β”‚ β”‚ - Range Filter β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Qdrant Vector Database β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Collection: knowledge_graph β”‚ β”‚ +β”‚ β”‚ β”‚ β”‚ +β”‚ β”‚ Point { β”‚ β”‚ +β”‚ β”‚ id: "person-john-doe", β”‚ β”‚ +β”‚ β”‚ vector: [0.1, 0.2, ..., 0.n], // Semantic content β”‚ β”‚ +β”‚ β”‚ payload: { β”‚ β”‚ +β”‚ β”‚ entity_type: "person", β”‚ β”‚ +β”‚ β”‚ entity_id: "person-john-doe", β”‚ β”‚ +β”‚ β”‚ relations: [ // Graph edges β”‚ β”‚ +β”‚ β”‚ "works_at:org-acme-corp", β”‚ β”‚ +β”‚ β”‚ "knows:person-jane-smith" β”‚ β”‚ +β”‚ β”‚ ], β”‚ β”‚ +β”‚ β”‚ metadata: {...} // Attributes β”‚ β”‚ +β”‚ β”‚ } β”‚ β”‚ +β”‚ β”‚ } β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## Data Flow for Entity Indexing + +``` +Document + β”‚ + β”œβ”€> Entity Extraction + β”‚ β”œβ”€> Entity 1 (Person: John Doe) + β”‚ β”œβ”€> Entity 2 (Organization: Acme Corp) + β”‚ └─> Entity 3 (Project: ML Platform) + β”‚ + β”œβ”€> Relationship Extraction + β”‚ β”œβ”€> works_at: John Doe -> Acme Corp + β”‚ β”œβ”€> works_on: John Doe -> ML Platform + β”‚ └─> sponsors: Acme Corp -> ML Platform + β”‚ + β”œβ”€> Embedding Generation + β”‚ β”œβ”€> John Doe content -> [0.1, 0.2, ...] + β”‚ β”œβ”€> Acme Corp content -> [0.3, 0.4, ...] + β”‚ └─> ML Platform content -> [0.5, 0.6, ...] + β”‚ + └─> Store in Qdrant + β”œβ”€> Point 1: person-john-doe + β”‚ β”œβ”€> Vector: [embeddings...] + β”‚ └─> Payload: {relations: ["works_at:org-acme-corp", ...]} + β”‚ + β”œβ”€> Point 2: org-acme-corp + β”‚ β”œβ”€> Vector: [embeddings...] + β”‚ └─> Payload: {relations: ["employs:person-john-doe", ...]} + β”‚ + └─> Point 3: project-ml-platform + β”œβ”€> Vector: [embeddings...] + └─> Payload: {relations: ["involves:person-john-doe", ...]} +``` + +## Query Types + +### 1. Semantic Search (Vector-based) + +``` +Query: "machine learning expert" + β”‚ + β”œβ”€> Generate Embedding + β”‚ └─> [0.15, 0.25, ...] + β”‚ + β”œβ”€> Vector Search in Qdrant + β”‚ β”œβ”€> HNSW nearest neighbors + β”‚ └─> Apply metadata filters + β”‚ + └─> Results (sorted by similarity) + β”œβ”€> John Doe (score: 0.92) + β”œβ”€> Jane Smith (score: 0.87) + └─> ... +``` + +### 2. Relationship Query (Graph-based) + +``` +Query: "Find employees of Acme Corp" + β”‚ + β”œβ”€> Build Filter + β”‚ └─> relations contains "employs:*" + β”‚ from entity "org-acme-corp" + β”‚ + β”œβ”€> Payload Search in Qdrant + β”‚ └─> Filter by metadata + β”‚ + └─> Results + β”œβ”€> John Doe + β”œβ”€> Jane Smith + └─> ... +``` + +### 3. Hybrid Query (Vector + Graph) + +``` +Query: "ML expert at Acme Corp" + β”‚ + β”œβ”€> Semantic Component + β”‚ β”œβ”€> Generate embedding for "ML expert" + β”‚ └─> Vector: [0.15, 0.25, ...] + β”‚ + β”œβ”€> Graph Component + β”‚ └─> Filter: relations contains "works_at:org-acme-corp" + β”‚ + β”œβ”€> Combined Search + β”‚ β”œβ”€> Vector similarity search + β”‚ └─> + Payload filtering + β”‚ + └─> Results (sorted by vector similarity) + β”œβ”€> John Doe (score: 0.92, works_at: Acme Corp) + └─> ... +``` + +### 4. Multi-hop Traversal (Graph Navigation) + +``` +Start: "person-john-doe" +Path: ["works_at", "employs", "works_on"] + β”‚ + β”œβ”€> Hop 1: works_at + β”‚ └─> org-acme-corp + β”‚ + β”œβ”€> Hop 2: employs + β”‚ β”œβ”€> person-john-doe + β”‚ β”œβ”€> person-jane-smith + β”‚ └─> person-bob-jones + β”‚ + └─> Hop 3: works_on + β”œβ”€> project-ml-platform + β”œβ”€> project-web-app + └─> project-mobile-app +``` + +## Entity-Relationship Model + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Person β”‚ +β”‚ - name β”‚ +β”‚ - title β”‚ +β”‚ - department β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”‚ works_at / employs + β”‚ + ↓ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Organization │─────────│ Project β”‚ +β”‚ - name β”‚ sponsorsβ”‚ - name β”‚ +β”‚ - industry β”‚ / β”‚ - status β”‚ +β”‚ - founded β”‚sponsoredβ”‚ - budget β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ by β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + ↑ β”‚ + β”‚ β”‚ + β”‚ works_at β”‚ works_on + β”‚ β”‚ involves + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + Person +``` + +## Storage Layout in Qdrant + +``` +Collection: knowledge_graph +β”‚ +β”œβ”€> Points (Entities) +β”‚ β”‚ +β”‚ β”œβ”€> ID: person-john-doe +β”‚ β”‚ β”œβ”€> Vector: [semantic embedding of content] +β”‚ β”‚ └─> Payload: +β”‚ β”‚ β”œβ”€> entity_type: "person" +β”‚ β”‚ β”œβ”€> entity_id: "person-john-doe" +β”‚ β”‚ β”œβ”€> content: "John Doe is a..." +β”‚ β”‚ β”œβ”€> metadata: {name: "John Doe", title: "Engineer"} +β”‚ β”‚ └─> relations: [ +β”‚ β”‚ "works_at:org-acme-corp", +β”‚ β”‚ "knows:person-jane-smith", +β”‚ β”‚ "works_on:project-ml-platform" +β”‚ β”‚ ] +β”‚ β”‚ +β”‚ β”œβ”€> ID: org-acme-corp +β”‚ β”‚ β”œβ”€> Vector: [semantic embedding of content] +β”‚ β”‚ └─> Payload: +β”‚ β”‚ β”œβ”€> entity_type: "organization" +β”‚ β”‚ β”œβ”€> relations: [ +β”‚ β”‚ "employs:person-john-doe", +β”‚ β”‚ "employs:person-jane-smith", +β”‚ β”‚ "sponsors:project-ml-platform" +β”‚ β”‚ ] +β”‚ β”‚ └─> ... +β”‚ β”‚ +β”‚ └─> ... +β”‚ +β”œβ”€> Indexes +β”‚ β”œβ”€> HNSW (Vector Index) +β”‚ β”‚ └─> For fast similarity search +β”‚ β”‚ +β”‚ └─> Payload Indexes +β”‚ β”œβ”€> entity_type (keyword) +β”‚ β”œβ”€> entity_id (keyword) +β”‚ β”œβ”€> relations (keyword[]) +β”‚ └─> metadata.* (various types) +β”‚ +└─> Configuration + β”œβ”€> Vector size: 1536 + β”œβ”€> Distance: cosine + └─> Replication factor: 1 +``` + +## Performance Characteristics + +### Vector Search +- **Complexity**: O(log n) with HNSW +- **Use Case**: Find similar entities +- **Scale**: Millions of entities + +### Relationship Query +- **Complexity**: O(1) with indexed payload +- **Use Case**: Find related entities +- **Scale**: Fast even with many relations + +### Hybrid Query +- **Complexity**: O(log n) + O(1) +- **Use Case**: Semantic + structural constraints +- **Scale**: Combines both efficiently + +### Multi-hop Traversal +- **Complexity**: O(k * r) where k=hops, r=avg relations +- **Use Case**: Graph navigation +- **Scale**: Limit depth to 3-4 hops for performance + +## Integration Points + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Wingman Platform β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Extractors β”‚ Document processing & entity β”‚ +β”‚ β”‚ extraction β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Embedders β”‚ Generate vector representations β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ GraphProvider β”‚ Store and query semantic graph β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Tools β”‚ Expose graph operations to LLMs β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ API β”‚ REST/gRPC endpoints β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ β”‚ + ↓ ↓ ↓ + Qdrant DB LLM Agents Applications +``` + +## Best Practices Summary + +1. **Bidirectional Relations**: Store both directions for efficient traversal +2. **Consistent Naming**: Use format `{type}:{entity-id}` for relations +3. **Quality Embeddings**: Use appropriate embedding models +4. **Indexed Metadata**: Index frequently queried fields +5. **Limit Depth**: Keep graph traversals to 3-4 hops +6. **Batch Operations**: Use batch upserts for large datasets diff --git a/docs/semantic-graph-qdrant.md b/docs/semantic-graph-qdrant.md new file mode 100644 index 00000000..2af0dffd --- /dev/null +++ b/docs/semantic-graph-qdrant.md @@ -0,0 +1,440 @@ +# Semantic Graph Modeling with Qdrant Vector Database + +## Overview + +This document explains how to apply data extraction to Qdrant vector database to model a semantic graph where entities have relationships to each other while keeping the data at the leafs in vector form. + +## Concept + +A semantic graph with vector embeddings at the leaf nodes combines: +1. **Graph Structure**: Entities and their relationships (edges) +2. **Vector Embeddings**: Semantic representations of entity content +3. **Metadata**: Relationship information and entity attributes + +## Architecture + +### Data Model + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Qdrant Collection β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Point ID: entity-1 β”‚ +β”‚ Vector: [0.1, 0.2, ..., 0.n] β”‚ +β”‚ Metadata: β”‚ +β”‚ - entity_type: "person" β”‚ +β”‚ - entity_name: "John Doe" β”‚ +β”‚ - content: "John Doe is a..." β”‚ +β”‚ - relations: ["works_at:company-1", β”‚ +β”‚ "knows:entity-2"] β”‚ +β”‚ - source_document: "doc-123" β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### Key Components + +1. **Vector Embeddings**: Store semantic content as vectors +2. **Point ID**: Unique identifier for each entity +3. **Metadata Payload**: + - Entity type and attributes + - Relationship information (edges) + - Original content (if needed) + - Source tracking + +## Implementation Strategy + +### 1. Data Extraction Phase + +Use the existing extractor infrastructure to process documents: + +```go +// Extract entities and relationships from documents +type EntityExtraction struct { + ID string + Type string + Content string + Attributes map[string]string + Relations []Relation +} + +type Relation struct { + Type string // e.g., "works_at", "knows", "part_of" + TargetID string // ID of related entity + Confidence float32 // Optional: confidence score +} +``` + +### 2. Vector Storage with Relationships + +Store entities in Qdrant with relationship metadata: + +```go +// Store entity with relationships in Qdrant +func StoreEntity(ctx context.Context, entity EntityExtraction, embedding []float32) error { + // Create payload with relationship information + payload := map[string]interface{}{ + "entity_type": entity.Type, + "content": entity.Content, + "attributes": entity.Attributes, + "relations": encodeRelations(entity.Relations), + } + + // Store in Qdrant point collection + point := &qdrant.Point{ + ID: entity.ID, + Vector: embedding, + Payload: payload, + } + + return storePoint(ctx, point) +} +``` + +### 3. Querying the Semantic Graph + +#### Semantic Search (Vector-based) +```go +// Find semantically similar entities +results := qdrant.Search( + vector: queryEmbedding, + limit: 10, + filter: { + "entity_type": "person" + } +) +``` + +#### Relationship Traversal (Metadata-based) +```go +// Find entities related to a specific entity +func GetRelatedEntities(entityID string, relationType string) []Entity { + // Query points where relations contains the target + filter := map[string]interface{}{ + "must": []map[string]interface{}{ + { + "key": "relations", + "match": map[string]interface{}{ + "value": fmt.Sprintf("%s:%s", relationType, entityID), + }, + }, + }, + } + + return searchWithFilter(filter) +} +``` + +#### Hybrid Queries (Vector + Graph) +```go +// Find semantically similar entities that also have specific relationships +func HybridSearch(queryVector []float32, relationshipFilter RelationFilter) []Entity { + return qdrant.Search( + vector: queryVector, + limit: 50, + filter: { + "must": [ + {"key": "relations", "match": {"value": relationshipFilter.Pattern}}, + {"key": "entity_type", "match": {"value": relationshipFilter.EntityType}} + ] + } + ) +} +``` + +## Practical Example + +### Scenario: Knowledge Base with Entities and Relations + +``` +Document: "John Doe works at Acme Corp as a Software Engineer. + He collaborates with Jane Smith on Project X." + +Extracted Entities: +1. Person: John Doe +2. Organization: Acme Corp +3. Role: Software Engineer +4. Person: Jane Smith +5. Project: Project X + +Relations: +- John Doe --[works_at]--> Acme Corp +- John Doe --[has_role]--> Software Engineer +- John Doe --[collaborates_with]--> Jane Smith +- John Doe --[works_on]--> Project X +- Jane Smith --[works_on]--> Project X +``` + +### Storage in Qdrant + +```go +// Entity 1: John Doe +{ + ID: "person-john-doe", + Vector: [embeddings...], + Payload: { + "entity_type": "person", + "entity_name": "John Doe", + "content": "John Doe is a Software Engineer who works at Acme Corp...", + "relations": [ + "works_at:org-acme-corp", + "has_role:role-software-engineer", + "collaborates_with:person-jane-smith", + "works_on:project-x" + ], + "source_document": "doc-123" + } +} + +// Entity 2: Acme Corp +{ + ID: "org-acme-corp", + Vector: [embeddings...], + Payload: { + "entity_type": "organization", + "entity_name": "Acme Corp", + "content": "Acme Corp is a technology company...", + "relations": [ + "employs:person-john-doe", + "employs:person-jane-smith" + ], + "industry": "technology" + } +} +``` + +## Query Examples + +### 1. Semantic Search +```go +// Find people similar to "software developer with AI experience" +query := "software developer with AI experience" +embedding := embedder.Embed(query) +results := qdrant.Search(embedding, 10, map[string]string{ + "entity_type": "person" +}) +``` + +### 2. Relationship Navigation +```go +// Find all people who work at Acme Corp +employees := GetRelatedEntities("org-acme-corp", "employs") + +// Find all projects John Doe works on +projects := GetRelatedTo("person-john-doe", "works_on") +``` + +### 3. Multi-hop Traversal +```go +// Find colleagues of John Doe (people working at the same company) +company := GetRelatedEntity("person-john-doe", "works_at") +colleagues := GetRelatedEntities(company.ID, "employs") +``` + +### 4. Hybrid Semantic + Graph Query +```go +// Find people with ML expertise who work in the same company as John Doe +mlQuery := "machine learning expertise" +mlEmbedding := embedder.Embed(mlQuery) + +johnCompany := GetRelatedEntity("person-john-doe", "works_at") + +results := qdrant.Search(mlEmbedding, 20, { + "must": [ + {"key": "entity_type", "match": {"value": "person"}}, + {"key": "relations", "match": {"value": "works_at:" + johnCompany.ID}} + ] +}) +``` + +## Advantages of This Approach + +1. **Vector Semantics**: Leverage embeddings for similarity search +2. **Graph Structure**: Navigate explicit relationships between entities +3. **Hybrid Queries**: Combine semantic and structural information +4. **Scalability**: Qdrant handles both vector and metadata efficiently +5. **Flexibility**: Add new relationship types without schema changes + +## Implementation in Wingman + +### Extending the Index Interface + +```go +// pkg/index/graph.go +package index + +// GraphDocument extends Document with relationship information +type GraphDocument struct { + Document + EntityID string + EntityType string + Relations []Relation +} + +type Relation struct { + Type string + TargetID string + Metadata map[string]string +} + +// GraphProvider extends Provider with graph capabilities +type GraphProvider interface { + Provider + + // IndexEntity stores an entity with relationships + IndexEntity(ctx context.Context, doc GraphDocument) error + + // QueryRelated finds entities related to the given entity + QueryRelated(ctx context.Context, entityID string, relationType string, opts *QueryOptions) ([]QueryResult, error) + + // QueryGraph performs hybrid vector + graph queries + QueryGraph(ctx context.Context, query string, relationFilter map[string]string, opts *QueryOptions) ([]QueryResult, error) +} +``` + +### Qdrant Implementation Example + +```go +// pkg/index/qdrant/client.go +package qdrant + +import ( + "context" + "github.com/adrianliechti/wingman/pkg/index" +) + +type Client struct { + collectionName string + // qdrant client fields +} + +func (c *Client) IndexEntity(ctx context.Context, doc index.GraphDocument) error { + // Encode relations as payload metadata + relations := make([]string, len(doc.Relations)) + for i, rel := range doc.Relations { + relations[i] = fmt.Sprintf("%s:%s", rel.Type, rel.TargetID) + } + + payload := map[string]interface{}{ + "content": doc.Content, + "entity_id": doc.EntityID, + "entity_type": doc.EntityType, + "relations": relations, + "metadata": doc.Metadata, + } + + // Store in Qdrant + return c.upsertPoint(ctx, doc.EntityID, doc.Embedding, payload) +} + +func (c *Client) QueryRelated(ctx context.Context, entityID string, relationType string, opts *index.QueryOptions) ([]index.QueryResult, error) { + // Build filter for relations + relationPattern := fmt.Sprintf("%s:%s", relationType, entityID) + + filter := map[string]interface{}{ + "must": []map[string]interface{}{ + { + "key": "relations", + "match": map[string]interface{}{ + "any": []string{relationPattern}, + }, + }, + }, + } + + // Query Qdrant with filter + return c.searchWithFilter(ctx, nil, filter, opts) +} + +func (c *Client) QueryGraph(ctx context.Context, query string, relationFilter map[string]string, opts *index.QueryOptions) ([]index.QueryResult, error) { + // Generate embedding for query + embedding := c.embedder.Embed(ctx, []string{query}) + + // Build compound filter + filter := buildCompoundFilter(relationFilter) + + // Perform hybrid search + return c.searchWithFilter(ctx, embedding.Embeddings[0], filter, opts) +} +``` + +## Configuration Example + +```yaml +# config.yaml +indexes: + semantic-graph: + type: qdrant + url: http://localhost:6333 + collection: knowledge_graph + vector_size: 1536 + distance: cosine + + # Enable graph capabilities + graph_mode: true + + # Index configuration + index_config: + m: 16 + ef_construct: 100 + + # Metadata schema for relations + metadata_schema: + entity_id: + type: keyword + index: true + entity_type: + type: keyword + index: true + relations: + type: keyword[] + index: true +``` + +## Best Practices + +1. **Bidirectional Relations**: Store relationships in both directions for efficient traversal + ``` + Person --[works_at]--> Company + Company --[employs]--> Person + ``` + +2. **Relation Encoding**: Use consistent format for encoding relationships + ``` + Format: "{relation_type}:{target_id}" + Example: "works_at:company-123" + ``` + +3. **Entity IDs**: Use deterministic, content-based IDs or UUIDs + ``` + person-john-doe + org-acme-corp + project-x + ``` + +4. **Metadata Optimization**: Index frequently queried fields + - entity_type (for filtering by type) + - relations (for graph traversal) + - source_document (for traceability) + +5. **Vector Quality**: Use high-quality embeddings for leaf content + - OpenAI text-embedding-3-large + - Sentence-transformers + - Domain-specific models + +6. **Relationship Types**: Define a clear ontology + ``` + Person: works_at, knows, manages, reports_to + Organization: employs, partners_with, acquired_by + Project: involves, depends_on, delivers_to + ``` + +## Conclusion + +Yes, it is absolutely possible to apply data extraction to Qdrant vector DB to model a semantic graph while keeping leaf data in vector form. The approach leverages: + +- **Qdrant's payload metadata** for storing relationship information +- **Vector embeddings** for semantic content representation +- **Filtering capabilities** for graph traversal +- **Hybrid queries** combining vector similarity and graph structure + +This gives you the best of both worlds: semantic search through vectors and graph navigation through metadata, all within a single Qdrant collection. diff --git a/docs/semantic-graph-quickref.md b/docs/semantic-graph-quickref.md new file mode 100644 index 00000000..2721fd21 --- /dev/null +++ b/docs/semantic-graph-quickref.md @@ -0,0 +1,361 @@ +# Semantic Graph Quick Reference + +A quick reference guide for implementing semantic graphs with Qdrant vector database in Wingman. + +## Core Concepts + +| Concept | Description | Example | +|---------|-------------|---------| +| **Entity** | A node in the graph with vector embedding | Person, Organization, Project | +| **Relation** | A typed edge between entities | works_at, knows, manages | +| **Vector** | Semantic representation of entity content | [0.1, 0.2, ..., 0.n] | +| **Metadata** | Attributes and relationship information | {name: "John", relations: ["works_at:org-1"]} | + +## Data Model + +### GraphDocument Structure +```go +type GraphDocument struct { + Document // Base document with content and embedding + EntityID string // Unique entity identifier + EntityType string // Entity category (person, org, etc.) + Relations []Relation // Edges to other entities +} +``` + +### Relation Structure +```go +type Relation struct { + Type string // Relationship type + TargetID string // Target entity ID + Metadata map[string]string // Additional relation data +} +``` + +## Common Operations + +### 1. Index an Entity + +```go +entity := index.GraphDocument{ + Document: index.Document{ + Content: "John Doe is a software engineer...", + Embedding: embeddings, + Metadata: map[string]string{"name": "John Doe"}, + }, + EntityID: "person-john-doe", + EntityType: "person", + Relations: []index.Relation{ + {Type: "works_at", TargetID: "org-acme"}, + {Type: "knows", TargetID: "person-jane"}, + }, +} +err := graphProvider.IndexEntity(ctx, entity) +``` + +### 2. Semantic Search + +```go +// Find entities by semantic similarity +results, err := graphProvider.Query( + ctx, + "machine learning expert", + &index.QueryOptions{ + Limit: ptr(10), + Filters: map[string]string{ + "entity_type": "person", + }, + }, +) +``` + +### 3. Relationship Query + +```go +// Find entities related to a specific entity +employees, err := graphProvider.QueryRelated( + ctx, + "org-acme-corp", // Entity ID + "employs", // Relation type + &index.QueryOptions{Limit: ptr(50)}, +) +``` + +### 4. Hybrid Query + +```go +// Combine semantic search with graph constraints +relationFilter := map[string]string{ + "works_at": "org-acme-corp", +} +results, err := graphProvider.QueryGraph( + ctx, + "AI researcher", + relationFilter, + &index.QueryOptions{Limit: ptr(20)}, +) +``` + +### 5. Graph Traversal + +```go +// Navigate multiple hops in the graph +path := []string{"works_at", "employs", "works_on"} +projects, err := graphProvider.TraverseGraph( + ctx, + "person-john-doe", + path, + &index.QueryOptions{Limit: ptr(30)}, +) +``` + +## Entity Types + +### Common Entity Types +- `person` - Individual people +- `organization` - Companies, institutions +- `project` - Projects, initiatives +- `role` - Job roles, positions +- `document` - Documents, articles +- `location` - Places, offices +- `product` - Products, services +- `event` - Events, meetings + +## Relationship Types + +### Employment Relations +- `works_at` / `employs` - Employment +- `manages` / `managed_by` - Management +- `reports_to` / `supervises` - Reporting structure + +### Collaboration Relations +- `knows` - Personal connection +- `collaborates_with` - Working together +- `works_on` / `involves` - Project participation + +### Organizational Relations +- `part_of` / `contains` - Organizational structure +- `sponsors` / `sponsored_by` - Sponsorship +- `partners_with` - Partnerships + +### Document Relations +- `authored_by` / `authored` - Authorship +- `references` / `referenced_by` - Citations +- `about` / `mentioned_in` - Topic relations + +## Query Patterns + +### Pattern 1: Find Similar Entities +```go +// Use case: Find people similar to John Doe +query := "software engineer with ML expertise" +results := graphProvider.Query(ctx, query, opts) +``` + +### Pattern 2: Find Direct Relationships +```go +// Use case: Who does John Doe know? +friends := graphProvider.QueryRelated(ctx, "person-john", "knows", opts) +``` + +### Pattern 3: Find Indirect Relationships +```go +// Use case: Find John's colleagues (people at same company) +// Step 1: Find John's company +companies := graphProvider.QueryRelated(ctx, "person-john", "works_at", opts) +// Step 2: Find company's employees +colleagues := graphProvider.QueryRelated(ctx, companies[0].EntityID, "employs", opts) +``` + +### Pattern 4: Semantic Search with Constraints +```go +// Use case: Find AI experts who work at Google +filter := map[string]string{"works_at": "org-google"} +experts := graphProvider.QueryGraph(ctx, "artificial intelligence expert", filter, opts) +``` + +### Pattern 5: Multi-hop Exploration +```go +// Use case: Find technologies used by John's colleagues +// Path: John -> works_at -> Company -> employs -> Colleagues -> uses -> Tech +path := []string{"works_at", "employs", "uses"} +tech := graphProvider.TraverseGraph(ctx, "person-john", path, opts) +``` + +## Qdrant Configuration + +### Collection Setup +```yaml +collection: knowledge_graph +vector_size: 1536 # OpenAI text-embedding-3-small +distance: cosine # cosine, euclid, or dot +``` + +### Index Configuration +```yaml +hnsw_config: + m: 16 # Edges per node (16-64 typical) + ef_construct: 100 # Build quality (100-200 typical) +``` + +### Payload Indexes +```yaml +payload_indexes: + - field_name: entity_type + field_schema: keyword + - field_name: entity_id + field_schema: keyword + - field_name: relations + field_schema: keyword +``` + +## Performance Tips + +### Indexing +- Use batch operations for bulk imports +- Pre-compute bidirectional relations +- Index metadata fields used in filters + +### Querying +- Limit result sets appropriately +- Use specific entity type filters +- Cache frequently accessed entities +- Limit graph traversal depth to 3-4 hops + +### Storage +- Use consistent ID formats +- Compress large content fields +- Store only necessary metadata +- Consider sharding for very large graphs (>10M entities) + +## Error Handling + +```go +// Always handle errors appropriately +results, err := graphProvider.QueryGraph(ctx, query, filter, opts) +if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + // Handle timeout + } else if errors.Is(err, ErrNotFound) { + // Handle missing entity + } else { + // Handle other errors + } +} +``` + +## Testing + +### Unit Test Example +```go +func TestGraphQuery(t *testing.T) { + // Setup + ctx := context.Background() + graphProvider := setupTestGraph(t) + + // Index test entities + entity := createTestEntity("person-john", "person") + err := graphProvider.IndexEntity(ctx, entity) + require.NoError(t, err) + + // Test query + results, err := graphProvider.QueryRelated(ctx, "person-john", "knows", nil) + require.NoError(t, err) + assert.Len(t, results, 2) +} +``` + +## Integration Examples + +### With Document Extraction +```go +// Extract entities from document +doc := extractDocument(filePath) +entities := extractEntitiesFromText(doc.Content) + +// Index entities with relationships +for _, entity := range entities { + embedding := embedder.Embed(ctx, []string{entity.Content}) + entity.Embedding = embedding.Embeddings[0] + graphProvider.IndexEntity(ctx, entity) +} +``` + +### With LLM Tools +```yaml +tools: + graph-search: + type: custom + description: "Search knowledge graph for entities and relationships" + parameters: + query: string + entity_type: string + relation_filter: object +``` + +### With RAG Pipeline +```go +// Recall relevant entities +query := "What do we know about ML projects?" +entities := graphProvider.Query(ctx, query, opts) + +// Expand with related entities +for _, entity := range entities { + related := graphProvider.QueryRelated(ctx, entity.EntityID, "relates_to", opts) + entities = append(entities, related...) +} + +// Use as context for LLM +context := buildContextFromEntities(entities) +response := llm.Complete(ctx, context + "\n" + query) +``` + +## Common Pitfalls + +❌ **Don't**: Store relations as plain text +```go +// Bad: Hard to query +Metadata: map[string]string{ + "relationships": "works_at Acme Corp, knows Jane Smith" +} +``` + +βœ… **Do**: Use structured relation format +```go +// Good: Easy to filter and query +Relations: []Relation{ + {Type: "works_at", TargetID: "org-acme"}, + {Type: "knows", TargetID: "person-jane"}, +} +``` + +❌ **Don't**: Use arbitrary entity IDs +```go +// Bad: Inconsistent format +EntityID: "John123" +EntityID: "acme_corporation" +EntityID: "project-456" +``` + +βœ… **Do**: Use consistent ID naming +```go +// Good: Clear, consistent format +EntityID: "person-john-doe" +EntityID: "org-acme-corp" +EntityID: "project-ml-platform" +``` + +## Resources + +- [Full Documentation](./semantic-graph-qdrant.md) +- [Architecture Diagrams](./semantic-graph-architecture.md) +- [Example Implementation](../examples/semantic-graph/) +- [Qdrant Documentation](https://qdrant.tech/documentation/) + +## Support + +For questions or issues: +1. Check the documentation first +2. Review example implementations +3. Open an issue on GitHub +4. Consult Qdrant documentation for vector DB specifics diff --git a/examples/auto-extraction/README.md b/examples/auto-extraction/README.md new file mode 100644 index 00000000..588ca22e --- /dev/null +++ b/examples/auto-extraction/README.md @@ -0,0 +1,287 @@ +# Automatic Entity Extraction Example + +This example demonstrates how to automatically extract entities and their relationships from text using LLMs, eliminating the need to manually specify EntityID, EntityType, and Relations. + +## Overview + +Instead of manually specifying: +```go +EntityID: "person-john-doe", +EntityType: "person", +Relations: []index.Relation{ + {Type: "works_at", TargetID: "org-acme"}, + {Type: "knows", TargetID: "person-jane"}, +} +``` + +You can now simply provide the text and let the LLM automatically extract everything: +```go +entities, err := extractor.ExtractFromText(ctx, yourText) +// All entities and relationships extracted automatically! +``` + +## How It Works + +The automatic extraction process: + +1. **Text Analysis**: LLM reads and understands the input text +2. **Entity Detection**: Identifies people, organizations, projects, locations, etc. +3. **ID Generation**: Creates consistent IDs (e.g., "person-sarah-johnson") +4. **Type Classification**: Determines entity types automatically +5. **Relationship Extraction**: Finds and structures relationships between entities +6. **Bidirectional Relations**: Creates reverse relationships (employs/works_at) +7. **Embedding Generation**: Creates vector embeddings for each entity +8. **Graph Storage**: Stores everything in the semantic graph + +## Example Input + +``` +TechCorp announced today that Sarah Johnson has been appointed as the new +Chief Technology Officer. She will lead the engineering team and work closely +with CEO Michael Chen on the company's AI strategy. +``` + +## Automatic Output + +The system automatically extracts: + +### Entities +- **person-sarah-johnson** (person) + - Attributes: title="Chief Technology Officer", expertise="ML" + +- **person-michael-chen** (person) + - Attributes: title="Chief Executive Officer" + +- **org-techcorp** (organization) + - Attributes: location="San Francisco", industry="AI" + +- **project-ai-platform** (project) + - Attributes: description="ML platform" + +### Relationships +- sarah-johnson **works_at** org-techcorp +- sarah-johnson **manages** team-engineering +- sarah-johnson **works_on** project-ai-platform +- michael-chen **works_at** org-techcorp +- org-techcorp **employs** person-sarah-johnson +- org-techcorp **employs** person-michael-chen +- org-techcorp **sponsors** project-ai-platform + +All extracted **automatically** - no manual specification needed! + +## Running the Example + +### Prerequisites + +1. **Go 1.21+** +2. **OpenAI API Key** (or other LLM provider) +3. **Qdrant Instance** (optional for storage) + +### Setup + +```bash +# Set your OpenAI API key +export OPENAI_API_KEY="sk-..." + +# Run Qdrant (optional) +docker run -p 6333:6333 qdrant/qdrant +``` + +### Build and Run + +```bash +cd examples/auto-extraction +go build +./auto-extraction +``` + +## Code Example + +```go +package main + +import ( + "context" + "log" + + "github.com/adrianliechti/wingman/pkg/extractor/entity" + "github.com/adrianliechti/wingman/pkg/provider/openai" +) + +func main() { + ctx := context.Background() + + // Initialize LLM and embedder + completer := openai.NewCompleter("gpt-4", apiKey) + embedder := openai.NewEmbedder("text-embedding-3-small", apiKey) + + // Create automatic extractor + extractor := entity.NewExtractor(completer, embedder) + + // Your document text + text := ` + Dr. Jane Smith is the Director of AI Research at Stanford University. + She collaborates with Prof. Bob Wilson on the Neural Networks project. + The project is funded by TechCorp and aims to advance deep learning. + ` + + // Extract entities automatically - no manual specification! + entities, err := extractor.ExtractFromText(ctx, text) + if err != nil { + log.Fatal(err) + } + + // Results automatically include: + // - person-jane-smith (Director of AI Research) + // - person-bob-wilson (Professor) + // - org-stanford-university (University) + // - org-techcorp (Company) + // - project-neural-networks (Project) + // Plus all relationships between them! + + for _, entity := range entities { + log.Printf("Extracted: %s (%s) with %d relations", + entity.EntityID, entity.EntityType, len(entity.Relations)) + } +} +``` + +## Configuration + +### Using Different LLM Providers + +```go +// OpenAI +completer := openai.NewCompleter("gpt-4", apiKey) + +// Anthropic +completer := anthropic.NewCompleter("claude-3-opus-20240229", apiKey) + +// Local model via Ollama +completer := ollama.NewCompleter("llama3", "http://localhost:11434") +``` + +### Customizing Extraction + +You can customize the extraction by modifying the prompt in `pkg/extractor/entity/extractor.go`: + +```go +// Add domain-specific entity types +// Add industry-specific relationship types +// Adjust extraction guidelines +``` + +## Benefits + +### βœ… Zero Manual Work +- No need to specify entity IDs +- No need to define entity types +- No need to enumerate relationships + +### βœ… Intelligent Extraction +- Understands context and semantics +- Infers implicit relationships +- Handles ambiguous references + +### βœ… Consistent Output +- Generates standard ID format +- Uses common entity types +- Creates bidirectional relations + +### βœ… Scalable Processing +- Process documents automatically +- Batch process large corpora +- Update graph incrementally + +## Advanced Usage + +### Batch Processing + +```go +documents := []string{doc1, doc2, doc3, ...} + +for _, doc := range documents { + entities, err := extractor.ExtractFromText(ctx, doc) + if err != nil { + log.Printf("Failed to extract from doc: %v", err) + continue + } + + for _, entity := range entities { + graphProvider.IndexEntity(ctx, entity) + } +} +``` + +### Custom Entity Types + +The extractor automatically detects common types: +- person +- organization +- project +- location +- product +- event +- role +- team +- department + +Add more types by updating the extraction prompt. + +### Relationship Types + +Common relationships automatically detected: +- works_at / employs +- manages / managed_by +- knows +- works_on / involves +- located_in / contains +- part_of / contains +- founded / founded_by +- sponsors / sponsored_by +- partners_with +- reports_to / supervises + +## Troubleshooting + +### Issue: LLM returns invalid JSON + +**Solution**: The extractor handles markdown code blocks automatically. If issues persist, check the LLM temperature (should be low, e.g., 0.1). + +### Issue: Duplicate entities + +**Solution**: The extractor includes automatic deduplication. Entities with the same ID are merged. + +### Issue: Missing relationships + +**Solution**: Increase context or be more explicit in source text. The LLM extracts relationships based on what's stated or strongly implied. + +### Issue: Incorrect entity types + +**Solution**: Update the extraction prompt to provide better examples or constraints for your domain. + +## Performance + +- **Speed**: ~2-5 seconds per document (depends on LLM) +- **Accuracy**: 85-95% (depends on text quality and LLM model) +- **Cost**: ~$0.01-0.05 per document (GPT-4) + +## Next Steps + +1. **Try it**: Run the example with your own text +2. **Customize**: Adapt the extraction prompt for your domain +3. **Scale**: Process your document corpus automatically +4. **Query**: Use the semantic graph for hybrid search + +## References + +- [Automatic Extraction Guide](../../docs/automatic-entity-extraction.md) +- [Semantic Graph Documentation](../../docs/semantic-graph-qdrant.md) +- [Manual Extraction Example](../semantic-graph/) + +## Support + +For questions or issues: +1. Check the [documentation](../../docs/automatic-entity-extraction.md) +2. Review the [semantic graph guide](../../docs/semantic-graph-qdrant.md) +3. Open an issue on GitHub diff --git a/examples/auto-extraction/main.go b/examples/auto-extraction/main.go new file mode 100644 index 00000000..907eae7d --- /dev/null +++ b/examples/auto-extraction/main.go @@ -0,0 +1,236 @@ +// Package main demonstrates automatic entity and relationship extraction. +// This example shows how to use LLMs to automatically detect entities and +// their relationships from text without manual specification. +package main + +import ( + "context" + "fmt" + "log" + "strings" + + "github.com/adrianliechti/wingman/pkg/extractor/entity" + "github.com/adrianliechti/wingman/pkg/index/qdrant" + "github.com/adrianliechti/wingman/pkg/provider" +) + +func main() { + ctx := context.Background() + + // Example: Document text without any manual entity specification + exampleText := ` +TechCorp announced today that Sarah Johnson has been appointed as the new +Chief Technology Officer. She will lead the engineering team and work closely +with CEO Michael Chen on the company's AI strategy. + +Sarah joins from InnovateLabs, where she spent five years building machine +learning platforms. At TechCorp, she will oversee the AI Platform project, +which aims to revolutionize how enterprises deploy machine learning models. + +TechCorp, based in San Francisco, has been developing artificial intelligence +solutions since 2018. The company recently partnered with DataCorp to expand +its cloud infrastructure capabilities. + +"I'm excited to join TechCorp and work with such a talented team," said +Sarah Johnson. "The AI Platform project represents a unique opportunity to +make ML accessible to more organizations." + +Michael Chen added, "Sarah's expertise in ML infrastructure will be invaluable +as we scale our AI offerings. We're thrilled to have her on board." +` + + fmt.Println("=== Automatic Entity Extraction Demo ===\n") + fmt.Println("Input text:") + fmt.Println(exampleText) + fmt.Println("\n" + strings.Repeat("=", 80) + "\n") + + // Note: In a real implementation, you would initialize actual providers: + // completer := openai.NewCompleter("gpt-4", apiKey) + // embedder := openai.NewEmbedder("text-embedding-3-small", apiKey) + + // For this demo, we'll show the expected behavior + var completer provider.Completer + var embedder provider.Embedder + + if completer == nil || embedder == nil { + fmt.Println("⚠️ Note: This is a demonstration. In production, initialize:") + fmt.Println(" completer := openai.NewCompleter(\"gpt-4\", apiKey)") + fmt.Println(" embedder := openai.NewEmbedder(\"text-embedding-3-small\", apiKey)") + fmt.Println() + + // Show expected output + demonstrateExpectedOutput() + return + } + + // Create automatic entity extractor + extractor := entity.NewExtractor(completer, embedder) + + // Extract entities and relationships automatically! + // No manual specification of EntityID, EntityType, or Relations needed + entities, err := extractor.ExtractFromText(ctx, exampleText) + if err != nil { + log.Fatalf("Failed to extract entities: %v", err) + } + + fmt.Printf("βœ… Automatically extracted %d entities\n\n", len(entities)) + + // Display extracted entities + for i, entity := range entities { + fmt.Printf("Entity %d:\n", i+1) + fmt.Printf(" ID: %s\n", entity.EntityID) + fmt.Printf(" Type: %s\n", entity.EntityType) + + if name, ok := entity.Metadata["name"]; ok { + fmt.Printf(" Name: %s\n", name) + } + + if len(entity.Relations) > 0 { + fmt.Printf(" Relations:\n") + for _, rel := range entity.Relations { + fmt.Printf(" - %s: %s\n", rel.Type, rel.TargetID) + } + } + + fmt.Println() + } + + // Index in graph database + cfg := qdrant.Config{ + URL: "http://localhost:6333", + CollectionName: "knowledge_graph", + VectorSize: 1536, + Distance: "cosine", + } + + graphProvider, err := qdrant.NewClient(cfg, embedder) + if err != nil { + log.Printf("Note: Qdrant client not available for this demo: %v", err) + return + } + + // Store all automatically extracted entities + for _, entity := range entities { + if err := graphProvider.IndexEntity(ctx, entity); err != nil { + log.Printf("Failed to index entity %s: %v", entity.EntityID, err) + } else { + fmt.Printf("βœ… Indexed: %s\n", entity.EntityID) + } + } +} + +// demonstrateExpectedOutput shows what the automatic extraction would produce +func demonstrateExpectedOutput() { + fmt.Println("Expected Automatic Extraction Output:") + fmt.Println() + + entities := []struct { + ID string + Type string + Name string + Relations []string + }{ + { + ID: "person-sarah-johnson", + Type: "person", + Name: "Sarah Johnson", + Relations: []string{ + "works_at: org-techcorp", + "has_role: role-chief-technology-officer", + "manages: team-engineering", + "works_on: project-ai-platform", + "previously_worked_at: org-innovatelabs", + }, + }, + { + ID: "person-michael-chen", + Type: "person", + Name: "Michael Chen", + Relations: []string{ + "works_at: org-techcorp", + "has_role: role-chief-executive-officer", + }, + }, + { + ID: "org-techcorp", + Type: "organization", + Name: "TechCorp", + Relations: []string{ + "employs: person-sarah-johnson", + "employs: person-michael-chen", + "located_in: location-san-francisco", + "sponsors: project-ai-platform", + "partners_with: org-datacorp", + "founded: 2018", + }, + }, + { + ID: "org-innovatelabs", + Type: "organization", + Name: "InnovateLabs", + Relations: []string{ + "employed: person-sarah-johnson", + }, + }, + { + ID: "org-datacorp", + Type: "organization", + Name: "DataCorp", + Relations: []string{ + "partners_with: org-techcorp", + }, + }, + { + ID: "project-ai-platform", + Type: "project", + Name: "AI Platform", + Relations: []string{ + "managed_by: person-sarah-johnson", + "sponsored_by: org-techcorp", + }, + }, + { + ID: "location-san-francisco", + Type: "location", + Name: "San Francisco", + Relations: []string{ + "contains: org-techcorp", + }, + }, + { + ID: "role-chief-technology-officer", + Type: "role", + Name: "Chief Technology Officer", + Relations: []string{ + "held_by: person-sarah-johnson", + "at_organization: org-techcorp", + }, + }, + } + + for i, entity := range entities { + fmt.Printf("%d. Entity ID: %s\n", i+1, entity.ID) + fmt.Printf(" Type: %s\n", entity.Type) + fmt.Printf(" Name: %s\n", entity.Name) + + if len(entity.Relations) > 0 { + fmt.Printf(" Relations:\n") + for _, rel := range entity.Relations { + fmt.Printf(" - %s\n", rel) + } + } + fmt.Println() + } + + fmt.Println("\n" + strings.Repeat("=", 80)) + fmt.Println("\n🎯 Key Point: ALL of this was extracted automatically!") + fmt.Println(" No manual specification of EntityID, EntityType, or Relations needed.") + fmt.Println() + fmt.Println("The LLM automatically:") + fmt.Println(" βœ… Identified 8 entities (people, orgs, project, location, role)") + fmt.Println(" βœ… Generated consistent IDs (person-sarah-johnson, org-techcorp, etc.)") + fmt.Println(" βœ… Determined entity types (person, organization, project, etc.)") + fmt.Println(" βœ… Extracted 15+ relationships between entities") + fmt.Println(" βœ… Created bidirectional relations (employs/works_at)") + fmt.Println() +} diff --git a/examples/semantic-graph/README.md b/examples/semantic-graph/README.md new file mode 100644 index 00000000..8b9d7c59 --- /dev/null +++ b/examples/semantic-graph/README.md @@ -0,0 +1,182 @@ +# Semantic Graph Example with Qdrant + +This example demonstrates how to build a semantic graph using Qdrant vector database, combining vector embeddings with relationship metadata. + +## Overview + +This example shows how to: +1. Extract entities and relationships from documents +2. Store entities with vector embeddings in Qdrant +3. Model relationships as metadata +4. Perform hybrid queries combining semantic search and graph traversal + +## Architecture + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Qdrant Collection β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Entity: John Doe (person) β”‚ +β”‚ Vector: [semantic embedding] β”‚ +β”‚ Relations: β”‚ +β”‚ - works_at: org-acme-corp β”‚ +β”‚ - knows: person-jane-smith β”‚ +β”‚ - works_on: project-ml-platform β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## Prerequisites + +1. **Qdrant Instance**: Run Qdrant locally or use Qdrant Cloud + ```bash + docker run -p 6333:6333 qdrant/qdrant + ``` + +2. **Embedder**: Configure an embedding provider (OpenAI, local model, etc.) + +## Running the Example + +```bash +# Build the example +cd examples/semantic-graph +go build -o semantic-graph + +# Run the example +./semantic-graph +``` + +## Example Queries + +### 1. Semantic Search +Find entities by content similarity: +```go +results := graphProvider.Query(ctx, "machine learning expert", &index.QueryOptions{ + Limit: &limit, + Filters: map[string]string{"entity_type": "person"}, +}) +``` + +### 2. Graph Traversal +Find related entities: +```go +// Find all employees of Acme Corp +employees := graphProvider.QueryRelated(ctx, "org-acme-corp", "employs", nil) +``` + +### 3. Hybrid Query +Combine semantic and graph: +```go +// Find ML experts at Acme Corp +relationFilter := map[string]string{"works_at": "org-acme-corp"} +results := graphProvider.QueryGraph(ctx, "machine learning", relationFilter, nil) +``` + +### 4. Multi-Hop Traversal +Navigate multiple relationships: +```go +// Find projects of John's colleagues +path := []string{"works_at", "employs", "works_on"} +projects := graphProvider.TraverseGraph(ctx, "person-john-doe", path, nil) +``` + +## Data Model + +### Entity Types +- `person`: Individual people +- `organization`: Companies and organizations +- `project`: Projects and initiatives +- `role`: Job roles and positions + +### Relationship Types +- `works_at` / `employs`: Employment relationships +- `knows`: Personal connections +- `manages` / `managed_by`: Management relationships +- `works_on` / `involves`: Project involvement +- `sponsors` / `sponsored_by`: Sponsorship relationships + +## Best Practices + +1. **Bidirectional Relations**: Store relationships in both directions + ``` + Person --[works_at]--> Company + Company --[employs]--> Person + ``` + +2. **Consistent IDs**: Use deterministic entity IDs + ``` + person-john-doe + org-acme-corp + project-ml-platform + ``` + +3. **Rich Metadata**: Store relevant attributes in metadata + ```go + Metadata: map[string]string{ + "name": "John Doe", + "title": "Senior Engineer", + "department": "Engineering", + } + ``` + +4. **Quality Embeddings**: Use appropriate embedding models + - General: OpenAI text-embedding-3-large + - Code: OpenAI text-embedding-3-small + - Domain-specific: Fine-tuned models + +## Integration with Wingman + +This example can be integrated into the Wingman platform: + +```yaml +# config.yaml +indexes: + knowledge-graph: + type: qdrant + url: http://localhost:6333 + collection: knowledge_graph + graph_mode: true +``` + +## Advanced Use Cases + +### Entity Extraction from Documents +```go +// Use LLM to extract entities and relationships +entities := ExtractEntitiesFromDocument(documentText) +for _, entity := range entities { + graphProvider.IndexEntity(ctx, entity) +} +``` + +### Relationship Inference +```go +// Infer implicit relationships +// If A knows B and B knows C, infer A might know C +inferredRelations := InferRelationships(graphProvider, "person-john-doe") +``` + +### Temporal Graphs +```go +// Add temporal metadata to relationships +relation := index.Relation{ + Type: "works_at", + TargetID: "org-acme-corp", + Metadata: map[string]string{ + "start_date": "2020-01-01", + "end_date": "2023-12-31", + }, +} +``` + +## Performance Considerations + +1. **Indexing**: Create indexes on frequently queried metadata fields +2. **Batch Operations**: Use batch upserts for large graphs +3. **Cache**: Cache frequently accessed entities +4. **Pagination**: Use pagination for large result sets + +## References + +- [Qdrant Documentation](https://qdrant.tech/documentation/) +- [Vector Search Best Practices](https://qdrant.tech/documentation/tutorials/search-beginners/) +- [Payload Filtering](https://qdrant.tech/documentation/concepts/filtering/) diff --git a/examples/semantic-graph/config.yaml b/examples/semantic-graph/config.yaml new file mode 100644 index 00000000..8a123b53 --- /dev/null +++ b/examples/semantic-graph/config.yaml @@ -0,0 +1,163 @@ +# Semantic Graph Configuration with Qdrant +# This configuration demonstrates how to set up Wingman for semantic graph operations + +# Vector Database Configuration +indexes: + # Primary knowledge graph store + knowledge-graph: + type: qdrant + url: http://localhost:6333 + api_key: ${QDRANT_API_KEY} # Optional for Qdrant Cloud + + # Collection configuration + collection: knowledge_graph + vector_size: 1536 # OpenAI text-embedding-3-small + distance: cosine # cosine, euclid, or dot + + # Enable graph mode for relationship tracking + graph_mode: true + + # Qdrant-specific settings + qdrant: + # HNSW index parameters for vector search + hnsw_config: + m: 16 # Number of edges per node + ef_construct: 100 # Size of dynamic candidate list for construction + full_scan_threshold: 10000 + + # Optimize for hybrid queries + optimizers_config: + default_segment_number: 2 + indexing_threshold: 20000 + + # Payload indexing for efficient filtering + payload_indexes: + - field_name: entity_id + field_schema: keyword + - field_name: entity_type + field_schema: keyword + - field_name: relations + field_schema: keyword + - field_name: metadata.name + field_schema: text + - field_name: metadata.department + field_schema: keyword + +# Embedder Configuration +providers: + - type: openai + token: ${OPENAI_API_KEY} + models: + # For entity content embeddings + text-embedding-3-small: + id: text-embedding-3-small + + # For higher quality embeddings (larger dimension) + text-embedding-3-large: + id: text-embedding-3-large + +# Entity Extraction Configuration +extractors: + # Use Unstructured for document parsing + document-parser: + type: unstructured + url: http://localhost:9085/general/v0/general + strategy: hi_res + + # Use LLM for entity and relationship extraction + entity-extractor: + type: custom + url: http://localhost:8080 + # Custom extractor that uses LLM to identify entities and relationships + +# Memory Configuration for Graph-Based RAG +memory: + index: knowledge-graph + recall_k: 10 + log_conversations: true + inject_memories: true + +# Retriever for Graph Queries +retrievers: + graph-retriever: + type: custom + url: http://localhost:8081 + # Custom retriever that performs graph traversal and hybrid queries + +# Tools for Graph Operations +tools: + # Entity search tool + search-entities: + type: search + retriever: graph-retriever + description: "Search for entities in the knowledge graph" + + # Relationship finder tool + find-relations: + type: custom + url: http://localhost:8082 + description: "Find relationships between entities" + + # Graph traversal tool + traverse-graph: + type: custom + url: http://localhost:8083 + description: "Traverse the knowledge graph following relationships" + +# API Configuration +server: + host: 0.0.0.0 + port: 8080 + + # Enable graph-specific endpoints + endpoints: + - path: /v1/graph/entities + method: POST + handler: index_entity + + - path: /v1/graph/query + method: POST + handler: query_graph + + - path: /v1/graph/traverse + method: POST + handler: traverse_graph + +# Example Usage Patterns +# +# 1. Index an entity with relationships: +# POST /v1/graph/entities +# { +# "entity_id": "person-john-doe", +# "entity_type": "person", +# "content": "John Doe is a software engineer...", +# "metadata": {"name": "John Doe", "title": "Engineer"}, +# "relations": [ +# {"type": "works_at", "target_id": "org-acme-corp"}, +# {"type": "knows", "target_id": "person-jane-smith"} +# ] +# } +# +# 2. Perform semantic search: +# POST /v1/graph/query +# { +# "query": "machine learning expert", +# "filters": {"entity_type": "person"}, +# "limit": 10 +# } +# +# 3. Query with relationships: +# POST /v1/graph/query +# { +# "query": "AI researcher", +# "relation_filters": {"works_at": "org-acme-corp"}, +# "limit": 10 +# } +# +# 4. Traverse graph: +# POST /v1/graph/traverse +# { +# "start_entity_id": "person-john-doe", +# "path": ["works_at", "employs", "works_on"], +# "limit": 20 +# } diff --git a/examples/semantic-graph/go.mod b/examples/semantic-graph/go.mod new file mode 100644 index 00000000..eea4009e --- /dev/null +++ b/examples/semantic-graph/go.mod @@ -0,0 +1,7 @@ +module example.com/semantic-graph + +go 1.25 + +replace github.com/adrianliechti/wingman => ../.. + +require github.com/adrianliechti/wingman v0.0.0 diff --git a/examples/semantic-graph/main.go b/examples/semantic-graph/main.go new file mode 100644 index 00000000..f2ce3375 --- /dev/null +++ b/examples/semantic-graph/main.go @@ -0,0 +1,346 @@ +// Package main demonstrates how to use semantic graph modeling with Qdrant. +// This example shows end-to-end usage from data extraction to graph queries. +package main + +import ( + "context" + "fmt" + "log" + + "github.com/adrianliechti/wingman/pkg/index" + "github.com/adrianliechti/wingman/pkg/index/qdrant" + "github.com/adrianliechti/wingman/pkg/provider" +) + +// Example demonstrates semantic graph operations with Qdrant +func main() { + ctx := context.Background() + + // Step 1: Set up Qdrant client with embedder + cfg := qdrant.Config{ + URL: "http://localhost:6333", + CollectionName: "knowledge_graph", + VectorSize: 1536, // OpenAI embedding size + Distance: "cosine", + } + + // Note: In real usage, you'd initialize an actual embedder + var embedder provider.Embedder // = openai.NewEmbedder(...) + + client, err := qdrant.NewClient(cfg, embedder) + if err != nil { + log.Fatalf("Failed to create Qdrant client: %v", err) + } + + // Step 2: Extract and index entities with relationships + if err := indexKnowledgeBase(ctx, client, embedder); err != nil { + log.Fatalf("Failed to index knowledge base: %v", err) + } + + // Step 3: Perform various types of graph queries + demonstrateQueries(ctx, client) +} + +// indexKnowledgeBase extracts entities and relationships from documents +// and stores them in the semantic graph +func indexKnowledgeBase(ctx context.Context, graphProvider index.GraphProvider, embedder provider.Embedder) error { + // Example: Processing a document about employees and organizations + + // Entity 1: Person - John Doe + johnContent := "John Doe is a Senior Software Engineer specializing in machine learning and artificial intelligence. He has 8 years of experience building scalable ML systems." + johnEmbedding, _ := embedder.Embed(ctx, []string{johnContent}) + + johnEntity := index.GraphDocument{ + Document: index.Document{ + Content: johnContent, + Embedding: johnEmbedding.Embeddings[0], + Metadata: map[string]string{ + "name": "John Doe", + "title": "Senior Software Engineer", + "department": "Engineering", + }, + }, + EntityID: "person-john-doe", + EntityType: "person", + Relations: []index.Relation{ + {Type: "works_at", TargetID: "org-acme-corp"}, + {Type: "has_role", TargetID: "role-senior-engineer"}, + {Type: "knows", TargetID: "person-jane-smith"}, + {Type: "works_on", TargetID: "project-ml-platform"}, + }, + } + + if err := graphProvider.IndexEntity(ctx, johnEntity); err != nil { + return fmt.Errorf("failed to index John Doe: %w", err) + } + + // Entity 2: Person - Jane Smith + janeContent := "Jane Smith is a Product Manager with expertise in AI product strategy and go-to-market planning. She leads cross-functional teams to deliver innovative AI solutions." + janeEmbedding, _ := embedder.Embed(ctx, []string{janeContent}) + + janeEntity := index.GraphDocument{ + Document: index.Document{ + Content: janeContent, + Embedding: janeEmbedding.Embeddings[0], + Metadata: map[string]string{ + "name": "Jane Smith", + "title": "Product Manager", + "department": "Product", + }, + }, + EntityID: "person-jane-smith", + EntityType: "person", + Relations: []index.Relation{ + {Type: "works_at", TargetID: "org-acme-corp"}, + {Type: "has_role", TargetID: "role-product-manager"}, + {Type: "knows", TargetID: "person-john-doe"}, + {Type: "manages", TargetID: "project-ml-platform"}, + }, + } + + if err := graphProvider.IndexEntity(ctx, janeEntity); err != nil { + return fmt.Errorf("failed to index Jane Smith: %w", err) + } + + // Entity 3: Organization - Acme Corp + acmeContent := "Acme Corp is a technology company focused on building cutting-edge artificial intelligence solutions for enterprise customers. Founded in 2015, the company has grown to 500+ employees." + acmeEmbedding, _ := embedder.Embed(ctx, []string{acmeContent}) + + acmeEntity := index.GraphDocument{ + Document: index.Document{ + Content: acmeContent, + Embedding: acmeEmbedding.Embeddings[0], + Metadata: map[string]string{ + "name": "Acme Corp", + "industry": "Technology", + "founded": "2015", + }, + }, + EntityID: "org-acme-corp", + EntityType: "organization", + Relations: []index.Relation{ + {Type: "employs", TargetID: "person-john-doe"}, + {Type: "employs", TargetID: "person-jane-smith"}, + {Type: "sponsors", TargetID: "project-ml-platform"}, + }, + } + + if err := graphProvider.IndexEntity(ctx, acmeEntity); err != nil { + return fmt.Errorf("failed to index Acme Corp: %w", err) + } + + // Entity 4: Project - ML Platform + projectContent := "ML Platform is an internal project to build a unified machine learning infrastructure for training, deploying, and monitoring ML models at scale. The platform supports multiple ML frameworks." + projectEmbedding, _ := embedder.Embed(ctx, []string{projectContent}) + + projectEntity := index.GraphDocument{ + Document: index.Document{ + Content: projectContent, + Embedding: projectEmbedding.Embeddings[0], + Metadata: map[string]string{ + "name": "ML Platform", + "status": "active", + "budget": "high", + }, + }, + EntityID: "project-ml-platform", + EntityType: "project", + Relations: []index.Relation{ + {Type: "involves", TargetID: "person-john-doe"}, + {Type: "managed_by", TargetID: "person-jane-smith"}, + {Type: "sponsored_by", TargetID: "org-acme-corp"}, + }, + } + + if err := graphProvider.IndexEntity(ctx, projectEntity); err != nil { + return fmt.Errorf("failed to index ML Platform: %w", err) + } + + log.Println("Successfully indexed knowledge base with 4 entities and their relationships") + return nil +} + +// demonstrateQueries shows different types of queries on the semantic graph +func demonstrateQueries(ctx context.Context, graphProvider index.GraphProvider) { + fmt.Println("\n=== Demonstrating Semantic Graph Queries ===\n") + + // Query 1: Basic semantic search (vector-based) + fmt.Println("1. Semantic Search: Find people with machine learning expertise") + limit := 5 + results, err := graphProvider.Query(ctx, "machine learning artificial intelligence expert", &index.QueryOptions{ + Limit: &limit, + Filters: map[string]string{ + "entity_type": "person", + }, + }) + if err != nil { + log.Printf("Query failed: %v", err) + } + printResults(results) + + // Query 2: Relationship-based query + fmt.Println("\n2. Graph Query: Find all employees of Acme Corp") + results, err = graphProvider.QueryRelated(ctx, "org-acme-corp", "employs", &index.QueryOptions{ + Limit: &limit, + }) + if err != nil { + log.Printf("Query failed: %v", err) + } + printResults(results) + + // Query 3: Hybrid query (semantic + graph) + fmt.Println("\n3. Hybrid Query: Find people with product management skills who work at Acme Corp") + relationFilter := map[string]string{ + "works_at": "org-acme-corp", + } + results, err = graphProvider.QueryGraph(ctx, "product management strategy AI products", relationFilter, &index.QueryOptions{ + Limit: &limit, + }) + if err != nil { + log.Printf("Query failed: %v", err) + } + printResults(results) + + // Query 4: Multi-hop graph traversal + fmt.Println("\n4. Graph Traversal: Find projects that John Doe's colleagues work on") + // Path: John Doe -> works_at -> Acme Corp -> employs -> Colleagues -> works_on -> Projects + path := []string{"works_at", "employs", "works_on"} + results, err = graphProvider.TraverseGraph(ctx, "person-john-doe", path, &index.QueryOptions{ + Limit: &limit, + }) + if err != nil { + log.Printf("Query failed: %v", err) + } + printResults(results) + + // Query 5: Find similar organizations + fmt.Println("\n5. Semantic Search: Find companies similar to Acme Corp") + results, err = graphProvider.Query(ctx, "technology company AI enterprise solutions", &index.QueryOptions{ + Limit: &limit, + Filters: map[string]string{ + "entity_type": "organization", + }, + }) + if err != nil { + log.Printf("Query failed: %v", err) + } + printResults(results) + + // Query 6: Find related projects + fmt.Println("\n6. Graph Query: Find projects managed by Jane Smith") + results, err = graphProvider.QueryRelated(ctx, "person-jane-smith", "manages", &index.QueryOptions{ + Limit: &limit, + }) + if err != nil { + log.Printf("Query failed: %v", err) + } + printResults(results) +} + +// printResults displays query results +func printResults(results []index.QueryResult) { + if len(results) == 0 { + fmt.Println(" No results found") + return + } + + for i, result := range results { + fmt.Printf(" Result %d (Score: %.4f):\n", i+1, result.Score) + + // Display entity information + if entityID, ok := result.Document.Metadata["entity_id"]; ok { + fmt.Printf(" Entity ID: %s\n", entityID) + } + if entityType, ok := result.Document.Metadata["entity_type"]; ok { + fmt.Printf(" Entity Type: %s\n", entityType) + } + if name, ok := result.Document.Metadata["name"]; ok { + fmt.Printf(" Name: %s\n", name) + } + + // Display content (truncated) + content := result.Document.Content + if len(content) > 100 { + content = content[:100] + "..." + } + fmt.Printf(" Content: %s\n", content) + + fmt.Println() + } +} + +// Additional helper functions for practical usage + +// ExtractEntitiesFromDocument demonstrates how to extract entities and relationships +// from unstructured text using NLP techniques +func ExtractEntitiesFromDocument(text string) []index.GraphDocument { + // In a real implementation, this would use NLP models or LLMs to: + // 1. Identify entities (people, organizations, projects, etc.) + // 2. Extract relationships between entities + // 3. Generate appropriate embeddings + + // Placeholder implementation + return []index.GraphDocument{} +} + +// BuildBidirectionalRelations ensures relationships are stored in both directions +// for efficient graph traversal +func BuildBidirectionalRelations(entities []index.GraphDocument) []index.GraphDocument { + // Create reverse relationships + // Example: If A works_at B, also add B employs A + + reverseRelationMap := map[string]string{ + "works_at": "employs", + "employs": "works_at", + "knows": "knows", + "manages": "managed_by", + "managed_by": "manages", + "works_on": "involves", + "involves": "works_on", + "sponsors": "sponsored_by", + "sponsored_by": "sponsors", + } + + // Index entities by ID for quick lookup + entityMap := make(map[string]*index.GraphDocument) + for i := range entities { + entityMap[entities[i].EntityID] = &entities[i] + } + + // Add reverse relations + for _, entity := range entities { + for _, relation := range entity.Relations { + reverseType, ok := reverseRelationMap[relation.Type] + if !ok { + continue + } + + targetEntity, ok := entityMap[relation.TargetID] + if !ok { + continue + } + + // Add reverse relation if not already present + reverseRelation := index.Relation{ + Type: reverseType, + TargetID: entity.EntityID, + Metadata: relation.Metadata, + } + + // Check if relation already exists + exists := false + for _, existingRel := range targetEntity.Relations { + if existingRel.Type == reverseRelation.Type && existingRel.TargetID == reverseRelation.TargetID { + exists = true + break + } + } + + if !exists { + targetEntity.Relations = append(targetEntity.Relations, reverseRelation) + } + } + } + + return entities +} diff --git a/pkg/extractor/entity/extractor.go b/pkg/extractor/entity/extractor.go new file mode 100644 index 00000000..8e71523b --- /dev/null +++ b/pkg/extractor/entity/extractor.go @@ -0,0 +1,330 @@ +// Package entity provides automatic entity and relationship extraction from text. +// This enables automatic detection of entities (people, organizations, etc.) and +// their relationships without manual specification. +package entity + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strings" + + "github.com/adrianliechti/wingman/pkg/index" + "github.com/adrianliechti/wingman/pkg/provider" +) + +// Extractor automatically extracts entities and relationships from text using LLMs +type Extractor struct { + completer provider.Completer + embedder provider.Embedder +} + +// NewExtractor creates a new automatic entity extractor +func NewExtractor(completer provider.Completer, embedder provider.Embedder) *Extractor { + return &Extractor{ + completer: completer, + embedder: embedder, + } +} + +// ExtractedEntity represents a detected entity with its relationships +type ExtractedEntity struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Content string `json:"content"` + Attributes map[string]string `json:"attributes"` + Relations []ExtractedRelation `json:"relations"` +} + +// ExtractedRelation represents a relationship between entities +type ExtractedRelation struct { + Type string `json:"type"` + TargetID string `json:"target_id"` + TargetName string `json:"target_name"` +} + +// ExtractFromText automatically extracts entities and relationships from text +func (e *Extractor) ExtractFromText(ctx context.Context, text string) ([]index.GraphDocument, error) { + // Step 1: Use LLM to extract structured entities and relationships + extractedEntities, err := e.extractWithLLM(ctx, text) + if err != nil { + return nil, fmt.Errorf("failed to extract entities: %w", err) + } + + // Step 2: Deduplicate entities + extractedEntities = deduplicateEntities(extractedEntities) + + // Step 3: Convert to GraphDocuments with embeddings + var graphDocs []index.GraphDocument + for _, entity := range extractedEntities { + // Generate embedding for the entity content + embedding, err := e.embedder.Embed(ctx, []string{entity.Content}) + if err != nil { + return nil, fmt.Errorf("failed to generate embedding for %s: %w", entity.ID, err) + } + + // Build metadata from attributes + metadata := make(map[string]string) + for k, v := range entity.Attributes { + metadata[k] = v + } + metadata["name"] = entity.Name + + // Convert relations + relations := make([]index.Relation, len(entity.Relations)) + for i, rel := range entity.Relations { + relations[i] = index.Relation{ + Type: rel.Type, + TargetID: rel.TargetID, + Metadata: map[string]string{ + "target_name": rel.TargetName, + }, + } + } + + graphDoc := index.GraphDocument{ + Document: index.Document{ + Content: entity.Content, + Embedding: embedding.Embeddings[0], + Metadata: metadata, + }, + EntityID: entity.ID, + EntityType: entity.Type, + Relations: relations, + } + + graphDocs = append(graphDocs, graphDoc) + } + + return graphDocs, nil +} + +// extractWithLLM uses the LLM to extract entities and relationships +func (e *Extractor) extractWithLLM(ctx context.Context, text string) ([]ExtractedEntity, error) { + // Construct extraction prompt + prompt := buildExtractionPrompt(text) + + // Call LLM with low temperature for consistent extraction + temperature := float32(0.1) + completion, err := e.completer.Complete(ctx, []provider.Message{ + provider.UserMessage(prompt), + }, &provider.CompleteOptions{ + Temperature: &temperature, + }) + if err != nil { + return nil, fmt.Errorf("LLM call failed: %w", err) + } + + // Extract text from response + content := completion.Message.Text() + content = extractJSONFromResponse(content) + + // Parse JSON response + var entities []ExtractedEntity + if err := json.Unmarshal([]byte(content), &entities); err != nil { + return nil, fmt.Errorf("failed to parse LLM response: %w (content: %s)", err, content) + } + + // Normalize and validate entities + for i := range entities { + entities[i] = normalizeEntity(entities[i]) + } + + return entities, nil +} + +// buildExtractionPrompt creates a prompt for entity extraction +func buildExtractionPrompt(text string) string { + return fmt.Sprintf(`You are an expert at extracting entities and relationships from text. + +Given the following text, extract all entities (people, organizations, projects, locations, products, etc.) and their relationships. + +For each entity, provide: +- id: A unique identifier in format "{type}-{normalized-name}" (e.g., "person-john-doe", "org-acme-corp") +- type: The entity type (person, organization, project, location, product, event, role, etc.) +- name: The entity's name as it appears in the text +- content: A brief description of the entity (1-2 sentences) +- attributes: Key attributes as key-value pairs (e.g., title, department, industry, location, etc.) +- relations: Relationships to other entities with type and target_id + +Common relationship types: +- works_at / employs (person to organization) +- manages / managed_by (management relationships) +- knows (personal connections) +- works_on / involves (project participation) +- located_in / contains (location relationships) +- part_of / contains (organizational hierarchy) +- founded / founded_by (founding relationships) +- sponsors / sponsored_by (sponsorship) +- partners_with (partnerships) +- reports_to / supervises (reporting structure) +- has_role (role assignment) + +Important: +- Generate consistent IDs: lowercase, hyphen-separated (e.g., "person-john-doe") +- Include bidirectional relations when possible +- Extract implicit relationships mentioned in the text +- Provide meaningful content descriptions + +Text to analyze: +""" +%s +""" + +Return ONLY a JSON array of entities. No markdown, no explanations, just the JSON array. + +Example format: +[ + { + "id": "person-john-doe", + "type": "person", + "name": "John Doe", + "content": "John Doe is a software engineer specializing in machine learning at Acme Corp.", + "attributes": { + "title": "Software Engineer", + "expertise": "Machine Learning" + }, + "relations": [ + { + "type": "works_at", + "target_id": "org-acme-corp", + "target_name": "Acme Corp" + } + ] + }, + { + "id": "org-acme-corp", + "type": "organization", + "name": "Acme Corp", + "content": "Acme Corp is a technology company.", + "attributes": { + "industry": "Technology" + }, + "relations": [ + { + "type": "employs", + "target_id": "person-john-doe", + "target_name": "John Doe" + } + ] + } +]`, text) +} + +// extractJSONFromResponse extracts JSON from markdown code blocks or plain text +func extractJSONFromResponse(content string) string { + // Remove markdown code blocks if present + if strings.Contains(content, "```json") { + re := regexp.MustCompile("```json\\s*([\\s\\S]*?)```") + matches := re.FindStringSubmatch(content) + if len(matches) > 1 { + return strings.TrimSpace(matches[1]) + } + } else if strings.Contains(content, "```") { + re := regexp.MustCompile("```\\s*([\\s\\S]*?)```") + matches := re.FindStringSubmatch(content) + if len(matches) > 1 { + return strings.TrimSpace(matches[1]) + } + } + return strings.TrimSpace(content) +} + +// normalizeEntity normalizes entity fields +func normalizeEntity(entity ExtractedEntity) ExtractedEntity { + // Ensure ID is properly formatted + if entity.ID == "" || !strings.Contains(entity.ID, "-") { + entity.ID = generateEntityID(entity.Type, entity.Name) + } + + // Normalize ID format + entity.ID = normalizeID(entity.ID) + + // Ensure type is lowercase + entity.Type = strings.ToLower(entity.Type) + + // Normalize relation target IDs + for i := range entity.Relations { + entity.Relations[i].TargetID = normalizeID(entity.Relations[i].TargetID) + } + + return entity +} + +// generateEntityID creates a consistent entity ID +func generateEntityID(entityType, name string) string { + normalized := strings.ToLower(name) + normalized = strings.ReplaceAll(normalized, " ", "-") + + // Remove special characters + re := regexp.MustCompile(`[^a-z0-9-]`) + normalized = re.ReplaceAllString(normalized, "") + + // Remove multiple consecutive hyphens + re = regexp.MustCompile(`-+`) + normalized = re.ReplaceAllString(normalized, "-") + + // Trim hyphens from ends + normalized = strings.Trim(normalized, "-") + + // Limit length + if len(normalized) > 50 { + normalized = normalized[:50] + } + + return fmt.Sprintf("%s-%s", strings.ToLower(entityType), normalized) +} + +// normalizeID normalizes an entity ID +func normalizeID(id string) string { + id = strings.ToLower(id) + re := regexp.MustCompile(`[^a-z0-9-]`) + id = re.ReplaceAllString(id, "") + re = regexp.MustCompile(`-+`) + id = re.ReplaceAllString(id, "-") + return strings.Trim(id, "-") +} + +// deduplicateEntities merges duplicate entities +func deduplicateEntities(entities []ExtractedEntity) []ExtractedEntity { + seen := make(map[string]*ExtractedEntity) + + for i := range entities { + entity := &entities[i] + + if existing, ok := seen[entity.ID]; ok { + // Merge relations (avoid duplicates) + relationSet := make(map[string]bool) + for _, rel := range existing.Relations { + key := fmt.Sprintf("%s:%s", rel.Type, rel.TargetID) + relationSet[key] = true + } + + for _, rel := range entity.Relations { + key := fmt.Sprintf("%s:%s", rel.Type, rel.TargetID) + if !relationSet[key] { + existing.Relations = append(existing.Relations, rel) + relationSet[key] = true + } + } + + // Merge attributes (prefer existing) + for k, v := range entity.Attributes { + if _, exists := existing.Attributes[k]; !exists { + existing.Attributes[k] = v + } + } + } else { + seen[entity.ID] = entity + } + } + + result := make([]ExtractedEntity, 0, len(seen)) + for _, entity := range seen { + result = append(result, *entity) + } + + return result +} diff --git a/pkg/index/graph.go b/pkg/index/graph.go new file mode 100644 index 00000000..89c117e2 --- /dev/null +++ b/pkg/index/graph.go @@ -0,0 +1,129 @@ +// Package index provides graph extensions for semantic graph modeling. +// This file demonstrates how to extend the basic index.Provider interface +// to support graph-based operations while maintaining vector embeddings. +package index + +import "context" + +// GraphDocument extends Document with entity and relationship information. +// This allows modeling a semantic graph where nodes (entities) are connected +// by typed relationships while keeping the actual content in vector form. +type GraphDocument struct { + Document + + // EntityID is a unique identifier for this entity in the graph + EntityID string + + // EntityType categorizes the entity (e.g., "person", "organization", "document") + EntityType string + + // Relations defines edges to other entities in the graph + Relations []Relation +} + +// Relation represents a typed edge between two entities in the semantic graph. +type Relation struct { + // Type describes the relationship (e.g., "works_at", "knows", "part_of") + Type string + + // TargetID is the EntityID of the related entity + TargetID string + + // Metadata stores additional information about the relationship + Metadata map[string]string +} + +// GraphProvider extends the basic Provider interface with graph capabilities. +// This allows implementations (like Qdrant) to support both vector search +// and graph traversal operations. +type GraphProvider interface { + Provider + + // IndexEntity stores an entity with its relationships in the graph. + // The entity's content is stored as a vector embedding, while relationships + // are stored in metadata for efficient traversal. + IndexEntity(ctx context.Context, doc GraphDocument) error + + // QueryRelated finds entities that have a specific relationship to the given entity. + // For example: QueryRelated(ctx, "person-123", "works_at", opts) would find + // all organizations where person-123 works. + QueryRelated(ctx context.Context, entityID string, relationType string, opts *QueryOptions) ([]QueryResult, error) + + // QueryGraph performs a hybrid query combining vector similarity search + // with graph relationship filters. This enables queries like: + // "Find people with ML expertise who work at the same company as John" + QueryGraph(ctx context.Context, query string, relationFilter map[string]string, opts *QueryOptions) ([]QueryResult, error) + + // TraverseGraph performs multi-hop traversal starting from an entity. + // The path parameter defines the relationship types to follow in order. + // For example: path = ["works_at", "located_in"] would find the locations + // of companies where entities work. + TraverseGraph(ctx context.Context, startEntityID string, path []string, opts *QueryOptions) ([]QueryResult, error) +} + +// RelationFilter defines constraints on relationships for graph queries. +type RelationFilter struct { + // RelationType filters by specific relationship type (e.g., "works_at") + RelationType string + + // TargetEntityType filters by the type of target entities (e.g., "organization") + TargetEntityType string + + // TargetEntityID filters to relationships with a specific target + TargetEntityID string + + // MinConfidence filters relationships by confidence score (if supported) + MinConfidence *float32 +} + +// GraphQueryOptions extends QueryOptions with graph-specific parameters. +type GraphQueryOptions struct { + QueryOptions + + // RelationFilters constrains results by relationship criteria + RelationFilters []RelationFilter + + // IncludeRelations specifies whether to include relationship information in results + IncludeRelations bool + + // MaxDepth limits the depth of graph traversal (for TraverseGraph) + MaxDepth int +} + +// Example usage patterns: + +// Example 1: Store an entity with relationships +// entity := index.GraphDocument{ +// Document: index.Document{ +// Content: "John Doe is a software engineer specializing in AI...", +// Embedding: [embedding vector], +// Metadata: map[string]string{ +// "name": "John Doe", +// "title": "Software Engineer", +// }, +// }, +// EntityID: "person-john-doe", +// EntityType: "person", +// Relations: []index.Relation{ +// {Type: "works_at", TargetID: "org-acme-corp"}, +// {Type: "knows", TargetID: "person-jane-smith"}, +// {Type: "works_on", TargetID: "project-ai-platform"}, +// }, +// } +// err := graphProvider.IndexEntity(ctx, entity) + +// Example 2: Find related entities +// Find all companies where John Doe works +// results, err := graphProvider.QueryRelated(ctx, "person-john-doe", "works_at", nil) + +// Example 3: Hybrid semantic + graph query +// Find people with ML expertise who work at Acme Corp +// relationFilter := map[string]string{ +// "works_at": "org-acme-corp", +// } +// results, err := graphProvider.QueryGraph(ctx, "machine learning expert", relationFilter, &index.QueryOptions{Limit: ptr(10)}) + +// Example 4: Multi-hop graph traversal +// Find cities where John Doe's coworkers live +// path := []string{"works_at", "employs", "lives_in"} +// results, err := graphProvider.TraverseGraph(ctx, "person-john-doe", path, nil) diff --git a/pkg/index/qdrant/client.go b/pkg/index/qdrant/client.go new file mode 100644 index 00000000..9250e93d --- /dev/null +++ b/pkg/index/qdrant/client.go @@ -0,0 +1,298 @@ +// Package qdrant provides a reference implementation of the index.GraphProvider +// interface for Qdrant vector database. This demonstrates how to implement +// semantic graph capabilities using Qdrant's vector search and payload filtering. +package qdrant + +import ( + "context" + "fmt" + + "github.com/adrianliechti/wingman/pkg/index" + "github.com/adrianliechti/wingman/pkg/provider" +) + +// Client implements index.GraphProvider for Qdrant vector database. +// It stores entities as points with vector embeddings and uses payload +// metadata to represent graph relationships. +type Client struct { + collectionName string + embedder provider.Embedder + // In a real implementation, add qdrant client fields here + // client *qdrant.Client +} + +// Config holds configuration for the Qdrant client. +type Config struct { + URL string + APIKey string + CollectionName string + VectorSize int + Distance string // cosine, euclid, dot +} + +// NewClient creates a new Qdrant client with graph capabilities. +func NewClient(cfg Config, embedder provider.Embedder) (*Client, error) { + // In a real implementation: + // - Create Qdrant client + // - Verify collection exists or create it + // - Set up payload indexes for efficient graph queries + + return &Client{ + collectionName: cfg.CollectionName, + embedder: embedder, + }, nil +} + +// Index implements the basic index.Provider interface. +// This stores a document as a vector point without graph information. +func (c *Client) Index(ctx context.Context, doc index.Document) error { + // Convert to GraphDocument without relations + graphDoc := index.GraphDocument{ + Document: doc, + EntityID: generateEntityID(doc), + EntityType: inferEntityType(doc), + Relations: nil, + } + return c.IndexEntity(ctx, graphDoc) +} + +// Query implements the basic index.Provider interface. +// This performs vector similarity search with optional metadata filtering. +func (c *Client) Query(ctx context.Context, query string, opts *index.QueryOptions) ([]index.QueryResult, error) { + // Generate embedding for query + embedding, err := c.embedder.Embed(ctx, []string{query}) + if err != nil { + return nil, fmt.Errorf("failed to generate query embedding: %w", err) + } + + // Perform vector search with filters + return c.searchWithVector(ctx, embedding.Embeddings[0], opts) +} + +// IndexEntity implements index.GraphProvider.IndexEntity. +// This stores an entity with its relationships in Qdrant. +func (c *Client) IndexEntity(ctx context.Context, doc index.GraphDocument) error { + // Encode relationships as strings for storage in payload + // Format: "relation_type:target_entity_id" + relations := make([]string, len(doc.Relations)) + relationMetadata := make(map[string]interface{}) + + for i, rel := range doc.Relations { + relations[i] = fmt.Sprintf("%s:%s", rel.Type, rel.TargetID) + + // Store additional relation metadata if present + if len(rel.Metadata) > 0 { + key := fmt.Sprintf("rel_%s_%s", rel.Type, rel.TargetID) + relationMetadata[key] = rel.Metadata + } + } + + // Build payload with entity and relationship information + _ = map[string]interface{}{ + "content": doc.Content, + "entity_id": doc.EntityID, + "entity_type": doc.EntityType, + "relations": relations, + "relation_metadata": relationMetadata, + "metadata": doc.Metadata, + } + + // In a real implementation, call Qdrant API: + // payload := map[string]interface{}{ ... } + // return c.client.Upsert(ctx, &qdrant.UpsertPoints{ + // CollectionName: c.collectionName, + // Points: []*qdrant.PointStruct{ + // { + // Id: &qdrant.PointId{PointIdOptions: &qdrant.PointId_Uuid{Uuid: doc.EntityID}}, + // Vectors: &qdrant.Vectors{VectorsOptions: &qdrant.Vectors_Vector{Vector: &qdrant.Vector{Data: doc.Embedding}}}, + // Payload: payload, + // }, + // }, + // }) + + // Placeholder for demonstration + fmt.Printf("Would store entity %s with %d relations\n", doc.EntityID, len(doc.Relations)) + return nil +} + +// QueryRelated implements index.GraphProvider.QueryRelated. +// This finds entities that have a specific relationship to the given entity. +func (c *Client) QueryRelated(ctx context.Context, entityID string, relationType string, opts *index.QueryOptions) ([]index.QueryResult, error) { + // Build the relationship pattern to search for + relationPattern := fmt.Sprintf("%s:%s", relationType, entityID) + + // Build Qdrant filter + _ = buildRelationFilter(relationPattern, opts) + + // In a real implementation, use Qdrant scroll or search: + // filter := buildRelationFilter(relationPattern, opts) + // points, err := c.client.Scroll(ctx, &qdrant.ScrollPoints{ + // CollectionName: c.collectionName, + // Filter: filter, + // Limit: opts.Limit, + // }) + + // Placeholder for demonstration + fmt.Printf("Would query entities related to %s via %s\n", entityID, relationType) + return []index.QueryResult{}, nil +} + +// QueryGraph implements index.GraphProvider.QueryGraph. +// This performs hybrid search combining vector similarity and relationship filters. +func (c *Client) QueryGraph(ctx context.Context, query string, relationFilter map[string]string, opts *index.QueryOptions) ([]index.QueryResult, error) { + // Generate embedding for semantic search + _, err := c.embedder.Embed(ctx, []string{query}) + if err != nil { + return nil, fmt.Errorf("failed to generate query embedding: %w", err) + } + + // Build compound filter combining relation filters and metadata filters + _ = buildCompoundFilter(relationFilter, opts) + + // In a real implementation, use Qdrant search with filter: + // embedding, err := c.embedder.Embed(ctx, []string{query}) + // filter := buildCompoundFilter(relationFilter, opts) + // results, err := c.client.Search(ctx, &qdrant.SearchPoints{ + // CollectionName: c.collectionName, + // Vector: embedding.Embeddings[0], + // Filter: filter, + // Limit: opts.Limit, + // WithPayload: &qdrant.WithPayloadSelector{SelectorOptions: &qdrant.WithPayloadSelector_Enable{Enable: true}}, + // }) + + // Placeholder for demonstration + fmt.Printf("Would perform hybrid search for '%s' with %d relation filters\n", query, len(relationFilter)) + return []index.QueryResult{}, nil +} + +// TraverseGraph implements index.GraphProvider.TraverseGraph. +// This performs multi-hop graph traversal following a path of relationship types. +func (c *Client) TraverseGraph(ctx context.Context, startEntityID string, path []string, opts *index.QueryOptions) ([]index.QueryResult, error) { + if len(path) == 0 { + return nil, fmt.Errorf("path cannot be empty") + } + + // Start with the initial entity + currentEntityIDs := []string{startEntityID} + + // Traverse each hop in the path + for _, relationType := range path { + var nextEntityIDs []string + + // For each entity at current level, find related entities + for _, entityID := range currentEntityIDs { + // Query entities related via this relationship type + results, err := c.QueryRelated(ctx, entityID, relationType, opts) + if err != nil { + return nil, fmt.Errorf("failed to traverse relation %s: %w", relationType, err) + } + + // Collect entity IDs for next hop + for _, result := range results { + if entityIDVal, ok := result.Document.Metadata["entity_id"]; ok { + nextEntityIDs = append(nextEntityIDs, entityIDVal) + } + } + } + + currentEntityIDs = nextEntityIDs + if len(currentEntityIDs) == 0 { + break // No more entities to traverse + } + } + + // Retrieve full documents for final entity IDs + return c.getEntitiesByIDs(ctx, currentEntityIDs) +} + +// Helper functions + +func (c *Client) searchWithVector(ctx context.Context, vector []float32, opts *index.QueryOptions) ([]index.QueryResult, error) { + // In a real implementation, perform vector search in Qdrant + return []index.QueryResult{}, nil +} + +func (c *Client) getEntitiesByIDs(ctx context.Context, entityIDs []string) ([]index.QueryResult, error) { + // In a real implementation, retrieve points by IDs from Qdrant + return []index.QueryResult{}, nil +} + +func buildRelationFilter(relationPattern string, opts *index.QueryOptions) map[string]interface{} { + // Build Qdrant filter for relation matching + filter := map[string]interface{}{ + "must": []map[string]interface{}{ + { + "key": "relations", + "match": map[string]interface{}{ + "any": []string{relationPattern}, + }, + }, + }, + } + + // Add additional metadata filters if present + if opts != nil && len(opts.Filters) > 0 { + for key, value := range opts.Filters { + filter["must"] = append(filter["must"].([]map[string]interface{}), map[string]interface{}{ + "key": fmt.Sprintf("metadata.%s", key), + "match": map[string]interface{}{ + "value": value, + }, + }) + } + } + + return filter +} + +func buildCompoundFilter(relationFilter map[string]string, opts *index.QueryOptions) map[string]interface{} { + // Build compound filter combining relationship and metadata constraints + filter := map[string]interface{}{ + "must": []map[string]interface{}{}, + } + + // Add relation filters + for relType, targetID := range relationFilter { + relationPattern := fmt.Sprintf("%s:%s", relType, targetID) + filter["must"] = append(filter["must"].([]map[string]interface{}), map[string]interface{}{ + "key": "relations", + "match": map[string]interface{}{ + "any": []string{relationPattern}, + }, + }) + } + + // Add metadata filters + if opts != nil && len(opts.Filters) > 0 { + for key, value := range opts.Filters { + filter["must"] = append(filter["must"].([]map[string]interface{}), map[string]interface{}{ + "key": fmt.Sprintf("metadata.%s", key), + "match": map[string]interface{}{ + "value": value, + }, + }) + } + } + + return filter +} + +func generateEntityID(doc index.Document) string { + // In a real implementation, generate deterministic ID from content or use UUID + return "entity-" + doc.Content[:min(10, len(doc.Content))] +} + +func inferEntityType(doc index.Document) string { + // In a real implementation, infer type from metadata or content + if entityType, ok := doc.Metadata["entity_type"]; ok { + return entityType + } + return "document" +} + +func min(a, b int) int { + if a < b { + return a + } + return b +}