diff --git a/README.md b/README.md index 83fb48e..b4a5821 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,18 @@ result, err := matcher.Find(ctx, "log in button", elements, semantic.FindOptions // result.BestScore = 0.82 ``` +Structured locators are also supported when descriptors include the corresponding fields: + +```go +result, err := matcher.Find(ctx, "role:button Sign In", elements, semantic.FindOptions{}) +result, err = matcher.Find(ctx, "placeholder:Search", elements, semantic.FindOptions{}) +result, err = matcher.Find(ctx, "nth:1:role:button", elements, semantic.FindOptions{}) +``` + +`nth:` is 1-based: `nth:1` selects the first ordered candidate, `nth:2` selects the second, and `nth:0` is not the first match. + +Use `find:` or `semantic:` to force natural-language matching for locator-like text. + ## Package Layout ``` diff --git a/cmd/semantic/main.go b/cmd/semantic/main.go index b99815c..d7ac931 100644 --- a/cmd/semantic/main.go +++ b/cmd/semantic/main.go @@ -81,9 +81,18 @@ type snapshotElement struct { Role string `json:"role"` Name string `json:"name"` Value string `json:"value"` + Label string `json:"label"` + Placeholder string `json:"placeholder"` + Alt string `json:"alt"` + Title string `json:"title"` + TestID string `json:"testid"` + TestIDAlt string `json:"test_id"` + Text string `json:"text"` + Tag string `json:"tag"` Interactive bool `json:"interactive"` Parent string `json:"parent"` Section string `json:"section"` + DocumentIdx int `json:"document_idx"` Depth int `json:"depth"` SiblingIdx int `json:"sibling_index"` SiblingCnt int `json:"sibling_count"` @@ -122,6 +131,11 @@ func loadSnapshot(path string) ([]semantic.ElementDescriptor, error) { descs := make([]semantic.ElementDescriptor, len(elements)) for i, e := range elements { + testID := e.TestID + if testID == "" { + testID = e.TestIDAlt + } + labelledBy := e.LabelledBy depth := e.Depth siblingIdx := e.SiblingIdx @@ -173,9 +187,17 @@ func loadSnapshot(path string) ([]semantic.ElementDescriptor, error) { Role: e.Role, Name: e.Name, Value: e.Value, + Label: e.Label, + Placeholder: e.Placeholder, + Alt: e.Alt, + Title: e.Title, + TestID: testID, + Text: e.Text, + Tag: e.Tag, Interactive: e.Interactive, Parent: e.Parent, Section: e.Section, + DocumentIdx: e.DocumentIdx, Positional: semantic.PositionalHints{ Depth: depth, SiblingIndex: siblingIdx, diff --git a/cmd/semantic/main_test.go b/cmd/semantic/main_test.go index 0a7d117..4e52474 100644 --- a/cmd/semantic/main_test.go +++ b/cmd/semantic/main_test.go @@ -12,8 +12,8 @@ func TestLoadSnapshot_PropagatesInteractiveFlag(t *testing.T) { } json := `[ - {"ref":"e1","role":"button","name":"Submit","interactive":true,"parent":"Login form","section":"Authentication","depth":3,"sibling_index":1,"sibling_count":2,"labelled_by":"Primary Action","x":20,"y":40,"width":120,"height":30}, - {"ref":"e2","role":"text","name":"Submit","interactive":false,"parent":"Payment form","section":"Checkout","positional":{"depth":2,"sibling_index":0,"sibling_count":1,"labelled_by":"Secondary Action","left":300,"top":640,"width":200,"height":44}} + {"ref":"e1","role":"button","name":"Submit","label":"Primary Action","placeholder":"Search","alt":"Submit icon","title":"Submit form","testid":"submit-button","text":"Submit","tag":"button","document_idx":7,"interactive":true,"parent":"Login form","section":"Authentication","depth":3,"sibling_index":1,"sibling_count":2,"labelled_by":"Primary Action","x":20,"y":40,"width":120,"height":30}, + {"ref":"e2","role":"text","name":"Submit","test_id":"legacy-submit","interactive":false,"parent":"Payment form","section":"Checkout","positional":{"depth":2,"sibling_index":0,"sibling_count":1,"labelled_by":"Secondary Action","left":300,"top":640,"width":200,"height":44}} ]` if _, err := f.WriteString(json); err != nil { t.Fatalf("WriteString failed: %v", err) @@ -38,6 +38,30 @@ func TestLoadSnapshot_PropagatesInteractiveFlag(t *testing.T) { if descs[0].Section != "Authentication" { t.Fatalf("expected first descriptor section=Authentication, got %q", descs[0].Section) } + if descs[0].Label != "Primary Action" { + t.Fatalf("expected first descriptor label=Primary Action, got %q", descs[0].Label) + } + if descs[0].Placeholder != "Search" { + t.Fatalf("expected first descriptor placeholder=Search, got %q", descs[0].Placeholder) + } + if descs[0].Alt != "Submit icon" { + t.Fatalf("expected first descriptor alt=Submit icon, got %q", descs[0].Alt) + } + if descs[0].Title != "Submit form" { + t.Fatalf("expected first descriptor title=Submit form, got %q", descs[0].Title) + } + if descs[0].TestID != "submit-button" { + t.Fatalf("expected first descriptor testid=submit-button, got %q", descs[0].TestID) + } + if descs[0].Text != "Submit" { + t.Fatalf("expected first descriptor text=Submit, got %q", descs[0].Text) + } + if descs[0].Tag != "button" { + t.Fatalf("expected first descriptor tag=button, got %q", descs[0].Tag) + } + if descs[0].DocumentIdx != 7 { + t.Fatalf("expected first descriptor document_idx=7, got %d", descs[0].DocumentIdx) + } if descs[0].Positional.Depth != 3 { t.Fatalf("expected first descriptor depth=3, got %d", descs[0].Positional.Depth) } @@ -65,6 +89,9 @@ func TestLoadSnapshot_PropagatesInteractiveFlag(t *testing.T) { if descs[1].Section != "Checkout" { t.Fatalf("expected second descriptor section=Checkout, got %q", descs[1].Section) } + if descs[1].TestID != "legacy-submit" { + t.Fatalf("expected second descriptor test_id fallback=legacy-submit, got %q", descs[1].TestID) + } if descs[1].Positional.Depth != 2 { t.Fatalf("expected second descriptor depth=2, got %d", descs[1].Positional.Depth) } diff --git a/docs/architecture/implementation.md b/docs/architecture/implementation.md index 9eb0518..6dbdb1e 100644 --- a/docs/architecture/implementation.md +++ b/docs/architecture/implementation.md @@ -8,14 +8,36 @@ The input unit. Each element from the accessibility tree becomes a descriptor: ```go type ElementDescriptor struct { - Ref string // "e4" — the element reference - Role string // "button" — ARIA role - Name string // "Sign In" — accessible name - Value string // "" — current value (for inputs) + Ref string + Role string + Name string + Value string + Label string + Placeholder string + Alt string + Title string + TestID string + Text string + Tag string } ``` -The `Composite()` method produces a single searchable string: `"button: Sign In"`. This is what matchers score against. +The `Composite()` method produces a single searchable string such as `"button: Sign In"`. Natural-language matchers score against this composite; structured locators score explicit fields directly. + +Structured locator queries are parsed before generic semantic matching: + +- `role: [name]` +- `text:` +- `label: