Skip to content

Commit 27af4e9

Browse files
jatezzzjatezzz
authored
ADFA-3912 | Fix fuzzy attribute parsing bounds and UI element tag filtering (#1275)
* fix: resolve bounds error in attribute parser and correct tag filtering Added bounds validation in FuzzyAttributeParser and updated UI element filter logic. * fix: restrict tag filtering to text boxes on `associateTextToWidgets` method
1 parent 69e03bf commit 27af4e9

3 files changed

Lines changed: 89 additions & 37 deletions

File tree

‎cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/FuzzyAttributeParser.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,8 @@ object FuzzyAttributeParser {
309309
annotation.length
310310
}
311311

312+
if (current.valueStart > valueEnd) continue
313+
312314
var rawValue = annotation.substring(current.valueStart, valueEnd).trim()
313315

314316
if (rawValue.isNotEmpty()) {

‎cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt‎

Lines changed: 73 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult
44
import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox
55
import org.appdevforall.codeonthego.computervision.domain.xml.AndroidXmlGenerator
66
import org.appdevforall.codeonthego.computervision.utils.MetadataDetector
7-
import org.appdevforall.codeonthego.computervision.utils.getSortedScaledPlaceholders
7+
import org.appdevforall.codeonthego.computervision.utils.buildPlaceholderOverrides
88
import kotlin.comparisons.compareBy
99

1010
class YoloToXmlConverter(
@@ -23,41 +23,31 @@ class YoloToXmlConverter(
2323
targetDpHeight: Int,
2424
wrapInScroll: Boolean = true
2525
): Pair<String, String> {
26-
val uiCandidates = detections
27-
.filter { (it.isYolo || it.label == "text") && it.label != "widget_tag" }
28-
.filterNot { MetadataDetector.isMetadataDetection(it.label, it.text) }
29-
.distinctBy {
30-
if (it.label.startsWith("switch")) {
31-
"${((it.boundingBox.top + it.boundingBox.bottom) / 2f).toInt() / 50}"
32-
} else {
33-
"${it.label}:${it.boundingBox.left}:${it.boundingBox.top}:${it.boundingBox.right}:${it.boundingBox.bottom}"
34-
}
35-
}
26+
// 1. Filter and prepare base UI candidates
27+
val uiCandidates = extractUiCandidates(detections)
3628

37-
var scaledBoxes = uiCandidates.map { geometryProcessor.scaleDetection(it, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight) }
29+
// 2. Scale detections to target DP dimensions
30+
val scaledBoxes = scaleDetections(uiCandidates, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight)
3831

39-
val parents = scaledBoxes.filter { it.label != "text" && !annotationMatcher.isTag(it.text) }
40-
var texts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
32+
// 3. Associate isolated text detections to their respective UI widgets
33+
val associatedBoxes = associateTextToWidgets(scaledBoxes)
4134

42-
scaledBoxes = geometryProcessor.assignTextToParents(parents, texts, scaledBoxes)
43-
texts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
44-
scaledBoxes = geometryProcessor.assignNearbyTextToWidgets(scaledBoxes, texts)
35+
// 4. Clean up and finalize the UI elements list
36+
val uiElements = finalizeUiElements(associatedBoxes)
4537

46-
val uiElements = scaledBoxes.filter {
47-
!annotationMatcher.isTag(it.text) && !MetadataDetector.isMetadataDetection(it.label, it.text)
48-
}
49-
val widgetTags = detections.filter { it.label == "widget_tag" || (!it.isYolo && annotationMatcher.isTag(it.text)) }
50-
val canvasTags = widgetTags.map { geometryProcessor.scaleDetection(it, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight) }
38+
// 5. Extract and scale reference tags (e.g., T-1, B-1) from the canvas
39+
val canvasTags = extractCanvasTags(detections, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight)
5140

41+
// 6. Match margin annotations with the extracted UI elements
5242
val finalAnnotations = annotationMatcher.matchAnnotationsToElements(canvasTags, uiElements, annotations)
5343

44+
// 7. Sort boxes top-to-bottom, left-to-right for sequential XML rendering
5445
val sortedBoxes = uiElements.sortedWith(compareBy({ it.y }, { it.x }))
5546

56-
val selectedImageOverrides = buildSelectedImageOverrides(
57-
uiElements = uiElements,
58-
selectedImagesByPlaceholderId = selectedImagesByPlaceholderId
59-
)
47+
// 8. Prepare local drawable resources overrides for image placeholders
48+
val selectedImageOverrides = uiElements.buildPlaceholderOverrides(selectedImagesByPlaceholderId)
6049

50+
// 9. Generate final XML output
6151
return xmlGenerator.buildXml(
6252
boxes = sortedBoxes,
6353
annotations = finalAnnotations,
@@ -67,17 +57,63 @@ class YoloToXmlConverter(
6757
)
6858
}
6959

70-
private fun buildSelectedImageOverrides(
71-
uiElements: List<ScaledBox>,
72-
selectedImagesByPlaceholderId: Map<String, String>
73-
): Map<ScaledBox, String> {
74-
val placeholders = uiElements.getSortedScaledPlaceholders()
75-
76-
return placeholders.mapIndexedNotNull { index, box ->
77-
val drawableReference = selectedImagesByPlaceholderId["ph_$index"]
78-
?: return@mapIndexedNotNull null
79-
box to drawableReference
80-
}.toMap()
60+
private fun extractUiCandidates(detections: List<DetectionResult>): List<DetectionResult> {
61+
return detections
62+
.filter { (it.isYolo || it.label == "text") && it.label != "widget_tag" }
63+
.filterNot { MetadataDetector.isMetadataDetection(it.label, it.text) }
64+
.distinctBy {
65+
if (it.label.startsWith("switch")) {
66+
// Deduplicate switches by grouping them within a 50px vertical band
67+
"${((it.boundingBox.top + it.boundingBox.bottom) / 2f).toInt() / 50}"
68+
} else {
69+
// Exact coordinate deduplication for other widgets
70+
"${it.label}:${it.boundingBox.left}:${it.boundingBox.top}:${it.boundingBox.right}:${it.boundingBox.bottom}"
71+
}
72+
}
73+
}
74+
75+
private fun scaleDetections(
76+
candidates: List<DetectionResult>,
77+
sourceWidth: Int,
78+
sourceHeight: Int,
79+
targetWidth: Int,
80+
targetHeight: Int
81+
): List<ScaledBox> {
82+
return candidates.map {
83+
geometryProcessor.scaleDetection(it, sourceWidth, sourceHeight, targetWidth, targetHeight)
84+
}
85+
}
86+
87+
private fun associateTextToWidgets(scaledBoxes: List<ScaledBox>): List<ScaledBox> {
88+
val parents = scaledBoxes.filter { it.label != "text" }
89+
val initialTexts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
90+
val textAssignedBoxes = geometryProcessor.assignTextToParents(parents, initialTexts, scaledBoxes)
91+
val remainingTexts = textAssignedBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) }
92+
93+
return geometryProcessor.assignNearbyTextToWidgets(textAssignedBoxes, remainingTexts)
94+
}
95+
96+
private fun finalizeUiElements(boxes: List<ScaledBox>): List<ScaledBox> {
97+
return boxes.filter {
98+
// Keep the widget if it's not pure text, or if it is text but not recognized as a tag.
99+
(it.label != "text" || !annotationMatcher.isTag(it.text)) &&
100+
!MetadataDetector.isMetadataDetection(it.label, it.text)
101+
}
102+
}
103+
104+
private fun extractCanvasTags(
105+
detections: List<DetectionResult>,
106+
sourceWidth: Int,
107+
sourceHeight: Int,
108+
targetWidth: Int,
109+
targetHeight: Int
110+
): List<ScaledBox> {
111+
val widgetTags = detections.filter {
112+
it.label == "widget_tag" || (!it.isYolo && annotationMatcher.isTag(it.text))
113+
}
114+
return widgetTags.map {
115+
geometryProcessor.scaleDetection(it, sourceWidth, sourceHeight, targetWidth, targetHeight)
116+
}
81117
}
82118

83119
companion object {

‎cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/utils/PlaceholderUtils.kt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,17 @@ fun List<ScaledBox>.getSortedScaledPlaceholders(): List<ScaledBox> {
1414
return this.filter { it.label == IMAGE_PLACEHOLDER_LABEL }
1515
.sortedWith(compareBy({ it.y }, { it.x }))
1616
}
17+
18+
/**
19+
* Associates ordered image placeholders with their selected drawable references.
20+
* Useful for mapping user-selected gallery images to the physical canvas bounding boxes.
21+
*/
22+
fun List<ScaledBox>.buildPlaceholderOverrides(selectedImagesByPlaceholderId: Map<String, String>): Map<ScaledBox, String> {
23+
val placeholders = this.getSortedScaledPlaceholders()
24+
25+
return placeholders.mapIndexedNotNull { index, box ->
26+
val drawableReference = selectedImagesByPlaceholderId["ph_$index"]
27+
?: return@mapIndexedNotNull null
28+
box to drawableReference
29+
}.toMap()
30+
}

0 commit comments

Comments
 (0)