From febb6c805b21f4caac3ba3ff63f02239f63857c7 Mon Sep 17 00:00:00 2001 From: qtjg <3.17509829e+08+qtjg@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:39:52 +0000 Subject: [PATCH] fix(parser): ignore language proficiency rows --- src/lib/heuristics/extract/skills.test.ts | 112 ++++++++++++++++++ src/lib/heuristics/extract/skills.ts | 69 +++++++++-- ...multiline-bullets-coursework.expected.json | 6 +- ...-proxy-role-first-experience.expected.json | 6 +- ...kia-proxy-role-first-experience.truth.json | 5 - ...rojects-activities-singlecol.expected.json | 6 +- ...enresume-laverne-word-quartz.expected.json | 6 +- 7 files changed, 186 insertions(+), 24 deletions(-) diff --git a/src/lib/heuristics/extract/skills.test.ts b/src/lib/heuristics/extract/skills.test.ts index 47bab79b..1dcda5c3 100644 --- a/src/lib/heuristics/extract/skills.test.ts +++ b/src/lib/heuristics/extract/skills.test.ts @@ -426,6 +426,68 @@ describe("tokenizeSkillLine — issue #221 non-skill sub-labels", () => { }); }); +describe("tokenizeSkillLine — issue #833 language proficiency rows", () => { + it("drops spoken-language proficiency rows across common label variants", () => { + expect(tokenizeSkillLine("Language: Fluent in Spanish")).toEqual([]); + expect(tokenizeSkillLine("Foreign Languages: Fluent in Spanish")).toEqual([]); + expect( + tokenizeSkillLine("Spoken Languages: Native German, conversational French"), + ).toEqual([]); + expect(tokenizeSkillLine("Languages : Fluent in Spanish")).toEqual([]); + }); + + it("drops standard proficiency-scale wording", () => { + expect( + tokenizeSkillLine("Languages: Full professional proficiency in German"), + ).toEqual([]); + expect(tokenizeSkillLine("Languages: Elementary proficiency in French")).toEqual( + [], + ); + expect(tokenizeSkillLine("Languages: JLPT N2 Japanese Proficiency")).toEqual( + [], + ); + }); + + it("keeps programming-language rows when one token contains proficiency wording", () => { + expect(tokenizeSkillLine("Languages: C, C++, Visual Basic")).toEqual([ + "C", + "C++", + "Visual Basic", + ]); + expect(tokenizeSkillLine("Languages: Kotlin, Swift, React Native")).toEqual([ + "Kotlin", + "Swift", + "React Native", + ]); + expect(tokenizeSkillLine("Languages: Proficient in Java, Python, Go")).toEqual([ + "Proficient in Java", + "Python", + "Go", + ]); + expect(tokenizeSkillLine("Languages: Java, Python (proficient), Go")).toEqual([ + "Java", + "Python (proficient)", + "Go", + ]); + }); + + it("keeps bare spoken-language lists and non-language proficiency skills", () => { + expect(tokenizeSkillLine("Languages: Python, Go, TypeScript")).toEqual([ + "Python", + "Go", + "TypeScript", + ]); + expect(tokenizeSkillLine("Languages: Spanish, French, Mandarin")).toEqual([ + "Spanish", + "French", + "Mandarin", + ]); + expect( + tokenizeSkillLine("Certifications: Certified in AWS Solutions Architecture"), + ).toEqual(["Certified in AWS Solutions Architecture"]); + }); +}); + describe("tokenizeSkillLine — issue #832 single-letter languages", () => { it("keeps C, R, and D as real programming languages", () => { expect(tokenizeSkillLine("Languages: C, R, D")).toEqual( @@ -820,6 +882,23 @@ describe("extractSkills — bulleted labelled single-column rows (#465)", () => }); }); +describe("extractSkills — language proficiency category boundary (#833)", () => { + it("does not misfile a wrapped tail under the previous category", () => { + const section = skillsLines([ + [{ x: 74, str: "Frameworks: React, Vue", w: 110 }], + [{ x: 74, str: "Languages: English (native), Spanish (fluent),", w: 180 }], + [{ x: 74, str: "French", w: 35 }], + ]); + const { value, categories } = extractSkills(section); + + expect(value).toEqual(["React", "Vue", "French"]); + expect(categories).toEqual([ + { label: "Frameworks", skills: ["React", "Vue"] }, + { label: "Languages", skills: ["French"] }, + ]); + }); +}); + describe("extractSkills — category capture (#473)", () => { it("captures each label as a category and keeps `skills` the flat union", () => { const section = skillsLines([ @@ -835,6 +914,39 @@ describe("extractSkills — category capture (#473)", () => { ]); }); + it("drops language proficiency rows without dropping language lists", () => { + const section = skillsLines([ + [...BULLET_RUNS, { x: 74, str: "Languages: Python, Go, TypeScript", w: 180 }], + [...BULLET_RUNS, { x: 74, str: "Language: Fluent in Spanish", w: 160 }], + [...BULLET_RUNS, { x: 74, str: "Languages: Spanish, French, Mandarin", w: 180 }], + [ + ...BULLET_RUNS, + { x: 74, str: "Certifications: Certified in AWS Solutions Architecture", w: 220 }, + ], + ]); + const { value, categories } = extractSkills(section); + + expect(value).toEqual([ + "Python", + "Go", + "TypeScript", + "Spanish", + "French", + "Mandarin", + "Certified in AWS Solutions Architecture", + ]); + expect(categories).toEqual([ + { label: "Languages", skills: ["Python", "Go", "TypeScript"] }, + { label: "Languages", skills: ["Spanish", "French", "Mandarin"] }, + { label: "Certifications", skills: ["Certified in AWS Solutions Architecture"] }, + ]); + expect(value).not.toContain("Fluent in Spanish"); + expect(categories).not.toContainEqual({ + label: "Language", + skills: ["Fluent in Spanish"], + }); + }); + it("INVARIANT 1: `skills` deep-equals `categories.flatMap((c) => c.skills)`", () => { // A wrapped Frontend list whose continuation flushed as its own bare cell // ("… HTML5," ⏎ "CSS3, JavaScript") must fold back into Frontend, not become diff --git a/src/lib/heuristics/extract/skills.ts b/src/lib/heuristics/extract/skills.ts index 1c719a41..59a19094 100644 --- a/src/lib/heuristics/extract/skills.ts +++ b/src/lib/heuristics/extract/skills.ts @@ -304,6 +304,36 @@ const SUBLABEL_PREFIX_RE = new RegExp(`^(${SUBLABEL_BODY}):\\s*`); * tore off its body. See the rejoin in `splitColumnCells`. */ const BARE_SUBLABEL_RE = new RegExp(`^${SUBLABEL_BODY}:$`); +/** A Skills sub-label that MAY head a spoken-language row. Deliberately NOT + * added to NON_SKILL_SUBLABEL_RE: on most engineering résumés `Languages:` + * heads the programming-language row, so the label is ambiguous and the body + * must decide whether this is a proficiency statement. Qualifiers are allowed + * because `Foreign Languages:` and `Spoken Languages:` are common variants. */ +const LANGUAGE_LABEL_RE = + /^(?:foreign\s+|spoken\s+|other\s+)?languages?\s*$/i; + +/** A spoken-language proficiency predication, distinguished from a delimited + * programming-language list by its proficiency wording rather than by the + * label. Keep this vocabulary broad enough to cover standard proficiency + * scales, but apply it per fragment so `Visual Basic` and `React Native` do + * not cause an entire programming-language row to disappear. */ +const LANGUAGE_PROFICIENCY_BODY_RE = + /\b(fluent|native|bilingual|conversational|proficient|proficiency|proficiencies|intermediate|elementary|limited|beginner|basic|working\s+proficiency|mother\s+tongue)\b/i; + +function isLanguageProficiencyCell(cell: string): boolean { + const debulleted = stripBullet(cell); + const match = debulleted.match(SUBLABEL_PREFIX_RE); + if (!match || !LANGUAGE_LABEL_RE.test(match[1])) return false; + + const fragments = splitRespectingParens(debulleted.slice(match[0].length)) + .map((fragment) => fragment.trim()) + .filter((fragment) => fragment !== ""); + return ( + fragments.length > 0 && + fragments.every((fragment) => LANGUAGE_PROFICIENCY_BODY_RE.test(fragment)) + ); +} + /** * Tokenizes a single column cell into valid skill tokens and adds them to * `out`. Drops the cell entirely when it looks like a contact/profile link — @@ -317,7 +347,11 @@ function tokenizeCell(cell: string, out: Set): void { // leading `Label:` prefix and drop the whole cell when it names a // hobbies/interests list — before the label is stripped and the items split. const labelMatch = debulleted.match(SUBLABEL_PREFIX_RE); - if (labelMatch && NON_SKILL_SUBLABEL_RE.test(labelMatch[1])) return; + if ( + labelMatch && + (NON_SKILL_SUBLABEL_RE.test(labelMatch[1]) || isLanguageProficiencyCell(debulleted)) + ) + return; const clean = debulleted.replace(SUBLABEL_PREFIX_RE, ""); // A whole cell that is a profile link ("github.com/janesmith") must be // dropped before splitting — a path slash would otherwise leave the path @@ -613,10 +647,16 @@ function upcomingContinuationTexts(lineCells: string[][], from: number): string[ * A non-skill sub-label (Interests/Hobbies) is not a category: `tokenizeCell` * drops such a cell whole, so it never contributes tokens and never reaches the * caller's category branch — but the guard here keeps the two decisions aligned. + * Language-proficiency rows retain their label here so a dropped row can still + * anchor a soft-wrapped continuation in `extractSkills`. */ function matchCellLabel(cell: string): string | undefined { const m = stripBullet(cell).match(SUBLABEL_PREFIX_RE); - if (!m || NON_SKILL_SUBLABEL_RE.test(m[1])) return undefined; + if ( + !m || + NON_SKILL_SUBLABEL_RE.test(m[1]) + ) + return undefined; return m[1].trim(); } @@ -647,11 +687,21 @@ export function extractSkills( value.push(tok); cellTokens.push(tok); } + const label = matchCellLabel(cell); + // A dropped language-proficiency row still opens a category boundary. Keep + // that anchor so a soft-wrapped tail cannot be misfiled under the previous + // category; the category is emitted only if a later continuation contributes + // a token. + if (cellTokens.length === 0) { + if (label !== undefined && isLanguageProficiencyCell(cell)) { + categories.push({ label, skills: [] }); + } + continue; + } + // A cell that produced nothing (dropped link/hobbies label, or all-duplicate) // is neither a category nor a bare contribution — skip it on both axes. - if (cellTokens.length === 0) continue; - const label = matchCellLabel(cell); if (label !== undefined) { categories.push({ label, skills: cellTokens }); } else if (categories.length > 0) { @@ -666,13 +716,18 @@ export function extractSkills( } } + const nonEmptyCategories = categories.filter( + (category) => category.skills.length > 0, + ); const confidence = value.length >= 5 ? 0.85 : value.length >= 2 ? 0.6 : 0.2; return { value, // Emit the structured view ONLY when the section is fully categorised: at - // least one label AND no bare head. Otherwise the field is absent (never - // `[]`), which reads as "uncategorised" per invariant (2). - ...(categories.length > 0 && !hasUncategorisedHead ? { categories } : {}), + // least one non-empty label AND no bare head. Otherwise the field is absent + // (never `[]`), which reads as "uncategorised" per invariant (2). + ...(nonEmptyCategories.length > 0 && !hasUncategorisedHead + ? { categories: nonEmptyCategories } + : {}), confidence, }; } diff --git a/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-multiline-bullets-coursework.expected.json b/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-multiline-bullets-coursework.expected.json index 011e9d8b..42694463 100644 --- a/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-multiline-bullets-coursework.expected.json +++ b/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-multiline-bullets-coursework.expected.json @@ -25,7 +25,7 @@ "skills", "website_url" ], - "skillsCount": 8, + "skillsCount": 7, "experienceCount": 3, "educationCount": 2, "projectsCount": 2, @@ -76,7 +76,7 @@ "sectionSource": "regex", "pageCount": 2, "rawCharCount": 2385, - "extractedCharCount": 1817, + "extractedCharCount": 1789, "sections": [ { "name": "profile", @@ -107,7 +107,7 @@ "hasSummary": false, "experienceCount": 3, "educationCount": 2, - "skillsCount": 8 + "skillsCount": 7 }, "linkAnnotationCount": 0, "disagreements": [] diff --git a/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-role-first-experience.expected.json b/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-role-first-experience.expected.json index 62e33e1a..1a9b07d0 100644 --- a/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-role-first-experience.expected.json +++ b/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-role-first-experience.expected.json @@ -22,7 +22,7 @@ "phoneIsValid", "skills" ], - "skillsCount": 12, + "skillsCount": 11, "experienceCount": 2, "educationCount": 1, "projectsCount": 0, @@ -74,7 +74,7 @@ "sectionSource": "regex", "pageCount": 1, "rawCharCount": 1428, - "extractedCharCount": 1147, + "extractedCharCount": 1130, "sections": [ { "name": "profile", @@ -101,7 +101,7 @@ "hasSummary": false, "experienceCount": 2, "educationCount": 1, - "skillsCount": 12 + "skillsCount": 11 }, "linkAnnotationCount": 0, "disagreements": [] diff --git a/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-role-first-experience.truth.json b/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-role-first-experience.truth.json index 163c6651..92c537b7 100644 --- a/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-role-first-experience.truth.json +++ b/tests/fixtures/pdfs/google-docs/google-docs-skia-proxy-role-first-experience.truth.json @@ -42,11 +42,6 @@ "issue": null, "status": "unfiled", "note": "Role 2's employer line reads “Multicultural Engineering Program – State Polytechnic University”; `company` comes back as just the university — the program half is not lost, the parser puts it on `team`, but `experience.company` scores the `company` field alone. The identical shape is measured on unknown/single-column-title-below-anchor. Possibly a defensible org/team split rather than a defect — recorded rather than assumed, because ground truth's job is to state what the page says and let a human adjudicate." - }, - "skills": { - "issue": 833, - "status": "open", - "note": "“Fluent in Spanish” is admitted as a skill from the “Language:” row; the Programming Languages row is now correct after #832." } } } diff --git a/tests/fixtures/pdfs/unknown/student-projects-activities-singlecol.expected.json b/tests/fixtures/pdfs/unknown/student-projects-activities-singlecol.expected.json index ae3d0b4b..1fbfd7c4 100644 --- a/tests/fixtures/pdfs/unknown/student-projects-activities-singlecol.expected.json +++ b/tests/fixtures/pdfs/unknown/student-projects-activities-singlecol.expected.json @@ -25,7 +25,7 @@ "skills", "website_url" ], - "skillsCount": 8, + "skillsCount": 7, "experienceCount": 3, "educationCount": 2, "projectsCount": 2, @@ -76,7 +76,7 @@ "sectionSource": "regex", "pageCount": 2, "rawCharCount": 2279, - "extractedCharCount": 1709, + "extractedCharCount": 1681, "sections": [ { "name": "profile", @@ -107,7 +107,7 @@ "hasSummary": false, "experienceCount": 3, "educationCount": 2, - "skillsCount": 8 + "skillsCount": 7 }, "linkAnnotationCount": 0, "disagreements": [] diff --git a/tests/fixtures/pdfs/word/openresume-laverne-word-quartz.expected.json b/tests/fixtures/pdfs/word/openresume-laverne-word-quartz.expected.json index 08bfa09c..8f262696 100644 --- a/tests/fixtures/pdfs/word/openresume-laverne-word-quartz.expected.json +++ b/tests/fixtures/pdfs/word/openresume-laverne-word-quartz.expected.json @@ -22,7 +22,7 @@ "skills", "summary" ], - "skillsCount": 8, + "skillsCount": 7, "experienceCount": 3, "educationCount": 1, "projectsCount": 0, @@ -74,7 +74,7 @@ "sectionSource": "regex", "pageCount": 1, "rawCharCount": 1336, - "extractedCharCount": 856, + "extractedCharCount": 839, "sections": [ { "name": "profile", @@ -105,7 +105,7 @@ "hasSummary": true, "experienceCount": 3, "educationCount": 1, - "skillsCount": 8 + "skillsCount": 7 }, "linkAnnotationCount": 0, "disagreements": []