Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a title guess method to get "better" title #12018

Merged
merged 24 commits into from
Oct 30, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.io.StringWriter;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
Expand Down Expand Up @@ -208,6 +209,70 @@ public ParserResult importDatabase(Path filePath) {
return new ParserResult(result);
}

private String guessBetterTitleInMetaData(List<String> metadata) {
String probableTitle = null;
int maxScore = 0;

for (String str : metadata) {
if (str == null) {
continue;
}
// Rule 1: Check for file type paths, ignore them
if (str.contains(".pdf") || str.contains(".docx") || str.contains(".doc")) {
continue;
}
// Rule 2: Abstract detection (too long for a title)
if (str.length() > 300) {
continue;
}
// Rule 3: Title length and academic keywords (heuristic)
int score = 0;
score += str.length(); // Titles tend to be longer
score += countAcademicKeywords(str); // Bonus for academic terms

if (score > maxScore) {
maxScore = score;
probableTitle = str;
}
}

return probableTitle;
}

// Count common academic keywords
private int countAcademicKeywords(String str) {
List<String> keywords = Arrays.asList("study", "exploring", "research", "development", "design", "learning");
int count = 0;
for (String keyword : keywords) {
if (str.toLowerCase().contains(keyword)) {
count++;
}
}
return count;
}

private List<String> buildMetaData(
String author, String editor, String abstractT, String keywords, String title,
String conference, String doi, String series, String volume, String number,
String pages, String year, String publisher) {
List<String> metadataList = new ArrayList<>();
metadataList.add(author);
metadataList.add(editor);
metadataList.add(abstractT);
metadataList.add(keywords);
metadataList.add(title);
metadataList.add(conference);
metadataList.add(doi);
metadataList.add(series);
metadataList.add(volume);
metadataList.add(number);
metadataList.add(pages);
metadataList.add(year);
metadataList.add(publisher);

return metadataList;
}

// make this method package visible so we can test it
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Change comment to @VisibleForTesting

Optional<BibEntry> getEntryFromPDFContent(String firstpageContents, String lineSeparator) {
// idea: split[] contains the different lines
Expand Down Expand Up @@ -437,6 +502,7 @@ Optional<BibEntry> getEntryFromPDFContent(String firstpageContents, String lineS
entry.setField(StandardField.KEYWORDS, keywords);
}
if (title != null) {
title = guessBetterTitleInMetaData(buildMetaData(author, editor, abstractT, keywords, title, conference, doi, series, volume, number, pages, year, publisher));
entry.setField(StandardField.TITLE, title);
}
if (conference != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.nio.file.Path;
import java.util.List;
import java.util.Objects;
import java.util.Optional;

import org.jabref.model.entry.BibEntry;
Expand All @@ -15,7 +16,7 @@

class PdfContentImporterTest {

private PdfContentImporter importer = new PdfContentImporter();
private final PdfContentImporter importer = new PdfContentImporter();

@Test
void doesNotHandleEncryptedPdfs() throws Exception {
Expand Down Expand Up @@ -123,4 +124,11 @@ British Journal of Nutrition (2008), 99, 1–11 doi: 10.1017/S0007114507795296

assertEquals(Optional.of(entry), importer.getEntryFromPDFContent(firstPageContent, "\n"));
}

@Test
void se2Pdfs() throws Exception {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
void se2Pdfs() throws Exception {
void se2Pdf() throws Exception {

Path file = Path.of(Objects.requireNonNull(PdfContentImporter.class.getResource("/pdfs/se2paper.pdf")).toURI());
List<BibEntry> result = importer.importDatabase(file).getDatabase().getEntries();
assertEquals("On How and We Can and Teach and Exploring New and Ways in and Professional Software and Development for Students", result.getFirst().getTitle().orElse(""));
leaf-soba marked this conversation as resolved.
Show resolved Hide resolved
}
}
Binary file added src/test/resources/pdfs/se2paper.pdf
Binary file not shown.
Loading