Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

NIFI-13988 For 1.x refactored number handling to account for a blank string and treat it as a null when the inferred type is a number. #9513

Merged
merged 1 commit into from
Nov 12, 2024
Merged

NIFI-13988 For 1.x refactored number handling to account for a blank string and treat it as a null when the inferred type is a number. #9513

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 27 additions & 7 deletions ...ns/nifi-record/src/main/java/org/apache/nifi/serialization/record/util/DataTypeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1476,7 +1476,8 @@ public static BigInteger toBigInt(final Object value, final String fieldName) {

if (value instanceof String) {
try {
return new BigInteger((String) value);
final String string = (String)value;
return isBlank(string) ? null : new BigInteger((String) value);
} catch (NumberFormatException nfe) {
throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to BigInteger for field " + fieldName
+ ", value is not a valid representation of BigInteger", nfe);
Expand Down Expand Up @@ -1562,7 +1563,8 @@ public static BigDecimal toBigDecimal(final Object value, final String fieldName

if (value instanceof String) {
try {
return new BigDecimal((String) value);
final String string = (String) value;
return isBlank(string) ? null : new BigDecimal(string);
} catch (NumberFormatException nfe) {
throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to BigDecimal for field " + fieldName
+ ", value is not a valid representation of BigDecimal", nfe);
Expand All @@ -1582,7 +1584,8 @@ public static Double toDouble(final Object value, final String fieldName) {
}

if (value instanceof String) {
return Double.parseDouble((String) value);
final String string = (String) value;
return isBlank(string) ? null : Double.parseDouble(string);
}

throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Double for field " + fieldName);
Expand Down Expand Up @@ -1618,7 +1621,8 @@ public static Float toFloat(final Object value, final String fieldName) {
}

if (value instanceof String) {
return Float.parseFloat((String) value);
final String string = (String) value;
return isBlank(string) ? null : Float.parseFloat(string);
}

throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Float for field " + fieldName);
Expand Down Expand Up @@ -1684,7 +1688,8 @@ public static Long toLong(final Object value, final String fieldName) {
}

if (value instanceof String) {
return Long.parseLong((String) value);
final String string = (String) value;
return isBlank(string) ? null : Long.parseLong(string);
}

if (value instanceof java.util.Date) {
Expand Down Expand Up @@ -1774,7 +1779,8 @@ public static Integer toInteger(final Object value, final String fieldName) {
}

if (value instanceof String) {
return Integer.parseInt((String) value);
final String string = (String) value;
return isBlank(string) ? null : Integer.parseInt(string);
}

throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Integer for field " + fieldName);
Expand Down Expand Up @@ -1803,7 +1809,8 @@ public static Short toShort(final Object value, final String fieldName) {
}

if (value instanceof String) {
return Short.parseShort((String) value);
final String string = (String) value;
return isBlank(string) ? null : Short.parseShort(string);
}

throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " to Short for field " + fieldName);
Expand Down Expand Up @@ -2386,4 +2393,17 @@ public static boolean isFittingNumberType(final Object value, final RecordFieldT

return false;
}

private static boolean isBlank(String string) {
final int strLen = string.length();
if (strLen == 0) {
return true;
}
for (int i = 0; i < strLen; i++) {
if (!Character.isWhitespace(string.charAt(i))) {
return false;
}
}
return true;
}
}
12 changes: 12 additions & 0 deletions ...ons/nifi-record/src/test/java/org/apache/nifi/serialization/record/TestDataTypeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1110,6 +1110,18 @@ public void testToLocalDateFromNumberEpochMillis() {
assertToLocalDateEquals(ISO_8601_YEAR_MONTH_DAY, epochMillis);
}

@Test
void testNumberParsingWhereStringBlank() {
final String fieldName = "someField";
assertNull(DataTypeUtils.toBigDecimal("", fieldName));
assertNull(DataTypeUtils.toBigInt("", fieldName));
assertNull(DataTypeUtils.toDouble("", fieldName));
assertNull(DataTypeUtils.toFloat("", fieldName));
assertNull(DataTypeUtils.toInteger("", fieldName));
assertNull(DataTypeUtils.toLong("", fieldName));
assertNull(DataTypeUtils.toShort("", fieldName));
}

private long toEpochMilliSystemDefaultZone(final LocalDate localDate) {
final LocalTime localTime = LocalTime.of(0, 0);
final Instant instantSystemDefaultZone = ZonedDateTime.of(localDate, localTime, SYSTEM_DEFAULT_ZONE_ID).toInstant();
Expand Down
76 changes: 50 additions & 26 deletions ...i-bundle/nifi-poi-services/src/test/java/org/apache/nifi/excel/TestExcelRecordReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import java.util.Collections;
import java.util.List;

import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertThrows;
Expand All @@ -78,45 +79,22 @@ public class TestExcelRecordReader {
@BeforeAll
static void setUpBeforeAll() throws Exception {
//Generate an Excel file and populate it with data
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
final XSSFSheet sheet = workbook.createSheet("User Info");
populateSheet(sheet);
workbook.write(outputStream);
}
final InputStream workbook = createWorkbook(DATA);

//Protect the Excel file with a password
try (POIFSFileSystem poifsFileSystem = new POIFSFileSystem()) {
EncryptionInfo encryptionInfo = new EncryptionInfo(EncryptionMode.agile);
Encryptor encryptor = encryptionInfo.getEncryptor();
encryptor.confirmPassword(PASSWORD);

try (OPCPackage opc = OPCPackage.open(new ByteArrayInputStream(outputStream.toByteArray()));
try (OPCPackage opc = OPCPackage.open(workbook);
OutputStream os = encryptor.getDataStream(poifsFileSystem)) {
opc.save(os);
}
poifsFileSystem.writeFilesystem(PASSWORD_PROTECTED);
}
}

private static void populateSheet(XSSFSheet sheet) {
//Adding the data to the Excel worksheet
int rowCount = 0;
for (Object[] dataRow : DATA) {
Row row = sheet.createRow(rowCount++);
int columnCount = 0;

for (Object field : dataRow) {
Cell cell = row.createCell(columnCount++);
if (field instanceof String) {
cell.setCellValue((String) field);
} else if (field instanceof Integer) {
cell.setCellValue((Integer) field);
}
}
}
}

@Test
void testNonExcelFile() {
ExcelRecordReaderConfiguration configuration = new ExcelRecordReaderConfiguration.Builder()
Expand Down Expand Up @@ -190,7 +168,7 @@ void testDropUnknownFields(boolean dropUnknownFields) throws MalformedRecordExce
List<Record> records = getRecords(recordReader, false, dropUnknownFields);

assertEquals(9, records.size());
if(dropUnknownFields) {
if (dropUnknownFields) {
records.forEach(record -> assertEquals(fields.size(), record.getRawFieldNames().size()));
} else {
records.forEach(record -> {
Expand Down Expand Up @@ -335,4 +313,50 @@ private RecordSchema getPasswordProtectedSchema() {
return new SimpleRecordSchema(Arrays.asList(new RecordField("id", RecordFieldType.INT.getDataType()),
new RecordField("name", RecordFieldType.STRING.getDataType())));
}

@Test
void testWithNumberColumnWhoseValueIsEmptyString() throws Exception {
final RecordSchema schema = new SimpleRecordSchema(Arrays.asList(new RecordField("first", RecordFieldType.STRING.getDataType()),
new RecordField("second", RecordFieldType.LONG.getDataType())));
final ExcelRecordReaderConfiguration configuration = new ExcelRecordReaderConfiguration.Builder()
.withSchema(schema)
.build();

final Object[][] data = {{"Manny", ""}};
final InputStream workbook = createWorkbook(data);
final ExcelRecordReader recordReader = new ExcelRecordReader(configuration, workbook, logger);

assertDoesNotThrow(() -> getRecords(recordReader, true, true));
}

private static InputStream createWorkbook(Object[][] data) throws Exception {
final ByteArrayOutputStream workbookOutputStream = new ByteArrayOutputStream();
try (XSSFWorkbook workbook = new XSSFWorkbook()) {
final XSSFSheet sheet = workbook.createSheet("SomeSheetName");
populateSheet(sheet, data);
workbook.write(workbookOutputStream);
}

return new ByteArrayInputStream(workbookOutputStream.toByteArray());
}

private static void populateSheet(XSSFSheet sheet, Object[][] data) {
//Adding the data to the Excel worksheet
int rowCount = 0;
for (Object[] dataRow : data) {
Row row = sheet.createRow(rowCount++);
int columnCount = 0;

for (Object field : dataRow) {
Cell cell = row.createCell(columnCount++);
if (field instanceof String) {
cell.setCellValue((String)field);
} else if (field instanceof Integer) {
cell.setCellValue(((Integer)field).doubleValue());
} else if (field instanceof Long) {
cell.setCellValue(((Long)field).doubleValue());
}
}
}
}
}
Loading