Skip to content

Commit

Permalink
feat(action-impl): sql action can read blob type (#220)
Browse files Browse the repository at this point in the history 
---------
Co-authored-by: karimGl <[email protected]>
Co-authored-by: boddissattva <[email protected]>
  • Loading branch information
@rbenyoussef
rbenyoussef authored Nov 25, 2024
1 parent 2effe53 commit a125ae5
Show file tree
Hide file tree
Showing 13 changed files with 564 additions and 422 deletions.
12 changes: 12 additions & 0 deletions .gitattributes
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,15 @@

# Declare files that will always have CRLF line endings on checkout.
*.bat text eol=crlf

# Declare files that will always have CRLF line endings on checkout.
*.sh text eol=lf

# Denote all files that are truly binary and should not be modified.
*.png binary
*.ico binary
*.gif binary
*.jks binary
*.db binary
*.db-shm binary
*.db-wal binary
14 changes: 14 additions & 0 deletions chutney/action-impl/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
<distribution>repo</distribution>
</license>
</licenses>
<properties>
<ojdbc11.version>23.6.0.24.10</ojdbc11.version>
</properties>

<dependencies>
<dependency>
Expand Down Expand Up @@ -300,6 +303,17 @@
<artifactId>selenium</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>oracle-free</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc11</artifactId>
<version>${ojdbc11.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
Expand Down
34 changes: 32 additions & 2 deletions chutney/action-impl/src/main/java/com/chutneytesting/action/sql/core/SqlClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@

import com.chutneytesting.tools.NotEnoughMemoryException;
import com.zaxxer.hikari.HikariDataSource;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.sql.Blob;
import java.sql.Connection;
import java.sql.Date;
import java.sql.ResultSet;
Expand All @@ -28,13 +32,16 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class SqlClient {

private final HikariDataSource dataSource;
private final int maxFetchSize;

private static final Logger LOGGER = LoggerFactory.getLogger(SqlClient.class);


public SqlClient(HikariDataSource dataSource, int maxFetchSize) {
this.dataSource = dataSource;
Expand Down Expand Up @@ -136,8 +143,11 @@ private static Object boxed(ResultSet rs, int i) throws SQLException {
if (isPrimitiveOrWrapper(type) || isJDBCNumericType(type) || isJDBCDateType(type)) {
return o;
}
if (o instanceof Blob) {
return readBlob((Blob) o);
}

return Optional.ofNullable(rs.getString(i)).orElse("null");
return String.valueOf(rs.getString(i));
}

private static boolean isJDBCNumericType(Class<?> type) {
Expand All @@ -160,5 +170,25 @@ private static boolean isJDBCDateType(Class<?> type) {
type.equals(Duration.class); // INTERVAL
}

private static String readBlob(Blob blob) {
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); InputStream inputStream = blob.getBinaryStream()) {
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
return outputStream.toString();
} catch (IOException | SQLException e) {
throw new RuntimeException(e);
}
finally {
try {
blob.free(); // (JDBC 4.0+)
} catch (SQLException e) {
LOGGER.warn("Failed to free Blob resources: {}", e.getMessage());
}
}
}

}
}
2 changes: 1 addition & 1 deletion chutney/action-impl/src/test/java/com/chutneytesting/action/sql/SqlActionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public void setUp() {
.setType(EmbeddedDatabaseType.H2)
.setScriptEncoding("UTF-8")
.ignoreFailedDrops(true)
.addScripts("db/sql/create_db.sql", "db/sql/insert_users.sql")
.addScripts("db/common/create_users.sql")
.build();
}

Expand Down
207 changes: 127 additions & 80 deletions chutney/action-impl/src/test/java/com/chutneytesting/action/sql/core/SqlClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,111 +19,158 @@
import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType;
import org.testcontainers.oracle.OracleContainer;
import org.testcontainers.utility.MountableFile;

public class SqlClientTest {

private static final String DB_NAME = "test_" + SqlClientTest.class;
private final Target sqlTarget = TestTarget.TestTargetBuilder.builder()
.withTargetId("sql")
.withUrl("jdbc:h2:mem")
.withProperty("jdbcUrl", "jdbc:h2:mem:" + DB_NAME)
.withProperty("user", "sa")
.build();

@BeforeEach
public void setUp() {
new EmbeddedDatabaseBuilder()
.setName(DB_NAME)
.setType(EmbeddedDatabaseType.H2)
.setScriptEncoding("UTF-8")
.ignoreFailedDrops(true)
.addScripts("db/sql/create_db.sql", "db/sql/insert_users.sql", "db/sql/insert_allsqltypes.sql")
.build();
}
@Nested
class H2SqlClientTest extends AllTests {
@BeforeAll
static void beforeAll() {
sqlTarget = TestTarget.TestTargetBuilder.builder()
.withTargetId("sql")
.withUrl("jdbc:h2:mem")
.withProperty("jdbcUrl", "jdbc:h2:mem:" + DB_NAME)
.withProperty("user", "sa")
.build();
}

@Test
public void should_return_headers_and_rows_on_select_query() throws SQLException {
Column c0 = new Column("ID", 0);
Column c1 = new Column("NAME", 1);
Column c2 = new Column("EMAIL", 2);
@BeforeEach
public void setUp() {
new EmbeddedDatabaseBuilder()
.setName(DB_NAME)
.setType(EmbeddedDatabaseType.H2)
.setScriptEncoding("UTF-8")
.ignoreFailedDrops(true)
.addScripts("db/common/create_users.sql", "db/h2/create_types.sql")
.build();
}

Row firstTuple = new Row(List.of(new Cell(c0, 1), new Cell(c1, "laitue"), new Cell(c2, "[email protected]")));
Row secondTuple = new Row(List.of(new Cell(c0, 2), new Cell(c1, "carotte"), new Cell(c2, "[email protected]")));
Row thirdTuple = new Row(List.of(new Cell(c0, 3), new Cell(c1, "tomate"), new Cell(c2, "null")));
}

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from users");
@Nested
class OracleSqlClientTest extends AllTests {
private static OracleContainer oracle = new OracleContainer("gvenzl/oracle-free:23.5-slim-faststart")
.withDatabaseName("testDB")
.withUsername("testUser")
.withPassword("testPassword")
.withCopyFileToContainer(MountableFile.forClasspathResource("db/oracle/init.sh"), "/container-entrypoint-initdb.d/init.sh")
.withCopyFileToContainer(MountableFile.forClasspathResource("db/oracle/create_types.sql"), "/sql/create_types.sql")
.withCopyFileToContainer(MountableFile.forClasspathResource("db/common/create_users.sql"), "/sql/create_users.sql");

@BeforeAll
static void beforeAll() {
oracle.start();
String address = oracle.getHost();
Integer port = oracle.getFirstMappedPort();
sqlTarget = TestTarget.TestTargetBuilder.builder()
.withTargetId("sql")
.withUrl("jdbc:oracle:thin:@" + address + ":" + port + "/testDB")
.withProperty("user", "testUser")
.withProperty("password", "testPassword")
.build();
}

assertThat(actual.getHeaders()).containsOnly("ID", "NAME", "EMAIL");
assertThat(actual.records).containsExactly(firstTuple, secondTuple, thirdTuple);
@AfterAll
static void afterAll() {
oracle.stop();
}
}

@Test
public void should_return_affected_rows_on_update_queries() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records records = sqlClient.execute("UPDATE USERS SET NAME = 'toto' WHERE ID = 1");
abstract static class AllTests {
protected static final String DB_NAME = "test_" + SqlClientTest.class;
protected static Target sqlTarget;

assertThat(records.affectedRows).isEqualTo(1);
}

@Test
public void should_return_count_on_count_queries() throws SQLException {
Column c0 = new Column("TOTAL", 0);
Row expectedTuple = new Row(Collections.singletonList(new Cell(c0, 3L)));
@Test
public void should_return_headers_and_rows_on_select_query() throws SQLException {

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("SELECT COUNT(*) as total FROM USERS");
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from users where ID = 1");

assertThat(actual.getHeaders()).containsOnly("ID", "NAME", "EMAIL");
assertThat(actual.records).hasSize(1);
assertThat(actual.records.get(0)).isNotNull();
List<Cell> firstRowCells = actual.records.get(0).cells;
assertThat(firstRowCells).hasSize(3);
assertThat(firstRowCells.get(0).column.name).isEqualTo("ID");
assertThat(((Number) firstRowCells.get(0).value).intValue()).isEqualTo(1);
assertThat(firstRowCells.get(1).column.name).isEqualTo("NAME");
assertThat(firstRowCells.get(1).value).isEqualTo("laitue");
assertThat(firstRowCells.get(2).column.name).isEqualTo("EMAIL");
assertThat(firstRowCells.get(2).value).isEqualTo("[email protected]");
}

assertThat(actual.records).containsExactly(expectedTuple);
}
@Test
public void should_return_affected_rows_on_update_queries() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records records = sqlClient.execute("UPDATE USERS SET NAME = 'toto' WHERE ID = 1");

@Test
public void should_retrieve_columns_as_string_but_for_date_and_numeric_sql_datatypes() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from allsqltypes");

Row firstRow = actual.rows().get(0);
assertThat(firstRow.get("COL_BOOLEAN")).isInstanceOf(Boolean.class);
assertThat(firstRow.get("COL_TINYINT")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_SMALLINT")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_MEDIUMINT")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_INTEGER")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_BIGINT")).isInstanceOf(Long.class);
assertThat(firstRow.get("COL_FLOAT")).isInstanceOf(Float.class);
assertThat(firstRow.get("COL_DOUBLE")).isInstanceOf(Double.class);
assertThat(firstRow.get("COL_DECIMAL")).isInstanceOf(BigDecimal.class);
assertThat(firstRow.get("COL_DECIMAL")).isInstanceOf(BigDecimal.class);
assertThat(firstRow.get("COL_DATE")).isInstanceOf(Date.class);
assertThat(firstRow.get("COL_TIME")).isInstanceOf(Time.class);
assertThat(firstRow.get("COL_TIMESTAMP")).isInstanceOf(Timestamp.class);
assertThat(firstRow.get("COL_CHAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_VARCHAR")).isInstanceOf(String.class);
// INTERVAL SQL types : cf. SqlClient.StatementConverter#isJDBCDateType(Class)
assertThat(firstRow.get("COL_INTERVAL_YEAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_INTERVAL_SECOND")).isInstanceOf(String.class);
}
assertThat(records.affectedRows).isEqualTo(1);
}

@Test
public void should_prevent_out_of_memory() {
try (MockedStatic<ChutneyMemoryInfo> chutneyMemoryInfoMockedStatic = Mockito.mockStatic(ChutneyMemoryInfo.class)) {
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::hasEnoughAvailableMemory).thenReturn(true, true, false);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::usedMemory).thenReturn(42L * 1024 * 1024);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::maxMemory).thenReturn(1337L * 1024 * 1024);
@Test
public void should_return_count_on_count_queries() throws SQLException {

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("SELECT COUNT(*) as total FROM USERS");

assertThat(actual.records).hasSize(1);
assertThat(actual.records.get(0)).isNotNull();
assertThat(actual.records.get(0).cells).hasSize(1);
assertThat(actual.records.get(0).cells.get(0)).isNotNull();
Number count = (Number) actual.records.get(0).cells.get(0).value;
assertThat(count.intValue()).isEqualTo(3);
assertThat(actual.records.get(0).cells.get(0).column.name).isEqualTo("TOTAL");
}

@Test
public void should_retrieve_columns_as_expected_datatypes() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from allsqltypes");

Row firstRow = actual.rows().get(0);
assertThat(firstRow.get("COL_BOOLEAN")).isInstanceOf(Boolean.class);
assertThat(firstRow.get("COL_INTEGER")).isInstanceOfAny(Integer.class, BigDecimal.class);
assertThat(firstRow.get("COL_FLOAT")).isInstanceOf(Float.class);
assertThat(firstRow.get("COL_DOUBLE")).isInstanceOf(Double.class);
assertThat(firstRow.get("COL_DECIMAL")).isInstanceOf(BigDecimal.class);
assertThat(firstRow.get("COL_DATE")).isInstanceOfAny(Date.class, Timestamp.class);
assertThat(firstRow.get("COL_TIME")).isInstanceOfAny(Time.class, String.class);
assertThat(firstRow.get("COL_TIMESTAMP")).isInstanceOfAny(Timestamp.class, String.class);
assertThat(firstRow.get("COL_CHAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_VARCHAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_INTERVAL_YEAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_INTERVAL_SECOND")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_BLOB")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_BLOB")).isEqualTo("Chutney is a funny tool.");
}

@Test
public void should_prevent_out_of_memory() {
try (MockedStatic<ChutneyMemoryInfo> chutneyMemoryInfoMockedStatic = Mockito.mockStatic(ChutneyMemoryInfo.class)) {
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::hasEnoughAvailableMemory).thenReturn(true, true, false);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::usedMemory).thenReturn(42L * 1024 * 1024);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::maxMemory).thenReturn(1337L * 1024 * 1024);

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);

Exception exception = assertThrows(NotEnoughMemoryException.class, () -> sqlClient.execute("select * from users"));
assertThat(exception.getMessage()).isEqualTo("Running step was stopped to prevent application crash. 42MB memory used of 1337MB max.\n" +
"Current step may not be the cause.\n" +
"Query fetched 2 rows");
Exception exception = assertThrows(NotEnoughMemoryException.class, () -> sqlClient.execute("select * from users"));
assertThat(exception.getMessage()).isEqualTo("Running step was stopped to prevent application crash. 42MB memory used of 1337MB max.\n" +
"Current step may not be the cause.\n" +
"Query fetched 2 rows");
}
}
}
}
7 changes: 7 additions & 0 deletions ...rc/test/resources/db/sql/insert_users.sql → ...test/resources/db/common/create_users.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,13 @@
* SPDX-License-Identifier: Apache-2.0
*
*/
DROP TABLE users;

CREATE TABLE users (
id INTEGER PRIMARY KEY,
name VARCHAR(30),
email VARCHAR(50)
);

INSERT INTO users VALUES (1, 'laitue', '[email protected]');
INSERT INTO users VALUES (2, 'carotte', '[email protected]');
Expand Down
Loading

0 comments on commit a125ae5

Please sign in to comment.