Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/sql action karim #221

Closed
wants to merge 2 commits into from
Closed

Feat/sql action karim #221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
edf5032
feat(action-impl): sql action can read blob type
rbenyoussef Nov 15, 2024
603d4f1
feat(action): sql client can handle blob (tested with h2 and oracle)
KarimGl Nov 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions chutney/action-impl/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
<distribution>repo</distribution>
</license>
</licenses>
<properties>
<ojdbc11.version>23.6.0.24.10</ojdbc11.version>
</properties>

<dependencies>
<dependency>
Expand Down Expand Up @@ -300,6 +303,17 @@
<artifactId>selenium</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>oracle-free</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc11</artifactId>
<version>${ojdbc11.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
Expand Down
34 changes: 32 additions & 2 deletions chutney/action-impl/src/main/java/com/chutneytesting/action/sql/core/SqlClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@

import com.chutneytesting.tools.NotEnoughMemoryException;
import com.zaxxer.hikari.HikariDataSource;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.sql.Blob;
import java.sql.Connection;
import java.sql.Date;
import java.sql.ResultSet;
Expand All @@ -28,13 +32,16 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class SqlClient {

private final HikariDataSource dataSource;
private final int maxFetchSize;

private static final Logger LOGGER = LoggerFactory.getLogger(SqlClient.class);


public SqlClient(HikariDataSource dataSource, int maxFetchSize) {
this.dataSource = dataSource;
Expand Down Expand Up @@ -136,8 +143,11 @@ private static Object boxed(ResultSet rs, int i) throws SQLException {
if (isPrimitiveOrWrapper(type) || isJDBCNumericType(type) || isJDBCDateType(type)) {
return o;
}
if (o instanceof Blob) {
return readBlob((Blob) o);
}

return Optional.ofNullable(rs.getString(i)).orElse("null");
return String.valueOf(rs.getString(i));
}

private static boolean isJDBCNumericType(Class<?> type) {
Expand All @@ -160,5 +170,25 @@ private static boolean isJDBCDateType(Class<?> type) {
type.equals(Duration.class); // INTERVAL
}

private static String readBlob(Blob blob) {
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); InputStream inputStream = blob.getBinaryStream()) {
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
return outputStream.toString();
} catch (IOException | SQLException e) {
throw new RuntimeException(e);
}
finally {
try {
blob.free(); // (JDBC 4.0+)
} catch (SQLException e) {
LOGGER.warn("Failed to free Blob resources: {}", e.getMessage());
}
}
}

}
}
2 changes: 1 addition & 1 deletion chutney/action-impl/src/test/java/com/chutneytesting/action/sql/SqlActionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public void setUp() {
.setType(EmbeddedDatabaseType.H2)
.setScriptEncoding("UTF-8")
.ignoreFailedDrops(true)
.addScripts("db/sql/create_db.sql", "db/sql/insert_users.sql")
.addScripts("db/common/create_users.sql")
.build();
}

Expand Down
207 changes: 127 additions & 80 deletions chutney/action-impl/src/test/java/com/chutneytesting/action/sql/core/SqlClientTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,111 +19,158 @@
import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType;
import org.testcontainers.oracle.OracleContainer;
import org.testcontainers.utility.MountableFile;

public class SqlClientTest {

private static final String DB_NAME = "test_" + SqlClientTest.class;
private final Target sqlTarget = TestTarget.TestTargetBuilder.builder()
.withTargetId("sql")
.withUrl("jdbc:h2:mem")
.withProperty("jdbcUrl", "jdbc:h2:mem:" + DB_NAME)
.withProperty("user", "sa")
.build();

@BeforeEach
public void setUp() {
new EmbeddedDatabaseBuilder()
.setName(DB_NAME)
.setType(EmbeddedDatabaseType.H2)
.setScriptEncoding("UTF-8")
.ignoreFailedDrops(true)
.addScripts("db/sql/create_db.sql", "db/sql/insert_users.sql", "db/sql/insert_allsqltypes.sql")
.build();
}
@Nested
class H2SqlClientTest extends AllTests {
@BeforeAll
static void beforeAll() {
sqlTarget = TestTarget.TestTargetBuilder.builder()
.withTargetId("sql")
.withUrl("jdbc:h2:mem")
.withProperty("jdbcUrl", "jdbc:h2:mem:" + DB_NAME)
.withProperty("user", "sa")
.build();
}

@Test
public void should_return_headers_and_rows_on_select_query() throws SQLException {
Column c0 = new Column("ID", 0);
Column c1 = new Column("NAME", 1);
Column c2 = new Column("EMAIL", 2);
@BeforeEach
public void setUp() {
new EmbeddedDatabaseBuilder()
.setName(DB_NAME)
.setType(EmbeddedDatabaseType.H2)
.setScriptEncoding("UTF-8")
.ignoreFailedDrops(true)
.addScripts("db/common/create_users.sql", "db/h2/create_types.sql")
.build();
}

Row firstTuple = new Row(List.of(new Cell(c0, 1), new Cell(c1, "laitue"), new Cell(c2, "laitue@fake.com")));
Row secondTuple = new Row(List.of(new Cell(c0, 2), new Cell(c1, "carotte"), new Cell(c2, "kakarot@fake.db")));
Row thirdTuple = new Row(List.of(new Cell(c0, 3), new Cell(c1, "tomate"), new Cell(c2, "null")));
}

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from users");
@Nested
class OracleSqlClientTest extends AllTests {
private static OracleContainer oracle = new OracleContainer("gvenzl/oracle-free:23.4-slim-faststart")
.withDatabaseName("testDB")
.withUsername("testUser")
.withPassword("testPassword")
.withCopyFileToContainer(MountableFile.forClasspathResource("db/oracle/init.sh"), "/container-entrypoint-initdb.d/init.sh")
.withCopyFileToContainer(MountableFile.forClasspathResource("db/oracle/create_types.sql"), "/sql/create_types.sql")
.withCopyFileToContainer(MountableFile.forClasspathResource("db/common/create_users.sql"), "/sql/create_users.sql");

@BeforeAll
static void beforeAll() {
oracle.start();
String address = oracle.getHost();
Integer port = oracle.getFirstMappedPort();
sqlTarget = TestTarget.TestTargetBuilder.builder()
.withTargetId("sql")
.withUrl("jdbc:oracle:thin:@" + address + ":" + port + "/testDB")
.withProperty("user", "testUser")
.withProperty("password", "testPassword")
.build();
}

assertThat(actual.getHeaders()).containsOnly("ID", "NAME", "EMAIL");
assertThat(actual.records).containsExactly(firstTuple, secondTuple, thirdTuple);
@AfterAll
static void afterAll() {
oracle.stop();
}
}

@Test
public void should_return_affected_rows_on_update_queries() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records records = sqlClient.execute("UPDATE USERS SET NAME = 'toto' WHERE ID = 1");
abstract static class AllTests {
protected static final String DB_NAME = "test_" + SqlClientTest.class;
protected static Target sqlTarget;

assertThat(records.affectedRows).isEqualTo(1);
}

@Test
public void should_return_count_on_count_queries() throws SQLException {
Column c0 = new Column("TOTAL", 0);
Row expectedTuple = new Row(Collections.singletonList(new Cell(c0, 3L)));
@Test
public void should_return_headers_and_rows_on_select_query() throws SQLException {

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("SELECT COUNT(*) as total FROM USERS");
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from users where ID = 1");

assertThat(actual.getHeaders()).containsOnly("ID", "NAME", "EMAIL");
assertThat(actual.records).hasSize(1);
assertThat(actual.records.get(0)).isNotNull();
List<Cell> firstRowCells = actual.records.get(0).cells;
assertThat(firstRowCells).hasSize(3);
assertThat(firstRowCells.get(0).column.name).isEqualTo("ID");
assertThat(((Number) firstRowCells.get(0).value).intValue()).isEqualTo(1);
assertThat(firstRowCells.get(1).column.name).isEqualTo("NAME");
assertThat(firstRowCells.get(1).value).isEqualTo("laitue");
assertThat(firstRowCells.get(2).column.name).isEqualTo("EMAIL");
assertThat(firstRowCells.get(2).value).isEqualTo("laitue@fake.com");
}

assertThat(actual.records).containsExactly(expectedTuple);
}
@Test
public void should_return_affected_rows_on_update_queries() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records records = sqlClient.execute("UPDATE USERS SET NAME = 'toto' WHERE ID = 1");

@Test
public void should_retrieve_columns_as_string_but_for_date_and_numeric_sql_datatypes() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from allsqltypes");

Row firstRow = actual.rows().get(0);
assertThat(firstRow.get("COL_BOOLEAN")).isInstanceOf(Boolean.class);
assertThat(firstRow.get("COL_TINYINT")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_SMALLINT")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_MEDIUMINT")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_INTEGER")).isInstanceOf(Integer.class);
assertThat(firstRow.get("COL_BIGINT")).isInstanceOf(Long.class);
assertThat(firstRow.get("COL_FLOAT")).isInstanceOf(Float.class);
assertThat(firstRow.get("COL_DOUBLE")).isInstanceOf(Double.class);
assertThat(firstRow.get("COL_DECIMAL")).isInstanceOf(BigDecimal.class);
assertThat(firstRow.get("COL_DECIMAL")).isInstanceOf(BigDecimal.class);
assertThat(firstRow.get("COL_DATE")).isInstanceOf(Date.class);
assertThat(firstRow.get("COL_TIME")).isInstanceOf(Time.class);
assertThat(firstRow.get("COL_TIMESTAMP")).isInstanceOf(Timestamp.class);
assertThat(firstRow.get("COL_CHAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_VARCHAR")).isInstanceOf(String.class);
// INTERVAL SQL types : cf. SqlClient.StatementConverter#isJDBCDateType(Class)
assertThat(firstRow.get("COL_INTERVAL_YEAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_INTERVAL_SECOND")).isInstanceOf(String.class);
}
assertThat(records.affectedRows).isEqualTo(1);
}

@Test
public void should_prevent_out_of_memory() {
try (MockedStatic<ChutneyMemoryInfo> chutneyMemoryInfoMockedStatic = Mockito.mockStatic(ChutneyMemoryInfo.class)) {
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::hasEnoughAvailableMemory).thenReturn(true, true, false);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::usedMemory).thenReturn(42L * 1024 * 1024);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::maxMemory).thenReturn(1337L * 1024 * 1024);
@Test
public void should_return_count_on_count_queries() throws SQLException {

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("SELECT COUNT(*) as total FROM USERS");

assertThat(actual.records).hasSize(1);
assertThat(actual.records.get(0)).isNotNull();
assertThat(actual.records.get(0).cells).hasSize(1);
assertThat(actual.records.get(0).cells.get(0)).isNotNull();
Number count = (Number) actual.records.get(0).cells.get(0).value;
assertThat(count.intValue()).isEqualTo(3);
assertThat(actual.records.get(0).cells.get(0).column.name).isEqualTo("TOTAL");
}

@Test
public void should_retrieve_columns_as_expected_datatypes() throws SQLException {
SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);
Records actual = sqlClient.execute("select * from allsqltypes");

Row firstRow = actual.rows().get(0);
assertThat(firstRow.get("COL_BOOLEAN")).isInstanceOf(Boolean.class);
assertThat(firstRow.get("COL_INTEGER")).isInstanceOfAny(Integer.class, BigDecimal.class);
assertThat(firstRow.get("COL_FLOAT")).isInstanceOf(Float.class);
assertThat(firstRow.get("COL_DOUBLE")).isInstanceOf(Double.class);
assertThat(firstRow.get("COL_DECIMAL")).isInstanceOf(BigDecimal.class);
assertThat(firstRow.get("COL_DATE")).isInstanceOfAny(Date.class, Timestamp.class);
assertThat(firstRow.get("COL_TIME")).isInstanceOfAny(Time.class, String.class);
assertThat(firstRow.get("COL_TIMESTAMP")).isInstanceOfAny(Timestamp.class, String.class);
assertThat(firstRow.get("COL_CHAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_VARCHAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_INTERVAL_YEAR")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_INTERVAL_SECOND")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_BLOB")).isInstanceOf(String.class);
assertThat(firstRow.get("COL_BLOB")).isEqualTo("Chutney is a funny tool.");
}

@Test
public void should_prevent_out_of_memory() {
try (MockedStatic<ChutneyMemoryInfo> chutneyMemoryInfoMockedStatic = Mockito.mockStatic(ChutneyMemoryInfo.class)) {
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::hasEnoughAvailableMemory).thenReturn(true, true, false);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::usedMemory).thenReturn(42L * 1024 * 1024);
chutneyMemoryInfoMockedStatic.when(ChutneyMemoryInfo::maxMemory).thenReturn(1337L * 1024 * 1024);

SqlClient sqlClient = new DefaultSqlClientFactory().create(sqlTarget);

Exception exception = assertThrows(NotEnoughMemoryException.class, () -> sqlClient.execute("select * from users"));
assertThat(exception.getMessage()).isEqualTo("Running step was stopped to prevent application crash. 42MB memory used of 1337MB max.\n" +
"Current step may not be the cause.\n" +
"Query fetched 2 rows");
Exception exception = assertThrows(NotEnoughMemoryException.class, () -> sqlClient.execute("select * from users"));
assertThat(exception.getMessage()).isEqualTo("Running step was stopped to prevent application crash. 42MB memory used of 1337MB max.\n" +
"Current step may not be the cause.\n" +
"Query fetched 2 rows");
}
}
}
}
7 changes: 7 additions & 0 deletions ...rc/test/resources/db/sql/insert_users.sql → ...test/resources/db/common/create_users.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,13 @@
* SPDX-License-Identifier: Apache-2.0
*
*/
DROP TABLE users;

CREATE TABLE users (
id INTEGER PRIMARY KEY,
name VARCHAR(30),
email VARCHAR(50)
);

INSERT INTO users VALUES (1, 'laitue', 'laitue@fake.com');
INSERT INTO users VALUES (2, 'carotte', 'kakarot@fake.db');
Expand Down
Loading
Loading