Skip to content

Commit

Permalink
Upgrading libraries, mainly the JSqlParser, to support more queries o…
Browse files Browse the repository at this point in the history
…ut of the box. (#41)

* Upgrading libraries, mainly the JSqlParser, to support more queries out of the box.
  • Loading branch information
onukristo authored Jan 26, 2024
1 parent 5dc0317 commit e841df2
Show file tree
Hide file tree
Showing 19 changed files with 185 additions and 174 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ jobs:
max-parallel: 100
matrix:
spring_boot_version:
- 3.1.2
- 3.0.7
- 2.7.13
- 2.6.15
- 3.2.2
- 3.1.6
- 3.0.13
- 2.7.18
env:
SPRING_BOOT_VERSION: ${{ matrix.spring_boot_version }}
services:
Expand Down
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres
to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [2.15.0] - 2024-01-25

* Upgrading libraries, mainly the JSqlParser, to support more queries out of the box.

## [2.14.0] - 2023-09-26

### Added
Expand Down
2 changes: 1 addition & 1 deletion build.common.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,8 @@ test {
}

tasks.findAll { it.name.startsWith("spotbugs") }*.configure {
effort = "max"
excludeFilter = file('../spotbugs-exclude.xml')

reports {
xml.required = true
html.required = true
Expand Down
15 changes: 11 additions & 4 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import com.github.spotbugs.snom.Confidence
import com.github.spotbugs.snom.Effort
import org.eclipse.jgit.api.errors.RefAlreadyExistsException

plugins {
id "com.github.spotbugs" version "5.0.13" apply false
id "com.github.spotbugs" version "6.0.7"
id "idea"
id 'org.ajoberstar.grgit' version '5.0.0'
id 'io.github.gradle-nexus.publish-plugin' version "1.1.0"
id 'com.github.johnrengelman.shadow' version '7.0.0' apply false
id 'org.ajoberstar.grgit' version '5.2.1'
id 'io.github.gradle-nexus.publish-plugin' version "1.3.0"
id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
}

idea.project {
Expand Down Expand Up @@ -39,3 +41,8 @@ nexusPublishing {
}
}
}

spotbugs {
effort = Effort.valueOf('MAX')
reportLevel = Confidence.valueOf('DEFAULT')
}
19 changes: 10 additions & 9 deletions build.libraries.gradle
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
ext {
springBootVersion = System.getenv("SPRING_BOOT_VERSION") ?: "2.6.15"
springBootVersion = System.getenv("SPRING_BOOT_VERSION") ?: "2.7.18"

libraries = [
// explicit versions
guava : "com.google.guava:guava:31.1-jre",
jsqlParser : "com.github.jsqlparser:jsqlparser:4.6",
guava : "com.google.guava:guava:33.0.0-jre",
jsqlParser : "com.github.jsqlparser:jsqlparser:4.8",
spotbugsAnnotations : "com.github.spotbugs:spotbugs-annotations:${spotbugs.toolVersion.get()}",
springBootDependencies : "org.springframework.boot:spring-boot-dependencies:${springBootVersion}",
testContainersMariaDb : "org.testcontainers:mariadb:1.17.6",
twBaseUtils : "com.transferwise.common:tw-base-utils:1.10.1",
testContainersMariaDb : "org.testcontainers:mariadb:1.19.3",
twBaseUtils : "com.transferwise.common:tw-base-utils:1.12.3",
twContext : "com.transferwise.common:tw-context:1.0.0",
twContextStarter : "com.transferwise.common:tw-context-starter:1.0.0",
twGracefulShutdown : 'com.transferwise.common:tw-graceful-shutdown:2.11.0',
twGracefulShutdownInterfaces : 'com.transferwise.common:tw-graceful-shutdown-interfaces:2.11.0',
twSpyqlCore : "com.transferwise.common:tw-spyql-core:1.6.0",
twSpyqlStarter : "com.transferwise.common:tw-spyql-starter:1.6.0",
twGracefulShutdown : 'com.transferwise.common:tw-graceful-shutdown:2.14.2',
twGracefulShutdownInterfaces: 'com.transferwise.common:tw-graceful-shutdown-interfaces:2.14.2',
twSpyqlCore : "com.transferwise.common:tw-spyql-core:1.6.1",
twSpyqlStarter : "com.transferwise.common:tw-spyql-starter:1.6.1",

// versions managed by spring-boot-dependencies platform
caffeine : "com.github.ben-manes.caffeine:caffeine",
Expand All @@ -24,6 +24,7 @@ ext {

hikariCp : "com.zaxxer:HikariCP",
junitJupiter : "org.junit.jupiter:junit-jupiter",
logbackClassic : "ch.qos.logback:logback-classic",
lombok : "org.projectlombok:lombok",
mariadbJavaClient : "org.mariadb.jdbc:mariadb-java-client",
micrometerCore : "io.micrometer:micrometer-core",
Expand Down
2 changes: 1 addition & 1 deletion gradle.properties
Original file line number Diff line number Diff line change
@@ -1 +1 @@
version=2.14.0
version=2.15.0
Binary file modified gradle/wrapper/gradle-wrapper.jar
Binary file not shown.
2 changes: 1 addition & 1 deletion gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
networkTimeout=10000
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
7 changes: 4 additions & 3 deletions gradlew
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,6 @@ done
APP_BASE_NAME=${0##*/}
APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit

# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'

# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum

Expand Down Expand Up @@ -197,6 +194,10 @@ if "$cygwin" || "$msys" ; then
done
fi


# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'

# Collect all arguments for the java command;
# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
# shell script including quotes and variable substitutions, so put them in
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public boolean canHandleInTransaction(Event event, Context context) {

@Override
public void handle(Event event, Context context) {
if (event == Event.BEFORE_MIGRATE) {
if (event == Event.BEFORE_VALIDATE) {
log.info("Disabling TAS query parsing before Flyway migration.");

var twContext = TwContext.current().createSubContext().asEntryPoint("Flyway", "Flyway");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public void setup() {
meterRegistry.getMeters().stream()
.filter(m -> (m.getId().getName().startsWith("EntryPoints") || m.getId().getName().startsWith("database")) && !(m instanceof Gauge))
.forEach(m -> {
log.info("Removing metric: " + m.getId());
log.debug("Removing metric: " + m.getId());
meterRegistry.remove(m);
});
meterCache.clear();
Expand Down
19 changes: 5 additions & 14 deletions tw-entrypoints/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,8 @@ dependencies {
implementation libraries.twSpyqlCore
implementation libraries.twGracefulShutdownInterfaces

shadow libraries.twContext
shadow libraries.commonsLang3
shadow libraries.caffeine
shadow libraries.guava
shadow libraries.micrometerCore
shadow libraries.slf4jApi
shadow libraries.twBaseUtils
shadow libraries.twSpyqlCore
shadow libraries.twGracefulShutdownInterfaces

testImplementation libraries.junitJupiter
testRuntimeOnly libraries.logbackClassic
}

apply from: "build-idea-fix.gradle.kts"
Expand All @@ -48,10 +39,10 @@ shadowJar {
manifest {
attributes 'Implementation-Version': "$project.version"
}
relocate('net.sf.jsqlparser', 'com.transferwise.common.entrypoints.shadow')
relocate('net.sf.jsqlparser', 'com.transferwise.common.entrypoints.shadow.net.sf.jsqlparser')
}

jar.enabled = true
jar.enabled = false
jar.dependsOn shadowJar

shadowJar {
Expand Down Expand Up @@ -105,8 +96,8 @@ publishing {
withXml { xml ->
def dependenciesNode = xml.asNode().get('dependencies') ?: xml.asNode().appendNode('dependencies')

project.configurations.getByName("shadow").resolvedConfiguration.firstLevelModuleDependencies.forEach {
if (it.configuration != "platform-runtime") {
project.configurations.getByName("runtimeClasspath").resolvedConfiguration.firstLevelModuleDependencies.forEach {
if (it.configuration != "platform-runtime" && it.moduleName != 'jsqlparser') {
def dependencyNode = dependenciesNode.appendNode('dependency')
dependencyNode.appendNode('groupId', it.moduleGroup)
dependencyNode.appendNode('artifactId', it.moduleName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,14 @@ protected void registerUnknownCalls() {
final long fetchedRows = das.getAndResetFetchedRowsCount();

TagsSet tagsSet = das.getTagsSet();
UnknownCallMeters meters = meterCache.metersContainer(METRIC_PREFIX_DAS + "unknownCallMetrics", tagsSet, () -> {
UnknownCallMeters meters = meterCache.metersContainer(METRIC_PREFIX_DAS + "unknownCallMetrics", tagsSet, (name, tags) -> {
UnknownCallMeters result = new UnknownCallMeters();
result.commits = meterCache.counter(COUNTER_UNREGISTERED_COMMITS, tagsSet);
result.rollbacks = meterCache.counter(COUNTER_UNREGISTERED_ROLLBACKS, tagsSet);
result.nonTransactionalQueries = meterCache.counter(COUNTER_UNREGISTERED_NT_QUERIES, tagsSet);
result.transactionalQueries = meterCache.counter(COUNTER_UNREGISTERED_T_QUERIES, tagsSet);
result.timeTakenNs = meterCache.counter(COUNTER_UNREGISTERED_TIME_TAKEN_NS, tagsSet);
result.emptyTransactions = meterCache.counter(COUNTER_UNREGISTERED_EMPTY_TRANSACTIONS, tagsSet);
result.commits = meterCache.counter(COUNTER_UNREGISTERED_COMMITS, tags);
result.rollbacks = meterCache.counter(COUNTER_UNREGISTERED_ROLLBACKS, tags);
result.nonTransactionalQueries = meterCache.counter(COUNTER_UNREGISTERED_NT_QUERIES, tags);
result.transactionalQueries = meterCache.counter(COUNTER_UNREGISTERED_T_QUERIES, tags);
result.timeTakenNs = meterCache.counter(COUNTER_UNREGISTERED_TIME_TAKEN_NS, tags);
result.emptyTransactions = meterCache.counter(COUNTER_UNREGISTERED_EMPTY_TRANSACTIONS, tags);
result.affectedRows = meterCache.counter(COUNTER_UNREGISTERED_AFFECTED_ROWS, tagsSet);
result.fetchedRows = meterCache.counter(COUNTER_UNREGISTERED_FETCHED_ROWS, tagsSet);
return result;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,18 +71,18 @@ private void registerCall(TwContext context, Map<String, DatabaseAccessStatistic
final long timeTakenInDatabaseNs = das.getTimeTakenInDatabaseNs();
final long fetchedRowsCount = das.getFetchedRowsCount();

CallMeters meters = meterCache.metersContainer(METRIC_PREFIX_DAS + "callMetrics", tagsSet, () -> {
CallMeters meters = meterCache.metersContainer(METRIC_PREFIX_DAS + "callMetrics", tagsSet, (name, tags) -> {
CallMeters result = new CallMeters();
result.commits = meterCache.summary(SUMMARY_REGISTERED_COMMITS, tagsSet);
result.rollbacks = meterCache.summary(SUMMARY_REGISTERED_ROLLBACKS, tagsSet);
result.nonTransactionalQueries = meterCache.summary(SUMMARY_REGISTERED_NT_QUERIES, tagsSet);
result.transactionalQueries = meterCache.summary(SUMMARY_REGISTERED_T_QUERIES, tagsSet);
result.maxConcurrentConnections = meterCache.summary(SUMMARY_REGISTERED_MAX_CONCURRENT_CONNECTIONS, tagsSet);
result.remainingOpenConnections = meterCache.summary(SUMMARY_REGISTERED_REMAINING_OPEN_CONNECTIONS, tagsSet);
result.emptyTransactions = meterCache.summary(SUMMARY_REGISTERED_EMPTY_TRANSACTIONS, tagsSet);
result.affectedRows = meterCache.summary(SUMMARY_REGISTERED_AFFECTED_ROWS, tagsSet);
result.fetchedRows = meterCache.summary(SUMMARY_REGISTERED_FETCHED_ROWS, tagsSet);
result.timeTakenNs = meterCache.timer(TIMER_REGISTERED_TIME_TAKEN, tagsSet);
result.commits = meterCache.summary(SUMMARY_REGISTERED_COMMITS, tags);
result.rollbacks = meterCache.summary(SUMMARY_REGISTERED_ROLLBACKS, tags);
result.nonTransactionalQueries = meterCache.summary(SUMMARY_REGISTERED_NT_QUERIES, tags);
result.transactionalQueries = meterCache.summary(SUMMARY_REGISTERED_T_QUERIES, tags);
result.maxConcurrentConnections = meterCache.summary(SUMMARY_REGISTERED_MAX_CONCURRENT_CONNECTIONS, tags);
result.remainingOpenConnections = meterCache.summary(SUMMARY_REGISTERED_REMAINING_OPEN_CONNECTIONS, tags);
result.emptyTransactions = meterCache.summary(SUMMARY_REGISTERED_EMPTY_TRANSACTIONS, tags);
result.affectedRows = meterCache.summary(SUMMARY_REGISTERED_AFFECTED_ROWS, tags);
result.fetchedRows = meterCache.summary(SUMMARY_REGISTERED_FETCHED_ROWS, tags);
result.timeTakenNs = meterCache.timer(TIMER_REGISTERED_TIME_TAKEN, tags);
return result;
});

Expand Down
Original file line number Diff line number Diff line change
@@ -1,56 +1,33 @@
package com.transferwise.common.entrypoints.tableaccessstatistics;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import lombok.Getter;
import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.SetStatement;
import net.sf.jsqlparser.statement.ShowColumnsStatement;
import net.sf.jsqlparser.statement.Statements;
import net.sf.jsqlparser.statement.alter.Alter;
import net.sf.jsqlparser.statement.create.index.CreateIndex;
import net.sf.jsqlparser.statement.create.view.CreateView;
import net.sf.jsqlparser.statement.drop.Drop;
import net.sf.jsqlparser.statement.execute.Execute;
import net.sf.jsqlparser.util.TablesNamesFinder;

public class CustomTablesNamesFinder extends TablesNamesFinder {

@Override
protected String extractTableName(Table table) {
return table.getName();
}

@Override
public void visit(Drop drop) {
visit(drop.getName());
}

@Override
public void visit(CreateIndex createIndex) {
visit(createIndex.getTable());
}

@Override
public void visit(CreateView createView) {
visit(createView.getView());
}
@Getter
private final List<String> tables = new ArrayList<>();
private final Set<String> uniqueTables = new HashSet<>();

@Override
public void visit(Alter alter) {
visit(alter.getTable());
}

@Override
public void visit(Statements stmts) {
}
/*
The super class loses the order of tables visited, as its tables list is based on Set.
We are providing here the option to get the ordered tables list.
*/
@Override
public void visit(Execute execute) {
}
public void visit(Table tableName) {
String tableWholeName = extractTableName(tableName);

@Override
public void visit(SetStatement set) {
}
if (!uniqueTables.contains(tableWholeName)) {
uniqueTables.add(tableWholeName);
tables.add(tableWholeName);
}

@Override
public void visit(ShowColumnsStatement set) {
super.visit(tableName);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,64 +4,24 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.parser.CCJSqlParser;
import net.sf.jsqlparser.parser.StringProvider;
import net.sf.jsqlparser.statement.Statements;

public class SqlParser {

/*
DATABASE is reserved keyword in sql parser, so having DATABASE() in sql will just throw error.
DATABASE() is however used by some of our internal libraries for MariaDb.
*/
private static final Pattern FUNCTION_REPLACEMENT_PATTERN = Pattern.compile("DATABASE()",
Pattern.LITERAL | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);

private static final Pattern ON_CONFLICT_REPLACEMENT_PATTERN = Pattern.compile("on\\s*conflict\\s*\\(.*?\\)",
Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.DOTALL);
private final ExecutorService executorService;

public SqlParser(ExecutorService executorService) {
this.executorService = executorService;
}

public Statements parse(String sql, Duration timeout) throws JSQLParserException {
sql = replaceFunctions(sql);
sql = replaceOnConflicts(sql);

CCJSqlParser parser = newParser(sql).withAllowComplexParsing(true);
return parseStatementAsync(parser, timeout);
}

// Sqlparser 4.6 does not support "on conflict" clause with multiple parameters.
// As a workaround, we change the query to a single parameter clause.
protected String replaceOnConflicts(String sql) {
var matcher = ON_CONFLICT_REPLACEMENT_PATTERN.matcher(sql);

// 99.99% of sqls don't have it, so let's avoid new string creation for those.
if (matcher.find()) {
matcher.reset();
return matcher.replaceAll(Matcher.quoteReplacement("on conflict (blah)"));
}

return sql;
}

protected String replaceFunctions(String sql) {
var matcher = FUNCTION_REPLACEMENT_PATTERN.matcher(sql);

// 99.99% of sqls don't have it, so let's avoid new string creation for those.
if (matcher.find()) {
matcher.reset();
return matcher.replaceAll(Matcher.quoteReplacement("UNSUPPORTED()"));
}

return sql;
}

protected Statements parseStatementAsync(CCJSqlParser parser, Duration timeout) throws JSQLParserException {
try {
var future = executorService.submit(parser::Statements);
Expand Down
Loading

0 comments on commit e841df2

Please sign in to comment.