Skip to content

Commit

Permalink
[PLAT-13860] Refactor TS storage graphs code to use date_bin function…
Browse files Browse the repository at this point in the history 
… and pre-aggregate data inside the DB.

Summary:
Right now we load all the raw data to build TS storage based graph in memory and aggregate in TS code.
This makes PG storage to perform aggregation and we only load results in memory.
We still perform topK filtering and average across nodes line calculation in TS.

Test Plan: Unit tested

Reviewers: rmadhavan

Reviewed By: rmadhavan

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D34873
  • Loading branch information
@anmalysh-yb
anmalysh-yb committed May 20, 2024
1 parent 6dd3725 commit f380502
Show file tree
Hide file tree
Showing 13 changed files with 1,493 additions and 1,512 deletions.
18 changes: 17 additions & 1 deletion troubleshoot/backend/src/main/java/com/yugabyte/troubleshoot/ts/models/GraphData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,18 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.Accessors;
import org.apache.commons.lang3.StringUtils;

@Data
@Accessors(chain = true)
@Builder(toBuilder = true)
@NoArgsConstructor
@AllArgsConstructor
public class GraphData {
private static DecimalFormat DF =
new DecimalFormat("0", DecimalFormatSymbols.getInstance(Locale.ENGLISH));
Expand All @@ -35,7 +41,17 @@ public class GraphData {
private String waitEventType;
private String waitEvent;
private Map<String, String> labels;
@JsonIgnore private List<GraphPoint> points = new ArrayList<>();
@JsonIgnore @Builder.Default private List<GraphPoint> points = new ArrayList<>();
@JsonIgnore private double total;

public void appendToTotal(Double value) {
total += value;
}

@JsonIgnore
public Double getAverage() {
return total / points.size();
}

@JsonProperty("x")
public List<Long> getX() {
Expand Down
2 changes: 1 addition & 1 deletion troubleshoot/backend/src/main/java/com/yugabyte/troubleshoot/ts/models/GraphLayout.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public static class GroupByLabel {
}

private String title;
private GraphType type = GraphType.COMMON;
@Builder.Default private GraphType type = GraphType.COMMON;
private Axis xaxis;
private Axis yaxis;
private Metadata metadata;
Expand Down
299 changes: 132 additions & 167 deletions ...oot/backend/src/main/java/com/yugabyte/troubleshoot/ts/service/TsStorageGraphService.java
View file

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions troubleshoot/backend/src/test/resources/anomaly/query_latency/anomalies.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
"name": "node1"
}
],
"endTime": "2024-01-18T16:31:12Z",
"graphEndTime": "2024-01-18T17:46:48Z",
"endTime": "2024-01-18T16:28:48Z",
"graphEndTime": "2024-01-18T17:44:24Z",
"graphStartTime": "2024-01-18T15:00:00Z",
"graphStepSeconds": 144,
"category": "SQL",
Expand Down Expand Up @@ -245,7 +245,7 @@
],
"title": "SQL query latency increase detected",
"type": "SQL_QUERY_LATENCY_INCREASE",
"startTime": "2024-01-18T15:40:48Z",
"startTime": "2024-01-18T15:38:24Z",
"summary": "Latencies increased for query 'select * from some_table' in database 'yugabyte'",
"universeUuid": "9ad06d1f-0355-4e3c-a42c-d052b38af7bc"
},
Expand All @@ -255,8 +255,8 @@
"name": "node3"
}
],
"endTime": "2024-01-18T17:21:36Z",
"graphEndTime": "2024-01-18T18:51:36Z",
"endTime": "2024-01-18T17:19:12Z",
"graphEndTime": "2024-01-18T18:49:12Z",
"graphStartTime": "2024-01-18T15:00:00Z",
"graphStepSeconds": 144,
"category": "SQL",
Expand Down Expand Up @@ -492,7 +492,7 @@
],
"title": "SQL query latency increase detected",
"type": "SQL_QUERY_LATENCY_INCREASE",
"startTime": "2024-01-18T16:21:36Z",
"startTime": "2024-01-18T16:19:12Z",
"summary": "Latencies increased for query 'select * from some_table' in database 'postgres'",
"universeUuid": "9ad06d1f-0355-4e3c-a42c-d052b38af7bc"
}
Expand Down
6 changes: 3 additions & 3 deletions troubleshoot/backend/src/test/resources/anomaly/query_lock_contention/anomalies.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
"name": "node1"
}
],
"endTime": "2024-01-18T15:40:48Z",
"graphEndTime": "2024-01-18T16:19:12Z",
"endTime": "2024-01-18T15:38:24Z",
"graphEndTime": "2024-01-18T16:16:48Z",
"graphStartTime": "2024-01-18T15:00:00Z",
"graphStepSeconds": 144,
"category": "SQL",
Expand Down Expand Up @@ -134,7 +134,7 @@
],
"title": "SQL query lock contention detected",
"type": "SQL_QUERY_LOCK_CONTENTION",
"startTime": "2024-01-18T15:21:36Z",
"startTime": "2024-01-18T15:19:12Z",
"summary": "Lock contention detected for query 'select * from some_table' in database 'yugabyte'",
"universeUuid": "9ad06d1f-0355-4e3c-a42c-d052b38af7bc"
}
Expand Down
240 changes: 120 additions & 120 deletions troubleshoot/backend/src/test/resources/query/outlier_ash_response.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,46 +7,46 @@
"waitEventComponent": "YSQL",
"waitEventType": "Network",
"x": [
1706272200000,
1706272560000,
1706272800000,
1706273160000,
1706273400000,
1706273760000,
1706274000000,
1706274360000,
1706274600000,
1706274960000,
1706275200000,
1706275560000,
1706275800000,
1706276160000,
1706276400000,
1706276760000,
1706277000000,
1706277360000,
1706277600000,
1706277960000,
1706278200000,
1706278560000,
1706278800000,
1706279160000,
1706279400000,
1706279760000,
1706280000000,
1706280360000,
1706280600000,
1706280960000,
1706281200000,
1706281560000,
1706281800000,
1706282160000,
1706282400000,
1706282760000,
1706283000000,
1706283360000,
1706283600000,
1706283960000
1706272080000,
1706272440000,
1706272680000,
1706273040000,
1706273280000,
1706273640000,
1706273880000,
1706274240000,
1706274480000,
1706274840000,
1706275080000,
1706275440000,
1706275680000,
1706276040000,
1706276280000,
1706276640000,
1706276880000,
1706277240000,
1706277480000,
1706277840000,
1706278080000,
1706278440000,
1706278680000,
1706279040000,
1706279280000,
1706279640000,
1706279880000,
1706280240000,
1706280480000,
1706280840000,
1706281080000,
1706281440000,
1706281680000,
1706282040000,
1706282280000,
1706282640000,
1706282880000,
1706283240000,
1706283480000,
1706283840000
],
"y": [
"1.5",
Expand Down Expand Up @@ -98,46 +98,46 @@
"waitEventComponent": "YSQL",
"waitEventType": "Network",
"x": [
1706272200000,
1706272560000,
1706272800000,
1706273160000,
1706273400000,
1706273760000,
1706274000000,
1706274360000,
1706274600000,
1706274960000,
1706275200000,
1706275560000,
1706275800000,
1706276160000,
1706276400000,
1706276760000,
1706277000000,
1706277360000,
1706277600000,
1706277960000,
1706278200000,
1706278560000,
1706278800000,
1706279160000,
1706279400000,
1706279760000,
1706280000000,
1706280360000,
1706280600000,
1706280960000,
1706281200000,
1706281560000,
1706281800000,
1706282160000,
1706282400000,
1706282760000,
1706283000000,
1706283360000,
1706283600000,
1706283960000
1706272080000,
1706272440000,
1706272680000,
1706273040000,
1706273280000,
1706273640000,
1706273880000,
1706274240000,
1706274480000,
1706274840000,
1706275080000,
1706275440000,
1706275680000,
1706276040000,
1706276280000,
1706276640000,
1706276880000,
1706277240000,
1706277480000,
1706277840000,
1706278080000,
1706278440000,
1706278680000,
1706279040000,
1706279280000,
1706279640000,
1706279880000,
1706280240000,
1706280480000,
1706280840000,
1706281080000,
1706281440000,
1706281680000,
1706282040000,
1706282280000,
1706282640000,
1706282880000,
1706283240000,
1706283480000,
1706283840000
],
"y": [
"1",
Expand Down Expand Up @@ -189,46 +189,46 @@
"waitEventComponent": "YSQL",
"waitEventType": "Network",
"x": [
1706272200000,
1706272560000,
1706272800000,
1706273160000,
1706273400000,
1706273760000,
1706274000000,
1706274360000,
1706274600000,
1706274960000,
1706275200000,
1706275560000,
1706275800000,
1706276160000,
1706276400000,
1706276760000,
1706277000000,
1706277360000,
1706277600000,
1706277960000,
1706278200000,
1706278560000,
1706278800000,
1706279160000,
1706279400000,
1706279760000,
1706280000000,
1706280360000,
1706280600000,
1706280960000,
1706281200000,
1706281560000,
1706281800000,
1706282160000,
1706282400000,
1706282760000,
1706283000000,
1706283360000,
1706283600000,
1706283960000
1706272080000,
1706272440000,
1706272680000,
1706273040000,
1706273280000,
1706273640000,
1706273880000,
1706274240000,
1706274480000,
1706274840000,
1706275080000,
1706275440000,
1706275680000,
1706276040000,
1706276280000,
1706276640000,
1706276880000,
1706277240000,
1706277480000,
1706277840000,
1706278080000,
1706278440000,
1706278680000,
1706279040000,
1706279280000,
1706279640000,
1706279880000,
1706280240000,
1706280480000,
1706280840000,
1706281080000,
1706281440000,
1706281680000,
1706282040000,
1706282280000,
1706282640000,
1706282880000,
1706283240000,
1706283480000,
1706283840000
],
"y": [
"2",
Expand Down
Loading

0 comments on commit f380502

Please sign in to comment.