[Improvement] add resposne ignore field config (#549)

apache · Jan 23, 2025 · 11aa928 · 11aa928
1 parent 869396b
commit 11aa928
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 10 deletions.
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java
@@ -37,6 +37,7 @@ public void setBackends(List<BackendRowV2> backends) {
         this.backends = backends;
     }
 
+    @JsonIgnoreProperties(ignoreUnknown = true)
     public static class BackendRowV2 {
         @JsonProperty("ip")
         public String ip;

diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Field.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Field.java
@@ -17,8 +17,10 @@
 
 package org.apache.doris.flink.rest.models;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class Field {
     @JsonProperty(value = "name")
     private String name;

diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java
@@ -17,10 +17,12 @@
 
 package org.apache.doris.flink.rest.models;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 import java.util.Map;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class QueryPlan {
     @JsonProperty(value = "status")
     private int status;

diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Schema.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Schema.java
@@ -17,9 +17,12 @@
 
 package org.apache.doris.flink.rest.models;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
 import java.util.ArrayList;
 import java.util.List;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class Schema {
     private int status = 0;
     private String keysType;

diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Tablet.java b/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Tablet.java
@@ -17,8 +17,11 @@
 
 package org.apache.doris.flink.rest.models;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
 import java.util.List;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class Tablet {
     private List<String> routings;
     private int version;

diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestRestService.java b/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestRestService.java
@@ -42,13 +42,15 @@
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_TABLET_SIZE_DEFAULT;
 import static org.apache.doris.flink.cfg.ConfigurationOptions.DORIS_TABLET_SIZE_MIN;
@@ -186,12 +188,11 @@ public void testChoiceFeNo() throws Exception {
     @Test
     public void testFeResponseToSchemaNotMap() throws Exception {
         String notSchemaRes =
-                "{\"property\":[{\"type\":\"TINYINT\",\"name\":\"k1\",\"comment\":\"\"},"
+                "{\"properties_error_key\":[{\"type\":\"TINYINT\",\"name\":\"k1\",\"comment\":\"\"},"
                         + "{\"name\":\"k5\",\"scale\":\"0\",\"comment\":\"\",\"type\":\"DECIMALV2\",\"precision\":\"9\"}],"
                         + "\"status\":200}";
-        thrown.expect(DorisException.class);
-        thrown.expectMessage(startsWith("Doris FE's response cannot map to schema. res: "));
-        RestService.parseSchema(notSchemaRes, logger);
+        Schema schema = RestService.parseSchema(notSchemaRes, logger);
+        Assert.assertTrue(schema.getProperties().isEmpty());
     }
 
     @Test
@@ -215,7 +216,7 @@ public void testFeResponseToSchemaShouldNotHappen() throws Exception {
     public void testFeResponseToQueryPlan() throws Exception {
         String res =
                 "{\"partitions\":{"
-                        + "\"11017\":{\"routings\":[\"be1\",\"be2\"],\"version\":3,\"versionHash\":1,\"schemaHash\":1},"
+                        + "\"11017\":{\"routings1\":[\"be1\",\"be2\"],\"version\":3,\"versionHash\":1,\"schemaHash\":1},"
                         + "\"11019\":{\"routings\":[\"be3\",\"be4\"],\"version\":3,\"versionHash\":1,\"schemaHash\":1}},"
                         + "\"opaqued_query_plan\":\"query_plan\",\"status\":200}";
 
@@ -407,12 +408,15 @@ public void testParseBackendV2() throws Exception {
     }
 
     @Test
-    public void testParseBackendV2Error() throws Exception {
+    public void testParseBackendV2Error() {
         String response =
-                "{\"backends\":[{\"ip1\":\"192.168.1.1\",\"http_port\":8042,\"is_alive\":true}, {\"ip\":\"192.168.1.2\",\"http_port\":8042,\"is_alive\":true}]}";
-        thrown.expect(DorisRuntimeException.class);
-        thrown.expectMessage(startsWith("Parse Doris BE's response to json failed"));
-        RestService.parseBackendV2(response, logger);
+                "{\"backends\":[{\"ip_error_key\":\"192.168.1.1\",\"http_port\":8042,\"is_alive\":true}, {\"ip\":\"192.168.1.2\",\"http_port\":8042,\"is_alive\":true}]}";
+        List<BackendV2.BackendRowV2> backendRowV2s = RestService.parseBackendV2(response, logger);
+        Assert.assertEquals(2, backendRowV2s.size());
+        List<String> actual = backendRowV2s.stream().map(m -> m.ip).collect(Collectors.toList());
+        List<String> excepted = Arrays.asList(null, "192.168.1.2");
+        Assert.assertEquals(actual.size(), excepted.size());
+        Assert.assertTrue(actual.containsAll(excepted));
     }
 
     @Test