Skip to content

Commit

Permalink
Add feature to exclude datatypes (#2548)
Browse files Browse the repository at this point in the history 
* Add feature to exclude datatypes

* Add unit tests for exclude data types feature

* Added data to the test data set, adjusted ShardQueryLogicTest to support the expectation of no results, and adjusted unit tests affected by the addition of new data.
  • Loading branch information
@jack-gitdev
jack-gitdev authored Oct 3, 2024
1 parent caa20fe commit d0ddd06
Show file tree
Hide file tree
Showing 6 changed files with 179 additions and 29 deletions.
25 changes: 24 additions & 1 deletion warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -785,7 +785,30 @@ protected void loadQueryParameters(ShardQueryConfiguration config, Query setting

if (StringUtils.isNotBlank(typeList)) {
HashSet<String> typeFilter = new HashSet<>();
typeFilter.addAll(Arrays.asList(StringUtils.split(typeList, Constants.PARAM_VALUE_SEP)));
HashSet<String> excludeSet = new HashSet<>();

for (String dataType : Arrays.asList(StringUtils.split(typeList, Constants.PARAM_VALUE_SEP))) {
if (dataType.charAt(0) == '!') {
excludeSet.add(StringUtils.substring(dataType, 1));
} else {
typeFilter.add(dataType);
}
}

if (!excludeSet.isEmpty()) {
if (typeFilter.isEmpty()) {
MetadataHelper metadataHelper = prepareMetadataHelper(config.getClient(), this.getMetadataTableName(), config.getAuthorizations(),
config.isRawTypes());

try {
typeFilter.addAll(metadataHelper.getDatatypes(null));
} catch (TableNotFoundException e) {
throw new RuntimeException(e);
}
}

typeFilter.removeAll(excludeSet);
}

if (log.isDebugEnabled()) {
log.debug("Type Filter: " + typeFilter);
Expand Down
16 changes: 8 additions & 8 deletions warehouse/query-core/src/test/java/datawave/query/CompositeFunctionsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,12 +453,12 @@ public void testNulls() throws Exception {
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO"),
Collections.emptyList(),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI", "TATTAGLIA"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI", "TATTAGLIA"),
Collections.emptyList(),
Collections.emptyList(),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI", "TATTAGLIA"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI", "TATTAGLIA"),
Collections.emptyList(),
Collections.emptyList(),
Collections.emptyList()
Expand Down Expand Up @@ -503,14 +503,14 @@ public void testNotNulls() throws Exception {
// @formatter:off
@SuppressWarnings("unchecked")
List<String>[] expectedLists = new List[] {
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI", "TATTAGLIA"),
Collections.emptyList(),
Collections.emptyList(),
Collections.emptyList(),
Collections.emptyList(),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI", "TATTAGLIA"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI", "TATTAGLIA"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO", "ANDOLINI", "TATTAGLIA"),
Arrays.asList("CORLEONE", "CAPONE", "SOPRANO"),
Collections.emptyList(),
Collections.emptyList(),
Expand Down
6 changes: 3 additions & 3 deletions warehouse/query-core/src/test/java/datawave/query/FunctionalSetTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,12 +257,12 @@ public void testMinMax() throws Exception {
};
@SuppressWarnings("unchecked")
List<String>[] expectedLists = new List[] {
Arrays.asList("ANDOLINI", "SOPRANO", "CORLEONE", "CAPONE"),
Arrays.asList("CORLEONE", "CAPONE"),
Arrays.asList("ANDOLINI", "SOPRANO", "CORLEONE", "CAPONE", "TATTAGLIA"),
Arrays.asList("CORLEONE", "CAPONE"),
Arrays.asList("CORLEONE", "CAPONE", "TATTAGLIA"),
Arrays.asList(),

Arrays.asList("CORLEONE", "CAPONE"),
Arrays.asList("CORLEONE", "CAPONE", "TATTAGLIA"),
Arrays.asList("CORLEONE", "CAPONE"),

Arrays.asList("CAPONE"),
Expand Down
2 changes: 1 addition & 1 deletion warehouse/query-core/src/test/java/datawave/query/LenientFieldsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ public void testLenientFields() throws Exception {
List<String>[] expectedLists = new List[] {
Arrays.asList(),
Arrays.asList("CORLEONE", "CAPONE"),
Arrays.asList("CORLEONE", "SOPRANO", "CAPONE"),
Arrays.asList("CORLEONE", "SOPRANO", "CAPONE", "TATTAGLIA"),
Arrays.asList()
};
// @formatter:on
Expand Down
84 changes: 68 additions & 16 deletions warehouse/query-core/src/test/java/datawave/query/tables/ShardQueryLogicTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,23 +193,27 @@ protected void runTestQuery(Set<Set<String>> expected, String querystr, Date sta
Assert.assertTrue(response instanceof DefaultEventQueryResponse);
DefaultEventQueryResponse eventQueryResponse = (DefaultEventQueryResponse) response;

for (Iterator<Set<String>> it = expected.iterator(); it.hasNext();) {
Set<String> expectedSet = it.next();
boolean found = false;

for (EventBase event : eventQueryResponse.getEvents()) {

if (expectedSet.contains("UID:" + event.getMetadata().getInternalId())) {
expectedSet.remove("UID:" + event.getMetadata().getInternalId());
((List<DefaultField>) event.getFields()).forEach((f) -> expectedSet.remove(f.getName() + ":" + f.getValueString()));
if (expectedSet.isEmpty()) {
found = true;
it.remove();
if (expected.isEmpty()) {
Assert.assertTrue(eventQueryResponse.getEvents() == null || eventQueryResponse.getEvents().isEmpty());
} else {
for (Iterator<Set<String>> it = expected.iterator(); it.hasNext();) {
Set<String> expectedSet = it.next();
boolean found = false;

for (EventBase event : eventQueryResponse.getEvents()) {

if (expectedSet.contains("UID:" + event.getMetadata().getInternalId())) {
expectedSet.remove("UID:" + event.getMetadata().getInternalId());
((List<DefaultField>) event.getFields()).forEach((f) -> expectedSet.remove(f.getName() + ":" + f.getValueString()));
if (expectedSet.isEmpty()) {
found = true;
it.remove();
}
break;
}
break;
}
Assert.assertTrue("field not found " + expectedSet, found);
}
Assert.assertTrue("field not found " + expectedSet, found);
}
}

Expand Down Expand Up @@ -312,8 +316,9 @@ public void testNegativeRegex() throws Exception {

String queryString = "UUID=='CAPONE' AND QUOTE!~'.*ind'";
Set<Set<String>> expected = new HashSet<>();
runTestQuery(expected, queryString, format.parse("20091231"), format.parse("20150101"), extraParameters);
expected.add(Sets.newHashSet("UID:" + WiseGuysIngest.caponeUID));

runTestQuery(expected, queryString, format.parse("20091231"), format.parse("20150101"), extraParameters);
}

@Test
Expand All @@ -323,8 +328,9 @@ public void testNegativeRegexV2() throws Exception {

String queryString = "UUID=='CAPONE' AND !(QUOTE=~'.*ind')";
Set<Set<String>> expected = new HashSet<>();
runTestQuery(expected, queryString, format.parse("20091231"), format.parse("20150101"), extraParameters);
expected.add(Sets.newHashSet("UID:" + WiseGuysIngest.caponeUID));

runTestQuery(expected, queryString, format.parse("20091231"), format.parse("20150101"), extraParameters);
}

@Test
Expand Down Expand Up @@ -360,4 +366,50 @@ public void testNegativeFilterRegexV2() throws Exception {

runTestQuery(expected, queryString, format.parse("20091231"), format.parse("20150101"), extraParameters);
}

@Test
public void testExcludeDataTypesBangDataType() throws Exception {
Map<String,String> extraParameters = new HashMap<>();
extraParameters.put("datatype.filter.set", "!test2");

Date startDate = format.parse("20091231");
Date endDate = format.parse("20150101");

String queryString = "UUID=='TATTAGLIA'";
Set<Set<String>> expected = new HashSet<>();
// No results expected

runTestQuery(expected, queryString, startDate, endDate, extraParameters);
}

@Test
public void testExcludeDataTypesNegateDataType() throws Exception {
Map<String,String> extraParameters = new HashMap<>();
extraParameters.put("datatype.filter.set", "test2,!test2");

Date startDate = format.parse("20091231");
Date endDate = format.parse("20150101");

String queryString = "UUID=='TATTAGLIA'";
Set<Set<String>> expected = new HashSet<>();
// Expect one result, since the negated data type results in empty set, which is treated by Datawave as all data types
expected.add(Sets.newHashSet("UID:" + WiseGuysIngest.tattagliaUID));

runTestQuery(expected, queryString, startDate, endDate, extraParameters);
}

@Test
public void testExcludeDataTypesIncludeOneTypeExcludeOneType() throws Exception {
Map<String,String> extraParameters = new HashMap<>();
extraParameters.put("datatype.filter.set", "test2,!test");

Date startDate = format.parse("20091231");
Date endDate = format.parse("20150101");

String queryString = "UUID=='TATTAGLIA' || UUID=='CAPONE'";
Set<Set<String>> expected = new HashSet<>();
expected.add(Sets.newHashSet("UID:" + WiseGuysIngest.tattagliaUID));

runTestQuery(expected, queryString, startDate, endDate, extraParameters);
}
}
Loading

0 comments on commit d0ddd06

Please sign in to comment.