[ALS-8098] Produce a list of all patients involved in query

- Add new expected result type PATIENTS - Add new PatientProcessor to produce result - Add to FileSharingService to enable uploader integration
hms-dbmi · Dec 23, 2024 · 5d83730 · 5d83730
1 parent 0d35b8c
commit 5d83730
Show file tree

Hide file tree

Showing 8 changed files with 176 additions and 37 deletions.
diff --git a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java
@@ -190,6 +190,7 @@ public String toString() {
 			break;
 		case DATAFRAME:
 		case SECRET_ADMIN_DATAFRAME:
+		case PATIENTS:
 			writePartFormat("Data Export Fields", fields, builder, true);
 			break;
 		case DATAFRAME_TIMESERIES:

diff --git a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/ResultType.java b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/ResultType.java
@@ -100,5 +100,10 @@ public enum ResultType {
      * Exports data as PFB, using avro
      * <a href="https://uc-cdis.github.io/pypfb/">https://uc-cdis.github.io/pypfb/</a>
      */
-	DATAFRAME_PFB
+	DATAFRAME_PFB,
+
+	/**
+	 * Patients associated with this query
+	 */
+	PATIENTS
 }
diff --git a/...src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessor.java b/...src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessor.java
@@ -0,0 +1,42 @@
+package edu.harvard.hms.dbmi.avillach.hpds.processing.patient;
+
+import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query;
+import edu.harvard.hms.dbmi.avillach.hpds.processing.AbstractProcessor;
+import edu.harvard.hms.dbmi.avillach.hpds.processing.AsyncResult;
+import edu.harvard.hms.dbmi.avillach.hpds.processing.HpdsProcessor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+@Component
+public class PatientProcessor implements HpdsProcessor {
+
+    private static final Logger LOG = LoggerFactory.getLogger(PatientProcessor.class);
+    private final AbstractProcessor abstractProcessor;
+
+    @Autowired
+    public PatientProcessor(AbstractProcessor abstractProcessor) {
+        this.abstractProcessor = abstractProcessor;
+    }
+
+    @Override
+    public String[] getHeaderRow(Query query) {
+        return new String[]{"PATIENT_NUM"};
+    }
+
+    @Override
+    public void runQuery(Query query, AsyncResult asyncResult) {
+        LOG.info("Pulling results for query {}", query.getId());
+        // floating all this in memory is a bit gross, but the whole list of
+        // patient IDs was already there, so I don't feel too bad
+        List<String[]> allPatients = abstractProcessor.getPatientSubsetForQuery(query).stream()
+            .map(patient -> new String[]{patient.toString()})
+            .toList();
+        LOG.info("Writing results for query {}", query.getId());
+        asyncResult.appendResults(allPatients);
+        LOG.info("Completed query {}", query.getId());
+    }
+}
diff --git a/...test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessorTest.java b/...test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessorTest.java
@@ -0,0 +1,47 @@
+package edu.harvard.hms.dbmi.avillach.hpds.processing.patient;
+
+import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query;
+import edu.harvard.hms.dbmi.avillach.hpds.data.query.ResultType;
+import edu.harvard.hms.dbmi.avillach.hpds.processing.AbstractProcessor;
+import edu.harvard.hms.dbmi.avillach.hpds.processing.AsyncResult;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+
+import java.util.List;
+import java.util.TreeSet;
+
+@EnableAutoConfiguration
+@SpringBootTest(classes = PatientProcessor.class)
+class PatientProcessorTest {
+
+    @MockBean
+    AbstractProcessor abstractProcessor;
+
+    @Autowired
+    PatientProcessor subject;
+
+    @Test
+    void shouldProcessPatientQuery() {
+        Query q = new Query();
+        q.setId("frank");
+        q.setPicSureId("frank");
+        q.setExpectedResultType(ResultType.PATIENTS);
+        AsyncResult writeToThis = Mockito.mock(AsyncResult.class);
+        Mockito.when(abstractProcessor.getPatientSubsetForQuery(q))
+            .thenReturn(new TreeSet<>(List.of(1, 2, 42)));
+
+        subject.runQuery(q, writeToThis);
+
+        Mockito.verify(writeToThis, Mockito.times(1))
+            .appendResults(Mockito.argThat(strings ->
+                strings.size() == 3 &&
+                strings.get(0)[0].equals("1") &&
+                strings.get(1)[0].equals("2") &&
+                strings.get(2)[0].equals("42"))
+            );
+    }
+}
diff --git a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java
@@ -14,7 +14,6 @@
 import edu.harvard.hms.dbmi.avillach.hpds.service.filesharing.TestDataService;
 import edu.harvard.hms.dbmi.avillach.hpds.service.util.Paginator;
 import edu.harvard.hms.dbmi.avillach.hpds.service.util.QueryDecorator;
-import org.apache.http.entity.ContentType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -32,14 +31,12 @@
 
 import edu.harvard.dbmi.avillach.domain.*;
 import edu.harvard.dbmi.avillach.util.UUIDv5;
-import edu.harvard.dbmi.avillach.service.IResourceRS;
 import edu.harvard.hms.dbmi.avillach.hpds.crypto.Crypto;
 import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.ColumnMeta;
 import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query;
 import edu.harvard.hms.dbmi.avillach.hpds.processing.*;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
-import org.springframework.stereotype.Component;
 import org.springframework.web.bind.annotation.RequestBody;
 
 @RequestMapping(value = "PIC-SURE", produces = "application/json")
@@ -308,7 +305,10 @@ public ResponseEntity writeQueryResult(
             success = fileSystemService.createPhenotypicData(query);
         } else if ("genomic".equals(datatype)) {
             success = fileSystemService.createGenomicData(query);
-        }
+        } else if ("patients".equals(datatype)) {
+			success = ResultType.PATIENTS.equals(query.getExpectedResultType()) &&
+				fileSystemService.createPatientList(query);
+		}
         return success ? ResponseEntity.ok().build() : ResponseEntity.internalServerError().build();
     }
 
@@ -391,6 +391,7 @@ private ResponseEntity _querySync(QueryRequest resultRequest) throws IOException
 		case DATAFRAME:
 		case SECRET_ADMIN_DATAFRAME:
 		case DATAFRAME_TIMESERIES:
+		case PATIENTS:
 			QueryStatus status = query(resultRequest).getBody();
 			while (status.getResourceStatus().equalsIgnoreCase("RUNNING")
 					|| status.getResourceStatus().equalsIgnoreCase("PENDING")) {

diff --git a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryService.java b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryService.java
@@ -8,6 +8,7 @@
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
+import edu.harvard.hms.dbmi.avillach.hpds.processing.patient.PatientProcessor;
 import edu.harvard.hms.dbmi.avillach.hpds.processing.timeseries.TimeseriesProcessor;
 import edu.harvard.hms.dbmi.avillach.hpds.data.query.ResultType;
 import edu.harvard.hms.dbmi.avillach.hpds.processing.dictionary.DictionaryService;
@@ -50,6 +51,7 @@ public class QueryService {
 	private final TimeseriesProcessor timeseriesProcessor;
 	private final CountProcessor countProcessor;
 	private final MultiValueQueryProcessor multiValueQueryProcessor;
+	private final PatientProcessor patientProcessor;
 
 	private final DictionaryService dictionaryService;
 	private final QueryDecorator queryDecorator;
@@ -59,15 +61,15 @@ public class QueryService {
 
 	@Autowired
 	public QueryService (AbstractProcessor abstractProcessor,
-						 QueryProcessor queryProcessor,
-						 TimeseriesProcessor timeseriesProcessor,
-						 CountProcessor countProcessor,
-						 MultiValueQueryProcessor multiValueQueryProcessor,
-						 @Autowired(required = false) DictionaryService dictionaryService,
+                         QueryProcessor queryProcessor,
+                         TimeseriesProcessor timeseriesProcessor,
+                         CountProcessor countProcessor,
+                         MultiValueQueryProcessor multiValueQueryProcessor,
+                         @Autowired(required = false) DictionaryService dictionaryService,
                          QueryDecorator queryDecorator,
-						 @Value("${SMALL_JOB_LIMIT}") Integer smallJobLimit,
-						 @Value("${SMALL_TASK_THREADS}") Integer smallTaskThreads,
-						 @Value("${LARGE_TASK_THREADS}") Integer largeTaskThreads) {
+                         @Value("${SMALL_JOB_LIMIT}") Integer smallJobLimit,
+                         @Value("${SMALL_TASK_THREADS}") Integer smallTaskThreads,
+                         @Value("${LARGE_TASK_THREADS}") Integer largeTaskThreads, PatientProcessor patientProcessor) {
 		this.abstractProcessor = abstractProcessor;
 		this.queryProcessor = queryProcessor;
 		this.timeseriesProcessor = timeseriesProcessor;
@@ -79,9 +81,10 @@ public QueryService (AbstractProcessor abstractProcessor,
 		SMALL_JOB_LIMIT = smallJobLimit;
 		SMALL_TASK_THREADS = smallTaskThreads;
 		LARGE_TASK_THREADS = largeTaskThreads;
+        this.patientProcessor = patientProcessor;
 
 
-		/* These have to be of type Runnable(nothing more specific) in order 
+        /* These have to be of type Runnable(nothing more specific) in order
 		 * to be compatible with ThreadPoolExecutor constructor prototype 
 		 */
 		largeTaskExecutionQueue = new PriorityBlockingQueue<Runnable>(1000);
@@ -124,6 +127,8 @@ private AsyncResult initializeResult(Query query) throws IOException {
 
 		HpdsProcessor p;
 		switch(query.getExpectedResultType()) {
+			case PATIENTS:
+				p = patientProcessor;
 			case SECRET_ADMIN_DATAFRAME:
 				p = queryProcessor;
 				break;

diff --git a/.../main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingService.java b/.../main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingService.java
@@ -1,18 +1,15 @@
 package edu.harvard.hms.dbmi.avillach.hpds.service.filesharing;
 
 import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query;
-import edu.harvard.hms.dbmi.avillach.hpds.data.query.ResultType;
 import edu.harvard.hms.dbmi.avillach.hpds.processing.AsyncResult;
 import edu.harvard.hms.dbmi.avillach.hpds.processing.VariantListProcessor;
 import edu.harvard.hms.dbmi.avillach.hpds.service.QueryService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
-import java.time.Duration;
-import java.time.temporal.ChronoUnit;
+import java.util.Optional;
 
 /**
  * Used for sharing data. Given a query, this service will write
@@ -23,21 +20,25 @@ public class FileSharingService {
 
     private static final Logger LOG = LoggerFactory.getLogger(FileSharingService.class);
 
-    @Autowired
-    private QueryService queryService;
-
-    @Autowired
-    private FileSystemService fileWriter;
-
-    @Autowired
-    private VariantListProcessor variantListProcessor;
+    private final QueryService queryService;
+    private final FileSystemService fileWriter;
+    private final VariantListProcessor variantListProcessor;
+
+    public FileSharingService(
+        QueryService queryService, FileSystemService fileWriter,
+        VariantListProcessor variantListProcessor
+    ) {
+        this.queryService = queryService;
+        this.fileWriter = fileWriter;
+        this.variantListProcessor = variantListProcessor;
+    }
 
     public boolean createPhenotypicData(Query query) {
-        AsyncResult result = queryService.getResultFor(query.getId());
-        if (result == null || result.getStatus() != AsyncResult.Status.SUCCESS) {
-            return false;
-        }
-        return fileWriter.writeResultToFile("phenotypic_data.csv", result, query.getPicSureId());
+        return createAndWriteData(query, "phenotypic_data.csv");
+    }
+
+    public boolean createPatientList(Query query) {
+        return createAndWriteData(query, "patients.txt");
     }
 
     public boolean createGenomicData(Query query) {
@@ -49,4 +50,12 @@ public boolean createGenomicData(Query query) {
             return false;
         }
     }
+
+    private boolean createAndWriteData(Query query, String fileName) {
+        AsyncResult result = queryService.getResultFor(query.getId());
+        if (result == null || result.getStatus() != AsyncResult.Status.SUCCESS) {
+            return false;
+        }
+        return fileWriter.writeResultToFile(fileName, result, query.getPicSureId());
+    }
 }
diff --git a/...t/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingServiceTest.java b/...t/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingServiceTest.java
@@ -1,38 +1,49 @@
 package edu.harvard.hms.dbmi.avillach.hpds.service.filesharing;
 
 import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query;
+import edu.harvard.hms.dbmi.avillach.hpds.data.query.ResultType;
 import edu.harvard.hms.dbmi.avillach.hpds.processing.AsyncResult;
 import edu.harvard.hms.dbmi.avillach.hpds.processing.VariantListProcessor;
 import edu.harvard.hms.dbmi.avillach.hpds.processing.io.ResultWriter;
+import edu.harvard.hms.dbmi.avillach.hpds.processing.patient.PatientProcessor;
 import edu.harvard.hms.dbmi.avillach.hpds.service.QueryService;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
 
 import java.io.IOException;
 
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-@ExtendWith(MockitoExtension.class)
+@EnableAutoConfiguration
+@SpringBootTest(classes = FileSharingService.class)
 public class FileSharingServiceTest {
 
-    @Mock
+    @MockBean
     QueryService queryService;
 
-    @Mock
+    @MockBean
     FileSystemService fileWriter;
 
-    @Mock
+    @MockBean
     VariantListProcessor variantListProcessor;
 
-    @Mock
+    @MockBean
+    PatientProcessor patientProcessor;
+
+    @MockBean
     ResultWriter resultWriter;
 
-    @InjectMocks
+    @Autowired
     FileSharingService subject;
 
     @Test
@@ -95,4 +106,22 @@ public void shouldNotCreateGenomicData() throws IOException {
 
         assertFalse(actual);
     }
+
+    @Test
+    void shouldCreatePatientsList() {
+        Query query = new Query();
+        query.setId("jasdijasd");
+        query.setPicSureId("jasdijasd");
+        query.setExpectedResultType(ResultType.PATIENTS);
+        AsyncResult result = new AsyncResult(query, patientProcessor, resultWriter);
+        result.setStatus(AsyncResult.Status.SUCCESS);
+        Mockito.when(queryService.getResultFor("jasdijasd"))
+            .thenReturn(result);
+        Mockito.when(fileWriter.writeResultToFile("patients.txt", result, "jasdijasd"))
+            .thenReturn(true);
+
+        boolean actual = subject.createPatientList(query);
+
+        Assertions.assertTrue(actual);
+    }
 }