diff --git a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java index e62013e4..f26749cd 100644 --- a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java +++ b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/Query.java @@ -190,6 +190,7 @@ public String toString() { break; case DATAFRAME: case SECRET_ADMIN_DATAFRAME: + case PATIENTS: writePartFormat("Data Export Fields", fields, builder, true); break; case DATAFRAME_TIMESERIES: diff --git a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/ResultType.java b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/ResultType.java index 1aceaa0e..28704b5d 100644 --- a/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/ResultType.java +++ b/client-api/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/data/query/ResultType.java @@ -100,5 +100,10 @@ public enum ResultType { * Exports data as PFB, using avro * https://uc-cdis.github.io/pypfb/ */ - DATAFRAME_PFB + DATAFRAME_PFB, + + /** + * Patients associated with this query + */ + PATIENTS } diff --git a/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessor.java b/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessor.java new file mode 100644 index 00000000..5e8db162 --- /dev/null +++ b/processing/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessor.java @@ -0,0 +1,42 @@ +package edu.harvard.hms.dbmi.avillach.hpds.processing.patient; + +import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query; +import edu.harvard.hms.dbmi.avillach.hpds.processing.AbstractProcessor; +import edu.harvard.hms.dbmi.avillach.hpds.processing.AsyncResult; +import edu.harvard.hms.dbmi.avillach.hpds.processing.HpdsProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class PatientProcessor implements HpdsProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(PatientProcessor.class); + private final AbstractProcessor abstractProcessor; + + @Autowired + public PatientProcessor(AbstractProcessor abstractProcessor) { + this.abstractProcessor = abstractProcessor; + } + + @Override + public String[] getHeaderRow(Query query) { + return new String[]{"PATIENT_NUM"}; + } + + @Override + public void runQuery(Query query, AsyncResult asyncResult) { + LOG.info("Pulling results for query {}", query.getId()); + // floating all this in memory is a bit gross, but the whole list of + // patient IDs was already there, so I don't feel too bad + List allPatients = abstractProcessor.getPatientSubsetForQuery(query).stream() + .map(patient -> new String[]{patient.toString()}) + .toList(); + LOG.info("Writing results for query {}", query.getId()); + asyncResult.appendResults(allPatients); + LOG.info("Completed query {}", query.getId()); + } +} diff --git a/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessorTest.java b/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessorTest.java new file mode 100644 index 00000000..a2cb4bc2 --- /dev/null +++ b/processing/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/processing/patient/PatientProcessorTest.java @@ -0,0 +1,47 @@ +package edu.harvard.hms.dbmi.avillach.hpds.processing.patient; + +import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query; +import edu.harvard.hms.dbmi.avillach.hpds.data.query.ResultType; +import edu.harvard.hms.dbmi.avillach.hpds.processing.AbstractProcessor; +import edu.harvard.hms.dbmi.avillach.hpds.processing.AsyncResult; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; + +import java.util.List; +import java.util.TreeSet; + +@EnableAutoConfiguration +@SpringBootTest(classes = PatientProcessor.class) +class PatientProcessorTest { + + @MockBean + AbstractProcessor abstractProcessor; + + @Autowired + PatientProcessor subject; + + @Test + void shouldProcessPatientQuery() { + Query q = new Query(); + q.setId("frank"); + q.setPicSureId("frank"); + q.setExpectedResultType(ResultType.PATIENTS); + AsyncResult writeToThis = Mockito.mock(AsyncResult.class); + Mockito.when(abstractProcessor.getPatientSubsetForQuery(q)) + .thenReturn(new TreeSet<>(List.of(1, 2, 42))); + + subject.runQuery(q, writeToThis); + + Mockito.verify(writeToThis, Mockito.times(1)) + .appendResults(Mockito.argThat(strings -> + strings.size() == 3 && + strings.get(0)[0].equals("1") && + strings.get(1)[0].equals("2") && + strings.get(2)[0].equals("42")) + ); + } +} \ No newline at end of file diff --git a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java index 749c5a36..dd150d7c 100644 --- a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java +++ b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/PicSureService.java @@ -14,7 +14,6 @@ import edu.harvard.hms.dbmi.avillach.hpds.service.filesharing.TestDataService; import edu.harvard.hms.dbmi.avillach.hpds.service.util.Paginator; import edu.harvard.hms.dbmi.avillach.hpds.service.util.QueryDecorator; -import org.apache.http.entity.ContentType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -32,14 +31,12 @@ import edu.harvard.dbmi.avillach.domain.*; import edu.harvard.dbmi.avillach.util.UUIDv5; -import edu.harvard.dbmi.avillach.service.IResourceRS; import edu.harvard.hms.dbmi.avillach.hpds.crypto.Crypto; import edu.harvard.hms.dbmi.avillach.hpds.data.phenotype.ColumnMeta; import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query; import edu.harvard.hms.dbmi.avillach.hpds.processing.*; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; -import org.springframework.stereotype.Component; import org.springframework.web.bind.annotation.RequestBody; @RequestMapping(value = "PIC-SURE", produces = "application/json") @@ -308,7 +305,10 @@ public ResponseEntity writeQueryResult( success = fileSystemService.createPhenotypicData(query); } else if ("genomic".equals(datatype)) { success = fileSystemService.createGenomicData(query); - } + } else if ("patients".equals(datatype)) { + success = ResultType.PATIENTS.equals(query.getExpectedResultType()) && + fileSystemService.createPatientList(query); + } return success ? ResponseEntity.ok().build() : ResponseEntity.internalServerError().build(); } @@ -391,6 +391,7 @@ private ResponseEntity _querySync(QueryRequest resultRequest) throws IOException case DATAFRAME: case SECRET_ADMIN_DATAFRAME: case DATAFRAME_TIMESERIES: + case PATIENTS: QueryStatus status = query(resultRequest).getBody(); while (status.getResourceStatus().equalsIgnoreCase("RUNNING") || status.getResourceStatus().equalsIgnoreCase("PENDING")) { diff --git a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryService.java b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryService.java index e316f0a5..bc4d4c9e 100644 --- a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryService.java +++ b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/QueryService.java @@ -8,6 +8,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import edu.harvard.hms.dbmi.avillach.hpds.processing.patient.PatientProcessor; import edu.harvard.hms.dbmi.avillach.hpds.processing.timeseries.TimeseriesProcessor; import edu.harvard.hms.dbmi.avillach.hpds.data.query.ResultType; import edu.harvard.hms.dbmi.avillach.hpds.processing.dictionary.DictionaryService; @@ -50,6 +51,7 @@ public class QueryService { private final TimeseriesProcessor timeseriesProcessor; private final CountProcessor countProcessor; private final MultiValueQueryProcessor multiValueQueryProcessor; + private final PatientProcessor patientProcessor; private final DictionaryService dictionaryService; private final QueryDecorator queryDecorator; @@ -59,15 +61,15 @@ public class QueryService { @Autowired public QueryService (AbstractProcessor abstractProcessor, - QueryProcessor queryProcessor, - TimeseriesProcessor timeseriesProcessor, - CountProcessor countProcessor, - MultiValueQueryProcessor multiValueQueryProcessor, - @Autowired(required = false) DictionaryService dictionaryService, + QueryProcessor queryProcessor, + TimeseriesProcessor timeseriesProcessor, + CountProcessor countProcessor, + MultiValueQueryProcessor multiValueQueryProcessor, + @Autowired(required = false) DictionaryService dictionaryService, QueryDecorator queryDecorator, - @Value("${SMALL_JOB_LIMIT}") Integer smallJobLimit, - @Value("${SMALL_TASK_THREADS}") Integer smallTaskThreads, - @Value("${LARGE_TASK_THREADS}") Integer largeTaskThreads) { + @Value("${SMALL_JOB_LIMIT}") Integer smallJobLimit, + @Value("${SMALL_TASK_THREADS}") Integer smallTaskThreads, + @Value("${LARGE_TASK_THREADS}") Integer largeTaskThreads, PatientProcessor patientProcessor) { this.abstractProcessor = abstractProcessor; this.queryProcessor = queryProcessor; this.timeseriesProcessor = timeseriesProcessor; @@ -79,9 +81,10 @@ public QueryService (AbstractProcessor abstractProcessor, SMALL_JOB_LIMIT = smallJobLimit; SMALL_TASK_THREADS = smallTaskThreads; LARGE_TASK_THREADS = largeTaskThreads; + this.patientProcessor = patientProcessor; - /* These have to be of type Runnable(nothing more specific) in order + /* These have to be of type Runnable(nothing more specific) in order * to be compatible with ThreadPoolExecutor constructor prototype */ largeTaskExecutionQueue = new PriorityBlockingQueue(1000); @@ -124,6 +127,8 @@ private AsyncResult initializeResult(Query query) throws IOException { HpdsProcessor p; switch(query.getExpectedResultType()) { + case PATIENTS: + p = patientProcessor; case SECRET_ADMIN_DATAFRAME: p = queryProcessor; break; diff --git a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingService.java b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingService.java index 1436b348..3fe8e1db 100644 --- a/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingService.java +++ b/service/src/main/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingService.java @@ -1,18 +1,15 @@ package edu.harvard.hms.dbmi.avillach.hpds.service.filesharing; import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query; -import edu.harvard.hms.dbmi.avillach.hpds.data.query.ResultType; import edu.harvard.hms.dbmi.avillach.hpds.processing.AsyncResult; import edu.harvard.hms.dbmi.avillach.hpds.processing.VariantListProcessor; import edu.harvard.hms.dbmi.avillach.hpds.service.QueryService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.IOException; -import java.time.Duration; -import java.time.temporal.ChronoUnit; +import java.util.Optional; /** * Used for sharing data. Given a query, this service will write @@ -23,21 +20,25 @@ public class FileSharingService { private static final Logger LOG = LoggerFactory.getLogger(FileSharingService.class); - @Autowired - private QueryService queryService; - - @Autowired - private FileSystemService fileWriter; - - @Autowired - private VariantListProcessor variantListProcessor; + private final QueryService queryService; + private final FileSystemService fileWriter; + private final VariantListProcessor variantListProcessor; + + public FileSharingService( + QueryService queryService, FileSystemService fileWriter, + VariantListProcessor variantListProcessor + ) { + this.queryService = queryService; + this.fileWriter = fileWriter; + this.variantListProcessor = variantListProcessor; + } public boolean createPhenotypicData(Query query) { - AsyncResult result = queryService.getResultFor(query.getId()); - if (result == null || result.getStatus() != AsyncResult.Status.SUCCESS) { - return false; - } - return fileWriter.writeResultToFile("phenotypic_data.csv", result, query.getPicSureId()); + return createAndWriteData(query, "phenotypic_data.csv"); + } + + public boolean createPatientList(Query query) { + return createAndWriteData(query, "patients.txt"); } public boolean createGenomicData(Query query) { @@ -49,4 +50,12 @@ public boolean createGenomicData(Query query) { return false; } } + + private boolean createAndWriteData(Query query, String fileName) { + AsyncResult result = queryService.getResultFor(query.getId()); + if (result == null || result.getStatus() != AsyncResult.Status.SUCCESS) { + return false; + } + return fileWriter.writeResultToFile(fileName, result, query.getPicSureId()); + } } diff --git a/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingServiceTest.java b/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingServiceTest.java index ff8a1149..37db8841 100644 --- a/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingServiceTest.java +++ b/service/src/test/java/edu/harvard/hms/dbmi/avillach/hpds/service/filesharing/FileSharingServiceTest.java @@ -1,38 +1,49 @@ package edu.harvard.hms.dbmi.avillach.hpds.service.filesharing; import edu.harvard.hms.dbmi.avillach.hpds.data.query.Query; +import edu.harvard.hms.dbmi.avillach.hpds.data.query.ResultType; import edu.harvard.hms.dbmi.avillach.hpds.processing.AsyncResult; import edu.harvard.hms.dbmi.avillach.hpds.processing.VariantListProcessor; import edu.harvard.hms.dbmi.avillach.hpds.processing.io.ResultWriter; +import edu.harvard.hms.dbmi.avillach.hpds.processing.patient.PatientProcessor; import edu.harvard.hms.dbmi.avillach.hpds.service.QueryService; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -@ExtendWith(MockitoExtension.class) +@EnableAutoConfiguration +@SpringBootTest(classes = FileSharingService.class) public class FileSharingServiceTest { - @Mock + @MockBean QueryService queryService; - @Mock + @MockBean FileSystemService fileWriter; - @Mock + @MockBean VariantListProcessor variantListProcessor; - @Mock + @MockBean + PatientProcessor patientProcessor; + + @MockBean ResultWriter resultWriter; - @InjectMocks + @Autowired FileSharingService subject; @Test @@ -95,4 +106,22 @@ public void shouldNotCreateGenomicData() throws IOException { assertFalse(actual); } + + @Test + void shouldCreatePatientsList() { + Query query = new Query(); + query.setId("jasdijasd"); + query.setPicSureId("jasdijasd"); + query.setExpectedResultType(ResultType.PATIENTS); + AsyncResult result = new AsyncResult(query, patientProcessor, resultWriter); + result.setStatus(AsyncResult.Status.SUCCESS); + Mockito.when(queryService.getResultFor("jasdijasd")) + .thenReturn(result); + Mockito.when(fileWriter.writeResultToFile("patients.txt", result, "jasdijasd")) + .thenReturn(true); + + boolean actual = subject.createPatientList(query); + + Assertions.assertTrue(actual); + } } \ No newline at end of file