feat: Grant Matching (#7)

* Grant matching object for FRP job
* Abstract base class for analysis
* Endpoints for Grant Upload webhook

packages/frp-server/src/app.module.ts

@@ -3,6 +3,8 @@ import { UploadModule } from './upload/upload.module';
 import { NocodbModule } from './nocodb/nocodb.module';
 import { ConfigModule } from '@nestjs/config';
 import { JobModule } from './job/job.module';
+import { PublicationsUploadModule } from './publications-upload/publications-upload.module';
+import { GrantsUploadModule } from './grants-upload/grants-upload.module';
 import configuration from './config/configuration';
 
 @Module({
@@ -13,7 +15,9 @@ import configuration from './config/configuration';
     }),
     UploadModule,
     NocodbModule,
-    JobModule
-  ],
+    JobModule,
+    PublicationsUploadModule,
+    GrantsUploadModule
+  ]
 })
 export class AppModule {}

packages/frp-server/src/config/configuration.ts

@@ -1,15 +1,32 @@
 export default () => ({
   nocodb: {
     baseUri: process.env.NOCODB_URI,
+    token: process.env.NOCODB_TOKEN,
+
+    // Faculty Table
     facultyTableID: process.env.FACULTY_TABLE_ID,
     facultyToFrpID: process.env.FACULTY_TO_FRP_ID,
-    token: process.env.NOCODB_TOKEN,
+
+    // FRP Table
     frpTableID: process.env.FRP_TABLE_ID,
+
+    // Publication Table
     publicationTableID: process.env.PUBLICATION_TABLE_ID,
     publicationToFacultyID: process.env.PUBLICATION_TO_FACULTY_ID,
     publicationToFRPID: process.env.PUBLICATION_TO_FRP_ID,
+
+    // Publication Upload Table
     publicationUploadTableID: process.env.PUBLICATION_UPLOAD_TABLE_ID,
-    publicationUploadToFacultyID: process.env.PUBLICATION_UPLOAD_TO_FACULTY_ID
+    publicationUploadToFacultyID: process.env.PUBLICATION_UPLOAD_TO_FACULTY_ID,
+
+    // Grant Table
+    grantsTableID: process.env.GRANT_TABLE_ID,
+    grantsToFacultyID: process.env.GRANT_TO_FACULTY_ID,
+    grantsToFRPID: process.env.GRANT_TO_FRP_ID,
+
+    // Grant Upload Table
+    grantUploadTableID: process.env.GRANT_UPLOAD_TABLE_ID,
+    grantUploadToFacultyID: process.env.GRANT_UPLOAD_TO_FACULTY_ID
   },
   kube: {
     jobImage: process.env.JOB_IMAGE,

packages/frp-server/src/grants-upload/dto/completion.dto.ts

@@ -0,0 +1,27 @@
+import { IsString, ValidateNested, IsArray, IsNumber } from 'class-validator';
+import { Type } from 'class-transformer';
+
+export class AnalysisResults {
+  @IsString()
+  title: string;
+
+  @IsNumber()
+  amount: number;
+}
+
+export class AnalysisCompletion {
+  @IsArray()
+  @ValidateNested({ each: true })
+  @Type(() => AnalysisResults)
+  /** The results from the matching process */
+  results: AnalysisResults[];
+
+  @IsString()
+  facultyID: string;
+
+  @IsString()
+  frpID: string;
+
+  @IsString()
+  uploadID: string;
+}

packages/frp-server/src/grants-upload/grants-upload.controller.ts

@@ -0,0 +1,27 @@
+import { Body, Controller, Post } from '@nestjs/common';
+import { GrantsUploadWebhookPayload } from '../nocodb/dto/grant-upload.dto';
+import { NocoDBInsertWebhookPayload } from '../nocodb/dto/webhook.dto';
+import { GrantsUploadService } from './grants-upload.service';
+import { AnalysisCompletion } from './dto/completion.dto';
+
+@Controller('grants')
+export class GrantsUploadController {
+  constructor(private readonly grantsUploadService: GrantsUploadService) {}
+
+  @Post('nocodbWebhook')
+  async handleNocodbWebhook(@Body() payload: NocoDBInsertWebhookPayload<GrantsUploadWebhookPayload>): Promise<void> {
+    const grantsUploadID = payload.data.rows[0].Id.toString();
+    const csvUrlStub = payload.data.rows[0].FAR[0].signedPath;
+
+    await this.grantsUploadService.handleUpload(grantsUploadID, csvUrlStub);
+  }
+
+  /**
+   * When the analysis process has complete, this will handle completing
+   * the analysis request.
+   */
+  @Post('complete')
+  async handleAnalysisComplete(@Body() analysisPayload: AnalysisCompletion): Promise<void> {
+    await this.grantsUploadService.handleAnalysisCompletion(analysisPayload);
+  }
+}

packages/frp-server/src/grants-upload/grants-upload.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+import { GrantsUploadService } from './grants-upload.service';
+import { GrantsUploadController } from './grants-upload.controller';
+import { JobModule } from '../job/job.module';
+import { NocodbModule } from '../nocodb/nocodb.module';
+
+@Module({
+  providers: [GrantsUploadService],
+  controllers: [GrantsUploadController],
+  imports: [JobModule, NocodbModule]
+})
+export class GrantsUploadModule {}

packages/frp-server/src/grants-upload/grants-upload.service.ts

@@ -0,0 +1,92 @@
+import { Injectable } from '@nestjs/common';
+import { FrpService } from '../nocodb/frp.service';
+import { FacultyService } from '../nocodb/faculty.service';
+import { GrantsService } from '../nocodb/grants.service';
+import { JobService } from '../job/job.service';
+import { ConfigService } from '@nestjs/config';
+import { AnalysisCompletion } from './dto/completion.dto';
+import { GrantsUploadService as NocoDBGrants } from '../nocodb/grants-upload.service';
+import { Grant } from '../nocodb/dto/grant.dto';
+
+@Injectable()
+export class GrantsUploadService {
+  private readonly backendUrl = this.configService.getOrThrow<string>('server.url');
+  private readonly nocodbBaseUrl = this.configService.getOrThrow<string>('nocodb.baseUri');
+
+  constructor(
+    private readonly facultyService: FacultyService,
+    private readonly frpService: FrpService,
+    private readonly grantsService: GrantsService,
+    private readonly jobService: JobService,
+    private readonly configService: ConfigService,
+    private readonly grantsUploadService: NocoDBGrants
+  ) {}
+
+  async handleUpload(grantsUploadID: string, csvUrlStub: string) {
+    // Get the faculty associated with the publication upload
+    // NOTE: only one faculty is allowed per upload
+    const facultyID = (await this.grantsUploadService.getFacultyLinks(grantsUploadID))[0].Id.toString();
+
+    // Get the FRP's associated with the faculty
+    const frpIDs = (await this.facultyService.getFRPLinks(facultyID)).map((frp) => frp.Id);
+    const frps = await Promise.all(frpIDs.map((frpID) => this.frpService.getFRP(frpID.toString())));
+
+    // TODO: In the future this information will be stored in a seperate DB and ID of
+    // that entry will be shared with the job
+
+    // Start the jobs
+    for (const frp of frps) {
+      await this.jobService.triggerJob(
+        `${this.nocodbBaseUrl}/${csvUrlStub}`,
+        frp.Title,
+        frp.Year.toString(),
+        `${this.backendUrl}/upload/complete`,
+        {
+          facultyID: facultyID.toString(),
+          frpID: frp.Id.toString(),
+          uploadID: grantsUploadID.toString()
+        },
+        'grant'
+      );
+    }
+  }
+
+  async handleAnalysisCompletion(analysisResults: AnalysisCompletion): Promise<void> {
+    console.log(analysisResults);
+    // Get all publications from the matching
+    const publications = await this.getOrCreatePublications(analysisResults);
+
+    // Link the publications with the associated faculty
+    await Promise.all(
+      publications.map(async (publication) => {
+        await this.grantsService.linkFaculty(publication.Id.toString(), analysisResults.facultyID.toString());
+      })
+    );
+
+    // Link the publications with the associated FRP
+    await Promise.all(
+      publications.map(async (publication) => {
+        await this.grantsService.linkFRP(publication.Id.toString(), analysisResults.frpID.toString());
+      })
+    );
+
+    // Make the upload as complete
+    await this.grantsUploadService.makeComplete(analysisResults.uploadID);
+  }
+
+  /**
+   * Will loop over all the publications from the analysis and either create new ones if they
+   * don't exist or return the existing ones
+   */
+  private async getOrCreatePublications(analysisResults: AnalysisCompletion): Promise<Grant[]> {
+    return Promise.all(
+      analysisResults.results.map(async (match) => {
+        const existing = await this.grantsService.findByTitle(match.title);
+        if (existing) {
+          return existing;
+        }
+        return this.grantsService.create(match.title, match.amount);
+      })
+    );
+  }
+}

packages/frp-server/src/job/job.service.ts

@@ -47,9 +47,19 @@ export class JobService {
     frpTitle: string,
     frpYear: string,
     webhookUrl: string,
-    webhookPayload: any
+    webhookPayload: any,
+    type: 'scholarly' | 'grant'
   ): Promise<void> {
-    const command = ['python', 'main.py', csvUrl, frpTitle, frpYear, webhookUrl, JSON.stringify(webhookPayload)];
+    const command = [
+      'python',
+      'main.py',
+      csvUrl,
+      frpTitle,
+      frpYear,
+      webhookUrl,
+      JSON.stringify(webhookPayload),
+      `--type=${type}`
+    ];
 
     this.job.spec!.template.spec!.containers[0].command = command;
 

packages/frp-server/src/nocodb/dto/file.dto.ts

@@ -0,0 +1,18 @@
+import { IsString, IsNumber } from 'class-validator';
+
+export class NocoDBFile {
+  @IsString()
+  title: string;
+
+  @IsString()
+  mimetype: string;
+
+  @IsNumber()
+  size: number;
+
+  @IsString()
+  path: string;
+
+  @IsString()
+  signedPath: string;
+}

packages/frp-server/src/nocodb/dto/grant-upload.dto.ts

@@ -0,0 +1,30 @@
+import { IsNumber, IsString, IsOptional, IsArray, ValidateNested } from 'class-validator';
+import { Type } from 'class-transformer';
+import { NocoDBFile } from './file.dto';
+
+export class GrantsUploadWebhookPayload {
+  @IsNumber()
+  Id: number;
+
+  @IsString()
+  @IsOptional()
+  Title: string;
+
+  @IsString()
+  CreatedAt: string;
+
+  @IsString()
+  @IsOptional()
+  UpdatedAt: string;
+
+  @IsArray()
+  @ValidateNested({ each: true })
+  @Type(() => NocoDBFile)
+  FAR: NocoDBFile[];
+
+  @IsNumber()
+  Faculty: number;
+
+  @IsString()
+  Status: string;
+}

packages/frp-server/src/nocodb/dto/grant.dto.ts

@@ -0,0 +1,4 @@
+export class Grant {
+  Id: number;
+  Amount: number;
+}

packages/frp-server/src/nocodb/dto/publication-upload.dto.ts

@@ -0,0 +1,30 @@
+import { IsNumber, IsString, IsOptional, IsArray, ValidateNested } from 'class-validator';
+import { Type } from 'class-transformer';
+import { NocoDBFile } from './file.dto';
+
+export class PublicationUploadWebhookPayload {
+  @IsNumber()
+  Id: number;
+
+  @IsString()
+  @IsOptional()
+  Title: string;
+
+  @IsString()
+  CreatedAt: string;
+
+  @IsString()
+  @IsOptional()
+  UpdatedAt: string;
+
+  @IsArray()
+  @ValidateNested({ each: true })
+  @Type(() => NocoDBFile)
+  FAR: NocoDBFile[];
+
+  @IsNumber()
+  Faculty: number;
+
+  @IsString()
+  Status: string;
+}

packages/frp-server/src/nocodb/dto/webhook.dto.ts

@@ -0,0 +1,30 @@
+/*
+ * The following DTOs are made based on the NocoDB documentation on
+ * the webhook payload. The names are set by NocoDB
+ */
+import { IsObject, IsString, ValidateNested, IsArray } from 'class-validator';
+import { Type } from 'class-transformer';
+
+export class NocoDBInsertWebhookData<T> {
+  @IsString()
+  table_id: string;
+
+  @IsString()
+  table_name: string;
+
+  @IsArray()
+  rows: T[];
+}
+
+export class NocoDBInsertWebhookPayload<T> {
+  @IsString()
+  type: string;
+
+  @IsString()
+  id: string;
+
+  @IsObject()
+  @ValidateNested()
+  @Type(() => NocoDBInsertWebhookData)
+  data: NocoDBInsertWebhookData<T>;
+}

packages/frp-server/src/nocodb/grants-upload.service.ts

@@ -0,0 +1,35 @@
+import { Injectable } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { Api } from 'nocodb-sdk';
+import { NocoDBLink } from './dto/link.dto';
+import { InjectNocoDB } from './nocodb.provider';
+import { requestAll } from './utils/pagination';
+
+@Injectable()
+export class GrantsUploadService {
+  private readonly grantUploadTableID = this.configService.getOrThrow<string>('nocodb.grantUploadTableID');
+  private readonly grantUploadToFacultyID = this.configService.getOrThrow<string>('nocodb.grantUploadToFacultyID');
+
+  constructor(
+    @InjectNocoDB() private readonly nocoDBService: Api<null>,
+    private readonly configService: ConfigService
+  ) {}
+
+  async getFacultyLinks(grantUploadID: string): Promise<NocoDBLink[]> {
+    return requestAll<NocoDBLink>((offset) => {
+      return this.nocoDBService.dbDataTableRow.nestedList(
+        this.grantUploadTableID,
+        this.grantUploadToFacultyID,
+        grantUploadID,
+        { offset }
+      );
+    });
+  }
+
+  async makeComplete(grantUploadID: string): Promise<void> {
+    await this.nocoDBService.dbDataTableRow.update(this.grantUploadTableID, {
+      Id: grantUploadID,
+      Status: 'Complete'
+    });
+  }
+}