Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add metrics api endpoint #619

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
feat: add metrics api endpoint
alko89 committed Aug 14, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 4c9f5a5c29cf63ccb4df866148aeb433354deceb
11 changes: 8 additions & 3 deletions example.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import * as Bull from 'bull';
import Queue3 from 'bull';
import { Queue as QueueMQ, Worker } from 'bullmq';
import { MetricsTime, Queue as QueueMQ, Worker } from 'bullmq';
import express from 'express';
import { BullMQAdapter } from '@bull-board/api/src/queueAdapters/bullMQ';
import { BullAdapter } from '@bull-board/api/src/queueAdapters/bull';
@@ -15,7 +15,7 @@ const redisOptions = {

const sleep = (t: number) => new Promise((resolve) => setTimeout(resolve, t * 1000));

const createQueue3 = (name: string) => new Queue3(name, { redis: redisOptions });
const createQueue3 = (name: string) => new Queue3(name, { redis: redisOptions, metrics: { maxDataPoints: MetricsTime.ONE_WEEK } });
const createQueueMQ = (name: string) => new QueueMQ(name, { connection: redisOptions });

function setupBullProcessor(bullQueue: Bull.Queue) {
@@ -45,7 +45,12 @@ async function setupBullMQProcessor(queueName: string) {

return { jobId: `This is the return value of job (${job.id})` };
},
{ connection: redisOptions }
{
connection: redisOptions,
metrics: {
maxDataPoints: MetricsTime.ONE_WEEK,
},
}
);
}

2 changes: 1 addition & 1 deletion packages/api/package.json
Original file line number Diff line number Diff line change
@@ -33,7 +33,7 @@
"devDependencies": {
"@types/redis-info": "^3.0.0",
"@types/supertest": "^2.0.12",
"bull": "^4.10.4",
"bull": "^4.11.3",
"bullmq": "^4.6.0",
"ioredis": "^5.3.2",
"supertest": "^6.3.3"
30 changes: 30 additions & 0 deletions packages/api/src/handlers/metrics.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import { BaseAdapter } from '../queueAdapters/base';
import {
BullBoardRequest,
ControllerHandlerReturnType,
} from '../../typings/app';
import { queueProvider } from '../providers/queue';

async function getMetrics(
req: BullBoardRequest,
queue: BaseAdapter
): Promise<ControllerHandlerReturnType> {
const type = req.query?.type || 'completed';
const metrics = await queue.getMetrics(type);

if (metrics) {
return {
status: 200,
body: metrics,
};
} else {
return {
status: 404,
body: { error: 'Metrics not available' },
};
}
}

export const metricsHandler = queueProvider(getMetrics, {
skipReadOnlyModeCheck: true,
});
4 changes: 4 additions & 0 deletions packages/api/src/queueAdapters/base.ts
Original file line number Diff line number Diff line change
@@ -2,9 +2,11 @@ import {
FormatterField,
JobCleanStatus,
JobCounts,
JobRetryStatus,
JobStatus,
QueueAdapterOptions,
QueueJob,
QueueMetrics,
} from '../../typings/app';

export abstract class BaseAdapter {
@@ -56,6 +58,8 @@ export abstract class BaseAdapter {

public abstract getJobLogs(id: string): Promise<string[]>;

public abstract getMetrics(type: JobRetryStatus, start?: number, end?: number): Promise<QueueMetrics | undefined | null>;

public abstract getName(): string;

public abstract getRedisInfo(): Promise<string>;
6 changes: 5 additions & 1 deletion packages/api/src/queueAdapters/bull.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Job, Queue } from 'bull';
import { JobCleanStatus, JobCounts, JobStatus, QueueAdapterOptions } from '../../typings/app';
import { JobCleanStatus, JobCounts, JobRetryStatus, JobStatus, QueueAdapterOptions } from '../../typings/app';
import { STATUSES } from '../constants/statuses';
import { BaseAdapter } from './base';

@@ -49,6 +49,10 @@ export class BullAdapter extends BaseAdapter {
public getJobLogs(id: string): Promise<string[]> {
return this.queue.getJobLogs(id).then(({ logs }) => logs);
}

public getMetrics(type: JobRetryStatus, start?: number, end?: number) {
return this.queue.getMetrics(type, start, end);
}

public isPaused(): Promise<boolean> {
return this.queue.isPaused();
6 changes: 5 additions & 1 deletion packages/api/src/queueAdapters/bullMQ.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Job, Queue } from 'bullmq';
import { JobCleanStatus, JobCounts, JobStatus, QueueAdapterOptions } from '../../typings/app';
import { JobCleanStatus, JobCounts, JobRetryStatus, JobStatus, QueueAdapterOptions } from '../../typings/app';
import { STATUSES } from '../constants/statuses';
import { BaseAdapter } from './base';

@@ -37,6 +37,10 @@ export class BullMQAdapter extends BaseAdapter {
return this.queue.getJobLogs(id).then(({ logs }) => logs);
}

public getMetrics(type: JobRetryStatus, start?: number, end?: number) {
return this.queue.getMetrics(type, start, end);
}

public isPaused(): Promise<boolean> {
return this.queue.isPaused();
}
6 changes: 6 additions & 0 deletions packages/api/src/routes.ts
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@ import { resumeQueueHandler } from './handlers/resumeQueue';
import { retryAllHandler } from './handlers/retryAll';
import { retryJobHandler } from './handlers/retryJob';
import { promoteAllHandler } from './handlers/promoteAll';
import { metricsHandler } from './handlers/metrics';

export const appRoutes: AppRouteDefs = {
entryPoint: {
@@ -78,5 +79,10 @@ export const appRoutes: AppRouteDefs = {
route: '/api/queues/:queueName/:jobId/promote',
handler: promoteJobHandler,
},
{
method: 'get',
route: '/api/metrics/:queueName',
handler: metricsHandler,
},
],
};
10 changes: 10 additions & 0 deletions packages/api/typings/app.ts
Original file line number Diff line number Diff line change
@@ -37,6 +37,16 @@ export interface QueueJob {
getState(): Promise<Status | 'stuck' | 'waiting-children' | 'prioritized' | 'unknown'>;
}

export interface QueueMetrics {
meta: {
count: number;
prevTS: number;
prevCount: number;
};
data: number[];
count: number;
}

export interface QueueJobJson {
// add properties as needed from real Bull/BullMQ jobs
id?: string | undefined | number | null;
13 changes: 13 additions & 0 deletions yarn.lock
Original file line number Diff line number Diff line change
@@ -5180,6 +5180,19 @@ bull@^4.10.4:
semver "^7.5.2"
uuid "^8.3.0"

bull@^4.11.3:
version "4.11.3"
resolved "https://registry.yarnpkg.com/bull/-/bull-4.11.3.tgz#466214c8d2aa6ac4fc3133c3616d31a37af4c099"
integrity sha512-DhS0XtiAuejkAY08iGOdDK35eex/yGNoezlWqGJTu9FqWFF/oBjUhpsusE9SXiI4culyDbOoFs+l3ar0VXhFqQ==
dependencies:
cron-parser "^4.2.1"
get-port "^5.1.1"
ioredis "^5.3.2"
lodash "^4.17.21"
msgpackr "^1.5.2"
semver "^7.5.2"
uuid "^8.3.0"

bullmq@^4.6.0:
version "4.7.0"
resolved "https://registry.yarnpkg.com/bullmq/-/bullmq-4.7.0.tgz#dd2d26219142cf76ab578386a0db75c4757f2380"