diff --git a/.github/workflows/meerkat.yml b/.github/workflows/meerkat.yml index d4155ceae..9ed0588a0 100644 --- a/.github/workflows/meerkat.yml +++ b/.github/workflows/meerkat.yml @@ -808,7 +808,7 @@ jobs: --set enable_dsp=true \ --set administrator_email=jonathan@wilbur.space \ --set administrator_email_public=true \ - --set vendor_version='3.2.1' \ + --set vendor_version='3.2.2' \ --set signing_required_for_chaining=false \ --set tcp_timeout_in_seconds=300 \ --set min_transfer_speed_bytes_per_minute=10 \ diff --git a/apps/meerkat-docs/docs/changelog-meerkat.md b/apps/meerkat-docs/docs/changelog-meerkat.md index 7b3d8269d..b077f016c 100644 --- a/apps/meerkat-docs/docs/changelog-meerkat.md +++ b/apps/meerkat-docs/docs/changelog-meerkat.md @@ -1,5 +1,12 @@ # Changelog for Meerkat DSA +## Version 3.2.2 + +This version **dramatically** improves the performance of the DSA for `search` +and `list` operations and for `addEntry` when performed in rapid succession, as +well as for a few other use cases. In general, you should see a 400% to 1000% +increase in performance. + ## Version 3.2.1 Fix bug with writing to IDM sockets. diff --git a/apps/meerkat-docs/docs/conformance.md b/apps/meerkat-docs/docs/conformance.md index 25b60b118..f4549ab37 100644 --- a/apps/meerkat-docs/docs/conformance.md +++ b/apps/meerkat-docs/docs/conformance.md @@ -1,7 +1,7 @@ # Conformance -In the statements below, the term "Meerkat DSA" refers to version 3.2.1 of -Meerkat DSA, hence these statements are only claimed for version 3.2.1 of +In the statements below, the term "Meerkat DSA" refers to version 3.2.2 of +Meerkat DSA, hence these statements are only claimed for version 3.2.2 of Meerkat DSA. ## X.519 Conformance Statement diff --git a/apps/meerkat/package.json b/apps/meerkat/package.json index 44abc54b9..6e39f3895 100644 --- a/apps/meerkat/package.json +++ b/apps/meerkat/package.json @@ -12,7 +12,7 @@ "url": "https://github.com/JonathanWilbur" } ], - "version": "3.2.1", + "version": "3.2.2", "license": "MIT", "bin": { "meerkat": "./main.js" diff --git a/apps/meerkat/src/app/authz/permittedToFindDSE.ts b/apps/meerkat/src/app/authz/permittedToFindDSE.ts index 4e458299e..7d7b56420 100644 --- a/apps/meerkat/src/app/authz/permittedToFindDSE.ts +++ b/apps/meerkat/src/app/authz/permittedToFindDSE.ts @@ -77,6 +77,7 @@ async function permittedToFindDSE ( const isMemberOfGroup = getIsGroupMember(ctx, NAMING_MATCHER); let accessControlScheme: OBJECT_IDENTIFIER | undefined; let authorizedToDiscloseOnError: boolean = false; + const subentriesCache: Map = new Map(); for (let i = 0; i < needleDN.length; i++) { const rdn = needleDN[i]; @@ -105,7 +106,14 @@ async function permittedToFindDSE ( ? [ ...admPoints, dse_i ] : [ ...admPoints ]; const relevantSubentries: Vertex[] = (await Promise.all( - relevantAdmPoints.map((ap) => getRelevantSubentries(ctx, dse_i, childDN, ap)), + relevantAdmPoints.map((ap) => getRelevantSubentries( + ctx, + dse_i, + childDN, + ap, + undefined, + subentriesCache, + )), )).flat(); const targetACI = await getACIItems( ctx, diff --git a/apps/meerkat/src/app/ctx.ts b/apps/meerkat/src/app/ctx.ts index f1fd2c033..af93642d9 100644 --- a/apps/meerkat/src/app/ctx.ts +++ b/apps/meerkat/src/app/ctx.ts @@ -1085,7 +1085,10 @@ const ctx: MeerkatContext = { app: "meerkat", }, }), - db: new PrismaClient(), + db: new PrismaClient({ + // log: ["query"], + // log: ['query', 'info', 'warn', 'error'], + }), telemetry: { init: async (): Promise => { try { diff --git a/apps/meerkat/src/app/distributed/OperationDispatcher.ts b/apps/meerkat/src/app/distributed/OperationDispatcher.ts index 4bfdbfcaa..fd263a644 100644 --- a/apps/meerkat/src/app/distributed/OperationDispatcher.ts +++ b/apps/meerkat/src/app/distributed/OperationDispatcher.ts @@ -267,6 +267,7 @@ async function search_procedures ( depth: 0, excludedById: new Set(), matching_rule_substitutions: new Map(), + subentriesCache: new Map(), }; const use_search_ii: boolean = ( @@ -816,6 +817,7 @@ class OperationDispatcher { effectiveHierarchySelections: data.hierarchySelections, effectiveSearchControls: data.searchControlOptions, effectiveServiceControls: data.serviceControls?.options, + subentriesCache: new Map(), }; const use_search_rule_check_ii: boolean = (nameResolutionPhase === completed); @@ -1498,6 +1500,7 @@ class OperationDispatcher { matching_rule_substitutions: new Map(), notification: [], effectiveEntryLimit: MAX_RESULTS, + subentriesCache: new Map(), }; await relatedEntryProcedure( ctx, diff --git a/apps/meerkat/src/app/distributed/addEntry.ts b/apps/meerkat/src/app/distributed/addEntry.ts index 2467fbcb2..2e567a484 100644 --- a/apps/meerkat/src/app/distributed/addEntry.ts +++ b/apps/meerkat/src/app/distributed/addEntry.ts @@ -91,7 +91,7 @@ import { id_opcode_addEntry, } from "@wildboar/x500/src/lib/modules/CommonProtocolSpecification/id-opcode-addEntry.va"; import establishSubordinate from "../dop/establishSubordinate"; -import { differenceInMilliseconds } from "date-fns"; +import { addSeconds, differenceInMilliseconds } from "date-fns"; import { ServiceProblem_timeLimitExceeded } from "@wildboar/x500/src/lib/modules/DirectoryAbstractService/ServiceProblem.ta"; @@ -260,6 +260,11 @@ async function addEntry ( assn: ClientAssociation, state: OperationDispatcherState, ): Promise { + const now = new Date(); + if (assn.subentriesCacheExpiration <= now) { + assn.subentriesCache.clear(); + assn.subentriesCacheExpiration = addSeconds(now, 10); + } const argument = _decode_AddEntryArgument(state.operationArgument); const data = getOptionallyProtectedValue(argument); const signErrors: boolean = ( @@ -393,7 +398,14 @@ async function addEntry ( const relevantSubentries: Vertex[] = ctx.config.bulkInsertMode ? [] : (await Promise.all( - state.admPoints.map((ap) => getRelevantSubentries(ctx, objectClasses, targetDN, ap)), + state.admPoints.map((ap) => getRelevantSubentries( + ctx, + objectClasses, + targetDN, + ap, + undefined, + assn.subentriesCache, + )), )).flat(); const accessControlScheme = [ ...state.admPoints ] // Array.reverse() works in-place, so we create a new array. .reverse() @@ -801,9 +813,23 @@ async function addEntry ( : [ ...relevantSubentries ]; // Must spread to create a new reference. Otherwise... if (existing.dse.admPoint?.administrativeRole.has(ID_AC_SPECIFIC)) { // ... (keep going) effectiveRelevantSubentries.length = 0; // ...this will modify the target-relevant subentries! - effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, existing, targetDN, existing))); + effectiveRelevantSubentries.push(...(await getRelevantSubentries( + ctx, + existing, + targetDN, + existing, + undefined, + assn.subentriesCache, + ))); } else if (existing.dse.admPoint?.administrativeRole.has(ID_AC_INNER)) { - effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, existing, targetDN, existing))); + effectiveRelevantSubentries.push(...(await getRelevantSubentries( + ctx, + existing, + targetDN, + existing, + undefined, + assn.subentriesCache, + ))); } const subordinateACI = await getACIItems( ctx, diff --git a/apps/meerkat/src/app/distributed/findDSE.ts b/apps/meerkat/src/app/distributed/findDSE.ts index 9cc62a1ae..b404b4877 100644 --- a/apps/meerkat/src/app/distributed/findDSE.ts +++ b/apps/meerkat/src/app/distributed/findDSE.ts @@ -327,6 +327,7 @@ export (errorProtection === ErrorProtectionRequest_signed) && (!assn || (assn.authorizedForSignedErrors)) ); + const subentriesCache: Map = new Map(); // Service controls const manageDSAIT: boolean = ( @@ -995,7 +996,14 @@ export ? [...state.admPoints, matchedVertex] : [...state.admPoints]; const relevantSubentries: Vertex[] = (await Promise.all( - relevantAdmPoints.map((ap) => getRelevantSubentries(ctx, matchedVertex, childDN, ap)), + relevantAdmPoints.map((ap) => getRelevantSubentries( + ctx, + matchedVertex, + childDN, + ap, + undefined, + subentriesCache, + )), )).flat(); const targetACI = await getACIItems( ctx, @@ -1177,7 +1185,14 @@ export ? [...state.admPoints, child] : [...state.admPoints]; const relevantSubentries: Vertex[] = (await Promise.all( - relevantAdmPoints.map((ap) => getRelevantSubentries(ctx, child, childDN, ap)), + relevantAdmPoints.map((ap) => getRelevantSubentries( + ctx, + child, + childDN, + ap, + undefined, + subentriesCache, + )), )).flat(); const targetACI = await getACIItems( ctx, @@ -1346,7 +1361,14 @@ export : undefined; const currentDN = getDistinguishedName(dse_i); const relevantSubentries: Vertex[] = (await Promise.all( - state.admPoints.map((ap) => getRelevantSubentries(ctx, dse_i, currentDN, ap)), + state.admPoints.map((ap) => getRelevantSubentries( + ctx, + dse_i, + currentDN, + ap, + undefined, + subentriesCache, + )), )).flat(); const targetACI = await getACIItems( ctx, diff --git a/apps/meerkat/src/app/distributed/list_i.ts b/apps/meerkat/src/app/distributed/list_i.ts index 686275e02..8156503c4 100644 --- a/apps/meerkat/src/app/distributed/list_i.ts +++ b/apps/meerkat/src/app/distributed/list_i.ts @@ -167,6 +167,13 @@ interface ListState extends Partial, Partial; } /** @@ -306,21 +313,6 @@ async function list_i ( : undefined; const NAMING_MATCHER = getNamingMatcherGetter(ctx); const subentries: boolean = (data.serviceControls?.options?.[subentriesBit] === TRUE_BIT); - const targetRelevantSubentries: Vertex[] = (await Promise.all( - state.admPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap)), - )).flat(); - const targetAccessControlScheme = [ ...state.admPoints ] // Array.reverse() works in-place, so we create a new array. - .reverse() - .find((ap) => ap.dse.admPoint!.accessControlScheme)?.dse.admPoint!.accessControlScheme; - const isMemberOfGroup = getIsGroupMember(ctx, NAMING_MATCHER); - - const isParent: boolean = target.dse.objectClass.has(PARENT); - const isChild: boolean = target.dse.objectClass.has(CHILD); - const isAncestor: boolean = (isParent && !isChild); - - let pagingRequest: PagedResultsRequest_newRequest | undefined; - let queryReference: string | undefined; - let cursorId: number | undefined; const signDSPResult: boolean = ( (state.chainingArguments.securityParameters?.errorProtection === ErrorProtectionRequest_signed) && (assn instanceof DSPAssociation) @@ -341,7 +333,28 @@ async function list_i ( chaining: chainingResults, results: [], resultSets: [], + subentriesCache: new Map(), }; + const targetRelevantSubentries: Vertex[] = (await Promise.all( + state.admPoints.map((ap) => getRelevantSubentries( + ctx, + target, + targetDN, + ap, + undefined, + ret.subentriesCache, + )), + )).flat(); + const targetAccessControlScheme = [ ...state.admPoints ] // Array.reverse() works in-place, so we create a new array. + .reverse() + .find((ap) => ap.dse.admPoint!.accessControlScheme)?.dse.admPoint!.accessControlScheme; + const isMemberOfGroup = getIsGroupMember(ctx, NAMING_MATCHER); + const isParent: boolean = target.dse.objectClass.has(PARENT); + const isChild: boolean = target.dse.objectClass.has(CHILD); + const isAncestor: boolean = (isParent && !isChild); + let pagingRequest: PagedResultsRequest_newRequest | undefined; + let queryReference: string | undefined; + let cursorId: number | undefined; if (data.pagedResults) { if ("newRequest" in data.pagedResults) { const nr = data.pagedResults.newRequest; @@ -581,9 +594,23 @@ async function list_i ( : [ ...targetRelevantSubentries ]; // Must spread to create a new reference. Otherwise... if (subordinate.dse.admPoint?.administrativeRole.has(ID_AC_SPECIFIC)) { // ... (keep going) effectiveRelevantSubentries.length = 0; // ...this will modify the target-relevant subentries! - effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, subordinate, subordinateDN, subordinate))); + effectiveRelevantSubentries.push(...(await getRelevantSubentries( + ctx, + subordinate, + subordinateDN, + subordinate, + undefined, + ret.subentriesCache, + ))); } else if (subordinate.dse.admPoint?.administrativeRole.has(ID_AC_INNER)) { - effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, subordinate, subordinateDN, subordinate))); + effectiveRelevantSubentries.push(...(await getRelevantSubentries( + ctx, + subordinate, + subordinateDN, + subordinate, + undefined, + ret.subentriesCache, + ))); } const subordinateACI = await getACIItems( ctx, diff --git a/apps/meerkat/src/app/distributed/list_ii.ts b/apps/meerkat/src/app/distributed/list_ii.ts index 44dff1c7c..1d38e48f2 100644 --- a/apps/meerkat/src/app/distributed/list_ii.ts +++ b/apps/meerkat/src/app/distributed/list_ii.ts @@ -192,6 +192,7 @@ async function list_ii ( (data.securityParameters?.errorProtection === ErrorProtectionRequest_signed) && (assn.authorizedForSignedErrors) ); + const subentriesCache: Map = new Map(); const requestor: DistinguishedName | undefined = data .securityParameters ?.certification_path @@ -300,7 +301,7 @@ async function list_ii ( ) : undefined; const targetRelevantSubentries: Vertex[] = (await Promise.all( - state.admPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap)), + state.admPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap, undefined, subentriesCache)), )).flat(); const targetAccessControlScheme = [ ...state.admPoints ] // Array.reverse() works in-place, so we create a new array. .reverse() @@ -553,9 +554,23 @@ async function list_ii ( : targetRelevantSubentries; if (subordinate.dse.admPoint?.administrativeRole.has(ID_AC_SPECIFIC)) { effectiveRelevantSubentries.length = 0; - effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, subordinate, subordinateDN, subordinate))); + effectiveRelevantSubentries.push(...(await getRelevantSubentries( + ctx, + subordinate, + subordinateDN, + subordinate, + undefined, + subentriesCache, + ))); } else if (subordinate.dse.admPoint?.administrativeRole.has(ID_AC_INNER)) { - effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, subordinate, subordinateDN, subordinate))); + effectiveRelevantSubentries.push(...(await getRelevantSubentries( + ctx, + subordinate, + subordinateDN, + subordinate, + undefined, + subentriesCache, + ))); } const subordinateACI = await getACIItems( ctx, diff --git a/apps/meerkat/src/app/distributed/search_i.ts b/apps/meerkat/src/app/distributed/search_i.ts index 154660013..55690fd5a 100644 --- a/apps/meerkat/src/app/distributed/search_i.ts +++ b/apps/meerkat/src/app/distributed/search_i.ts @@ -412,6 +412,13 @@ interface SearchState extends Partial, Partial; } /** @@ -1677,7 +1684,7 @@ async function search_i_ex ( const targetDN = getDistinguishedName(target); const NAMING_MATCHER = getNamingMatcherGetter(ctx); const relevantSubentries: Vertex[] = (await Promise.all( - state.admPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap)), + state.admPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap, undefined, searchState.subentriesCache)), )).flat(); const accessControlScheme = [ ...state.admPoints ] // Array.reverse() works in-place, so we create a new array. .reverse() @@ -3271,7 +3278,7 @@ async function search_i_ex ( ) ) { const relevantSubentries: Vertex[] = (await Promise.all( - adminPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap)), + adminPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap, undefined, searchState.subentriesCache)), )).flat(); const targetACI = await getACIItems( ctx, diff --git a/apps/meerkat/src/app/distributed/search_ii.ts b/apps/meerkat/src/app/distributed/search_ii.ts index 32c13af4d..391de337d 100644 --- a/apps/meerkat/src/app/distributed/search_ii.ts +++ b/apps/meerkat/src/app/distributed/search_ii.ts @@ -78,8 +78,6 @@ import getEntryExistsFilter from "../database/entryExistsFilter"; import { searchRules } from "@wildboar/x500/src/lib/collections/attributes"; import { attributeValueFromDB } from "../database/attributeValueFromDB"; import { MAX_RESULTS } from "../constants"; -import accessControlSchemesThatUseRBAC from "../authz/accessControlSchemesThatUseRBAC"; -import { get_security_labels_for_rdn } from "../authz/get_security_labels_for_rdn"; const BYTES_IN_A_UUID: number = 16; diff --git a/apps/meerkat/src/app/dit/getRelevantSubentries.ts b/apps/meerkat/src/app/dit/getRelevantSubentries.ts index bd58a20b2..e5c3c3e33 100644 --- a/apps/meerkat/src/app/dit/getRelevantSubentries.ts +++ b/apps/meerkat/src/app/dit/getRelevantSubentries.ts @@ -37,12 +37,20 @@ async function getRelevantSubentries ( entryDN: DistinguishedName, admPoint: Vertex, where?: Prisma.EntryWhereInput, + subentryCache?: Map, ): Promise { const NAMING_MATCHER = getNamingMatcherGetter(ctx); - const subentries = await readSubordinates(ctx, admPoint, undefined, undefined, undefined, { - ...where, - subentry: true, - }); + const subentries = subentryCache?.get(admPoint.dse.id) ?? await readSubordinates( + ctx, + admPoint, + undefined, + undefined, + undefined, + { + ...where, + subentry: true, + }, + ); const objectClasses = Array.isArray(entry) ? entry : Array.from(entry.dse.objectClass.values()).map(ObjectIdentifier.fromString); @@ -83,6 +91,7 @@ async function getRelevantSubentries ( relevant_sub_ids.delete(subentry.dse.id); } } + subentryCache?.set(admPoint.dse.id, subentries); return ret; } diff --git a/apps/meerkat/src/assets/static/conformance.md b/apps/meerkat/src/assets/static/conformance.md index 1dc53a063..d9638c6bb 100644 --- a/apps/meerkat/src/assets/static/conformance.md +++ b/apps/meerkat/src/assets/static/conformance.md @@ -1,7 +1,7 @@ # Conformance -In the statements below, the term "Meerkat DSA" refers to version 3.2.1 of -Meerkat DSA, hence these statements are only claimed for version 3.2.1 of +In the statements below, the term "Meerkat DSA" refers to version 3.2.2 of +Meerkat DSA, hence these statements are only claimed for version 3.2.2 of Meerkat DSA. ## X.519 Conformance Statement diff --git a/k8s/charts/meerkat-dsa/Chart.yaml b/k8s/charts/meerkat-dsa/Chart.yaml index 47e2d52b3..263791a93 100644 --- a/k8s/charts/meerkat-dsa/Chart.yaml +++ b/k8s/charts/meerkat-dsa/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: meerkat-dsa description: X.500 Directory Server (DSA) and LDAP Server by Wildboar Software. type: application -version: 2.17.1 -appVersion: 3.2.1 +version: 2.17.2 +appVersion: 3.2.2 home: https://wildboarsoftware.com keywords: - directory diff --git a/libs/meerkat-types/package.json b/libs/meerkat-types/package.json index 56fdba6b5..e3dacac63 100644 --- a/libs/meerkat-types/package.json +++ b/libs/meerkat-types/package.json @@ -1,5 +1,5 @@ { "name": "@wildboar/meerkat-types", - "version": "1.12.0", + "version": "1.13.0", "license": "MIT" } diff --git a/libs/meerkat-types/src/lib/types.ts b/libs/meerkat-types/src/lib/types.ts index 913479e0a..6759b2031 100644 --- a/libs/meerkat-types/src/lib/types.ts +++ b/libs/meerkat-types/src/lib/types.ts @@ -3403,6 +3403,12 @@ interface WithIntegerProtocolVersion { protocolVersion?: number; } +export +interface WithSubentryCache { + subentriesCache: Map; + subentriesCacheExpiration: Date; +} + /** * @summary An application association with this DSA. * @description @@ -3413,7 +3419,7 @@ interface WithIntegerProtocolVersion { * @class */ export -abstract class ClientAssociation implements WithIntegerProtocolVersion { +abstract class ClientAssociation implements WithIntegerProtocolVersion, WithSubentryCache { /** The version number of the protocol in use. */ public protocolVersion?: number | undefined; @@ -3424,6 +3430,9 @@ abstract class ClientAssociation implements WithIntegerProtocolVersion { /** A UUID that uniquely identifies this association */ public readonly id = randomUUID(); + public subentriesCache: Map = new Map(); + public subentriesCacheExpiration: Date = new Date(); + /** * The vertex of the DIT to which the remote host bound. * diff --git a/pkg/control b/pkg/control index 5c82f4b4e..a285c4403 100644 --- a/pkg/control +++ b/pkg/control @@ -1,5 +1,5 @@ Package: meerkat-dsa -Version: 3.2.1 +Version: 3.2.2 Section: database Priority: optional Architecture: i386 diff --git a/pkg/docker-compose.yaml b/pkg/docker-compose.yaml index 51ea3931a..9868cf325 100644 --- a/pkg/docker-compose.yaml +++ b/pkg/docker-compose.yaml @@ -20,7 +20,7 @@ services: labels: author: Wildboar Software app: meerkat - version: "3.2.1" + version: "3.2.2" ports: - '1389:389/tcp' # LDAP TCP Port - '4632:4632/tcp' # IDM Socket diff --git a/pkg/meerkat-dsa.rb b/pkg/meerkat-dsa.rb index 16aab99e4..58f28c8dc 100644 --- a/pkg/meerkat-dsa.rb +++ b/pkg/meerkat-dsa.rb @@ -2,7 +2,7 @@ class MeerkatDSA < Formula desc "X.500 Directory Server (DSA) and LDAP Server by Wildboar Software" homepage "https://github.com/Wildboar-Software/directory" url "https://github.com/Wildboar-Software/directory/archive/v1.1.0.tar.gz" - version = "3.2.1" + version = "3.2.2" # sha256 "e86694b2e15d8d4da2477c44e584fb5e860666787d010801199a0a77bcf28a2d" def install diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 8409750f0..25d6df06a 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: meerkat-dsa base: core20 -version: '3.2.1' +version: '3.2.2' summary: X.500 Directory (DSA) and LDAP Server description: | Fully-featured X.500 directory server / directory system agent (DSA)