Merge pull request #27 from Wildboar-Software/cache_subentries

Cache subentries

.github/workflows/meerkat.yml

@@ -808,7 +808,7 @@ jobs:
             --set enable_dsp=true \
             --set administrator_email=jonathan@wilbur.space \
             --set administrator_email_public=true \
-            --set vendor_version='3.2.1' \
+            --set vendor_version='3.2.2' \
             --set signing_required_for_chaining=false \
             --set tcp_timeout_in_seconds=300 \
             --set min_transfer_speed_bytes_per_minute=10 \

apps/meerkat-docs/docs/changelog-meerkat.md

@@ -1,5 +1,12 @@
 # Changelog for Meerkat DSA
 
+## Version 3.2.2
+
+This version **dramatically** improves the performance of the DSA for `search`
+and `list` operations and for `addEntry` when performed in rapid succession, as
+well as for a few other use cases. In general, you should see a 400% to 1000%
+increase in performance.
+
 ## Version 3.2.1
 
 Fix bug with writing to IDM sockets.

apps/meerkat-docs/docs/conformance.md

@@ -1,7 +1,7 @@
 # Conformance
 
-In the statements below, the term "Meerkat DSA" refers to version 3.2.1 of
-Meerkat DSA, hence these statements are only claimed for version 3.2.1 of
+In the statements below, the term "Meerkat DSA" refers to version 3.2.2 of
+Meerkat DSA, hence these statements are only claimed for version 3.2.2 of
 Meerkat DSA.
 
 ## X.519 Conformance Statement

apps/meerkat/package.json

@@ -12,7 +12,7 @@
             "url": "https://github.com/JonathanWilbur"
         }
     ],
-    "version": "3.2.1",
+    "version": "3.2.2",
     "license": "MIT",
     "bin": {
         "meerkat": "./main.js"

apps/meerkat/src/app/authz/permittedToFindDSE.ts

@@ -77,6 +77,7 @@ async function permittedToFindDSE (
     const isMemberOfGroup = getIsGroupMember(ctx, NAMING_MATCHER);
     let accessControlScheme: OBJECT_IDENTIFIER | undefined;
     let authorizedToDiscloseOnError: boolean = false;
+    const subentriesCache: Map<number, Vertex[]> = new Map();
 
     for (let i = 0; i < needleDN.length; i++) {
         const rdn = needleDN[i];
@@ -105,7 +106,14 @@ async function permittedToFindDSE (
                 ? [ ...admPoints, dse_i ]
                 : [ ...admPoints ];
             const relevantSubentries: Vertex[] = (await Promise.all(
-                relevantAdmPoints.map((ap) => getRelevantSubentries(ctx, dse_i, childDN, ap)),
+                relevantAdmPoints.map((ap) => getRelevantSubentries(
+                    ctx,
+                    dse_i,
+                    childDN,
+                    ap,
+                    undefined,
+                    subentriesCache,
+                )),
             )).flat();
             const targetACI = await getACIItems(
                 ctx,

apps/meerkat/src/app/ctx.ts

@@ -1085,7 +1085,10 @@ const ctx: MeerkatContext = {
             app: "meerkat",
         },
     }),
-    db: new PrismaClient(),
+    db: new PrismaClient({
+        // log: ["query"],
+        // log: ['query', 'info', 'warn', 'error'],
+    }),
     telemetry: {
         init: async (): Promise<void> => {
             try {

apps/meerkat/src/app/distributed/OperationDispatcher.ts

@@ -267,6 +267,7 @@ async function search_procedures (
         depth: 0,
         excludedById: new Set(),
         matching_rule_substitutions: new Map(),
+        subentriesCache: new Map(),
     };
 
     const use_search_ii: boolean = (
@@ -816,6 +817,7 @@ class OperationDispatcher {
                 effectiveHierarchySelections: data.hierarchySelections,
                 effectiveSearchControls: data.searchControlOptions,
                 effectiveServiceControls: data.serviceControls?.options,
+                subentriesCache: new Map(),
             };
 
             const use_search_rule_check_ii: boolean = (nameResolutionPhase === completed);
@@ -1498,6 +1500,7 @@ class OperationDispatcher {
             matching_rule_substitutions: new Map(),
             notification: [],
             effectiveEntryLimit: MAX_RESULTS,
+            subentriesCache: new Map(),
         };
         await relatedEntryProcedure(
             ctx,

apps/meerkat/src/app/distributed/addEntry.ts

@@ -91,7 +91,7 @@ import {
     id_opcode_addEntry,
 } from "@wildboar/x500/src/lib/modules/CommonProtocolSpecification/id-opcode-addEntry.va";
 import establishSubordinate from "../dop/establishSubordinate";
-import { differenceInMilliseconds } from "date-fns";
+import { addSeconds, differenceInMilliseconds } from "date-fns";
 import {
     ServiceProblem_timeLimitExceeded
 } from "@wildboar/x500/src/lib/modules/DirectoryAbstractService/ServiceProblem.ta";
@@ -260,6 +260,11 @@ async function addEntry (
     assn: ClientAssociation,
     state: OperationDispatcherState,
 ): Promise<OperationReturn> {
+    const now = new Date();
+    if (assn.subentriesCacheExpiration <= now) {
+        assn.subentriesCache.clear();
+        assn.subentriesCacheExpiration = addSeconds(now, 10);
+    }
     const argument = _decode_AddEntryArgument(state.operationArgument);
     const data = getOptionallyProtectedValue(argument);
     const signErrors: boolean = (
@@ -393,7 +398,14 @@ async function addEntry (
     const relevantSubentries: Vertex[] = ctx.config.bulkInsertMode
         ? []
         : (await Promise.all(
-            state.admPoints.map((ap) => getRelevantSubentries(ctx, objectClasses, targetDN, ap)),
+            state.admPoints.map((ap) => getRelevantSubentries(
+                ctx,
+                objectClasses,
+                targetDN,
+                ap,
+                undefined,
+                assn.subentriesCache,
+            )),
         )).flat();
     const accessControlScheme = [ ...state.admPoints ] // Array.reverse() works in-place, so we create a new array.
         .reverse()
@@ -801,9 +813,23 @@ async function addEntry (
                 : [ ...relevantSubentries ]; // Must spread to create a new reference. Otherwise...
             if (existing.dse.admPoint?.administrativeRole.has(ID_AC_SPECIFIC)) { // ... (keep going)
                 effectiveRelevantSubentries.length = 0; // ...this will modify the target-relevant subentries!
-                effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, existing, targetDN, existing)));
+                effectiveRelevantSubentries.push(...(await getRelevantSubentries(
+                    ctx,
+                    existing,
+                    targetDN,
+                    existing,
+                    undefined,
+                    assn.subentriesCache,
+                )));
             } else if (existing.dse.admPoint?.administrativeRole.has(ID_AC_INNER)) {
-                effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, existing, targetDN, existing)));
+                effectiveRelevantSubentries.push(...(await getRelevantSubentries(
+                    ctx,
+                    existing,
+                    targetDN,
+                    existing,
+                    undefined,
+                    assn.subentriesCache,
+                )));
             }
             const subordinateACI = await getACIItems(
                 ctx,

apps/meerkat/src/app/distributed/findDSE.ts

@@ -327,6 +327,7 @@ export
         (errorProtection === ErrorProtectionRequest_signed)
         && (!assn || (assn.authorizedForSignedErrors))
     );
+    const subentriesCache: Map<number, Vertex[]> = new Map();
 
     // Service controls
     const manageDSAIT: boolean = (
@@ -995,7 +996,14 @@ export
                         ? [...state.admPoints, matchedVertex]
                         : [...state.admPoints];
                     const relevantSubentries: Vertex[] = (await Promise.all(
-                        relevantAdmPoints.map((ap) => getRelevantSubentries(ctx, matchedVertex, childDN, ap)),
+                        relevantAdmPoints.map((ap) => getRelevantSubentries(
+                            ctx,
+                            matchedVertex,
+                            childDN,
+                            ap,
+                            undefined,
+                            subentriesCache,
+                        )),
                     )).flat();
                     const targetACI = await getACIItems(
                         ctx,
@@ -1177,7 +1185,14 @@ export
                             ? [...state.admPoints, child]
                             : [...state.admPoints];
                         const relevantSubentries: Vertex[] = (await Promise.all(
-                            relevantAdmPoints.map((ap) => getRelevantSubentries(ctx, child, childDN, ap)),
+                            relevantAdmPoints.map((ap) => getRelevantSubentries(
+                                ctx,
+                                child,
+                                childDN,
+                                ap,
+                                undefined,
+                                subentriesCache,
+                            )),
                         )).flat();
                         const targetACI = await getACIItems(
                             ctx,
@@ -1346,7 +1361,14 @@ export
                 : undefined;
             const currentDN = getDistinguishedName(dse_i);
             const relevantSubentries: Vertex[] = (await Promise.all(
-                state.admPoints.map((ap) => getRelevantSubentries(ctx, dse_i, currentDN, ap)),
+                state.admPoints.map((ap) => getRelevantSubentries(
+                    ctx,
+                    dse_i,
+                    currentDN,
+                    ap,
+                    undefined,
+                    subentriesCache,
+                )),
             )).flat();
             const targetACI = await getACIItems(
                 ctx,

apps/meerkat/src/app/distributed/list_i.ts

@@ -167,6 +167,13 @@ interface ListState extends Partial<WithRequestStatistics>, Partial<WithOutcomeS
     resultSets: ListResult[];
     poq?: PartialOutcomeQualifier;
     queryReference?: string;
+
+    /**
+     * A mapping of the administrative point's database ID to all of its
+     * subentries. This avoids the expensive process of loading all subentries
+     * for an admin point for every search result that is to be evaluated.
+     */
+    subentriesCache: Map<number, Vertex[]>;
 }
 
 /**
@@ -306,21 +313,6 @@ async function list_i (
         : undefined;
     const NAMING_MATCHER = getNamingMatcherGetter(ctx);
     const subentries: boolean = (data.serviceControls?.options?.[subentriesBit] === TRUE_BIT);
-    const targetRelevantSubentries: Vertex[] = (await Promise.all(
-        state.admPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap)),
-    )).flat();
-    const targetAccessControlScheme = [ ...state.admPoints ] // Array.reverse() works in-place, so we create a new array.
-        .reverse()
-        .find((ap) => ap.dse.admPoint!.accessControlScheme)?.dse.admPoint!.accessControlScheme;
-    const isMemberOfGroup = getIsGroupMember(ctx, NAMING_MATCHER);
-
-    const isParent: boolean = target.dse.objectClass.has(PARENT);
-    const isChild: boolean = target.dse.objectClass.has(CHILD);
-    const isAncestor: boolean = (isParent && !isChild);
-
-    let pagingRequest: PagedResultsRequest_newRequest | undefined;
-    let queryReference: string | undefined;
-    let cursorId: number | undefined;
     const signDSPResult: boolean = (
         (state.chainingArguments.securityParameters?.errorProtection === ErrorProtectionRequest_signed)
         && (assn instanceof DSPAssociation)
@@ -341,7 +333,28 @@ async function list_i (
         chaining: chainingResults,
         results: [],
         resultSets: [],
+        subentriesCache: new Map(),
     };
+    const targetRelevantSubentries: Vertex[] = (await Promise.all(
+        state.admPoints.map((ap) => getRelevantSubentries(
+            ctx,
+            target,
+            targetDN,
+            ap,
+            undefined,
+            ret.subentriesCache,
+        )),
+    )).flat();
+    const targetAccessControlScheme = [ ...state.admPoints ] // Array.reverse() works in-place, so we create a new array.
+        .reverse()
+        .find((ap) => ap.dse.admPoint!.accessControlScheme)?.dse.admPoint!.accessControlScheme;
+    const isMemberOfGroup = getIsGroupMember(ctx, NAMING_MATCHER);
+    const isParent: boolean = target.dse.objectClass.has(PARENT);
+    const isChild: boolean = target.dse.objectClass.has(CHILD);
+    const isAncestor: boolean = (isParent && !isChild);
+    let pagingRequest: PagedResultsRequest_newRequest | undefined;
+    let queryReference: string | undefined;
+    let cursorId: number | undefined;
     if (data.pagedResults) {
         if ("newRequest" in data.pagedResults) {
             const nr = data.pagedResults.newRequest;
@@ -581,9 +594,23 @@ async function list_i (
                     : [ ...targetRelevantSubentries ]; // Must spread to create a new reference. Otherwise...
                 if (subordinate.dse.admPoint?.administrativeRole.has(ID_AC_SPECIFIC)) { // ... (keep going)
                     effectiveRelevantSubentries.length = 0; // ...this will modify the target-relevant subentries!
-                    effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, subordinate, subordinateDN, subordinate)));
+                    effectiveRelevantSubentries.push(...(await getRelevantSubentries(
+                        ctx,
+                        subordinate,
+                        subordinateDN,
+                        subordinate,
+                        undefined,
+                        ret.subentriesCache,
+                    )));
                 } else if (subordinate.dse.admPoint?.administrativeRole.has(ID_AC_INNER)) {
-                    effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, subordinate, subordinateDN, subordinate)));
+                    effectiveRelevantSubentries.push(...(await getRelevantSubentries(
+                        ctx,
+                        subordinate,
+                        subordinateDN,
+                        subordinate,
+                        undefined,
+                        ret.subentriesCache,
+                    )));
                 }
                 const subordinateACI = await getACIItems(
                     ctx,

apps/meerkat/src/app/distributed/list_ii.ts

@@ -192,6 +192,7 @@ async function list_ii (
         (data.securityParameters?.errorProtection === ErrorProtectionRequest_signed)
         && (assn.authorizedForSignedErrors)
     );
+    const subentriesCache: Map<number, Vertex[]> = new Map();
     const requestor: DistinguishedName | undefined = data
         .securityParameters
         ?.certification_path
@@ -300,7 +301,7 @@ async function list_ii (
         )
         : undefined;
     const targetRelevantSubentries: Vertex[] = (await Promise.all(
-        state.admPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap)),
+        state.admPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap, undefined, subentriesCache)),
     )).flat();
     const targetAccessControlScheme = [ ...state.admPoints ] // Array.reverse() works in-place, so we create a new array.
         .reverse()
@@ -553,9 +554,23 @@ async function list_ii (
                     : targetRelevantSubentries;
                 if (subordinate.dse.admPoint?.administrativeRole.has(ID_AC_SPECIFIC)) {
                     effectiveRelevantSubentries.length = 0;
-                    effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, subordinate, subordinateDN, subordinate)));
+                    effectiveRelevantSubentries.push(...(await getRelevantSubentries(
+                        ctx,
+                        subordinate,
+                        subordinateDN,
+                        subordinate,
+                        undefined,
+                        subentriesCache,
+                    )));
                 } else if (subordinate.dse.admPoint?.administrativeRole.has(ID_AC_INNER)) {
-                    effectiveRelevantSubentries.push(...(await getRelevantSubentries(ctx, subordinate, subordinateDN, subordinate)));
+                    effectiveRelevantSubentries.push(...(await getRelevantSubentries(
+                        ctx,
+                        subordinate,
+                        subordinateDN,
+                        subordinate,
+                        undefined,
+                        subentriesCache,
+                    )));
                 }
                 const subordinateACI = await getACIItems(
                     ctx,

apps/meerkat/src/app/distributed/search_i.ts

@@ -412,6 +412,13 @@ interface SearchState extends Partial<WithRequestStatistics>, Partial<WithOutcom
     effectiveEntryLimit: number;
     effectiveFamilyGrouping?: FamilyGrouping;
     effectiveFamilyReturn?: FamilyReturn;
+
+    /**
+     * A mapping of the administrative point's database ID to all of its
+     * subentries. This avoids the expensive process of loading all subentries
+     * for an admin point for every search result that is to be evaluated.
+     */
+    subentriesCache: Map<number, Vertex[]>;
 }
 
 /**
@@ -1677,7 +1684,7 @@ async function search_i_ex (
     const targetDN = getDistinguishedName(target);
     const NAMING_MATCHER = getNamingMatcherGetter(ctx);
     const relevantSubentries: Vertex[] = (await Promise.all(
-        state.admPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap)),
+        state.admPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap, undefined, searchState.subentriesCache)),
     )).flat();
     const accessControlScheme = [ ...state.admPoints ] // Array.reverse() works in-place, so we create a new array.
         .reverse()
@@ -3271,7 +3278,7 @@ async function search_i_ex (
                     )
                 ) {
                     const relevantSubentries: Vertex[] = (await Promise.all(
-                        adminPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap)),
+                        adminPoints.map((ap) => getRelevantSubentries(ctx, target, targetDN, ap, undefined, searchState.subentriesCache)),
                     )).flat();
                     const targetACI = await getACIItems(
                         ctx,

apps/meerkat/src/app/distributed/search_ii.ts

@@ -78,8 +78,6 @@ import getEntryExistsFilter from "../database/entryExistsFilter";
 import { searchRules } from "@wildboar/x500/src/lib/collections/attributes";
 import { attributeValueFromDB } from "../database/attributeValueFromDB";
 import { MAX_RESULTS } from "../constants";
-import accessControlSchemesThatUseRBAC from "../authz/accessControlSchemesThatUseRBAC";
-import { get_security_labels_for_rdn } from "../authz/get_security_labels_for_rdn";
 
 const BYTES_IN_A_UUID: number = 16;