Add a possibility to re-add deleted elements with preserved ids

change/@itwin-imodel-transformer-635c5a66-27ec-4eae-9501-ca020a3b90aa.json

@@ -0,0 +1,7 @@
+{
+  "type": "prerelease",
+  "comment": "add more cases to support preserveElementIdsForFiltering options",
+  "packageName": "@itwin/imodel-transformer",
+  "email": "Daniel.Erbynn@bentley.com",
+  "dependentChangeType": "patch"
+}

packages/transformer/src/IModelImporter.ts

@@ -63,6 +63,11 @@ export interface IModelImportOptions {
    * @see [IModelImporter Options]($docs/learning/transformer/index.md#IModelImporter)
    */
   autoExtendProjectExtents?: boolean | { excludeOutliers: boolean };
+  /**
+   * Indicates whether the importer is used in a transformation process.
+   * If `true`, the importer is passed into the transformer.
+   */
+  fromTransformation?: boolean;
   /** See [IModelTransformOptions]($transformer) */
   preserveElementIdsForFiltering?: boolean;
   /** If `true`, simplify the element geometry for visualization purposes. For example, convert b-reps into meshes.
@@ -103,6 +108,20 @@ export class IModelImporter {
    */
   private _duplicateCodeValueMap: Map<Id64String, string>;
 
+  /**
+   * The set of elements added during export of an element in transformation to track which elements to update during import.
+   * Defaults to an empty set.
+   * @note
+   *
+   * This is used as an optimization when `preserveElementIdsForFiltering` is set to `true`
+   * In normal cases where this option set to `false`,
+   * the importer determines whether to insert or update based off of whether the ID is defined on `elementProps`.
+   * However, with `preserveElementIdsForFiltering` set to `true`, IDs are always set, so we can't determine insert/update like the normal case.
+   * The transformer already knows if an element exists or not by the time `importElement` is called and pushes to this set with `markElementAsUpdate`.
+   *
+   */
+  private _elementsToUpdate = new Set<Id64String>([]);
+
   /** The set of elements that should not be updated by this IModelImporter.
    * Defaults to an empty set.
    * @note Adding an element to this set is typically necessary when remapping a source element to one that already exists in the target and already has the desired properties.
@@ -133,6 +152,7 @@ export class IModelImporter {
     this.targetDb = targetDb;
     this.options = {
       autoExtendProjectExtents: options?.autoExtendProjectExtents ?? true,
+      fromTransformation: options?.fromTransformation ?? false,
       preserveElementIdsForFiltering:
         options?.preserveElementIdsForFiltering ?? false,
       simplifyElementGeometry: options?.simplifyElementGeometry ?? false,
@@ -155,6 +175,16 @@ export class IModelImporter {
     return false;
   }
 
+  /**
+   * Marks an element so that it can be updated during import when [[IModelImportOptions.preserveElementIdsForFiltering]] is set to true
+   */
+  public markElementAsUpdate(elementId: Id64String) {
+    if (this._rootElementIds.has(elementId)) {
+      return;
+    }
+    this._elementsToUpdate.add(elementId);
+  }
+
   /** Import the specified ModelProps (either as an insert or an update) into the target iModel. */
   public importModel(modelProps: ModelProps): void {
     if (undefined === modelProps.id || !Id64.isValidId64(modelProps.id))
@@ -229,6 +259,25 @@ export class IModelImporter {
 
   /** Import the specified ElementProps (either as an insert or an update) into the target iModel. */
   public importElement(elementProps: ElementProps): Id64String {
+    const tryUpdateElement = (elemProps: ElementProps): void => {
+      try {
+        this.onUpdateElement(elemProps);
+      } catch (err) {
+        if ((err as IModelError).errorNumber === IModelStatus.DuplicateCode) {
+          assert(
+            elemProps.code.value !== undefined,
+            "NULL code values are always considered unique and cannot clash"
+          );
+          this._duplicateCodeValueMap.set(elemProps.id!, elemProps.code.value);
+          // Using NULL code values as an alternative is not valid because definition elements cannot have NULL code values.
+          elemProps.code.value = Guid.createValue();
+          this.onUpdateElement(elemProps);
+        } else {
+          throw err;
+        }
+      }
+    };
+
     if (
       undefined !== elementProps.id &&
       this.doNotUpdateElement(elementProps.id)
@@ -246,6 +295,7 @@ export class IModelImporter {
           "elementProps.id must be defined during a preserveIds operation"
         );
       }
+
       // Categories are the only element that onInserted will immediately insert a new element (their default subcategory)
       // since default subcategories always exist and always will be inserted after their categories, we treat them as an update
       // to prevent duplicate inserts.
@@ -257,29 +307,23 @@ export class IModelImporter {
       ) {
         this.onUpdateElement(elementProps);
       } else {
-        this.onInsertElement(elementProps);
+        // if [[IModelImportOptions.fromTransformation]] is false, try and get the element (update if it exists)
+        const shouldUpdateElement =
+          !this.options.fromTransformation &&
+          this.targetDb.elements.tryGetElement(elementProps.id);
+        if (
+          this._elementsToUpdate.has(elementProps.id) ||
+          shouldUpdateElement
+        ) {
+          tryUpdateElement(elementProps);
+          this._elementsToUpdate.delete(elementProps.id);
+        } else {
+          this.onInsertElement(elementProps);
+        }
       }
     } else {
       if (undefined !== elementProps.id) {
-        try {
-          this.onUpdateElement(elementProps);
-        } catch (err) {
-          if ((err as IModelError).errorNumber === IModelStatus.DuplicateCode) {
-            assert(
-              elementProps.code.value !== undefined,
-              "NULL code values are always considered unique and cannot clash"
-            );
-            this._duplicateCodeValueMap.set(
-              elementProps.id,
-              elementProps.code.value
-            );
-            // Using NULL code values as an alternative is not valid because definition elements cannot have NULL code values.
-            elementProps.code.value = Guid.createValue();
-            this.onUpdateElement(elementProps);
-          } else {
-            throw err;
-          }
-        }
+        tryUpdateElement(elementProps);
       } else {
         this.onInsertElement(elementProps); // targetElementProps.id assigned by insertElement
       }

packages/transformer/src/IModelTransformer.ts

@@ -668,6 +668,7 @@ export class IModelTransformer extends IModelExportHandler {
       this.validateSharedOptionsMatch();
     }
     this.targetDb = this.importer.targetDb;
+    this.importer.options.fromTransformation = true;
     // create the IModelCloneContext, it must be initialized later
     this.context = new IModelCloneContext(this.sourceDb, this.targetDb);
 
@@ -1882,10 +1883,7 @@ export class IModelTransformer extends IModelExportHandler {
   public override onExportElement(sourceElement: Element): void {
     let targetElementId: Id64String;
     let targetElementProps: ElementProps;
-    if (this._options.preserveElementIdsForFiltering) {
-      targetElementId = sourceElement.id;
-      targetElementProps = this.onTransformElement(sourceElement);
-    } else if (this._options.wasSourceIModelCopiedToTarget) {
+    if (this._options.wasSourceIModelCopiedToTarget) {
       targetElementId = sourceElement.id;
       targetElementProps =
         this.targetDb.elements.getElementProps(targetElementId);
@@ -1944,6 +1942,34 @@ export class IModelTransformer extends IModelExportHandler {
       ? targetElementId
       : undefined;
 
+    if (this._options.preserveElementIdsForFiltering) {
+      const isInvalid = !Id64.isValid(targetElementId);
+
+      if (!isInvalid && targetElementId !== sourceElement.id) {
+        // Element found with different id
+        throw new Error(
+          `Element id(${sourceElement.id}) cannot be preserved. Found a different mapping(${targetElementId}) from source element`
+        );
+      } else if (!isInvalid && targetElementId === sourceElement.id) {
+        // targetElementId is valid (indicating update)
+        this.importer.markElementAsUpdate(sourceElement.id);
+      } else if (isInvalid) {
+        const sourceInTargetElemProps =
+          this.targetDb.elements.tryGetElementProps(sourceElement.id);
+
+        // if we don't find mapping for source element in target(invalid) but another element with source id exists in target
+        if (sourceInTargetElemProps) {
+          // Element id is already taken by another element
+          throw new Error(
+            `Element id(${sourceElement.id}) cannot be preserved. An unrelated element in the target already uses id: ${sourceElement.id}`
+          );
+        } else {
+          // Element id in target is available to be remapped
+          targetElementProps.id = sourceElement.id;
+        }
+      }
+    }
+
     if (!this._options.wasSourceIModelCopiedToTarget) {
       this.importer.importElement(targetElementProps); // don't need to import if iModel was copied
     }

packages/transformer/src/test/standalone/IModelTransformer.test.ts

@@ -2655,6 +2655,122 @@ describe("IModelTransformer", () => {
     }
   }
 
+  it("process() with preserveElementIdsForFiltering set to true should re-add deleted element with same id", async () => {
+    const sourceDbPath = IModelTransformerTestUtils.prepareOutputFile(
+      "IModelTransformer",
+      "PreserveIdOnTestModel-Source.bim"
+    );
+    const sourceDb = SnapshotDb.createEmpty(sourceDbPath, {
+      rootSubject: { name: "iModelA" },
+    });
+    Subject.insert(sourceDb, IModel.rootSubjectId, "Subject1");
+    Subject.insert(sourceDb, IModel.rootSubjectId, "Subject2");
+    sourceDb.saveChanges();
+
+    const targetDbPath = IModelTransformerTestUtils.prepareOutputFile(
+      "IModelTransformer",
+      "PreserveIdOnTestModel-Target.bim"
+    );
+    const targetDb = SnapshotDb.createEmpty(targetDbPath, {
+      rootSubject: sourceDb.rootSubject,
+    });
+
+    // Execute process() so that elements from source are copied to target
+    const transformer = new IModelTransformer(sourceDb, targetDb, {
+      preserveElementIdsForFiltering: true,
+    });
+    await transformer.process();
+    targetDb.saveChanges();
+
+    let sourceContent = await getAllElementsInvariants(sourceDb);
+    let targetContent = await getAllElementsInvariants(targetDb);
+    expect(targetContent).to.deep.equal(sourceContent);
+
+    // Delete subject 1 from target
+    const code1 = Subject.createCode(
+      targetDb,
+      IModel.rootSubjectId,
+      "Subject1"
+    );
+    const targetSubjectId1 = targetDb.elements.queryElementIdByCode(code1);
+    expect(targetSubjectId1).to.not.be.undefined;
+    targetDb.elements.deleteElement(targetSubjectId1!);
+    targetDb.saveChanges();
+
+    // update subject 2 in source
+    const code2 = Subject.createCode(
+      sourceDb,
+      IModel.rootSubjectId,
+      "Subject2"
+    );
+    const targetSubjectId2 = sourceDb.elements.queryElementIdByCode(code2);
+    const subject2 = sourceDb.elements.getElement<Subject>(targetSubjectId2!);
+    subject2.description = "Subject2 Updated Description";
+    sourceDb.elements.updateElement(subject2.toJSON());
+    sourceDb.saveChanges();
+
+    // Calling process() for second time with option to preserve elements in hopes of restoring deleted element
+    const secondTransformer = new IModelTransformer(sourceDb, targetDb, {
+      preserveElementIdsForFiltering: true,
+    });
+    await secondTransformer.process(); // should not throw error: duplicate code (65547) and should re-add deleted element
+    targetDb.saveChanges();
+
+    sourceContent = await getAllElementsInvariants(sourceDb);
+    targetContent = await getAllElementsInvariants(targetDb);
+    expect(targetContent).to.deep.equal(sourceContent);
+
+    sourceDb.close();
+    targetDb.close();
+  });
+
+  it("process() with preserveElementIdsForFiltering set to true should not throw when called on 2 identical iModels", async () => {
+    const seedDb = SnapshotDb.openFile(
+      TestUtils.IModelTestUtils.resolveAssetFile("CompatibilityTestSeed.bim")
+    );
+    const sourceDbPath = IModelTransformerTestUtils.prepareOutputFile(
+      "IModelTransformer",
+      "PreserveIdOnTestModel-Source.bim"
+    );
+    // transforming the seed to an empty will update it to the latest bis from the new target
+    // which minimizes differences we'd otherwise need to filter later
+    const sourceDb = SnapshotDb.createEmpty(sourceDbPath, {
+      rootSubject: seedDb.rootSubject,
+    });
+    const seedTransformer = new IModelTransformer(seedDb, sourceDb);
+    await seedTransformer.process();
+    sourceDb.saveChanges();
+
+    const targetDbPath = IModelTransformerTestUtils.prepareOutputFile(
+      "IModelTransformer",
+      "PreserveIdOnTestModel-Target.bim"
+    );
+    const targetDb = SnapshotDb.createEmpty(targetDbPath, {
+      rootSubject: sourceDb.rootSubject,
+    });
+
+    // Calling process() for first time will add all elements from source to target
+    const transformer = new IModelTransformer(sourceDb, targetDb, {
+      preserveElementIdsForFiltering: true,
+    });
+    await transformer.process();
+    targetDb.saveChanges();
+
+    // should not throw error: duplicate code (65547)
+    const thirdTransformer = new IModelTransformer(sourceDb, targetDb, {
+      preserveElementIdsForFiltering: true,
+    });
+    await thirdTransformer.process();
+    targetDb.saveChanges();
+
+    const sourceContent = await getAllElementsInvariants(sourceDb);
+    const targetContent = await getAllElementsInvariants(targetDb);
+    expect(targetContent).to.deep.equal(sourceContent);
+
+    sourceDb.close();
+    targetDb.close();
+  });
+
   it("reference deletion is considered invalid when danglingReferencesBehavior='reject' and that is the default", async () => {
     const sourceDbPath = IModelTransformerTestUtils.prepareOutputFile(
       "IModelTransformer",