Adds transfers between stores to external attachments (#1358)

This PR follows on from #1320, and adds support for:
- Moving attachments between stores
- Basic configuration of external attachments using environment variables

This allows a user to change the document's store, then transfer all of the attachments from their current store(s) to the new default.

This includes transfers from internal (SQLite) storage to external storage, external to internal, and external to external (e.g MinIO to filesystem).

This PR also introduces the concept of "labels", which are an admin-friendly way to refer to a store, and map 1-to-1 with store IDs. Labels don't need to be unique between instances, only within an instance.

### User-facing changes:
-  Adds API endpoints to:
  - Migrate all attachments from their current store to the store set for that document
  - Check on the status of transfers
  - Get and set the store for a document
- Adds an environment variable for setting external attachments behaviour `GRIST_EXTERNAL_ATTACHMENTS_MODE`
  - `test` mode sets Grist to use a temporary folder in the filesystem.
  - `snapshots` mode sets Grist to use the external storage currently used for snapshots, to also be used for attachments.

### Internal changes:
- Adds methods to AttachmentFileManager to facilitate transfers from one storage to another.
- Exposes new methods on ActiveDoc for triggering transfers, retrieving transfer status and setting the store.
- Refactors how DocStorage provides access to attachment details
- Adds a way to retrieve attachment config from env vars, and use them to decide which stores will be available.
- Adds a `snapshot` external storage provider, that uses an attachment-compatible external storage for attachments.

All of the logic behind these changes should be documented in the source code with comments.

Author: Spoffy

app/common/UserAPI.ts

@@ -25,6 +25,7 @@ import {
 import {addCurrentOrgToPath, getGristConfig} from 'app/common/urlUtils';
 import { AxiosProgressEvent } from 'axios';
 import omitBy from 'lodash/omitBy';
+import {StringUnion} from 'app/common/StringUnion';
 
 
 export type {FullUser, UserProfile};
@@ -481,6 +482,11 @@ interface SqlResult extends TableRecordValuesWithoutIds {
   statement: string;
 }
 
+export const DocAttachmentsLocation = StringUnion(
+  "none", "internal", "mixed", "external"
+);
+export type DocAttachmentsLocation = typeof DocAttachmentsLocation.type;
+
 /**
  * Collect endpoints related to the content of a single document that we've been thinking
  * of as the (restful) "Doc API".  A few endpoints that could be here are not, for historical

app/plugin/DocApiTypes-ti.ts

@@ -90,6 +90,10 @@ export const SqlPost = t.iface([], {
   "timeout": t.opt("number"),
 });
 
+export const SetAttachmentStorePost = t.iface([], {
+  "type": t.union(t.lit("internal"), t.lit("external")),
+});
+
 const exportedTypeSuite: t.ITypeSuite = {
   NewRecord,
   NewRecordWithStringId,
@@ -108,5 +112,6 @@ const exportedTypeSuite: t.ITypeSuite = {
   TablesPost,
   TablesPatch,
   SqlPost,
+  SetAttachmentStorePost,
 };
 export default exportedTypeSuite;

app/plugin/DocApiTypes.ts

@@ -120,3 +120,7 @@ export interface SqlPost {
                      // other queued queries on same document, because of
                      // limitations of API node-sqlite3 exposes.
 }
+
+export interface SetAttachmentStorePost {
+  type: "internal" | "external"
+}

app/server/generateInitialDocSql.ts

@@ -34,11 +34,13 @@ export async function main(baseName: string) {
     if (await fse.pathExists(fname)) {
       await fse.remove(fname);
     }
-    const docManager = new DocManager(storageManager, pluginManager, null as any, new AttachmentStoreProvider([], ""), {
-      create,
-      getAuditLogger() { return createNullAuditLogger(); },
-      getTelemetry() { return createDummyTelemetry(); },
-    } as any);
+    const docManager = new DocManager(storageManager, pluginManager, null as any,
+      new AttachmentStoreProvider([], ""), {
+        create,
+        getAuditLogger() { return createNullAuditLogger(); },
+        getTelemetry() { return createDummyTelemetry(); },
+      } as any
+    );
     const activeDoc = new ActiveDoc(docManager, baseName);
     const session = makeExceptionalDocSession('nascent');
     await activeDoc.createEmptyDocWithDataEngine(session);

app/server/lib/ActiveDoc.ts

@@ -286,7 +286,7 @@ export class ActiveDoc extends EventEmitter {
   constructor(
     private readonly _docManager: DocManager,
     private _docName: string,
-    externalAttachmentStoreProvider?: IAttachmentStoreProvider,
+    private _attachmentStoreProvider?: IAttachmentStoreProvider,
     private _options?: ICreateActiveDocOptions
   ) {
     super();
@@ -392,11 +392,11 @@ export class ActiveDoc extends EventEmitter {
       loadTable: this._rawPyCall.bind(this, 'load_table'),
     });
 
-    // This will throw errors if _options?.doc or externalAttachmentStoreProvider aren't provided,
+    // This will throw errors if _options?.doc or _attachmentStoreProvider aren't provided,
     // and ActiveDoc tries to use an external attachment store.
     this._attachmentFileManager = new AttachmentFileManager(
       this.docStorage,
-      externalAttachmentStoreProvider,
+      _attachmentStoreProvider,
       _options?.doc,
     );
 
@@ -871,8 +871,12 @@ export class ActiveDoc extends EventEmitter {
           }
         }
       );
-      const userActions: UserAction[] = await Promise.all(
-        upload.files.map(file => this._prepAttachment(docSession, file)));
+      const userActions: UserAction[] = [];
+      // Process uploads sequentially to reduce risk of race conditions.
+      // Minimal performance impact due to the main async operation being serialized SQL queries.
+      for (const file of upload.files) {
+        userActions.push(await this._prepAttachment(docSession, file));
+      }
       const result = await this._applyUserActionsWithExtendedOptions(docSession, userActions, {
         attachment: true,
       });
@@ -945,6 +949,57 @@ export class ActiveDoc extends EventEmitter {
     return data;
   }
 
+  @ActiveDoc.keepDocOpen
+  public async startTransferringAllAttachmentsToDefaultStore() {
+    const attachmentStoreId = (await this._getDocumentSettings()).attachmentStoreId;
+    // If no attachment store is set on the doc, it should transfer everything to internal storage
+    await this._attachmentFileManager.startTransferringAllFilesToOtherStore(attachmentStoreId);
+  }
+
+  /**
+   * Returns a summary of pending attachment transfers between attachment stores.
+   */
+  public attachmentTransferStatus() {
+    return this._attachmentFileManager.transferStatus();
+  }
+
+  /**
+   * Returns a summary of where attachments on this doc are stored.
+   */
+  public async attachmentLocationSummary() {
+    return await this._attachmentFileManager.locationSummary();
+  }
+
+  /*
+   * Wait for all attachment transfers to be finished, keeping the doc open
+   * for as long as possible.
+   */
+  @ActiveDoc.keepDocOpen
+  public async allAttachmentTransfersCompleted() {
+      await this._attachmentFileManager.allTransfersCompleted();
+  }
+
+
+  public async setAttachmentStore(docSession: OptDocSession, id: string | undefined): Promise<void> {
+    const docSettings = await this._getDocumentSettings();
+    docSettings.attachmentStoreId = id;
+    await this._updateDocumentSettings(docSession, docSettings);
+  }
+
+  /**
+   * Sets the document attachment store using the store's label.
+   * This avoids needing to know the exact store ID, which can be challenging to calculate in all
+   * the places we might want to set the store.
+   */
+  public async setAttachmentStoreFromLabel(docSession: OptDocSession, label: string | undefined): Promise<void> {
+    const id = label === undefined ? undefined : this._attachmentStoreProvider?.getStoreIdFromLabel(label);
+    return this.setAttachmentStore(docSession, id);
+  }
+
+  public async getAttachmentStore(): Promise<string | undefined> {
+    return (await this._getDocumentSettings()).attachmentStoreId;
+  }
+
   /**
    * Fetches the meta tables to return to the client when first opening a document.
    */
@@ -2857,15 +2912,24 @@ export class ActiveDoc extends EventEmitter {
   }
 
   private async _getDocumentSettings(): Promise<DocumentSettings> {
-    const docInfo = await this.docStorage.get('SELECT documentSettings FROM _grist_DocInfo');
-    const docSettingsString = docInfo?.documentSettings;
-    const docSettings = docSettingsString ? safeJsonParse(docSettingsString, undefined) : undefined;
+    const docSettings = this.docData?.docSettings();
     if (!docSettings) {
       throw new Error("No document settings found");
     }
     return docSettings;
   }
 
+  private async _updateDocumentSettings(docSessions: OptDocSession, settings: DocumentSettings): Promise<void> {
+    const docInfo = this.docData?.docInfo();
+    if (!docInfo) {
+      throw new Error("No document info found");
+    }
+    await this.applyUserActions(docSessions, [
+      // Use docInfo.id to avoid hard-coding a reference to a specific row id, in case it changes.
+      ["UpdateRecord", "_grist_DocInfo", docInfo.id, { documentSettings: JSON.stringify(settings) }]
+    ]);
+  }
+
   private async _makeEngine(): Promise<ISandbox> {
     // Figure out what kind of engine we need for this document.
     let preferredPythonVersion: '2' | '3' = process.env.PYTHON_VERSION === '2' ? '2' : '3';