feat(downloadSequence): add download at runtime

src/browser.ts

@@ -14,6 +14,7 @@ import HttpError from "./download/download-engine/streams/download-engine-fetch-
 import BaseDownloadEngine from "./download/download-engine/engine/base-download-engine.js";
 import {InvalidOptionError} from "./download/download-engine/engine/error/InvalidOptionError.js";
 import {DownloadFlags, DownloadStatus} from "./download/download-engine/download-file/progress-status-file.js";
+import {NoDownloadEngineProvidedError} from "./download/download-engine/engine/error/no-download-engine-provided-error.js";
 
 export {
     DownloadFlags,
@@ -28,7 +29,8 @@ export {
     FetchStreamError,
     IpullError,
     EngineError,
-    InvalidOptionError
+    InvalidOptionError,
+    NoDownloadEngineProvidedError
 };
 
 export type {

src/download/browser-download.ts

@@ -1,5 +1,6 @@
 import DownloadEngineBrowser, {DownloadEngineOptionsBrowser} from "./download-engine/engine/download-engine-browser.js";
 import DownloadEngineMultiDownload from "./download-engine/engine/download-engine-multi-download.js";
+import {NoDownloadEngineProvidedError} from "./download-engine/engine/error/no-download-engine-provided-error.js";
 
 const DEFAULT_PARALLEL_STREAMS_FOR_BROWSER = 3;
 
@@ -25,5 +26,9 @@ export async function downloadFileBrowser(options: DownloadFileBrowserOptions) {
  * Download multiple files in the browser environment.
  */
 export async function downloadSequenceBrowser(...downloads: (DownloadEngineBrowser | Promise<DownloadEngineBrowser>)[]) {
+    if (downloads.length === 0) {
+        throw new NoDownloadEngineProvidedError();
+    }
+
     return await DownloadEngineMultiDownload.fromEngines(downloads);
 }

src/download/download-engine/download-file/progress-status-file.ts

@@ -13,6 +13,7 @@ export type ProgressStatus = {
 };
 
 export enum DownloadStatus {
+    Loading = "Loading",
     Active = "Active",
     Paused = "Paused",
     NotStarted = "NotStarted",
@@ -33,7 +34,7 @@ export default class ProgressStatusFile {
     public readonly downloadPart: number;
     public readonly transferredBytes: number;
     public readonly transferAction: string;
-    public readonly downloadStatus: DownloadStatus = DownloadStatus.Active;
+    public downloadStatus: DownloadStatus = DownloadStatus.Active;
     public downloadFlags: DownloadFlags[] = [];
     public totalBytes: number = 0;
     public startTime: number = 0;

src/download/download-engine/engine/download-engine-multi-download.ts

@@ -5,6 +5,7 @@ import BaseDownloadEngine, {BaseDownloadEngineEvents} from "./base-download-engi
 import DownloadAlreadyStartedError from "./error/download-already-started-error.js";
 import {concurrency} from "./utils/concurrency.js";
 import {DownloadFlags, DownloadStatus} from "../download-file/progress-status-file.js";
+import {NoDownloadEngineProvidedError} from "./error/no-download-engine-provided-error.js";
 
 const DEFAULT_PARALLEL_DOWNLOADS = 1;
 
@@ -27,7 +28,8 @@ export default class DownloadEngineMultiDownload<Engine extends DownloadEngineMu
     protected _progressStatisticsBuilder = new ProgressStatisticsBuilder();
     protected _downloadStatues: (ProgressStatusWithIndex | FormattedStatus)[] = [];
     protected _closeFiles: (() => Promise<void>)[] = [];
-
+    protected _lastStatus?: ProgressStatusWithIndex;
+    protected _loadingDownloads = 0;
 
     protected constructor(engines: (DownloadEngineMultiAllowedEngines | DownloadEngineMultiDownload)[], options: DownloadEngineMultiDownloadOptions) {
         super();
@@ -40,30 +42,68 @@ export default class DownloadEngineMultiDownload<Engine extends DownloadEngineMu
         return this._downloadStatues;
     }
 
+    public get status() {
+        if (!this._lastStatus) {
+            throw new NoDownloadEngineProvidedError();
+        }
+        return this._lastStatus;
+    }
+
     public get downloadSize(): number {
         return this.downloads.reduce((acc, engine) => acc + engine.downloadSize, 0);
     }
 
     protected _init() {
         this._progressStatisticsBuilder.downloadStatus = DownloadStatus.NotStarted;
-
-        this._changeEngineFinishDownload();
-        for (const [index, engine] of Object.entries(this.downloads)) {
-            const numberIndex = Number(index);
-            this._downloadStatues[numberIndex] = engine.status;
-            engine.on("progress", (progress) => {
-                this._downloadStatues[numberIndex] = progress;
-            });
-        }
-
-        this._progressStatisticsBuilder.add(...this.downloads);
         this._progressStatisticsBuilder.on("progress", progress => {
             progress = {
                 ...progress,
                 downloadFlags: progress.downloadFlags.concat([DownloadFlags.DownloadSequence])
             };
+            this._lastStatus = progress;
             this.emit("progress", progress);
         });
+
+        let index = 0;
+        for (const engine of this.downloads) {
+            this._addEngine(engine, index++);
+        }
+
+        // Prevent multiple progress events on adding engines
+        this._progressStatisticsBuilder.add(...this.downloads);
+    }
+
+    private _addEngine(engine: Engine, index: number) {
+        this._downloadStatues[index] = engine.status;
+        engine.on("progress", (progress) => {
+            this._downloadStatues[index] = progress;
+        });
+
+        this._changeEngineFinishDownload(engine);
+    }
+
+    public async addDownload(engine: Engine | DownloadEngineMultiDownload<any> | Promise<Engine | DownloadEngineMultiDownload<any>>) {
+        const index = this.downloads.length + this._loadingDownloads;
+        this._downloadStatues[index] = ProgressStatisticsBuilder.loadingStatusEmptyStatistics();
+
+        this._loadingDownloads++;
+        this._progressStatisticsBuilder._totalDownloadParts++;
+        const awaitEngine = engine instanceof Promise ? await engine : engine;
+        this._progressStatisticsBuilder._totalDownloadParts--;
+        this._loadingDownloads--;
+
+        if (awaitEngine instanceof DownloadEngineMultiDownload) {
+            let countEngines = 0;
+            for (const subEngine of awaitEngine.downloads) {
+                this._addEngine(subEngine, index + countEngines++);
+                this.downloads.push(subEngine);
+            }
+            this._progressStatisticsBuilder.add(...awaitEngine.downloads);
+        } else {
+            this._addEngine(awaitEngine, index);
+            this.downloads.push(awaitEngine);
+            this._progressStatisticsBuilder.add(awaitEngine);
+        }
     }
 
     public async download(): Promise<void> {
@@ -93,20 +133,18 @@ export default class DownloadEngineMultiDownload<Engine extends DownloadEngineMu
         await this.close();
     }
 
-    private _changeEngineFinishDownload() {
-        for (const engine of this.downloads) {
-            const options = engine._fileEngineOptions;
-            const onFinishAsync = options.onFinishAsync;
-            const onCloseAsync = options.onCloseAsync;
-
-            options.onFinishAsync = undefined;
-            options.onCloseAsync = undefined;
-            this._closeFiles.push(async () => {
-                await onFinishAsync?.();
-                await options.writeStream.close();
-                await onCloseAsync?.();
-            });
-        }
+    private _changeEngineFinishDownload(engine: Engine) {
+        const options = engine._fileEngineOptions;
+        const onFinishAsync = options.onFinishAsync;
+        const onCloseAsync = options.onCloseAsync;
+
+        options.onFinishAsync = undefined;
+        options.onCloseAsync = undefined;
+        this._closeFiles.push(async () => {
+            await onFinishAsync?.();
+            await options.writeStream.close();
+            await onCloseAsync?.();
+        });
     }
 
     private async _finishEnginesDownload() {

src/download/download-engine/engine/error/no-download-engine-provided-error.ts

@@ -0,0 +1,7 @@
+import EngineError from "./engine-error.js";
+
+export class NoDownloadEngineProvidedError extends EngineError {
+    constructor(error = "No download engine provided for download sequence") {
+        super(error);
+    }
+}

src/download/node-download.ts

@@ -3,6 +3,7 @@ import BaseDownloadEngine from "./download-engine/engine/base-download-engine.js
 import DownloadEngineMultiDownload, {DownloadEngineMultiDownloadOptions} from "./download-engine/engine/download-engine-multi-download.js";
 import CliAnimationWrapper, {CliProgressDownloadEngineOptions} from "./transfer-visualize/transfer-cli/cli-animation-wrapper.js";
 import {CLI_LEVEL} from "./transfer-visualize/transfer-cli/transfer-cli.js";
+import {NoDownloadEngineProvidedError} from "./download-engine/engine/error/no-download-engine-provided-error.js";
 
 const DEFAULT_PARALLEL_STREAMS_FOR_NODEJS = 3;
 export type DownloadFileOptions = DownloadEngineOptionsNodejs & CliProgressDownloadEngineOptions & {
@@ -45,6 +46,10 @@ export async function downloadSequence(options?: DownloadSequenceOptions | Downl
         downloadOptions = options;
     }
 
+    if (downloads.length === 0) {
+        throw new NoDownloadEngineProvidedError();
+    }
+
     downloadOptions.cliLevel = CLI_LEVEL.HIGH;
     const downloader = DownloadEngineMultiDownload.fromEngines(downloads, downloadOptions);
     const wrapper = new CliAnimationWrapper(downloader, downloadOptions);

src/download/transfer-visualize/progress-statistics-builder.ts

@@ -3,7 +3,8 @@ import {EventEmitter} from "eventemitter3";
 import TransferStatistics from "./transfer-statistics.js";
 import {createFormattedStatus, FormattedStatus} from "./format-transfer-status.js";
 import DownloadEngineFile from "../download-engine/download-file/download-engine-file.js";
-import {DownloadStatus, ProgressStatus} from "../download-engine/download-file/progress-status-file.js";
+import ProgressStatusFile, {DownloadStatus, ProgressStatus} from "../download-engine/download-file/progress-status-file.js";
+import DownloadEngineMultiDownload from "../download-engine/engine/download-engine-multi-download.js";
 
 export type ProgressStatusWithIndex = FormattedStatus & {
     index: number,
@@ -13,35 +14,47 @@ interface CliProgressBuilderEvents {
     progress: (progress: ProgressStatusWithIndex) => void;
 }
 
-export type AnyEngine = DownloadEngineFile | BaseDownloadEngine;
+export type AnyEngine = DownloadEngineFile | BaseDownloadEngine | DownloadEngineMultiDownload;
 export default class ProgressStatisticsBuilder extends EventEmitter<CliProgressBuilderEvents> {
     private _engines: AnyEngine[] = [];
     private _activeTransfers: { [index: number]: number } = {};
     private _totalBytes = 0;
     private _transferredBytes = 0;
-    private _totalDownloadParts = 0;
+    /**
+     * @internal
+     */
+    _totalDownloadParts = 0;
     private _activeDownloadPart = 0;
     private _startTime = 0;
-    private statistics = new TransferStatistics();
+    private _statistics = new TransferStatistics();
+    private _lastStatus?: ProgressStatusWithIndex;
     public downloadStatus: DownloadStatus = null!;
 
     public get totalBytes() {
         return this._totalBytes;
     }
 
-
     public get transferredBytesWithActiveTransfers() {
         return this._transferredBytes + Object.values(this._activeTransfers)
             .reduce((acc, bytes) => acc + bytes, 0);
     }
 
+    public get status() {
+        return this._lastStatus;
+    }
+
+    /**
+     * Add engines to the progress statistics builder, will only add engines once
+     */
     public add(...engines: AnyEngine[]) {
         for (const engine of engines) {
-            this._initEvents(engine);
+            if (!this._engines.includes(engine)) {
+                this._initEvents(engine, engines.at(-1) === engine);
+            }
         }
     }
 
-    private _initEvents(engine: AnyEngine) {
+    private _initEvents(engine: AnyEngine, sendProgress = false) {
         this._engines.push(engine);
         this._totalBytes += engine.downloadSize;
         const index = this._engines.length - 1;
@@ -56,6 +69,10 @@ export default class ProgressStatisticsBuilder extends EventEmitter<CliProgressB
             delete this._activeTransfers[index];
             this._transferredBytes += engine.downloadSize;
         });
+
+        if (sendProgress) {
+            this._sendProgress(engine.status, index, downloadPartStart);
+        }
     }
 
 
@@ -66,10 +83,10 @@ export default class ProgressStatisticsBuilder extends EventEmitter<CliProgressB
             this._activeDownloadPart = downloadPartStart + data.downloadPart;
         }
 
-        const progress = this.statistics.updateProgress(this.transferredBytesWithActiveTransfers, this.totalBytes);
+        const progress = this._statistics.updateProgress(this.transferredBytesWithActiveTransfers, this.totalBytes);
         const activeDownloads = Object.keys(this._activeTransfers).length;
 
-        this.emit("progress", {
+        this._lastStatus = {
             ...createFormattedStatus({
                 ...progress,
                 downloadPart: this._activeDownloadPart,
@@ -83,7 +100,9 @@ export default class ProgressStatisticsBuilder extends EventEmitter<CliProgressB
                 downloadFlags: data.downloadFlags
             }),
             index
-        });
+        };
+
+        this.emit("progress", this._lastStatus);
     }
 
     static oneStatistics(engine: DownloadEngineFile) {
@@ -95,4 +114,15 @@ export default class ProgressStatisticsBuilder extends EventEmitter<CliProgressB
             ...statistics
         });
     }
+
+    static loadingStatusEmptyStatistics() {
+        const statistics = TransferStatistics.oneStatistics(0, 0);
+        const status = new ProgressStatusFile(0, "???");
+        status.downloadStatus = DownloadStatus.Loading;
+
+        return createFormattedStatus({
+            ...status,
+            ...statistics
+        });
+    }
 }

src/download/transfer-visualize/transfer-cli/cli-animation-wrapper.ts

@@ -4,6 +4,7 @@ import switchCliProgressStyle, {AvailableCLIProgressStyle} from "./progress-bars
 import {CliFormattedStatus} from "./progress-bars/base-transfer-cli-progress-bar.js";
 import TransferCli, {CLI_LEVEL, TransferCliOptions} from "./transfer-cli.js";
 import {BaseMultiProgressBar} from "./multiProgressBars/BaseMultiProgressBar.js";
+import cliSpinners from "cli-spinners";
 
 const DEFAULT_CLI_STYLE: AvailableCLIProgressStyle = "auto";
 type AllowedDownloadEngines = DownloadEngineNodejs | DownloadEngineMultiDownload;
@@ -17,6 +18,7 @@ export type CliProgressDownloadEngineOptions = {
     cliName?: string;
     cliAction?: string;
     fetchStrategy?: "localFile" | "fetch";
+    loadingAnimation?: cliSpinners.SpinnerName;
     /** @internal */
     cliLevel?: CLI_LEVEL;
 };
@@ -51,7 +53,10 @@ export default class CliAnimationWrapper {
                 createStatusLine: this._options.cliStyle,
                 multiProgressBar: this._options.createMultiProgressBar ?? BaseMultiProgressBar
             } :
-            switchCliProgressStyle(this._options.cliStyle ?? DEFAULT_CLI_STYLE, {truncateName: this._options.truncateName});
+            switchCliProgressStyle(this._options.cliStyle ?? DEFAULT_CLI_STYLE, {
+                truncateName: this._options.truncateName,
+                loadingSpinner: this._options.loadingAnimation
+            });
 
         this._activeCLI = new TransferCli(cliOptions, this._options.cliLevel);
     }

src/download/transfer-visualize/transfer-cli/loading-animation/base-loading-animation.ts

@@ -48,7 +48,7 @@ export default abstract class BaseLoadingAnimation {
         this._animationActive = true;
         while (this._animationActive) {
             this._render();
-            await sleep(this.options.updateIntervalMs ?? DEFAULT_UPDATE_INTERVAL_MS);
+            await sleep(this.options.updateIntervalMs || DEFAULT_UPDATE_INTERVAL_MS);
         }
     }
 

src/download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.ts

@@ -32,16 +32,18 @@ export class BaseMultiProgressBar {
      */
     protected recorderStatusByImportance(statuses: FormattedStatus[]) {
         const activeTasks = statuses.filter(status => status.downloadStatus === DownloadStatus.Active);
-        const remaining = statuses.filter(status => status.downloadStatus === DownloadStatus.Paused || status.downloadStatus === DownloadStatus.NotStarted);
+        const remaining = statuses.filter(status => [DownloadStatus.Paused, DownloadStatus.NotStarted].includes(status.downloadStatus));
+        const loading = statuses.filter(status => status.downloadStatus === DownloadStatus.Loading);
         const finishedTasks = statuses.filter(status => status.downloadStatus === DownloadStatus.Finished)
             .sort((a, b) => b.endTime - a.endTime);
 
-        const showTotalTasks = activeTasks.concat(remaining);
+        const showTotalTasks = activeTasks.concat(remaining)
+            .concat(loading);
         const showTotalTasksWithFinished = showTotalTasks.concat(finishedTasks);
 
         return {
             notFinished: showTotalTasks.length > 0,
-            remaining: remaining.length,
+            remaining: remaining.length + loading.length,
             allStatusesSorted: showTotalTasksWithFinished
         };
     }