Add native loading progress tracking (#180)

src/engine/runtime.js

@@ -518,6 +518,16 @@ class Runtime extends EventEmitter {
          * @type {Object.<string, object>}
          */
         this.extensionStorage = {};
+
+        /**
+         * Total number of scratch-storage load() requests since the runtime was created or cleared.
+         */
+        this.totalAssetRequests = 0;
+
+        /**
+         * Total number of finished or errored scratch-storage load() requests since the runtime was created or cleared.
+         */
+        this.finishedAssetRequests = 0;
     }
 
     /**
@@ -657,6 +667,14 @@ class Runtime extends EventEmitter {
         return 'AFTER_EXECUTE';
     }
 
+    /**
+     * Event name for reporting asset download progress. Fired with finished, total
+     * @const {string}
+     */
+    static get ASSET_PROGRESS () {
+        return 'ASSET_PROGRESS';
+    }
+
     /**
      * Event name when the project is started (threads may not necessarily be
      * running).
@@ -2256,6 +2274,8 @@ class Runtime extends EventEmitter {
         this.getNumberOfCloudVariables = newCloudDataManager.getNumberOfCloudVariables;
         this.addCloudVariable = this._initializeAddCloudVariable(newCloudDataManager);
         this.removeCloudVariable = this._initializeRemoveCloudVariable(newCloudDataManager);
+
+        this.resetProgress();
     }
 
     /**
@@ -3393,6 +3413,39 @@ class Runtime extends EventEmitter {
         this.externalCommunicationMethods[method] = enabled;
         this.updatePrivacy();
     }
+
+    emitAssetProgress () {
+        this.emit(Runtime.ASSET_PROGRESS, this.finishedAssetRequests, this.totalAssetRequests);
+    }
+
+    resetProgress () {
+        this.finishedAssetRequests = 0;
+        this.totalAssetRequests = 0;
+        this.emitAssetProgress();
+    }
+
+    /**
+     * Wrap an asset loading promise with progress support.
+     * @template T
+     * @param {Promise<T>} promise
+     * @returns {Promise<T>}
+     */
+    wrapAssetRequest (promise) {
+        this.totalAssetRequests++;
+        this.emitAssetProgress();
+
+        return promise
+            .then(result => {
+                this.finishedAssetRequests++;
+                this.emitAssetProgress();
+                return result;
+            })
+            .catch(error => {
+                this.finishedAssetRequests++;
+                this.emitAssetProgress();
+                throw error;
+            });
+    }
 }
 
 /**

src/serialization/sb2.js

@@ -500,9 +500,10 @@ const parseScratchAssets = function (object, runtime, topLevel, zip) {
             // the file name of the costume should be the baseLayerID followed by the file ext
             const assetFileName = `${costumeSource.baseLayerID}.${ext}`;
             const textLayerFileName = costumeSource.textLayerID ? `${costumeSource.textLayerID}.png` : null;
-            costumePromises.push(deserializeCostume(costume, runtime, zip, assetFileName, textLayerFileName)
-                .then(() => loadCostume(costume.md5, costume, runtime, 2 /* optVersion */))
-            );
+            costumePromises.push(runtime.wrapAssetRequest(
+                deserializeCostume(costume, runtime, zip, assetFileName, textLayerFileName)
+                    .then(() => loadCostume(costume.md5, costume, runtime, 2 /* optVersion */))
+            ));
         }
     }
     // Sounds from JSON
@@ -535,10 +536,10 @@ const parseScratchAssets = function (object, runtime, topLevel, zip) {
             // the file name of the sound should be the soundID (provided from the project.json)
             // followed by the file ext
             const assetFileName = `${soundSource.soundID}.${ext}`;
-            soundPromises.push(
+            soundPromises.push(runtime.wrapAssetRequest(
                 deserializeSound(sound, runtime, zip, assetFileName)
                     .then(() => loadSound(sound, runtime, soundBank))
-            );
+            ));
         }
     }
 

src/serialization/sb3.js

@@ -1111,8 +1111,8 @@ const parseScratchAssets = function (object, runtime, zip) {
         // we're always loading the 'sb3' representation of the costume
         // any translation that needs to happen will happen in the process
         // of building up the costume object into an sb3 format
-        return deserializeCostume(costume, runtime, zip)
-            .then(() => loadCostume(costumeMd5Ext, costume, runtime));
+        return runtime.wrapAssetRequest(deserializeCostume(costume, runtime, zip)
+            .then(() => loadCostume(costumeMd5Ext, costume, runtime)));
         // Only attempt to load the costume after the deserialization
         // process has been completed
     });
@@ -1136,8 +1136,8 @@ const parseScratchAssets = function (object, runtime, zip) {
         // we're always loading the 'sb3' representation of the costume
         // any translation that needs to happen will happen in the process
         // of building up the costume object into an sb3 format
-        return deserializeSound(sound, runtime, zip)
-            .then(() => loadSound(sound, runtime, assets.soundBank));
+        return runtime.wrapAssetRequest(deserializeSound(sound, runtime, zip)
+            .then(() => loadSound(sound, runtime, assets.soundBank)));
         // Only attempt to load the sound after the deserialization
         // process has been completed.
     });

src/util/tw-asset-util.js

@@ -9,7 +9,6 @@ class AssetUtil {
      * @returns {Promise<Storage.Asset>} scratch-storage asset object
      */
     static getByMd5ext (runtime, zip, assetType, md5ext) {
-        const storage = runtime.storage;
         const idParts = StringUtil.splitFirst(md5ext, '.');
         const md5 = idParts[0];
         const ext = idParts[1].toLowerCase();
@@ -26,17 +25,17 @@ class AssetUtil {
             }
 
             if (file) {
-                return file.async('uint8array').then(data => runtime.storage.createAsset(
+                return runtime.wrapAssetRequest(file.async('uint8array').then(data => runtime.storage.createAsset(
                     assetType,
                     ext,
                     data,
                     md5,
                     false
-                ));
+                )));
             }
         }
 
-        return storage.load(assetType, md5, ext);
+        return runtime.wrapAssetRequest(runtime.storage.load(assetType, md5, ext));
     }
 }
 

src/virtual-machine.js

@@ -190,6 +190,9 @@ class VirtualMachine extends EventEmitter {
         this.runtime.on(Runtime.COMPILE_ERROR, (target, error) => {
             this.emit(Runtime.COMPILE_ERROR, target, error);
         });
+        this.runtime.on(Runtime.ASSET_PROGRESS, (finished, total) => {
+            this.emit(Runtime.ASSET_PROGRESS, finished, total);
+        });
         this.runtime.on(Runtime.TURBO_MODE_OFF, () => {
             this.emit(Runtime.TURBO_MODE_OFF);
         });

test/integration/tw_asset_progress.js

@@ -0,0 +1,140 @@
+const {test} = require('tap');
+const fs = require('fs');
+const path = require('path');
+const Runtime = require('../../src/engine/runtime');
+const VirtualMachine = require('../../src/virtual-machine');
+const makeTestStorage = require('../fixtures/make-test-storage');
+const AssetUtil = require('../../src/util/tw-asset-util');
+
+test('emitAssetProgress', t => {
+    const vm = new VirtualMachine();
+
+    let runtimeOK = false;
+    let vmOK = false;
+    vm.runtime.on('ASSET_PROGRESS', (finished, total) => {
+        t.equal(finished, 1, 'runtime finished');
+        t.equal(total, 2, 'runtime total');
+        runtimeOK = true;
+    });
+    vm.on('ASSET_PROGRESS', (finished, total) => {
+        t.equal(finished, 1, 'vm finished');
+        t.equal(total, 2, 'vm total');
+        vmOK = true;
+    });
+
+    vm.runtime.totalAssetRequests = 2;
+    vm.runtime.finishedAssetRequests = 1;
+    vm.runtime.emitAssetProgress();
+
+    t.ok(runtimeOK, 'runtime');
+    t.ok(vmOK, 'vm');
+    t.end();
+});
+
+test('resetProgress', t => {
+    t.plan(4);
+
+    const runtime = new Runtime();
+    runtime.finishedAssetRequests = 10;
+    runtime.totalAssetRequests = 10;
+
+    runtime.on('ASSET_PROGRESS', (finished, total) => {
+        t.equal(finished, 0, 'event finished');
+        t.equal(total, 0, 'event total');
+    });
+
+    runtime.resetProgress();
+
+    t.equal(runtime.finishedAssetRequests, 0, 'property finishedAssetRequests');
+    t.equal(runtime.totalAssetRequests, 0, 'property totalAssetRequests');
+    t.end();
+});
+
+test('dispose', t => {
+    t.plan(1);
+    const runtime = new Runtime();
+    runtime.resetProgress = () => {
+        t.pass();
+    };
+    runtime.dispose();
+    t.end();
+});
+
+test('wrapAssetRequest', t => {
+    const runtime = new Runtime();
+
+    const log = [];
+    runtime.on('ASSET_PROGRESS', (finished, total) => {
+        log.push([finished, total]);
+    });
+
+    Promise.all([
+        runtime.wrapAssetRequest(Promise.resolve(1)),
+        runtime.wrapAssetRequest(Promise.resolve(2))
+    ]).then(results => {
+        t.same(results, [1, 2]);
+
+        // eslint-disable-next-line prefer-promise-reject-errors
+        runtime.wrapAssetRequest(Promise.reject(3)).catch(error => {
+            t.equal(error, 3);
+            t.same(log, [
+                [0, 1],
+                [0, 2],
+                [1, 2],
+                [2, 2],
+                [2, 3],
+                [3, 3]
+            ]);
+            t.end();
+        });
+    });
+});
+
+test('asset util emits progress', t => {
+    const runtime = new Runtime();
+
+    const storage = makeTestStorage();
+    storage.load = (assetType, assetId) => Promise.resolve({
+        assetId
+    });
+    runtime.attachStorage(storage);
+
+    const log = [];
+    runtime.on('ASSET_PROGRESS', (finished, total) => {
+        log.push([finished, total]);
+    });
+
+    AssetUtil.getByMd5ext(runtime, null, runtime.storage.AssetType.SVG, 'abcdef.svg').then(() => {
+        t.same(log, [
+            [0, 1],
+            [1, 1]
+        ]);
+        t.end();
+    });
+});
+
+//  For the next tests, we have some fixtures that contain 2 assets: 1 sound + 1 costume
+// We'll just load them and make sure that each deserializer emits reasonable progress events
+for (const format of ['sb', 'sb2', 'sb3']) {
+    test(format, t => {
+        const fixture = fs.readFileSync(path.join(__dirname, `../fixtures/tw-asset-progress.${format}`));
+        const vm = new VirtualMachine();
+
+        const log = [];
+        vm.on('ASSET_PROGRESS', (finished, total) => {
+            log.push([finished, total]);
+        });
+
+        vm.loadProject(fixture)
+            .then(() => {
+                t.same(log, [
+                    [0, 0], // loadProject() implies dispose()
+                    [0, 1],
+                    [0, 2],
+                    [1, 2],
+                    [2, 2]
+                ]);
+                t.end();
+            });
+    });
+}

test/unit/tw_asset_util.js

@@ -35,15 +35,23 @@ test('getByMd5ext from zip subdirectory', t => {
 });
 
 test('getByMd5ext from storage with null zip', t => {
+    t.plan(4);
+
     const rt = new Runtime();
     rt.attachStorage(makeTestStorage());
 
     rt.storage.load = (assetType, md5, ext) => {
         t.equal(assetType, rt.storage.AssetType.SVG);
         t.equal(md5, '00000000000000000000000000000000');
         t.equal(ext, 'svg');
-        t.end();
+        return Promise.resolve({
+            fromStorage: true
+        });
     };
 
-    AssetUtil.getByMd5ext(rt, null, rt.storage.AssetType.SVG, '00000000000000000000000000000000.svg');
+    AssetUtil.getByMd5ext(rt, null, rt.storage.AssetType.SVG, '00000000000000000000000000000000.svg')
+        .then(asset => {
+            t.ok(asset.fromStorage);
+            t.end();
+        });
 });