Make imports happen in chunks.

BrunoBernardino · Nov 29, 2020 · fc711df · fc711df
1 parent cb63b7c
commit fc711df
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 5 deletions.
diff --git a/lib/data-utils.ts b/lib/data-utils.ts
@@ -26,7 +26,12 @@ import { RxDBAttachmentsPlugin } from 'rxdb/plugins/attachments';
 import { RxDBLocalDocumentsPlugin } from 'rxdb/plugins/local-documents';
 import { RxDBQueryBuilderPlugin } from 'rxdb/plugins/query-builder';
 
-import { sortByName, sortByDate, showNotification } from './utils';
+import {
+  sortByName,
+  sortByDate,
+  showNotification,
+  splitArrayInChunks,
+} from './utils';
 import * as T from './types';
 
 const localDbName = './BudgetZen__data__v0';
@@ -596,8 +601,29 @@ export const importData = async (
       });
     }
 
-    await db.budgets.bulkInsert(budgets);
-    await db.expenses.bulkInsert(expenses);
+    const chunkLength = 200;
+
+    if (budgets.length > chunkLength) {
+      const chunkedBudgets = splitArrayInChunks(budgets, chunkLength);
+      for (const budgetsChunk of chunkedBudgets) {
+        await db.budgets.bulkInsert(budgetsChunk);
+        // Wait a second, to avoid hitting rate limits
+        await new Promise((resolve) => setTimeout(resolve, 1000));
+      }
+    } else {
+      await db.budgets.bulkInsert(budgets);
+    }
+
+    if (expenses.length > chunkLength) {
+      const chunkedExpenses = splitArrayInChunks(expenses, chunkLength);
+      for (const expensesChunk of chunkedExpenses) {
+        await db.expenses.bulkInsert(expensesChunk);
+        // Wait a second, to avoid hitting rate limits
+        await new Promise((resolve) => setTimeout(resolve, 1000));
+      }
+    } else {
+      await db.expenses.bulkInsert(expenses);
+    }
 
     return true;
   } catch (error) {

diff --git a/lib/utils.test.ts b/lib/utils.test.ts
@@ -1,4 +1,4 @@
-import { formatNumber } from './utils';
+import { formatNumber, splitArrayInChunks } from './utils';
 
 describe('lib/utils', () => {
   it('.formatNumber', () => {
@@ -16,4 +16,69 @@ describe('lib/utils', () => {
       expect(result).toEqual(test.expected);
     }
   });
+
+  it('.splitArrayInChunks', () => {
+    const tests = [
+      {
+        input: {
+          array: [
+            { number: 1 },
+            { number: 2 },
+            { number: 3 },
+            { number: 4 },
+            { number: 5 },
+            { number: 6 },
+          ],
+          chunkLength: 2,
+        },
+        expected: [
+          [{ number: 1 }, { number: 2 }],
+          [{ number: 3 }, { number: 4 }],
+          [{ number: 5 }, { number: 6 }],
+        ],
+      },
+      {
+        input: {
+          array: [
+            { number: 1 },
+            { number: 2 },
+            { number: 3 },
+            { number: 4 },
+            { number: 5 },
+          ],
+          chunkLength: 2,
+        },
+        expected: [
+          [{ number: 1 }, { number: 2 }],
+          [{ number: 3 }, { number: 4 }],
+          [{ number: 5 }],
+        ],
+      },
+      {
+        input: {
+          array: [
+            { number: 1 },
+            { number: 2 },
+            { number: 3 },
+            { number: 4 },
+            { number: 5 },
+            { number: 6 },
+          ],
+          chunkLength: 3,
+        },
+        expected: [
+          [{ number: 1 }, { number: 2 }, { number: 3 }],
+          [{ number: 4 }, { number: 5 }, { number: 6 }],
+        ],
+      },
+    ];
+
+    for (const test of tests) {
+      const result = splitArrayInChunks(
+        test.input.array,
+        test.input.chunkLength,
+      );
+      expect(result).toEqual(test.expected);
+    }
+  });
 });
diff --git a/lib/utils.ts b/lib/utils.ts
@@ -51,6 +51,18 @@ export const sortByMissingBudget = (
   return valueB - valueA;
 };
 
+export const splitArrayInChunks = (array: any[], chunkLength: number) => {
+  const chunks = [];
+  let chunkIndex = 0;
+  const arrayLength = array.length;
+
+  while (chunkIndex < arrayLength) {
+    chunks.push(array.slice(chunkIndex, (chunkIndex += chunkLength)));
+  }
+
+  return chunks;
+};
+
 export const showNotification = (
   message: string,
   type: 'success' | 'error' = 'success',

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "budgetzen-app",
   "version": "1.0.0",
-  "build": "L7E8tF",
+  "build": "WxpKARxZ",
   "author": "Bruno Bernardino <me@brunobernardino.com>",
   "license": "UNLICENSED",
   "repository": {