Add file cache, increase test coverage

.env.sample

@@ -9,4 +9,4 @@ DEBUG_LOGGING = false
 
 # Set prefered exchange if you have securities that are traded at multiple exchanges, like Vanguard FTSE All-World UCITS ETF (VWRL.AS, VWCE.DE, VRRA.L)
 # Leave commented if you don't want to use this.
-#DEGIRO_PREFERED_EXCHANGE_POSTFIX = ".AS"
+#DEGIRO_PREFERED_EXCHANGE_POSTFIX = ".AS"

.github/workflows/frameworkTesting.yml

@@ -5,7 +5,6 @@ on:
   push:
     branches:
       - main
-      - feature/*
     paths:
       - "src/**"
   pull_request:
@@ -72,7 +71,7 @@ jobs:
           {
             "schemaVersion": 1,
             "label": "Code Coverage",
-            "message": "$coverage",
+            "message": "$coverage%",
             "style": "for-the-badge",
             "color": "$color"
           }

GitVersion.yml

@@ -1,4 +1,4 @@
-next-version: 0.4.0
+next-version: 0.5.0
 assembly-informational-format: "{NuGetVersion}"
 mode: ContinuousDeployment
 branches:

README.md

@@ -88,11 +88,16 @@ The following parameters can be given to the Docker run command.
 | `--env USE_POLLING=true` | Y | When set to true, the container will continously look for new files to process and the container will not stop. |
 | `--env DEBUG_LOGGING=true` | Y | When set to true, the container will show logs in more detail, useful for error tracing. |
 | `--env FORCE_DEGIRO_V2=true` | Y | When set to true, the converter will use the DEGIRO V2 converter (currently in beta) when a DEGIRO file was found. |
+| `--env PURGE_CACHE=true` | Y | When set to true, the file cache will be purged on start. |
 
 1: You can retrieve your Ghostfolio account ID by going to Accounts > select your account and copying the ID from the URL.
 
 ![image](https://user-images.githubusercontent.com/5620002/203353840-f5db7323-fb2f-4f4f-befc-e4e340466a74.png)
 
+### Caching
+
+The tool uses `cacache` to store data retrieved from Yahoo Finance inside the container. This way the load on Yahoo Finance is reduced and the tool should run faster. The cached data is stored inside the container in `tmp/e2g-cache`. If you feel you need to invalidate your cache, you can do so by adding `--env PURGE_CACHE=true` to your run command. This will clear the cache on container start, and the tool will recreate the cache the next time it has to retrieve data from Yahoo Finance.
+
 </details>
 
 ## Run locally
@@ -129,6 +134,10 @@ You can now run `npm run start [exporttype]`. See the table with run commands be
 | Swissquote  | `run start swissquote` (or `sq`)    |
 | Schwab      | `run start schwab`                  |
 
+### Caching
+
+The tool uses `cacache` to store data retrieved from Yahoo Finance on disk. This way the load on Yahoo Finance is reduced and the tool should run faster. The cached data is stored in `tmp/e2g-cache`. If you feel you need to invalidate your cache, you can do so by removing the folder and the tool will recreate the cache when you run it the next time.
+
 </details>
 
 ## Import to Ghostfolio

jest.config.ts

@@ -3,12 +3,17 @@ import type { Config } from '@jest/types';
 // Sync object
 const config: Config.InitialOptions = {
   verbose: true,
-  testTimeout: 30000,
+  testTimeout: 15000,
   transform: {
     '^.+\\.tsx?$': 'ts-jest'
   },
   coverageDirectory: 'coverage',
   collectCoverageFrom: ['src/**/*.ts'],
+  coveragePathIgnorePatterns: [
+    '/src/models',
+    '/src/manual.ts',
+    '/src/watcher.ts',
+    '/src/converter.ts'],
   coverageReporters: ['text', 'cobertura', 'html']
 };
 

package.json

@@ -13,6 +13,7 @@
   },
   "license": "Apache-2.0",
   "devDependencies": {
+    "@types/cacache": "^17.0.2",
     "@types/jest": "^29.5.11",
     "@types/node": "^20.10.4",
     "jest": "^29.7.0",
@@ -22,6 +23,7 @@
   },
   "dependencies": {
     "@types/cli-progress": "^3.11.5",
+    "cacache": "^18.0.2",
     "chokidar": "^3.5.3",
     "cli-progress": "^3.12.0",
     "closest-match": "^1.3.3",

sample-degiro-export.csv

@@ -40,4 +40,8 @@ Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id
 25-07-2023,14:31,25-07-2023,ISHARES KOREA,IE00B0M63391,"Koop 1 @ 42,53 EUR",,EUR,-42.53,EUR,45.92,d4f45240-d763-4b95-8590-2ba158d38207
 15-05-2019,09:05,15-05-2019,ISHARES MSCI WOR A,IE00B4L5Y983,"Compra 6 ISHARES MSCI WOR A@49,785 EUR (IE00B4L5Y983)",,EUR,-298.71,EUR,0.64,a47e2746-bfbd-4654-bd6c-5e58e470d32f
 02-01-2024,14:42,02-01-2024,ISHARES MSCI WOR A,IE00B4L5Y983,Comissões de transação DEGIRO e/ou taxas de terceiros,,EUR,-1.00,EUR,2.54,7b377a93-5695-4131-8954-5c78996fbed4
-02-01-2024,14:42,02-01-2024,ISHARES MSCI WOR A,IE00B4L5Y983,"Compra 1 ISHARES MSCI WOR A@82,055 EUR (IE00B4L5Y983)",,EUR,-82.06,EUR,3.54,7b377a93-5695-4131-8954-5c78996fbed4
+02-01-2024,14:42,02-01-2024,ISHARES MSCI WOR A,IE00B4L5Y983,"Compra 1 ISHARES MSCI WOR A@82,055 EUR (IE00B4L5Y983)",,EUR,-82.06,EUR,3.54,7b377a93-5695-4131-8954-5c78996fbed4
+05-02-2024,07:16,31-01-2024,,,DEGIRO Aansluitingskosten 2024 (Nasdaq - NDQ),,EUR,-2.50,EUR,-6.28,
+05-02-2024,07:16,31-01-2024,,,DEGIRO Aansluitingskosten 2024 (Euronext Milan - MIL),,EUR,-2.50,EUR,-3.78,
+05-02-2024,07:16,31-01-2024,,,DEGIRO Aansluitingskosten 2024 (Xetra - XET),,EUR,-2.50,EUR,-1.28,
+02-02-2024,08:00,01-02-2024,AT&T INC.,US00206R1023,Dividend,,USD,1.39,USD,1.39,

sample-schwab-export.csv

@@ -98,4 +98,5 @@ Date,Action,Symbol,Description,Quantity,Price,Fees & Comm,Amount
 01/03/2023,Reinvest Shares,NMFC,NEW MOUNTAIN FIN CO,10.9694,$12.45,,-$136.56
 01/03/2023,Reinvest Shares,TCPC,BLACKROCK TCP CAPITAL CO,10.0443,$13.17,,-$132.25
 "09/08/2023","Sell","SNAXX","SCHWAB VALUE ADVANTAGE MONEY ULTRA","500,135","$1.00","","$500135.00"
+"10/18/2023","Wire Sent","","WIRED FUNDS DISBURSED","","","","-$100000.00"
 Transactions Total,,,,,,,"-$26,582.91"

src/converter.ts

@@ -11,8 +11,7 @@ import { SwissquoteConverter } from "./converters/swissquoteConverter";
 import { FinpensionConverter } from "./converters/finpensionConverter";
 import { YahooFinanceService } from "./yahooFinanceService";
 
-
-export function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) {
+export async function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) {
 
     // Verify if Ghostolio account ID is set (because without it there can be no valid output).
     if (!process.env.GHOSTFOLIO_ACCOUNT_ID) {
@@ -22,7 +21,7 @@ export function createAndRunConverter(converterType: string, inputFilePath: stri
     const converterTypeLc = converterType.toLocaleLowerCase();
 
     // Determine convertor type.
-    const converter = createConverter(converterTypeLc);
+    const converter = await createConverter(converterTypeLc);
 
     // Map the file to a Ghostfolio import.
     converter.readAndProcessFile(inputFilePath, (result: GhostfolioExport) => {
@@ -41,10 +40,13 @@ export function createAndRunConverter(converterType: string, inputFilePath: stri
     }, (error) => errorCallback(error));
 }
 
-function createConverter(converterType: string): AbstractConverter {
+async function createConverter(converterType: string): Promise<AbstractConverter> {
 
     const yahooFinanceService = new YahooFinanceService();
 
+    const cacheSize = await yahooFinanceService.loadCache();
+    console.log(`[i] Restored ${cacheSize[0]} ISIN-symbol pairs and ${cacheSize[1]} symbols from cache..`);
+
     let converter: AbstractConverter;
 
     switch (converterType) {

src/converters/degiroConverter.test.ts

@@ -1,14 +1,16 @@
-import { YahooFinanceService } from "../yahooFinanceService";
 import { DeGiroConverter } from "./degiroConverter";
+import { YahooFinanceService } from "../yahooFinanceService";
 
 describe("degiroConverter", () => {
 
-    it("should construct", () => {
+  it("should construct", () => {
+
+    // Act
+    const sut = new DeGiroConverter(new YahooFinanceService());
 
-      // Act
-      const sut = new DeGiroConverter(new YahooFinanceService());
+    // Assert
+    expect(sut).toBeTruthy();
+  });
 
-      // Asssert
-      expect(sut).toBeTruthy();
-    });
+  // This converter is replaced by V2, so no sense in unit testing this any further.
 });

src/converters/degiroConverterV2.test.ts

@@ -1,14 +1,120 @@
-import { YahooFinanceService } from "../yahooFinanceService";
 import { DeGiroConverterV2 } from "./degiroConverterV2";
+import { YahooFinanceService } from "../yahooFinanceService";
+import { GhostfolioExport } from "../models/ghostfolioExport";
+
+fdescribe("degiroConverterV2", () => {
+
+  it("should construct", () => {
+
+    // Act
+    const sut = new DeGiroConverterV2(new YahooFinanceService());
+
+    // Assert
+    expect(sut).toBeTruthy();
+  });
+
+  it("should process sample CSV file", (done) => {
+
+    // Arange
+    const sut = new DeGiroConverterV2(new YahooFinanceService());
+    const inputFile = "sample-degiro-export.csv";
+
+    // Act      
+    sut.readAndProcessFile(inputFile, (actualExport: GhostfolioExport) => {
 
-describe("degiroConverterV2", () => {
+      // Assert
+      expect(actualExport).toBeTruthy();
+      expect(actualExport.activities.length).toBeGreaterThan(0);
+      expect(actualExport.activities.length).toBe(15);
 
-    it("should construct", () => {
+      done();
+    }, () => { done.fail("Should not have an error!"); });
+  });
+
+  describe("should throw an error if", () => {
+    it("the input file does not exist", (done) => {
+
+      // Arrange
+      const sut = new DeGiroConverterV2(new YahooFinanceService());
+
+      let tempFileName = "tmp/testinput/degiro-filedoesnotexist.csv";
 
       // Act
+      sut.readAndProcessFile(tempFileName, () => { done.fail("Should not succeed!"); }, (err: Error) => {
+
+        // Assert
+        expect(err).toBeTruthy();
+
+        done();
+      });
+    });
+
+    it("the input file is empty", (done) => {
+
+      // Arrange
       const sut = new DeGiroConverterV2(new YahooFinanceService());
 
-      // Asssert
-      expect(sut).toBeTruthy();
+      let tempFileContent = "";
+      tempFileContent += "Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id\n";
+
+      // Act
+      sut.processFileContents(tempFileContent, () => { done.fail("Should not succeed!"); }, (err: Error) => {
+
+        // Assert
+        expect(err).toBeTruthy();
+        expect(err.message).toContain("An error ocurred while parsing");
+
+        done();
+      });
+    });
+
+    it("Yahoo Finance throws an error", (done) => {
+
+      // Arrange
+      let tempFileContent = "";
+      tempFileContent += "Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id\n";
+      tempFileContent += `15-12-2022,16:55,15-12-2022,VICI PROPERTIES INC. C,US9256521090,DEGIRO Transactiekosten en/of kosten van derden,,EUR,"-1,00",EUR,"31,98",5925d76b-eb36-46e3-b017-a61a6d03c3e7\n`;
+      tempFileContent += `15-12-2022,16:55,15-12-2022,VICI PROPERTIES INC. C,US9256521090,"Koop 1 @ 33,9 USD",,USD,"-33,90",USD,"-33,90",5925d76b-eb36-46e3-b017-a61a6d03c3e7`;
+
+      // Mock Yahoo Finance service to throw error.
+      const yahooFinanceService = new YahooFinanceService();
+      jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { throw new Error("Unit test error"); });
+      const sut = new DeGiroConverterV2(yahooFinanceService);
+
+      // Act      
+      sut.processFileContents(tempFileContent, () => { done.fail("Should not succeed!"); }, (err: Error) => {
+
+        // Assert
+        expect(err).toBeTruthy();
+        expect(err.message).toContain("Unit test error");
+
+        done();
+      });
     });
+  });
+
+  it("should log when Yahoo Finance returns no symbol", (done) => {
+
+    // Arrange
+    let tempFileContent = "";
+    tempFileContent += "Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id\n";
+    tempFileContent += `15-12-2022,16:55,15-12-2022,VICI PROPERTIES INC. C,US9256521090,DEGIRO Transactiekosten en/of kosten van derden,,EUR,"-1,00",EUR,"31,98",5925d76b-eb36-46e3-b017-a61a6d03c3e7\n`;
+    tempFileContent += `15-12-2022,16:55,15-12-2022,VICI PROPERTIES INC. C,US9256521090,"Koop 1 @ 33,9 USD",,USD,"-33,90",USD,"-33,90",5925d76b-eb36-46e3-b017-a61a6d03c3e7`;
+
+    // Mock Yahoo Finance service to return null.
+    const yahooFinanceService = new YahooFinanceService();
+    jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { return null });
+    const sut = new DeGiroConverterV2(yahooFinanceService);
+
+    // Bit hacky, but it works.
+    const consoleSpy = jest.spyOn((sut as any).progress, "log");
+
+    // Act      
+    sut.processFileContents(tempFileContent, () => {
+
+      expect(consoleSpy).toHaveBeenCalledWith("[i] No result found for US9256521090 with currency EUR! Please add this manually..\n");
+
+      done();
+    }, () => done.fail("Should not have an error!"));
+  });
 });

src/converters/etoroConverter.test.ts

@@ -28,6 +28,9 @@ describe("etoroConverter", () => {
 
       // Assert
       expect(actualExport).toBeTruthy();
+      expect(actualExport.activities.length).toBeGreaterThan(0);
+      expect(actualExport.activities.length).toBe(18);
+
       done();
     }, () => { done.fail("Should not have an error!"); });
   });
@@ -45,6 +48,7 @@ describe("etoroConverter", () => {
 
         // Assert
         expect(err).toBeTruthy();
+
         done();
       });
     });
@@ -62,15 +66,15 @@ describe("etoroConverter", () => {
 
         // Assert
         expect(err).toBeTruthy();
-        expect(err.message).toContain("An error ocurred while parsing")
+        expect(err.message).toContain("An error ocurred while parsing");
+
         done();
       });
     });
 
     it("Yahoo Finance throws an error", (done) => {
 
       // Arrange
-
       let tempFileContent = "";
       tempFileContent += "Date,Type,Details,Amount,Units,Realized Equity Change,Realized Equity,Balance,Position ID,Asset type,NWA\n";
       tempFileContent += `02/01/2024 00:10:33,Dividend,NKE/USD,0.17,-,0.17,"4,581.91",99.60,2272508626,Stocks,0.00`;
@@ -85,7 +89,8 @@ describe("etoroConverter", () => {
 
         // Assert
         expect(err).toBeTruthy();
-        expect(err.message).toContain("Unit test error")
+        expect(err.message).toContain("Unit test error");
+
         done();
       });
     });
@@ -94,7 +99,6 @@ describe("etoroConverter", () => {
   it("should log when Yahoo Finance returns no symbol", (done) => {
 
     // Arrange
-
     let tempFileContent = "";
     tempFileContent += "Date,Type,Details,Amount,Units,Realized Equity Change,Realized Equity,Balance,Position ID,Asset type,NWA\n";
     tempFileContent += `02/01/2024 00:10:33,Dividend,NKE/USD,0.17,-,0.17,"4,581.91",99.60,2272508626,Stocks,0.00`;
@@ -111,6 +115,7 @@ describe("etoroConverter", () => {
     sut.processFileContents(tempFileContent, () => {
 
       expect(consoleSpy).toHaveBeenCalledWith("[i] No result found for dividend action for NKE/USD! Please add this manually..\n");
+
       done();
     }, () => done.fail("Should not have an error!"));
   });

src/converters/etoroConverter.ts

@@ -139,14 +139,16 @@ export class EtoroConverter extends AbstractConverter {
                     continue;
                 }
 
+                const unitPrice = parseFloat((record.amount / record.units).toFixed(6));
+
                 // Add record to export.
                 result.activities.push({
                     accountId: process.env.GHOSTFOLIO_ACCOUNT_ID,
                     comment: "",
                     fee: 0,
                     quantity: record.units,
                     type: GhostfolioOrderType[record.type],
-                    unitPrice: record.amount,
+                    unitPrice: unitPrice,
                     currency: currency,
                     dataSource: "YAHOO",
                     date: date.format("YYYY-MM-DDTHH:mm:ssZ"),