diff --git a/.env.sample b/.env.sample index a7efbedc..f561b2bd 100644 --- a/.env.sample +++ b/.env.sample @@ -9,4 +9,4 @@ DEBUG_LOGGING = false # Set prefered exchange if you have securities that are traded at multiple exchanges, like Vanguard FTSE All-World UCITS ETF (VWRL.AS, VWCE.DE, VRRA.L) # Leave commented if you don't want to use this. -#DEGIRO_PREFERED_EXCHANGE_POSTFIX = ".AS" \ No newline at end of file +#DEGIRO_PREFERED_EXCHANGE_POSTFIX = ".AS" diff --git a/.github/workflows/frameworkTesting.yml b/.github/workflows/frameworkTesting.yml index d60f4df5..a080c462 100644 --- a/.github/workflows/frameworkTesting.yml +++ b/.github/workflows/frameworkTesting.yml @@ -5,7 +5,6 @@ on: push: branches: - main - - feature/* paths: - "src/**" pull_request: @@ -72,7 +71,7 @@ jobs: { "schemaVersion": 1, "label": "Code Coverage", - "message": "$coverage", + "message": "$coverage%", "style": "for-the-badge", "color": "$color" } diff --git a/GitVersion.yml b/GitVersion.yml index 14cb296f..b25bf3af 100644 --- a/GitVersion.yml +++ b/GitVersion.yml @@ -1,4 +1,4 @@ -next-version: 0.4.0 +next-version: 0.5.0 assembly-informational-format: "{NuGetVersion}" mode: ContinuousDeployment branches: diff --git a/README.md b/README.md index 6f11fad3..ec40367e 100644 --- a/README.md +++ b/README.md @@ -88,11 +88,16 @@ The following parameters can be given to the Docker run command. | `--env USE_POLLING=true` | Y | When set to true, the container will continously look for new files to process and the container will not stop. | | `--env DEBUG_LOGGING=true` | Y | When set to true, the container will show logs in more detail, useful for error tracing. | | `--env FORCE_DEGIRO_V2=true` | Y | When set to true, the converter will use the DEGIRO V2 converter (currently in beta) when a DEGIRO file was found. | +| `--env PURGE_CACHE=true` | Y | When set to true, the file cache will be purged on start. | 1: You can retrieve your Ghostfolio account ID by going to Accounts > select your account and copying the ID from the URL. ![image](https://user-images.githubusercontent.com/5620002/203353840-f5db7323-fb2f-4f4f-befc-e4e340466a74.png) +### Caching + +The tool uses `cacache` to store data retrieved from Yahoo Finance inside the container. This way the load on Yahoo Finance is reduced and the tool should run faster. The cached data is stored inside the container in `tmp/e2g-cache`. If you feel you need to invalidate your cache, you can do so by adding `--env PURGE_CACHE=true` to your run command. This will clear the cache on container start, and the tool will recreate the cache the next time it has to retrieve data from Yahoo Finance. + ## Run locally @@ -129,6 +134,10 @@ You can now run `npm run start [exporttype]`. See the table with run commands be | Swissquote | `run start swissquote` (or `sq`) | | Schwab | `run start schwab` | +### Caching + +The tool uses `cacache` to store data retrieved from Yahoo Finance on disk. This way the load on Yahoo Finance is reduced and the tool should run faster. The cached data is stored in `tmp/e2g-cache`. If you feel you need to invalidate your cache, you can do so by removing the folder and the tool will recreate the cache when you run it the next time. + ## Import to Ghostfolio diff --git a/jest.config.ts b/jest.config.ts index 1d9a50fa..462e1d16 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -3,12 +3,17 @@ import type { Config } from '@jest/types'; // Sync object const config: Config.InitialOptions = { verbose: true, - testTimeout: 30000, + testTimeout: 15000, transform: { '^.+\\.tsx?$': 'ts-jest' }, coverageDirectory: 'coverage', collectCoverageFrom: ['src/**/*.ts'], + coveragePathIgnorePatterns: [ + '/src/models', + '/src/manual.ts', + '/src/watcher.ts', + '/src/converter.ts'], coverageReporters: ['text', 'cobertura', 'html'] }; diff --git a/package.json b/package.json index d1ef4207..6d99d9aa 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ }, "license": "Apache-2.0", "devDependencies": { + "@types/cacache": "^17.0.2", "@types/jest": "^29.5.11", "@types/node": "^20.10.4", "jest": "^29.7.0", @@ -22,6 +23,7 @@ }, "dependencies": { "@types/cli-progress": "^3.11.5", + "cacache": "^18.0.2", "chokidar": "^3.5.3", "cli-progress": "^3.12.0", "closest-match": "^1.3.3", diff --git a/sample-degiro-export.csv b/sample-degiro-export.csv index 4f650151..1a5ae62a 100644 --- a/sample-degiro-export.csv +++ b/sample-degiro-export.csv @@ -40,4 +40,8 @@ Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id 25-07-2023,14:31,25-07-2023,ISHARES KOREA,IE00B0M63391,"Koop 1 @ 42,53 EUR",,EUR,-42.53,EUR,45.92,d4f45240-d763-4b95-8590-2ba158d38207 15-05-2019,09:05,15-05-2019,ISHARES MSCI WOR A,IE00B4L5Y983,"Compra 6 ISHARES MSCI WOR A@49,785 EUR (IE00B4L5Y983)",,EUR,-298.71,EUR,0.64,a47e2746-bfbd-4654-bd6c-5e58e470d32f 02-01-2024,14:42,02-01-2024,ISHARES MSCI WOR A,IE00B4L5Y983,Comissões de transação DEGIRO e/ou taxas de terceiros,,EUR,-1.00,EUR,2.54,7b377a93-5695-4131-8954-5c78996fbed4 -02-01-2024,14:42,02-01-2024,ISHARES MSCI WOR A,IE00B4L5Y983,"Compra 1 ISHARES MSCI WOR A@82,055 EUR (IE00B4L5Y983)",,EUR,-82.06,EUR,3.54,7b377a93-5695-4131-8954-5c78996fbed4 \ No newline at end of file +02-01-2024,14:42,02-01-2024,ISHARES MSCI WOR A,IE00B4L5Y983,"Compra 1 ISHARES MSCI WOR A@82,055 EUR (IE00B4L5Y983)",,EUR,-82.06,EUR,3.54,7b377a93-5695-4131-8954-5c78996fbed4 +05-02-2024,07:16,31-01-2024,,,DEGIRO Aansluitingskosten 2024 (Nasdaq - NDQ),,EUR,-2.50,EUR,-6.28, +05-02-2024,07:16,31-01-2024,,,DEGIRO Aansluitingskosten 2024 (Euronext Milan - MIL),,EUR,-2.50,EUR,-3.78, +05-02-2024,07:16,31-01-2024,,,DEGIRO Aansluitingskosten 2024 (Xetra - XET),,EUR,-2.50,EUR,-1.28, +02-02-2024,08:00,01-02-2024,AT&T INC.,US00206R1023,Dividend,,USD,1.39,USD,1.39, \ No newline at end of file diff --git a/sample-schwab-export.csv b/sample-schwab-export.csv index 96411802..62976754 100644 --- a/sample-schwab-export.csv +++ b/sample-schwab-export.csv @@ -98,4 +98,5 @@ Date,Action,Symbol,Description,Quantity,Price,Fees & Comm,Amount 01/03/2023,Reinvest Shares,NMFC,NEW MOUNTAIN FIN CO,10.9694,$12.45,,-$136.56 01/03/2023,Reinvest Shares,TCPC,BLACKROCK TCP CAPITAL CO,10.0443,$13.17,,-$132.25 "09/08/2023","Sell","SNAXX","SCHWAB VALUE ADVANTAGE MONEY ULTRA","500,135","$1.00","","$500135.00" +"10/18/2023","Wire Sent","","WIRED FUNDS DISBURSED","","","","-$100000.00" Transactions Total,,,,,,,"-$26,582.91" \ No newline at end of file diff --git a/src/converter.ts b/src/converter.ts index 428a77a7..c05aa13e 100644 --- a/src/converter.ts +++ b/src/converter.ts @@ -11,8 +11,7 @@ import { SwissquoteConverter } from "./converters/swissquoteConverter"; import { FinpensionConverter } from "./converters/finpensionConverter"; import { YahooFinanceService } from "./yahooFinanceService"; - -export function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) { +export async function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) { // Verify if Ghostolio account ID is set (because without it there can be no valid output). if (!process.env.GHOSTFOLIO_ACCOUNT_ID) { @@ -22,7 +21,7 @@ export function createAndRunConverter(converterType: string, inputFilePath: stri const converterTypeLc = converterType.toLocaleLowerCase(); // Determine convertor type. - const converter = createConverter(converterTypeLc); + const converter = await createConverter(converterTypeLc); // Map the file to a Ghostfolio import. converter.readAndProcessFile(inputFilePath, (result: GhostfolioExport) => { @@ -41,10 +40,13 @@ export function createAndRunConverter(converterType: string, inputFilePath: stri }, (error) => errorCallback(error)); } -function createConverter(converterType: string): AbstractConverter { +async function createConverter(converterType: string): Promise { const yahooFinanceService = new YahooFinanceService(); + const cacheSize = await yahooFinanceService.loadCache(); + console.log(`[i] Restored ${cacheSize[0]} ISIN-symbol pairs and ${cacheSize[1]} symbols from cache..`); + let converter: AbstractConverter; switch (converterType) { diff --git a/src/converters/degiroConverter.test.ts b/src/converters/degiroConverter.test.ts index 870a686d..cf09eb6e 100644 --- a/src/converters/degiroConverter.test.ts +++ b/src/converters/degiroConverter.test.ts @@ -1,14 +1,16 @@ -import { YahooFinanceService } from "../yahooFinanceService"; import { DeGiroConverter } from "./degiroConverter"; +import { YahooFinanceService } from "../yahooFinanceService"; describe("degiroConverter", () => { - it("should construct", () => { + it("should construct", () => { + + // Act + const sut = new DeGiroConverter(new YahooFinanceService()); - // Act - const sut = new DeGiroConverter(new YahooFinanceService()); + // Assert + expect(sut).toBeTruthy(); + }); - // Asssert - expect(sut).toBeTruthy(); - }); + // This converter is replaced by V2, so no sense in unit testing this any further. }); diff --git a/src/converters/degiroConverterV2.test.ts b/src/converters/degiroConverterV2.test.ts index c844d654..b1b4e84a 100644 --- a/src/converters/degiroConverterV2.test.ts +++ b/src/converters/degiroConverterV2.test.ts @@ -1,14 +1,120 @@ -import { YahooFinanceService } from "../yahooFinanceService"; import { DeGiroConverterV2 } from "./degiroConverterV2"; +import { YahooFinanceService } from "../yahooFinanceService"; +import { GhostfolioExport } from "../models/ghostfolioExport"; + +fdescribe("degiroConverterV2", () => { + + it("should construct", () => { + + // Act + const sut = new DeGiroConverterV2(new YahooFinanceService()); + + // Assert + expect(sut).toBeTruthy(); + }); + + it("should process sample CSV file", (done) => { + + // Arange + const sut = new DeGiroConverterV2(new YahooFinanceService()); + const inputFile = "sample-degiro-export.csv"; + + // Act + sut.readAndProcessFile(inputFile, (actualExport: GhostfolioExport) => { -describe("degiroConverterV2", () => { + // Assert + expect(actualExport).toBeTruthy(); + expect(actualExport.activities.length).toBeGreaterThan(0); + expect(actualExport.activities.length).toBe(15); - it("should construct", () => { + done(); + }, () => { done.fail("Should not have an error!"); }); + }); + + describe("should throw an error if", () => { + it("the input file does not exist", (done) => { + + // Arrange + const sut = new DeGiroConverterV2(new YahooFinanceService()); + + let tempFileName = "tmp/testinput/degiro-filedoesnotexist.csv"; // Act + sut.readAndProcessFile(tempFileName, () => { done.fail("Should not succeed!"); }, (err: Error) => { + + // Assert + expect(err).toBeTruthy(); + + done(); + }); + }); + + it("the input file is empty", (done) => { + + // Arrange const sut = new DeGiroConverterV2(new YahooFinanceService()); - // Asssert - expect(sut).toBeTruthy(); + let tempFileContent = ""; + tempFileContent += "Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id\n"; + + // Act + sut.processFileContents(tempFileContent, () => { done.fail("Should not succeed!"); }, (err: Error) => { + + // Assert + expect(err).toBeTruthy(); + expect(err.message).toContain("An error ocurred while parsing"); + + done(); + }); + }); + + it("Yahoo Finance throws an error", (done) => { + + // Arrange + let tempFileContent = ""; + tempFileContent += "Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id\n"; + tempFileContent += `15-12-2022,16:55,15-12-2022,VICI PROPERTIES INC. C,US9256521090,DEGIRO Transactiekosten en/of kosten van derden,,EUR,"-1,00",EUR,"31,98",5925d76b-eb36-46e3-b017-a61a6d03c3e7\n`; + tempFileContent += `15-12-2022,16:55,15-12-2022,VICI PROPERTIES INC. C,US9256521090,"Koop 1 @ 33,9 USD",,USD,"-33,90",USD,"-33,90",5925d76b-eb36-46e3-b017-a61a6d03c3e7`; + + // Mock Yahoo Finance service to throw error. + const yahooFinanceService = new YahooFinanceService(); + jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { throw new Error("Unit test error"); }); + const sut = new DeGiroConverterV2(yahooFinanceService); + + // Act + sut.processFileContents(tempFileContent, () => { done.fail("Should not succeed!"); }, (err: Error) => { + + // Assert + expect(err).toBeTruthy(); + expect(err.message).toContain("Unit test error"); + + done(); + }); }); + }); + + it("should log when Yahoo Finance returns no symbol", (done) => { + + // Arrange + let tempFileContent = ""; + tempFileContent += "Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id\n"; + tempFileContent += `15-12-2022,16:55,15-12-2022,VICI PROPERTIES INC. C,US9256521090,DEGIRO Transactiekosten en/of kosten van derden,,EUR,"-1,00",EUR,"31,98",5925d76b-eb36-46e3-b017-a61a6d03c3e7\n`; + tempFileContent += `15-12-2022,16:55,15-12-2022,VICI PROPERTIES INC. C,US9256521090,"Koop 1 @ 33,9 USD",,USD,"-33,90",USD,"-33,90",5925d76b-eb36-46e3-b017-a61a6d03c3e7`; + + // Mock Yahoo Finance service to return null. + const yahooFinanceService = new YahooFinanceService(); + jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { return null }); + const sut = new DeGiroConverterV2(yahooFinanceService); + + // Bit hacky, but it works. + const consoleSpy = jest.spyOn((sut as any).progress, "log"); + + // Act + sut.processFileContents(tempFileContent, () => { + + expect(consoleSpy).toHaveBeenCalledWith("[i] No result found for US9256521090 with currency EUR! Please add this manually..\n"); + + done(); + }, () => done.fail("Should not have an error!")); + }); }); diff --git a/src/converters/etoroConverter.test.ts b/src/converters/etoroConverter.test.ts index eeea48d9..08fc3fa7 100644 --- a/src/converters/etoroConverter.test.ts +++ b/src/converters/etoroConverter.test.ts @@ -28,6 +28,9 @@ describe("etoroConverter", () => { // Assert expect(actualExport).toBeTruthy(); + expect(actualExport.activities.length).toBeGreaterThan(0); + expect(actualExport.activities.length).toBe(18); + done(); }, () => { done.fail("Should not have an error!"); }); }); @@ -45,6 +48,7 @@ describe("etoroConverter", () => { // Assert expect(err).toBeTruthy(); + done(); }); }); @@ -62,7 +66,8 @@ describe("etoroConverter", () => { // Assert expect(err).toBeTruthy(); - expect(err.message).toContain("An error ocurred while parsing") + expect(err.message).toContain("An error ocurred while parsing"); + done(); }); }); @@ -70,7 +75,6 @@ describe("etoroConverter", () => { it("Yahoo Finance throws an error", (done) => { // Arrange - let tempFileContent = ""; tempFileContent += "Date,Type,Details,Amount,Units,Realized Equity Change,Realized Equity,Balance,Position ID,Asset type,NWA\n"; tempFileContent += `02/01/2024 00:10:33,Dividend,NKE/USD,0.17,-,0.17,"4,581.91",99.60,2272508626,Stocks,0.00`; @@ -85,7 +89,8 @@ describe("etoroConverter", () => { // Assert expect(err).toBeTruthy(); - expect(err.message).toContain("Unit test error") + expect(err.message).toContain("Unit test error"); + done(); }); }); @@ -94,7 +99,6 @@ describe("etoroConverter", () => { it("should log when Yahoo Finance returns no symbol", (done) => { // Arrange - let tempFileContent = ""; tempFileContent += "Date,Type,Details,Amount,Units,Realized Equity Change,Realized Equity,Balance,Position ID,Asset type,NWA\n"; tempFileContent += `02/01/2024 00:10:33,Dividend,NKE/USD,0.17,-,0.17,"4,581.91",99.60,2272508626,Stocks,0.00`; @@ -111,6 +115,7 @@ describe("etoroConverter", () => { sut.processFileContents(tempFileContent, () => { expect(consoleSpy).toHaveBeenCalledWith("[i] No result found for dividend action for NKE/USD! Please add this manually..\n"); + done(); }, () => done.fail("Should not have an error!")); }); diff --git a/src/converters/etoroConverter.ts b/src/converters/etoroConverter.ts index ec1f6bf0..16983a68 100644 --- a/src/converters/etoroConverter.ts +++ b/src/converters/etoroConverter.ts @@ -139,6 +139,8 @@ export class EtoroConverter extends AbstractConverter { continue; } + const unitPrice = parseFloat((record.amount / record.units).toFixed(6)); + // Add record to export. result.activities.push({ accountId: process.env.GHOSTFOLIO_ACCOUNT_ID, @@ -146,7 +148,7 @@ export class EtoroConverter extends AbstractConverter { fee: 0, quantity: record.units, type: GhostfolioOrderType[record.type], - unitPrice: record.amount, + unitPrice: unitPrice, currency: currency, dataSource: "YAHOO", date: date.format("YYYY-MM-DDTHH:mm:ssZ"), diff --git a/src/converters/finpensionConverter.test.ts b/src/converters/finpensionConverter.test.ts index 5e496200..c91d0084 100644 --- a/src/converters/finpensionConverter.test.ts +++ b/src/converters/finpensionConverter.test.ts @@ -1,14 +1,122 @@ -import { YahooFinanceService } from "../yahooFinanceService"; import { FinpensionConverter } from "./finpensionConverter"; +import { YahooFinanceService } from "../yahooFinanceService"; +import { GhostfolioExport } from "../models/ghostfolioExport"; describe("finpensionConverter", () => { - it("should construct", () => { + afterEach(() => { + jest.clearAllMocks(); + }) + + it("should construct", () => { + + // Act + const sut = new FinpensionConverter(new YahooFinanceService()); + + // Assert + expect(sut).toBeTruthy(); + }); + + it("should process sample CSV file", (done) => { + + // Arange + const sut = new FinpensionConverter(new YahooFinanceService()); + const inputFile = "sample-finpension-export.csv"; + + // Act + sut.readAndProcessFile(inputFile, (actualExport: GhostfolioExport) => { + + // Assert + expect(actualExport).toBeTruthy(); + expect(actualExport.activities.length).toBeGreaterThan(0); + expect(actualExport.activities.length).toBe(24); + + done(); + }, () => { done.fail("Should not have an error!"); }); + }); + + describe("should throw an error if", () => { + it("the input file does not exist", (done) => { + + // Arrange + const sut = new FinpensionConverter(new YahooFinanceService()); + + let tempFileName = "tmp/testinput/finpension-filedoesnotexist.csv"; // Act + sut.readAndProcessFile(tempFileName, () => { done.fail("Should not succeed!"); }, (err: Error) => { + + // Assert + expect(err).toBeTruthy(); + + done(); + }); + }); + + it("the input file is empty", (done) => { + + // Arrange const sut = new FinpensionConverter(new YahooFinanceService()); - // Asssert - expect(sut).toBeTruthy(); + let tempFileContent = ""; + tempFileContent += `Date;Category;"Asset Name";ISIN;"Number of Shares";"Asset Currency";"Currency Rate";"Asset Price in CHF";"Cash Flow";Balance\n`; + + // Act + sut.processFileContents(tempFileContent, () => { done.fail("Should not succeed!"); }, (err: Error) => { + + // Assert + expect(err).toBeTruthy(); + expect(err.message).toContain("An error ocurred while parsing"); + + done(); + }); + }); + + it("Yahoo Finance throws an error", (done) => { + + // Arrange + let tempFileContent = ""; + tempFileContent += `Date;Category;"Asset Name";ISIN;"Number of Shares";"Asset Currency";"Currency Rate";"Asset Price in CHF";"Cash Flow";Balance\n`; + tempFileContent += `2023-07-11;Buy;"CSIF (CH) Bond Corporate Global ex CHF Blue ZBH";CH0189956813;0.001000;CHF;1.000000;821.800000;-0.821800;16.484551`; + + // Mock Yahoo Finance service to throw error. + const yahooFinanceService = new YahooFinanceService(); + jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { throw new Error("Unit test error"); }); + const sut = new FinpensionConverter(yahooFinanceService); + + // Act + sut.processFileContents(tempFileContent, () => { done.fail("Should not succeed!"); }, (err: Error) => { + + // Assert + expect(err).toBeTruthy(); + expect(err.message).toContain("Unit test error"); + + done(); + }); }); + }); + + it("should log when Yahoo Finance returns no symbol", (done) => { + + // Arrange + let tempFileContent = ""; + tempFileContent += `Date;Category;"Asset Name";ISIN;"Number of Shares";"Asset Currency";"Currency Rate";"Asset Price in CHF";"Cash Flow";Balance\n`; + tempFileContent += `2023-07-11;Buy;"CSIF (CH) Bond Corporate Global ex CHF Blue ZBH";CH0189956813;0.001000;CHF;1.000000;821.800000;-0.821800;16.484551`; + + // Mock Yahoo Finance service to return null. + const yahooFinanceService = new YahooFinanceService(); + jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { return null }); + const sut = new FinpensionConverter(yahooFinanceService); + + // Bit hacky, but it works. + const consoleSpy = jest.spyOn((sut as any).progress, "log"); + + // Act + sut.processFileContents(tempFileContent, () => { + + expect(consoleSpy).toHaveBeenCalledWith("[i] No result found for buy action for CH0189956813 with currency CHF! Please add this manually..\n"); + + done(); + }, () => done.fail("Should not have an error!")); + }); }); diff --git a/src/converters/finpensionConverter.ts b/src/converters/finpensionConverter.ts index 740aa1af..ad313536 100644 --- a/src/converters/finpensionConverter.ts +++ b/src/converters/finpensionConverter.ts @@ -19,7 +19,7 @@ export class FinpensionConverter extends AbstractConverter { public processFileContents(input: string, successCallback: any, errorCallback: any): void { // Parse the CSV and convert to Ghostfolio import format. - const parser = parse(input, { + parse(input, { delimiter: ";", fromLine: 2, columns: this.processHeaders(input, ";"), @@ -57,7 +57,7 @@ export class FinpensionConverter extends AbstractConverter { }, async (_, records: FinpensionRecord[]) => { // If records is empty, parsing failed.. - if (records === undefined) { + if (records === undefined || records.length === 0) { return errorCallback(new Error("An error ocurred while parsing!")); } @@ -121,7 +121,7 @@ export class FinpensionConverter extends AbstractConverter { // Log whenever there was no match found. if (!security) { - this.progress.log(`[i]\tNo result found for ${record.category} action for ${record.isin || record.assetName} with currency ${record.assetCurrency}! Please add this manually..\n`); + this.progress.log(`[i] No result found for ${record.category} action for ${record.isin || record.assetName} with currency ${record.assetCurrency}! Please add this manually..\n`); bar1.increment(); continue; } @@ -157,12 +157,6 @@ export class FinpensionConverter extends AbstractConverter { successCallback(result); }); - - // Catch any error. - parser.on('error', function (err) { - console.log("[i] An error ocurred while processing the input file! See error below:") - console.error("[e]", err.message); - }); } /** diff --git a/src/converters/schwabConverter.test.ts b/src/converters/schwabConverter.test.ts index 275f2733..417aaaab 100644 --- a/src/converters/schwabConverter.test.ts +++ b/src/converters/schwabConverter.test.ts @@ -1,14 +1,124 @@ -import { YahooFinanceService } from "../yahooFinanceService"; import { SchwabConverter } from "./schwabConverter"; +import { YahooFinanceService } from "../yahooFinanceService"; +import { GhostfolioExport } from "../models/ghostfolioExport"; + +describe("schwabConverter", () => { + + afterEach(() => { + jest.clearAllMocks(); + }) + + it("should construct", () => { + + // Act + const sut = new SchwabConverter(new YahooFinanceService()); + + // Assert + expect(sut).toBeTruthy(); + }); + + it("should process sample CSV file", (done) => { + + // Arange + const sut = new SchwabConverter(new YahooFinanceService()); + const inputFile = "sample-schwab-export.csv"; + + // Act + sut.readAndProcessFile(inputFile, (actualExport: GhostfolioExport) => { -describe("SchwabConverter", () => { + // Assert + expect(actualExport).toBeTruthy(); + expect(actualExport.activities.length).toBeGreaterThan(0); + expect(actualExport.activities.length).toBe(98); - it("should construct", () => { + done(); + }, () => { done.fail("Should not have an error!"); }); + }); + + describe("should throw an error if", () => { + it("the input file does not exist", (done) => { + + // Arrange + const sut = new SchwabConverter(new YahooFinanceService()); + + let tempFileName = "tmp/testinput/schwab-filedoesnotexist.csv"; // Act + sut.readAndProcessFile(tempFileName, () => { done.fail("Should not succeed!"); }, (err: Error) => { + + // Assert + expect(err).toBeTruthy(); + + done(); + }); + }); + + it("the input file is empty", (done) => { + + // Arrange const sut = new SchwabConverter(new YahooFinanceService()); - // Asssert - expect(sut).toBeTruthy(); + let tempFileContent = ""; + tempFileContent += `Date,Action,Symbol,Description,Quantity,Price,Fees & Comm,Amount\n`; + + // Act + sut.processFileContents(tempFileContent, () => { done.fail("Should not succeed!"); }, (err: Error) => { + + // Assert + expect(err).toBeTruthy(); + expect(err.message).toContain("An error ocurred while parsing"); + + done(); + }); + }); + + it("Yahoo Finance throws an error", (done) => { + + // Arrange + let tempFileContent = ""; + tempFileContent += `Date,Action,Symbol,Description,Quantity,Price,Fees & Comm,Amount\n`; + tempFileContent += `08/22/2023,Sell,FIHBX,FEDERATED HERMES INSTL HIGH YIELD BD IS,592.199,$8.46,$10.00,"$5,000.00"\n`; + tempFileContent += `Transactions Total,,,,,,,"-$26,582.91"`; + + // Mock Yahoo Finance service to throw error. + const yahooFinanceService = new YahooFinanceService(); + jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { throw new Error("Unit test error"); }); + const sut = new SchwabConverter(yahooFinanceService); + + // Act + sut.processFileContents(tempFileContent, (e) => { done.fail("Should not succeed!"); }, (err: Error) => { + + // Assert + expect(err).toBeTruthy(); + expect(err.message).toContain("Unit test error"); + + done(); + }); }); + }); + + it("should log when Yahoo Finance returns no symbol", (done) => { + + // Arrange + let tempFileContent = ""; + tempFileContent += `Date,Action,Symbol,Description,Quantity,Price,Fees & Comm,Amount\n`; + tempFileContent += `08/22/2023,Sell,FIHBX,FEDERATED HERMES INSTL HIGH YIELD BD IS,592.199,$8.46,$10.00,"$5,000.00"\n`; + tempFileContent += `Transactions Total,,,,,,,"-$26,582.91"`; + + // Mock Yahoo Finance service to return null. + const yahooFinanceService = new YahooFinanceService(); + jest.spyOn(yahooFinanceService, "getSecurity").mockImplementation(() => { return null }); + const sut = new SchwabConverter(yahooFinanceService); + + // Bit hacky, but it works. + const consoleSpy = jest.spyOn((sut as any).progress, "log"); + + // Act + sut.processFileContents(tempFileContent, () => { + + expect(consoleSpy).toHaveBeenCalledWith("[i] No result found for sell action for FIHBX with currency USD! Please add this manually..\n"); + + done(); + }, () => done.fail("Should not have an error!")); + }); }); diff --git a/src/converters/schwabConverter.ts b/src/converters/schwabConverter.ts index 652836f9..aa4b2ce3 100644 --- a/src/converters/schwabConverter.ts +++ b/src/converters/schwabConverter.ts @@ -87,15 +87,15 @@ export class SchwabConverter extends AbstractConverter { }, activities: [] } - + // Populate the progress bar. const bar1 = this.progress.create(records.length - 1, 0); - // Skip last line of export ( stats). + // Skip last line of export (stats). for (let idx = 0; idx < records.length - 1; idx++) { const record = records[idx]; - - // Skip administrative fee/deposit/withdraw transactions. + + // Skip administrative deposit/withdraw transactions. if (this.isIgnoredRecord(record)) { bar1.increment(); continue; @@ -142,7 +142,7 @@ export class SchwabConverter extends AbstractConverter { // Log whenever there was no match found. if (!security) { - this.progress.log(`[i]\tNo result found for ${record.action} action for ${record.symbol || record.description} with currency USD! Please add this manually..\n`); + this.progress.log(`[i] No result found for ${record.action} action for ${record.symbol || record.description} with currency USD! Please add this manually..\n`); bar1.increment(); continue; } diff --git a/src/converters/swissquoteConverter.test.ts b/src/converters/swissquoteConverter.test.ts index 7483fae1..cfce4fbb 100644 --- a/src/converters/swissquoteConverter.test.ts +++ b/src/converters/swissquoteConverter.test.ts @@ -3,16 +3,16 @@ import { GhostfolioExport } from "../models/ghostfolioExport"; import { YahooFinanceService } from "../yahooFinanceService"; describe("swissquoteConverter", () => { - + it("should construct", () => { // Act const sut = new SwissquoteConverter(new YahooFinanceService()); - // Asssert + // Assert expect(sut).toBeTruthy(); }); - + it("should process sample CSV file", (done) => { // Act @@ -20,14 +20,15 @@ describe("swissquoteConverter", () => { const inputFile = "sample-swissquote-export.csv"; // Act - sut.readAndProcessFile(inputFile, (actualExport: GhostfolioExport) => { + sut.readAndProcessFile(inputFile, (actualExport: GhostfolioExport) => { // Assert expect(actualExport).toBeTruthy(); - - // Finish the test + expect(actualExport.activities.length).toBeGreaterThan(0); + expect(actualExport.activities.length).toBe(14); + done(); - }, () => { fail("Should not have an error!"); }); + }, () => { fail("Should not have an error!"); }); }); describe("should throw an error if", () => { @@ -37,14 +38,15 @@ describe("swissquoteConverter", () => { const sut = new SwissquoteConverter(new YahooFinanceService()); let tempFileName = "tmp/testinput/swissquote-filedoesnotexist.csv"; - + // Act - sut.readAndProcessFile(tempFileName, () => { fail("Should not succeed!"); }, (err: Error) => { + sut.readAndProcessFile(tempFileName, () => { fail("Should not succeed!"); }, (err: Error) => { // Assert expect(err).toBeTruthy(); + done(); - }); + }); }); it("the input file is empty", (done) => { @@ -54,16 +56,17 @@ describe("swissquoteConverter", () => { // Create temp file. let tempFileContent = ""; - tempFileContent += "Date;Order #;Transaction;Symbol;Name;ISIN;Quantity;Unit price;Costs;Accrued Interest;Net Amount;Balance;Currency\n"; - + tempFileContent += "Date;Order #;Transaction;Symbol;Name;ISIN;Quantity;Unit price;Costs;Accrued Interest;Net Amount;Balance;Currency\n"; + // Act - sut.processFileContents(tempFileContent, () => { fail("Should not succeed!"); }, (err: Error) => { + sut.processFileContents(tempFileContent, () => { fail("Should not succeed!"); }, (err: Error) => { // Assert expect(err).toBeTruthy(); - expect(err.message).toContain("An error ocurred while parsing") + expect(err.message).toContain("An error ocurred while parsing"); + done(); - }); + }); }); it("Yahoo Finance got empty input for query", (done) => { @@ -73,15 +76,16 @@ describe("swissquoteConverter", () => { // Create temp file. let tempFileContent = ""; - tempFileContent += "Date;Order #;Transaction;Symbol;Name;ISIN;Quantity;Unit price;Costs;Accrued Interest;Net Amount;Balance;Currency\n"; + tempFileContent += "Date;Order #;Transaction;Symbol;Name;ISIN;Quantity;Unit price;Costs;Accrued Interest;Net Amount;Balance;Currency\n"; tempFileContent += "10-08-2022 15:30:02;113947121;Buy;;;;200.0;19.85;5.96;0.00;-3975.96;168660.08;USD"; - + // Act - sut.processFileContents(tempFileContent, () => { fail("Should not succeed!"); }, (err) => { + sut.processFileContents(tempFileContent, () => { fail("Should not succeed!"); }, (err) => { // Assert expect(err).toBeTruthy(); - done(); + + done(); }); }); }); diff --git a/src/converters/swissquoteConverter.ts b/src/converters/swissquoteConverter.ts index 67988ca0..ca6efe05 100644 --- a/src/converters/swissquoteConverter.ts +++ b/src/converters/swissquoteConverter.ts @@ -131,7 +131,7 @@ export class SwissquoteConverter extends AbstractConverter { // Log whenever there was no match found. if (!security) { - this.progress.log(`[i]\tNo result found for ${record.transaction} action for ${record.isin || record.symbol || record.name} with currency ${record.currency}! Please add this manually..\n`); + this.progress.log(`[i] No result found for ${record.transaction} action for ${record.isin || record.symbol || record.name} with currency ${record.currency}! Please add this manually..\n`); bar1.increment(); continue; } diff --git a/src/converters/trading212Converter.test.ts b/src/converters/trading212Converter.test.ts index 2cd562b1..7241f69e 100644 --- a/src/converters/trading212Converter.test.ts +++ b/src/converters/trading212Converter.test.ts @@ -28,6 +28,9 @@ describe("trading212Converter", () => { // Assert expect(actualExport).toBeTruthy(); + expect(actualExport.activities.length).toBeGreaterThan(0); + expect(actualExport.activities.length).toBe(7); + done(); }, () => { done.fail("Should not have an error!"); }); }); @@ -45,6 +48,7 @@ describe("trading212Converter", () => { // Assert expect(err).toBeTruthy(); + done(); }); }); @@ -62,7 +66,8 @@ describe("trading212Converter", () => { // Assert expect(err).toBeTruthy(); - expect(err.message).toContain("An error ocurred while parsing") + expect(err.message).toContain("An error ocurred while parsing"); + done(); }); }); @@ -85,7 +90,8 @@ describe("trading212Converter", () => { // Assert expect(err).toBeTruthy(); - expect(err.message).toContain("Unit test error") + expect(err.message).toContain("Unit test error"); + done(); }); }); @@ -111,6 +117,7 @@ describe("trading212Converter", () => { sut.processFileContents(tempFileContent, () => { expect(consoleSpy).toHaveBeenCalledWith("[i] No result found for buy action for US17275R1023 with currency USD! Please add this manually..\n"); + done(); }, () => done.fail("Should not have an error!")); }); diff --git a/src/testUtils.ts b/src/testUtils.ts deleted file mode 100644 index 18d02558..00000000 --- a/src/testUtils.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* istanbul ignore */ - -import * as fs from "fs"; -import { GhostfolioExport } from "./models/ghostfolioExport"; - -export function getResultFile(fileName: string): GhostfolioExport { - - const contents = fs.readFileSync(fileName, "utf-8"); - - return JSON.parse(contents); -} diff --git a/src/watcher.ts b/src/watcher.ts index a678602e..baef91d0 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -1,9 +1,20 @@ import path from "path"; import * as fs from "fs"; import chokidar from "chokidar"; +import * as cacache from "cacache"; import * as matcher from "closest-match"; import { createAndRunConverter } from "./converter"; +// Check if the cache should be purged. +if (Boolean(process.env.PURGE_CACHE)) { + + console.log("[i] Purging cache (PURGE_CACHE set to true).."); + Promise.all([ + cacache.rm("tmp/e2g-cache", "isinSymbolCache"), + cacache.rm("tmp/e2g-cache", "symbolCache") + ]).then(() => console.log("[i] Cache purged!")); +} + // Define input and output. const inputFolder = process.env.E2G_INPUT_FOLDER || "/var/e2g-input"; const outputFolder = process.env.E2G_OUTPUT_FOLDER || "/var/e2g-output"; diff --git a/src/yahooFinanceService.ts b/src/yahooFinanceService.ts index f5f42b96..6c09f376 100644 --- a/src/yahooFinanceService.ts +++ b/src/yahooFinanceService.ts @@ -1,6 +1,9 @@ +import * as cacache from "cacache"; import yahooFinance from 'yahoo-finance2'; import { YahooFinanceRecord } from './models/yahooFinanceRecord'; +const cachePath = "tmp/e2g-cache"; + export class YahooFinanceService { // Local cache of earlier retrieved symbols. @@ -40,32 +43,24 @@ export class YahooFinanceService { // When isin was given, check wether there is a symbol conversion cached. Then change map. if (isin && this.isinSymbolCache.has(isin)) { - symbol = this.isinSymbolCache[isin]; + symbol = this.isinSymbolCache.get(isin); } - - // Second, check if the requested security is known by symbol (if given). - if (symbol) { - - const symbolMatch = this.symbolCache.has(symbol); - // If a match was found, return the security. - if (symbolMatch) { - this.logDebug(`Retrieved symbol ${symbol} from cache!`, progress); - return symbolMatch[1]; - } + // Second, check if the requested security is known by symbol (if given). + // If a match was found, return the security. + if (symbol && this.symbolCache.has(symbol)) { + this.logDebug(`Retrieved symbol ${symbol} from cache!`, progress); + return this.symbolCache.get(symbol); } - // The security is not known. Try to find is. - - // First try by ISIN. - let symbols = await this.getSymbolsByQuery(isin, progress); - this.logDebug(`getSecurity(): Found ${symbols.length} match${symbols.length === 1 ? "" : "es"} by ISIN ${isin}`, progress); + // The security is not known. Try to find it + let symbols: YahooFinanceRecord[] = []; // First try by ISIN. // If no ISIN was given as a parameter, just skip this part. if (isin) { symbols = await this.getSymbolsByQuery(isin, progress); - this.logDebug(`getSecurity(): Found ${symbols.length} matches by ISIN ${isin}`, progress); + this.logDebug(`getSecurity(): Found ${symbols.length} match${symbols.length === 1 ? "" : "es"} by ISIN ${isin}`, progress); // If no result found by ISIN, try by symbol. if (symbols.length == 0 && symbol) { @@ -112,13 +107,15 @@ export class YahooFinanceService { this.logDebug(`getSecurity(): Match found for ${isin ?? symbol ?? name}`, progress); - // If there was an isin given, place it in the isin-symbol mapping cache. - if (isin) { - this.isinSymbolCache[isin] = symbolMatch.symbol; + // If there was an isin given, place it in the isin-symbol mapping cache (if it wasn't there before). + if (isin && !this.isinSymbolCache.has(isin)) { + await this.saveInCache(isin, null, symbolMatch.symbol); } - // Store the record in cache by symbol. - this.symbolCache[symbolMatch.symbol] = symbolMatch; + // Store the record in cache by symbol (if it wasn't there before). + if (!this.symbolCache.has(symbolMatch.symbol)) { + await this.saveInCache(null, symbolMatch.symbol, symbolMatch); + } return symbolMatch; } @@ -126,6 +123,33 @@ export class YahooFinanceService { return null; } + /** + * Load the cache with ISIN and symbols. + * + * @returns The size of the loaded cache + */ + public async loadCache(): Promise<[number, number]> { + + // Verify if there is data in the ISIN-Symbol cache. If so, restore to the local variable. + const isinSymbolCacheExist = await cacache.get.info(cachePath, "isinSymbolCache"); + if (isinSymbolCacheExist) { + const cache = await cacache.get(cachePath, "isinSymbolCache"); + const cacheAsJson = JSON.parse(cache.data.toString(), this.mapReviver); + this.isinSymbolCache = cacheAsJson; + } + + // Verify if there is data in the Symbol cache. If so, restore to the local variable. + const symbolCacheExists = await cacache.get.info(cachePath, "symbolCache"); + if (symbolCacheExists) { + const cache = await cacache.get(cachePath, "symbolCache"); + const cacheAsJson = JSON.parse(cache.data.toString(), this.mapReviver); + this.symbolCache = cacheAsJson; + } + + // Return cache sizes. + return [this.isinSymbolCache.size, this.symbolCache.size]; + } + /** * Get symbols for a security by a given key. * @@ -133,7 +157,7 @@ export class YahooFinanceService { * @returns The symbols that are retrieved from Yahoo Finance, if any. */ private async getSymbolsByQuery(query: string, progress?: any): Promise { - + // First get quotes for the query. let queryResult = await yahooFinance.search(query, { @@ -229,13 +253,28 @@ export class YahooFinanceService { return symbolMatch; } + private async saveInCache(isin?: string, symbol?: string, value?: any) { + + // Save ISIN-value combination to cache if given. + if (isin && value) { + this.isinSymbolCache.set(isin, value); + await cacache.put(cachePath, "isinSymbolCache", JSON.stringify(this.isinSymbolCache, this.mapReplacer)); + } + + // Save symbol-value combination to cache if given. + if (symbol && value) { + this.symbolCache.set(symbol, value); + await cacache.put(cachePath, "symbolCache", JSON.stringify(this.symbolCache, this.mapReplacer)); + } + } + private logDebug(message, progress?, additionalTabs?: boolean) { const messageToLog = (additionalTabs ? '\t' : '') + `\t${message}` - if (process.env.DEBUG_LOGGING == "true") { + if (Boolean(process.env.DEBUG_LOGGING) == true) { if (!progress) { - console.log(`[i] ${messageToLog}`); + console.log(`[d] ${messageToLog}`); } else { progress.log(`[d] ${messageToLog}\n`); @@ -244,4 +283,25 @@ export class YahooFinanceService { } private sink() { } + + private mapReplacer(_, value) { + if (value instanceof Map) { + return { + dataType: 'Map', + value: Array.from(value.entries()), // or with spread: value: [...value] + }; + } else { + return value; + } + } + + private mapReviver(_, value) { + if (typeof value === 'object' && value !== null) { + if (value.dataType === 'Map') { + return new Map(value.value); + } + } + + return value; + } }