Desktop,CLI: Fixes #10653: Change Resource filetype detecting strategy (

#10907)

Co-authored-by: Laurent Cozic <laurent22@users.noreply.github.com>

.eslintignore

@@ -1338,6 +1338,7 @@ packages/lib/services/trash/permanentlyDeleteOldItems.test.js
 packages/lib/services/trash/permanentlyDeleteOldItems.js
 packages/lib/services/trash/restoreItems.test.js
 packages/lib/services/trash/restoreItems.js
+packages/lib/shim-init-node.test.js
 packages/lib/shim-init-node.js
 packages/lib/shim.js
 packages/lib/string-utils.test.js

.gitignore

@@ -1315,6 +1315,7 @@ packages/lib/services/trash/permanentlyDeleteOldItems.test.js
 packages/lib/services/trash/permanentlyDeleteOldItems.js
 packages/lib/services/trash/restoreItems.test.js
 packages/lib/services/trash/restoreItems.js
+packages/lib/shim-init-node.test.js
 packages/lib/shim-init-node.js
 packages/lib/shim.js
 packages/lib/string-utils.test.js

packages/app-cli/app/command-e2ee.ts

@@ -8,8 +8,7 @@ import shim from '@joplin/lib/shim';
 import * as pathUtils from '@joplin/lib/path-utils';
 import { getEncryptionEnabled, localSyncInfo } from '@joplin/lib/services/synchronizer/syncInfoUtils';
 import { generateMasterKeyAndEnableEncryption, loadMasterKeysFromSettings, masterPasswordIsValid, setupAndDisableEncryption } from '@joplin/lib/services/e2ee/utils';
-const imageType = require('image-type');
-const readChunk = require('read-chunk');
+import { fromFile as fileTypeFromFile } from 'file-type';
 
 class Command extends BaseCommand {
 	public usage() {
@@ -136,8 +135,7 @@ class Command extends BaseCommand {
 					const outputDir = options.output ? options.output : require('os').tmpdir();
 					let outFile = `${outputDir}/${pathUtils.filename(args.path)}.${Date.now()}.bin`;
 					await EncryptionService.instance().decryptFile(args.path, outFile);
-					const buffer = await readChunk(outFile, 0, 64);
-					const detectedType = imageType(buffer);
+					const detectedType = await fileTypeFromFile(outFile);
 
 					if (detectedType) {
 						const newOutFile = `${outFile}.${detectedType.ext}`;

packages/app-cli/package.json

@@ -47,15 +47,14 @@
     "aws-sdk": "2.1340.0",
     "chalk": "4.1.2",
     "compare-version": "0.1.2",
+    "file-type": "16.5.4",
     "fs-extra": "11.2.0",
     "html-entities": "1.4.0",
-    "image-type": "3.1.0",
     "keytar": "7.9.0",
     "md5": "2.3.0",
     "node-rsa": "1.1.1",
     "open": "8.4.2",
     "proper-lockfile": "4.1.2",
-    "read-chunk": "2.1.0",
     "server-destroy": "1.0.1",
     "sharp": "0.33.4",
     "sprintf-js": "1.1.3",

packages/app-cli/tests/support/valid_pdf_without_ext

@@ -0,0 +1,198 @@
+%PDF-1.3
+%����
+
+1 0 obj
+<<
+/Type /Catalog
+/Outlines 2 0 R
+/Pages 3 0 R
+>>
+endobj
+
+2 0 obj
+<<
+/Type /Outlines
+/Count 0
+>>
+endobj
+
+3 0 obj
+<<
+/Type /Pages
+/Count 2
+/Kids [ 4 0 R 6 0 R ] 
+>>
+endobj
+
+4 0 obj
+<<
+/Type /Page
+/Parent 3 0 R
+/Resources <<
+/Font <<
+/F1 9 0 R 
+>>
+/ProcSet 8 0 R
+>>
+/MediaBox [0 0 612.0000 792.0000]
+/Contents 5 0 R
+>>
+endobj
+
+5 0 obj
+<< /Length 1074 >>
+stream
+2 J
+BT
+0 0 0 rg
+/F1 0027 Tf
+57.3750 722.2800 Td
+( A Simple PDF File ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 688.6080 Td
+( This is a small demonstration .pdf file - ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 664.7040 Td
+( just for use in the Virtual Mechanics tutorials. More text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 652.7520 Td
+( text. And more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 628.8480 Td
+( And more text. And more text. And more text. And more text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 616.8960 Td
+( text. And more text. Boring, zzzzz. And more text. And more text. And ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 604.9440 Td
+( more text. And more text. And more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 592.9920 Td
+( And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 569.0880 Td
+( And more text. And more text. And more text. And more text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 557.1360 Td
+( text. And more text. And more text. Even more. Continued on page 2 ...) Tj
+ET
+endstream
+endobj
+
+6 0 obj
+<<
+/Type /Page
+/Parent 3 0 R
+/Resources <<
+/Font <<
+/F1 9 0 R 
+>>
+/ProcSet 8 0 R
+>>
+/MediaBox [0 0 612.0000 792.0000]
+/Contents 7 0 R
+>>
+endobj
+
+7 0 obj
+<< /Length 676 >>
+stream
+2 J
+BT
+0 0 0 rg
+/F1 0027 Tf
+57.3750 722.2800 Td
+( Simple PDF File 2 ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 688.6080 Td
+( ...continued from page 1. Yet more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 676.6560 Td
+( And more text. And more text. And more text. And more text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 664.7040 Td
+( text. Oh, how boring typing this stuff. But not as boring as watching ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 652.7520 Td
+( paint dry. And more text. And more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 640.8000 Td
+( Boring.  More, a little more text. The end, and just as well. ) Tj
+ET
+endstream
+endobj
+
+8 0 obj
+[/PDF /Text]
+endobj
+
+9 0 obj
+<<
+/Type /Font
+/Subtype /Type1
+/Name /F1
+/BaseFont /Helvetica
+/Encoding /WinAnsiEncoding
+>>
+endobj
+
+10 0 obj
+<<
+/Creator (Rave \(http://www.nevrona.com/rave\))
+/Producer (Nevrona Designs)
+/CreationDate (D:20060301072826)
+>>
+endobj
+
+xref
+0 11
+0000000000 65535 f
+0000000019 00000 n
+0000000093 00000 n
+0000000147 00000 n
+0000000222 00000 n
+0000000390 00000 n
+0000001522 00000 n
+0000001690 00000 n
+0000002423 00000 n
+0000002456 00000 n
+0000002574 00000 n
+
+trailer
+<<
+/Size 11
+/Root 1 0 R
+/Info 10 0 R
+>>
+
+startxref
+2714
+%%EOF

packages/lib/package.json

@@ -59,14 +59,14 @@
     "diff-match-patch": "1.0.5",
     "fast-deep-equal": "3.1.3",
     "fast-xml-parser": "3.21.1",
+    "file-type": "16.5.4",
     "follow-redirects": "1.15.6",
     "form-data": "4.0.0",
     "fs-extra": "11.2.0",
     "hpagent": "1.2.0",
     "html-entities": "1.4.0",
     "html-minifier": "4.0.0",
     "image-data-uri": "2.0.1",
-    "image-type": "3.1.0",
     "immer": "7.0.15",
     "js-yaml": "4.1.0",
     "markdown-it": "13.0.2",
@@ -83,7 +83,6 @@
     "promise": "8.3.0",
     "query-string": "7.1.3",
     "re-reselect": "4.0.1",
-    "read-chunk": "2.1.0",
     "redux": "4.2.1",
     "relative": "3.0.2",
     "reselect": "4.1.8",

packages/lib/shim-init-node.test.ts

@@ -0,0 +1,19 @@
+
+const { shimInit } = require('./shim-init-node');
+import shim from './shim';
+import { setupDatabaseAndSynchronizer, supportDir } from './testing/test-utils';
+
+describe('shim-init-node', () => {
+
+	beforeEach(async () => {
+		await setupDatabaseAndSynchronizer(1);
+		shimInit();
+	});
+
+	test('should set mime the correct mime for a PDF file even if the extension is missing', async () => {
+		const filePath = `${supportDir}/valid_pdf_without_ext`;
+		const resource = await shim.createResourceFromPath(filePath);
+
+		expect(resource.mime).toBe('application/pdf');
+	});
+});

packages/lib/shim-init-node.ts

@@ -12,6 +12,7 @@ import { ResourceEntity } from './services/database/types';
 import { TextItem } from 'pdfjs-dist/types/src/display/api';
 import replaceUnsupportedCharacters from './utils/replaceUnsupportedCharacters';
 import { FetchBlobOptions } from './types';
+import { fromFile as fileTypeFromFile } from 'file-type';
 import crypto from './services/e2ee/crypto';
 
 import FileApiDriverLocal from './file-api-driver-local';
@@ -306,9 +307,6 @@ function shimInit(options: ShimInitOptions = null) {
 			...options,
 		};
 
-		const readChunk = require('read-chunk');
-		const imageType = require('image-type');
-
 		const isUpdate = !!options.destinationResourceId;
 
 		const uuid = require('./uuid').default;
@@ -332,8 +330,7 @@ function shimInit(options: ShimInitOptions = null) {
 		let fileExt = safeFileExtension(fileExtension(filePath));
 
 		if (!resource.mime) {
-			const buffer = await readChunk(filePath, 0, 64);
-			const detectedType = imageType(buffer);
+			const detectedType = await fileTypeFromFile(filePath);
 
 			if (detectedType) {
 				fileExt = detectedType.ext;