Add load/export support for TSV (#2916)

apps/zui/src/components/data-format-select.tsx

@@ -9,6 +9,7 @@ export function DataFormatOptions() {
       <option value="json">JSON</option>
       <option value="line">Line</option>
       <option value="parquet">Parquet</option>
+      <option value="tsv">TSV</option>
       <option value="vng">VNG</option>
       <option value="zeek">Zeek</option>
       <option value="zjson">ZJSON</option>

apps/zui/src/domain/results/utils/prep-export-query.ts

@@ -22,6 +22,7 @@ function cutColumns(query: string, api: ZuiApi) {
 }
 
 function maybeFuse(query: string, format: string) {
-  if (format === "csv" || format === "arrows") query += " | fuse"
+  if (format === "csv" || format == "tsv" || format === "arrows")
+    query += " | fuse"
   return query
 }

apps/zui/src/js/components/ExportModal.tsx

@@ -75,6 +75,10 @@ const ExportModal = ({onClose}) => {
               <input type="radio" id="ndjson" value="ndjson" name="format" />
               <label htmlFor="ndjson">NDJSON</label>
             </div>
+            <div className={forms.radioInput}>
+              <input type="radio" id="tsv" value="tsv" name="format" />
+              <label htmlFor="tsv">TSV</label>
+            </div>
             <div className={forms.radioInput}>
               <input type="radio" id="vng" value="vng" name="format" />
               <label htmlFor="vng">VNG</label>

packages/zed-js/src/client/types.ts

@@ -7,6 +7,7 @@ export type ResponseFormat =
   | 'csv'
   | 'json'
   | 'ndjson'
+  | 'tsv'
   | 'vng'
   | 'zeek'
   | 'zjson'
@@ -69,6 +70,7 @@ export type LoadFormat =
   | 'json'
   | 'line'
   | 'parquet'
+  | 'tsv'
   | 'vng'
   | 'zeek'
   | 'zjson'
@@ -82,6 +84,7 @@ export type LoadContentType =
   | 'application/json'
   | 'application/x-line'
   | 'application/x-parquet'
+  | 'text/tab-separated-values'
   | 'application/x-vng'
   | 'application/x-zeek'
   | 'application/x-zjson'

packages/zed-js/src/client/utils.ts

@@ -36,6 +36,7 @@ export function accept(format: ResponseFormat) {
     csv: 'text/csv',
     json: 'application/json',
     ndjson: 'application/x-ndjson',
+    tsv: 'text/tab-separated-values',
     vng: 'application/x-vng',
     zeek: 'application/x-zeek',
     zjson: 'application/x-zjson',

packages/zui-player/README.md

@@ -51,7 +51,7 @@ import { getPath } from 'zui-test-data';
 
 play('Preview & Load', (app, test) => {
   test('create new pool, change key, type <enter>', async () => {
-    await app.dropFile(getPath('sample.tsv'));
+    await app.dropFile(getPath('sample.zeektsv'));
     await app.click('button', 'Pool Settings');
     await app.fill('Pool Key', 'my_new_key');
     await app.press('Enter');

packages/zui-player/tests/export.spec.ts

@@ -11,6 +11,7 @@ const formats = [
   { label: 'CSV', expectedSize: 10851 },
   { label: 'JSON', expectedSize: 13659 },
   { label: 'NDJSON', expectedSize: 13657 },
+  { label: 'TSV', expectedSize: 10797 },
   { label: 'VNG', expectedSize: 8053 },
   { label: 'Zeek', expectedSize: 10138 },
   { label: 'ZJSON', expectedSize: 18007 },
@@ -23,7 +24,7 @@ test.describe('Export tests', () => {
 
   test.beforeAll(async () => {
     await app.init();
-    await app.createPool([getPath('sample.tsv')]);
+    await app.createPool([getPath('sample.zeektsv')]);
     await app.click('button', 'Query Pool');
     await app.query('sort ts, _path');
   });

packages/zui-player/tests/ingest.spec.ts

@@ -31,7 +31,7 @@ test.describe('Ingest tests', () => {
 
   test.beforeAll(async () => {
     await app.init();
-    await app.createPool([getPath('sample.tsv')]);
+    await app.createPool([getPath('sample.zeektsv')]);
     await app.click('button', 'Query Pool');
   });
 

packages/zui-player/tests/preview-and-load.spec.ts

@@ -3,7 +3,7 @@ import { getPath } from 'zui-test-data';
 
 play('Preview & Load', (app, test) => {
   test('create new pool, change key, type <enter>', async () => {
-    await app.dropFile(getPath('sample.tsv'));
+    await app.dropFile(getPath('sample.zeektsv'));
     await app.click('button', 'Pool Settings');
     await app.fill('Pool Key', 'my_new_key');
     await app.press('Enter');

packages/zui-player/tests/queries.spec.ts

@@ -7,7 +7,7 @@ test.describe('Query tests', () => {
 
   test.beforeAll(async () => {
     await app.init();
-    await app.createPool([getPath('sample.tsv')]);
+    await app.createPool([getPath('sample.zeektsv')]);
     await app.click('button', 'Query Pool');
   });
 
@@ -26,9 +26,9 @@ test.describe('Query tests', () => {
       nodes.map((n) => n.innerText.trim().replaceAll(/\s+/g, ' '))
     );
     const expected = [
-      "from 'sample.tsv' | 3 now",
-      "from 'sample.tsv' | 2 now",
-      "from 'sample.tsv' | 1 now",
+      "from 'sample.zeektsv' | 3 now",
+      "from 'sample.zeektsv' | 2 now",
+      "from 'sample.zeektsv' | 1 now",
     ];
     expect(entries).toEqual(expected);
   });