feat(vis-type: bar): switch to ECharts bar chart (#497)

* feat: bar chart with echarts (wip)

* feat: support all bar chart config options

* feat: compute data table for each facet

* perf: compose bar data table once and filter them for each facet

* feat: make focus facet selector working

* style: calculate chart height

* feat: scrollable facet views

* feat: select items with click on a bar

* feat: add tooltip on hover

* feat: assign color scale

* feat: binned color scale for numerical group columns

* fix: selection of items on bar click

* feat: highlight incoming selection

* fix: bar height for non-grouped bars

* style: truncate axis labels after 50 chars

* fix: add scrollbar

* fix: if agg column or cat column is undefined

* fix: account for null values in the config

- make group value default
- harmonize bar width/height

* !feat: don't exclude already selected columns in sidebar

* fix: scatter vis for selected points

(cherry picked from commit 8e4a0ea)

* fix: check width and height of bar chart container

Fixes datavisyn/communication_xaira_ordino#26

The ReactECharts component will check the resize of the parent element and update accordingly

* fix: bar chart facets horizontal scrollbar (#480)

* fix: make resizing work for facets

* feat: add a loading overlay when the viewport resizes

* fix: scroll flickering

* fix: remove pos relative

* chore: remove unnecessary overflow hidden

* chore: address PR review

* fix: storybook error

* feat: implement bar plot sorting (#482)

* feat: add sorting controls
wip: add sorting for y axis
wip: add property in config to preload sorting
wip: use in reprovisyn

* feat: improvements in sorting

* feat: handle categorical column change

* feat: implement initial sorting

* feat: move sort into the header

* fix: sort for normalized mode

* chore: sort facets by name

* fix: add null check for sortState

* fix: add null checks for sort button

* fix: fallback value

* fix: remove circular dependency

* fix: address PR feedback

* chore: use color for unknown values

* fix: storybook error

* fix: title of bar plots overlaps with legend when in 5050 view with volcano (#486)

* feat: add sorting controls
wip: add sorting for y axis
wip: add property in config to preload sorting
wip: use in reprovisyn

* feat: improvements in sorting

* feat: handle categorical column change

* feat: implement initial sorting

* feat: move sort into the header

* fix: sort for normalized mode

* chore: sort facets by name

* fix: add null check for sortState

* fix: add null checks for sort button

* fix: fallback value

* fix: remove circular dependency

* fix: address PR feedback

* chore: use color for unknown values

* fix: storybook error

* fix: overlapping legends in bar chart

* chore: update SingleEChartsBarChart.tsx

* fix: aggregations in bar chart

* fix(wip): select bars in sorted chart
wip: the current config in main app has a bug which needs to be resolved

* feat: add shift + select to add to the selection

* fix: temporarily disable bar chart selection

* fix: wip show categories for all the facets

* chore: update config

* fix: add padding top and bottom to bar chart height

* chore: rename variables

* fix: update bars on sorting
wip: currently only ascending sort works for horizontal bars
wip: grouping with the same column does not work
wip: ⚠️ major performance issue

* fix: display same grouped bars in sorted order correctly
wip: colors do not match
wip: selection in sorting does not work as expected

* fix: sorting using copilot powered development

* chore: cleanup after feature

* feat(wip): add condition to sort unknown at the end of the list

wip: the behavior is inconsistent for vertical and horizontal orientation

* chore: minor improvements

* fix: add react-window virtualizer

* docs: add documentation for matrixSort method

* fix: handle NaN values in matrix aggregation

* feat: improve shift + click to select usability

* fix: remove control key event listener

* chore: remove internal sorting state

* chore: remove redundant code

* Fixed virtualization remount by having a stable renderer and using the itemData property of react-window

* fix: adjust max label length

* fix: bar label overlap + add optimization for large data

* fix: rotate x-axis labels by 45 degrees

* fix: emphasize and blur options

* fix: make barWidth responsive when there are a lot of bars

* fix: horizontal scrollbar issue

* refactor: optimize sorting for performance

* Fixed rerenders caused by new settings object, incorrect height of facets, added hook with setRef function to use instead of react echarts component

* added useChart to exported hooks in visyn_core

* linted hook

* removed option and converted it to a usememo

* Combined state of vis to one state object

* cleaned up

* chore: cleanup + minor fixes

* fix: build error

* fix: build errors

* chore: rename variables

* fix: infinite loop

* feat: add axis label

* fix: min and max aggregations

* fix: lint error

* chore: cleanup

* chore: move function to proper utils file

* chore: cleanup

* fix: battle with strict mode for affected files

* chore: move calculate chart height function to bar chart

* feat: use mantine scrollarea for variable size list

* feat: use mantine scrollarea for variable size list

* chore: continuing battle with strict mode

* fix: empty bar plot on facets

* fix: color and selection

* fix: use higher opacity for unselected

* fix: set opacity back to 0.5

* fix: selection broke

* fix: selection again

* fix: add temp xAxis domain

* fix: selection for same group and facet combination

* fix: modify domain for aggregation axis

* fix: add higher opacity to to unselected  bar label items in grouped mode

* fix: reverse condition

* fix: adapt opacity and colors

* fix: reaggregate bar chart data + add global min and max
wip: selection does not work

* fix: selection

* feat: sort categories

* fix: tooltip clipping

* fix: follow up fix for selection

* feat: add useFullHeight flag to use full height if available

* fix: labels when direction is changed

* fix: aggregation type sensitive global min and max

* feat: implement sorting along both axes

* feat: add DOM sensitive label truncate

* fix: recalculate height

* fix: single value in facet

* fix: minor improvements

* style: make chart responsive to the container

* feat: add responsive labels for y axis for horizontal bar chart

* chore: cleanup unnecessary ref

* fix: click same bar to deselect

* fix: labels width be min of max label width or container / 3

* refactor: add selection information in the header

* fix: categoriesList null check

* fix: invalidate item size cache on config change

* fix: invalidate item size cache on datatable change as well

* feat: sync grid left across facets

* feat(wip): show axis label tooltip
wip: position the tooltip correctly
wip: add a flag to enable / disable the tooltip

* chore: sync colors

* fix: tooltip position + refactoring

* fix: storybook

* chore: select bars when label clicked

* fix: tooltip position for horizontal as well as vertical bars

* fix: strict mode issues in useChart.ts

* fix: add min-width to the bars and make the chart scrollable

* style: make title in sync with other charts

* style: add descriptive title for facets

* style: change axis title for normalized

* style: fix cursor pointer for non interactive elements

* style: round to 4 digits in legend  for numerical group

* fix: add custom binning

* fix: always show tooltip for vertical bars

* fix: axis label distance

* chore: restructure bar plot files + remove legacy bar plot components

* feat: add axis label click for sort

* fix: color scale for faceted plots

* chore: expose isBarConfig function

* chore: restructure files again

* fix: do not compute legend when no grouping is available

* fix: min max calculation

* chore: deprecated bar chart sort buttons

* chore: make useFullHeight as default

* fix: build error

* chore: add comments

* fix: label of aggregation axis

* fix: reduce unnecessary recalculation of aggregatedData

* fix: numerical color scale

* chore: remove optional check

* chore: improve tooltip

* style: improve tooltip

* style: sync tooltip appearance

* style: make axis well spaced

* fix: make loader appear only once

* chore: cleanup

* fix: scrollbar overlap on legend

* fix: tooltip rounding for numerical scales

* chore: update label helper

* Adjusted imports for echarts

* feat: implement responsive width flag

* chore: update title as per the feedback

* chore: minor improvements

* fix: hide tooltip labels in stacked mode and show only when hovered

* fix: display facets in the tooltip first

* fix: htmlize tooltip

* fix: bug when same facet and category selected for numerical group

* fix: color scale for numerical grouping and faceting

* fix: color of legend and tooltip

* fix: dimensions

* chore: remove bar chart sort button from the header

* chore: add placeholder tests

* fix: improve median logic

* fix: bar chart color scale for facet and group combinations

* chore: display labels for grouped bars faceted by same column

* fix: bar chart story

* fix: tooltip label for group having only one bar

* fix: selection when only one bar is present in grouped mode

* chore: remove group.ts

* chore: add group.ts

* chore: improve diversity of test data

* fix: group legend and tooltip for single numerical value grouping

* fix: iris story

* fix: generalize the fix

* fix: generalize the fix 2

* chore: optimizations on previous fix

* fix: limit min and max to 0

* refactor: updates to aggregation types + reduce complexity

* fix: global domain calculation of min max and median

* fix: global domain calculation of average

* fix: use the correct median function import

* fix: label visibility

* fix: median logic

* chore: reduce number of looping for median calculation

* fix: tooltip content for same group and facet column

* test: add tests for bar dimensions

* test: add tests for bin lookup

* chore: cleanup in singleechartbarchart

* test: add tests for generated-aggregated-data-lookup

* docs: add comment about `merge` flag in BaseVisConfig

* chore: cleanup main app

* docs: update comment

* fix: median aggregation

* fix: storybook

* feat(vis-type: bar): download chart is clipped for long charts (#561)

Co-authored-by: Holger Stitz <holger.stitz@datavisyn.io>

* chore: extend bar tests

* chore: extend bar tests

* chore: add comment

* chore: extend bar tests

* chore: update storybook

* feat: add tooltip for disabled segmented control

* fix: color scale order for numerical groupings

* fix: main app for the PW tests to pass

---------

Co-authored-by: Holger Stitz <holger.stitz@datavisyn.io>
Co-authored-by: oltionchampari <oltion.champari@datavisyn.io>
Co-authored-by: Moritz Heckmann <moritz.heckmann@datavisyn.io>
Co-authored-by: Champari Oltion <51322092+oltionchampari@users.noreply.github.com>

package.json

@@ -106,10 +106,12 @@
     "d3-force-boundary": "^0.0.3",
     "d3-hexbin": "^0.2.2",
     "d3v7": "npm:d3@^7.9.0",
+    "echarts": "^5.5.1",
     "fit-curve": "^0.2.0",
     "html-to-image": "^1.11.11",
     "i18next": "^23.14.0",
     "jstat": "^1.9.6",
+    "jszip": "^3.10.1",
     "lineupjs": "4.11.0",
     "lodash": "~4.17.21",
     "plotly.js-dist-min": "~2.12.0",
@@ -118,6 +120,7 @@
     "react-highlight-words": "^0.20.0",
     "react-plotly.js": "^2.6.0",
     "react-spring": "^9.7.4",
+    "react-window": "^1.8.10",
     "use-deep-compare-effect": "^1.8.1",
     "visyn_scripts": "^11.1.0"
   },
@@ -135,6 +138,7 @@
     "@storybook/react": "^7.6.20",
     "@storybook/react-webpack5": "^7.6.20",
     "@storybook/testing-library": "0.2.2",
+    "@types/react-window": "^1.8.8",
     "chromatic": "^11.7.1",
     "storybook": "^7.6.20",
     "storybook-addon-swc": "^1.2.0"

src/demo/MainApp.tsx

@@ -34,7 +34,7 @@ export function MainApp() {
       },
     ],
     color: {
-      description: null,
+      description: '',
       id: 'cellularity',
       name: 'Cellularity',
     },
@@ -66,6 +66,11 @@ export function MainApp() {
             aboutAppModal: {
               content: <Text>This is the demo app for visyn core.</Text>,
             },
+            center: (
+              <Text c="white" size="sm">
+                {breastCancerData.length} data points / {selection.length} points selected
+              </Text>
+            ),
           }}
         />
       }

src/utils/index.ts

@@ -3,3 +3,4 @@ export * from './getCssValue';
 export * from './initializeLibrary';
 export * from './fromNow';
 export * from './colors';
+export * from './sanitize-filename';

src/utils/sanitize-filename.ts

@@ -0,0 +1,69 @@
+// NOTE: @dv-usama-ansari: Referenced from https://github.com/parshap/truncate-utf8-bytes/blob/master/lib/truncate.js
+import { Buffer } from 'buffer';
+
+function isHighSurrogate(codePoint: number) {
+  return codePoint >= 0xd800 && codePoint <= 0xdbff;
+}
+
+function isLowSurrogate(codePoint: number) {
+  return codePoint >= 0xdc00 && codePoint <= 0xdfff;
+}
+
+// Truncate string by size in bytes
+function truncate(getLength: (str: string) => number, input: string, byteLength: number) {
+  let curByteLength = 0;
+  let codePoint = 0;
+  let segment = '';
+
+  for (let i = 0; i < input.length; i += 1) {
+    codePoint = input.charCodeAt(i);
+    segment = input[i] as string;
+
+    if (isHighSurrogate(codePoint) && isLowSurrogate(input.charCodeAt(i + 1))) {
+      i += 1;
+      segment += input[i];
+    }
+
+    curByteLength += getLength(segment);
+
+    if (curByteLength === byteLength) {
+      return input.slice(0, i + 1);
+    }
+    if (curByteLength > byteLength) {
+      return input.slice(0, i - segment.length + 1);
+    }
+  }
+
+  return input;
+}
+
+// NOTE: @dv-usama-ansari: Referenced from https://github.com/parshap/truncate-utf8-bytes/blob/master/index.js
+const getLength = Buffer.byteLength.bind(Buffer);
+const boundTruncate = truncate.bind(null, getLength);
+
+// NOTE: @dv-usama-ansari: Referenced from https://github.com/parshap/node-sanitize-filename/blob/master/index.js
+const illegalRe = /[/?<>\\:*|"]/g;
+// eslint-disable-next-line no-control-regex
+const controlRe = /[\x00-\x1f\x80-\x9f]/g;
+const reservedRe = /^\.+$/;
+const windowsReservedRe = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])(\..*)?$/i;
+const windowsTrailingRe = /[. ]+$/;
+
+function sanitizeHelper(input: string, replacement: string) {
+  const sanitized = input
+    .replace(illegalRe, replacement)
+    .replace(controlRe, replacement)
+    .replace(reservedRe, replacement)
+    .replace(windowsReservedRe, replacement)
+    .replace(windowsTrailingRe, replacement);
+  return boundTruncate(sanitized, 255);
+}
+
+export function sanitize(input: string, options?: { replacement?: '' }) {
+  const replacement = options?.replacement ?? '';
+  const output = sanitizeHelper(input, replacement);
+  if (replacement === '') {
+    return output;
+  }
+  return sanitizeHelper(output, '');
+}

src/vis/EagerVis.tsx

@@ -23,10 +23,7 @@ import {
 
 import { VisSidebar } from './VisSidebar';
 import { VisSidebarOpenButton } from './VisSidebarOpenButton';
-import { BarVis } from './bar/BarVis';
-import { BarVisSidebar } from './bar/BarVisSidebar';
-import { EBarDirection, EBarDisplayType, EBarGroupingType, IBarConfig } from './bar/interfaces';
-import { barMergeDefaultConfig } from './bar/utils';
+import { BarVis, BarVisSidebar, EBarDirection, EBarDisplayType, EBarGroupingType, IBarConfig, barMergeDefaultConfig } from './bar';
 import { correlationMergeDefaultConfig } from './correlation';
 import { CorrelationVis } from './correlation/CorrelationVis';
 import { CorrelationVisSidebar } from './correlation/CorrelationVisSidebar';
@@ -256,7 +253,7 @@ export function EagerVis({
     if (isSelectedVisTypeRegistered && (!visConfig?.merged || prevVisConfig?.type !== visConfig?.type)) {
       // TODO: I would prefer this to be not in a useEffect, as then we wouldn't have the render-flicker: https://react.dev/learn/you-might-not-need-an-effect#adjusting-some-state-when-a-prop-changes
       setPrevVisConfig(visConfig);
-      _setVisConfig?.(getVisByType(visConfig.type)?.mergeConfig(columns, { ...visConfig, merged: true }));
+      _setVisConfig?.(getVisByType(visConfig.type)?.mergeConfig(columns, { ...visConfig, merged: true }) as BaseVisConfig);
     }
   }, [_setVisConfig, columns, getVisByType, isSelectedVisTypeRegistered, prevVisConfig?.type, visConfig]);