Skip to content

Commit

Permalink
feat: data fetching (qlik-oss#73)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cbt1
cbt1 authored Apr 21, 2022
1 parent b4e5ddb commit 877dfec
Show file tree
Hide file tree
Showing 31 changed files with 2,564 additions and 416 deletions.
1 change: 1 addition & 0 deletions .eslintrc.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ overrides:
semi: 2
quotes: [2, 'single']
react/jsx-indent-props: 2
no-plusplus: 0

# Unit test files
- files:
Expand Down
198 changes: 159 additions & 39 deletions src/hooks/use-data-model.ts
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,91 @@
/* eslint-disable no-param-reassign */
import { useMemo, usePromise, useState } from '@nebula.js/stardust';
import createData from '../pivot-table/data';
import { DataModel, FetchNextPage, PivotData } from '../types/types';
import createData, { addDataPage, addPage } from '../pivot-table/data';
import { DataModel, FetchMoreData, FetchNextPage, PivotData, ViewService } from '../types/types';
import useExpandOrCollapser from './use-expand-or-collapser';
import { DEFAULT_PAGE_SIZE, Q_PATH } from '../constants';
import { DEFAULT_PAGE_SIZE, PSEUDO_DIMENSION_INDEX, Q_PATH } from '../constants';
import useNebulaCallback from './use-nebula-callback';
import { NxSelectionCellType } from '../types/QIX';

const NOOP_PIVOT_DATA = {} as PivotData;

const getNextArea = (qWidth: number, qHeight: number) => ({
qLeft: 0,
qTop: 0,
qWidth,
qHeight
const MAX_GRID_SIZE = 10000;

const getPagesToTheTop = (scrollService: ViewService, maxHeight: number): EngineAPI.IRect[] => {
const { gridColumnStartIndex, gridRowStartIndex, gridWidth, gridHeight } = scrollService;
const pages = [] as EngineAPI.IRect[];

if (gridWidth === 0 && gridHeight === 0) {
return pages;
}

const totalHeight = Math.min(gridRowStartIndex + (gridHeight * 2), maxHeight);
const batchSize = Math.floor(MAX_GRID_SIZE / gridWidth);
let batchTop = Math.max(0, totalHeight - batchSize);
let batchHeight = totalHeight - batchTop;

do {
pages.unshift({
qLeft: gridColumnStartIndex,
qTop: Math.max(0, batchTop),
qWidth: gridWidth,
qHeight: batchHeight,
});

batchHeight = Math.min(batchHeight, batchTop);
batchTop -= batchSize;

} while (pages[0]?.qTop > 0);

return pages;
};

const getPagesToTheLeft = (scrollService: ViewService, maxWidth: number): EngineAPI.IRect[] => {
const { gridColumnStartIndex, gridRowStartIndex, gridWidth, gridHeight } = scrollService;
const pages = [] as EngineAPI.IRect[];

if (gridWidth === 0 && gridHeight === 0) {
return pages;
}

const totalWidth = Math.min(gridColumnStartIndex + (gridWidth * 2), maxWidth);
const batchSize = Math.floor(MAX_GRID_SIZE / gridHeight);
let batchLeft = Math.max(0, totalWidth - batchSize);
let batchWidth = totalWidth - batchLeft;

do {
pages.unshift({
qLeft: Math.max(0, batchLeft),
qTop: gridRowStartIndex,
qWidth: batchWidth,
qHeight: gridHeight,
});

batchWidth = Math.min(batchWidth, batchLeft);
batchLeft -= batchSize;

} while (pages[0]?.qLeft > 0);

return pages;
};

const getNextPage = (qLeft: number, qTop: number) => ({
qLeft,
qTop,
qWidth: DEFAULT_PAGE_SIZE,
qHeight: DEFAULT_PAGE_SIZE
});

export default function useDataModel(layout: EngineAPI.IGenericHyperCubeLayout, model: EngineAPI.IGenericObject | undefined ): DataModel {
export default function useDataModel(
layout: EngineAPI.IGenericHyperCubeLayout,
model: EngineAPI.IGenericObject | undefined,
viewService: ViewService
): DataModel {
const [loading, setLoading] = useState<boolean>(false);
const [pivotData, setPivotData] = useState<PivotData>(NOOP_PIVOT_DATA);
const [loading, setLoading] = useState(false);
const [hasMoreRows, setHasMoreRows] = useState(false);
const [hasMoreColumns, setHasMoreColumns] = useState(false);
const [qHyperCube, setHyperCube] = useState<EngineAPI.IHyperCube>({} as EngineAPI.IHyperCube);
const [maxAreaWidth, setMaxAreaWidth] = useState(DEFAULT_PAGE_SIZE);
const [maxAreaHeight, setMaxAreaHeight] = useState(DEFAULT_PAGE_SIZE);
const {
collapseLeft,
collapseTop,
Expand All @@ -32,50 +95,93 @@ export default function useDataModel(layout: EngineAPI.IGenericHyperCubeLayout,

const newLayoutHandler = useNebulaCallback(async () => {
if (layout && model) {
const { qLastExpandedPos, qPivotDataPages } = layout.qHyperCube;
let pivotPage = qPivotDataPages[0];
const { qLastExpandedPos, qPivotDataPages, qSize } = layout.qHyperCube;
const pivotPage = qPivotDataPages[0];
let nextPivotData = createData(pivotPage, layout.qHyperCube);

if (qLastExpandedPos) {
const width = (qLastExpandedPos?.qx || 0) + DEFAULT_PAGE_SIZE;
const height = (qLastExpandedPos?.qy || 0) + DEFAULT_PAGE_SIZE;
const area = getNextArea(Math.max(maxAreaWidth, width), Math.max(maxAreaHeight, height));

[pivotPage] = await model.getHyperCubePivotData(Q_PATH, [area]);

setMaxAreaWidth(prev => Math.max(prev, width));
setMaxAreaHeight(prev => Math.max(prev, height));
const pages = [
...getPagesToTheLeft(viewService, qSize.qcx),
...getPagesToTheTop(viewService, qSize.qcy)
];
const fetchPageQueries = pages.map(async (page) => {
const [nextPivotPage] = await model.getHyperCubePivotData(Q_PATH, [page]);
nextPivotData = addPage(nextPivotData, nextPivotPage);
});
await Promise.all(fetchPageQueries)
.catch(e => {
// TODO handle error
console.error(e);
});

viewService.shouldResetScroll = false;
} else {
// Layout was received because of a property change or selection change (confirmed or cancelled)
viewService.shouldResetScroll = true;
}

setHasMoreRows(pivotPage.qArea.qHeight < layout.qHyperCube.qSize.qcy);
setHasMoreColumns(pivotPage.qArea.qWidth < layout.qHyperCube.qSize.qcx);
setHasMoreRows(nextPivotData.size.data.y < layout.qHyperCube.qSize.qcy);
setHasMoreColumns(nextPivotData.size.data.x < layout.qHyperCube.qSize.qcx);
setHyperCube(layout.qHyperCube);
setPivotData(createData(pivotPage, layout.qHyperCube));
setPivotData(nextPivotData);
}
}, [layout, model]);
}, [layout, model, viewService]);

usePromise(() => newLayoutHandler(), [newLayoutHandler]);

const fetchNextPage = useNebulaCallback<FetchNextPage>(async (isRow: boolean) => {
const fetchNextPage = useNebulaCallback<FetchNextPage>(async (isRow: boolean, startIndex: number) => {
if (loading || !model) return;
if (isRow && !hasMoreRows) return;
if (!isRow && !hasMoreColumns) return;

setLoading(true);

try {
const width = isRow ? maxAreaWidth : maxAreaWidth + DEFAULT_PAGE_SIZE;
const height = isRow ? maxAreaHeight + DEFAULT_PAGE_SIZE : maxAreaHeight;
const [pivotPage] = await model.getHyperCubePivotData(Q_PATH, [getNextArea(width, height)]);
let nextPivotPage: EngineAPI.INxPivotPage;
let nextPivotData: PivotData;
if (isRow) {
const nextArea = getNextPage(startIndex, pivotData.size.data.y);
[nextPivotPage] = await model.getHyperCubePivotData(Q_PATH, [nextArea]);
nextPivotData = addPage(pivotData, nextPivotPage);
setHasMoreRows(nextPivotData.size.data.y < qHyperCube.qSize.qcy);
} else {
const nextArea = getNextPage(pivotData.size.data.x, startIndex);
[nextPivotPage] = await model.getHyperCubePivotData(Q_PATH, [nextArea]);
nextPivotData = addPage(pivotData, nextPivotPage);
setHasMoreColumns(nextPivotData.size.data.x < qHyperCube.qSize.qcx);
}

setLoading(false);
setPivotData(nextPivotData);
viewService.shouldResetScroll = false;
} catch (error) {
console.error(error);
setLoading(false);
}
}, [model, qHyperCube, loading, pivotData, viewService]);

const fetchMoreData = useNebulaCallback<FetchMoreData>(async (left: number, top: number, width: number, height: number) => {
if (loading || !model) return;

setLoading(true);

setMaxAreaWidth(prev => Math.max(prev, width));
setMaxAreaHeight(prev => Math.max(prev, height));
try {
const nextArea = {
qLeft: left,
qTop: top,
qWidth: Math.min(width, pivotData.size.data.x - left),
qHeight: Math.min(height, pivotData.size.data.y - top)
};

const [nextPivotPage] = await model.getHyperCubePivotData(Q_PATH, [nextArea]);
const nextPivotData = addDataPage(pivotData, nextPivotPage);
setLoading(false);
setHasMoreRows(pivotPage.qArea.qHeight < qHyperCube.qSize.qcy);
setHasMoreColumns(pivotPage.qArea.qWidth < qHyperCube.qSize.qcx);
setPivotData(createData(pivotPage, qHyperCube));
setPivotData(nextPivotData);
} catch (error) {
console.log('ERROR', error);
console.error(error);
setLoading(false);
}
}, [maxAreaWidth, maxAreaHeight, model, qHyperCube]);
}, [model, qHyperCube, loading, pivotData]);

const isDimensionLocked = useNebulaCallback((qType: EngineAPI.NxSelectionCellType, qRow: number, qCol: number) => {
if (qType === NxSelectionCellType.NX_CELL_LEFT) {
Expand All @@ -95,8 +201,19 @@ export default function useDataModel(layout: EngineAPI.IGenericHyperCubeLayout,

const getNoLeftDims = useNebulaCallback(() => qHyperCube.qNoOfLeftDims, [qHyperCube]);

const getMeasureInfoIndexFromCellIndex = useNebulaCallback((index: number) => {
const { qNoOfLeftDims, qEffectiveInterColumnSortOrder, qMeasureInfo } = qHyperCube;
const pIndex = qEffectiveInterColumnSortOrder.findIndex((num) => num === PSEUDO_DIMENSION_INDEX);
if (pIndex < qNoOfLeftDims) {
return 0;
}

return index % qMeasureInfo.length;
}, [qHyperCube]);

const dataModel = useMemo<DataModel>(() => ({
fetchNextPage,
fetchMoreData,
hasMoreColumns,
hasMoreRows,
collapseLeft,
Expand All @@ -108,8 +225,10 @@ export default function useDataModel(layout: EngineAPI.IGenericHyperCubeLayout,
isDimensionLocked,
getDimensionInfo,
getMeasureInfo,
getNoLeftDims
getNoLeftDims,
getMeasureInfoIndexFromCellIndex,
}),[fetchNextPage,
fetchMoreData,
hasMoreColumns,
hasMoreRows,
collapseLeft,
Expand All @@ -120,7 +239,8 @@ export default function useDataModel(layout: EngineAPI.IGenericHyperCubeLayout,
isDimensionLocked,
getDimensionInfo,
getMeasureInfo,
getNoLeftDims
getNoLeftDims,
getMeasureInfoIndexFromCellIndex,
]);

return dataModel;
Expand Down
13 changes: 13 additions & 0 deletions src/hooks/use-view-service.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import { useMemo, useState } from '@nebula.js/stardust';
import createViewService from '../services/view-service';
import { ViewService } from '../types/types';

const useViewService = (): ViewService => {
const [instance, setInstance] = useState({} as ViewService);

useMemo(() => setInstance(createViewService()), []);

return instance;
};

export default useViewService;
13 changes: 8 additions & 5 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import ext from './ext';
import { render, teardown } from './pivot-table/Root';
import useDataModel from './hooks/use-data-model';
import { ExtendedSelections } from './types/types';
import useViewService from './hooks/use-view-service';

// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
export default function supernova() {
Expand All @@ -22,20 +23,22 @@ export default function supernova() {
const model = useModel();
const rect = useRect();
const constraints = useConstraints();
const dataModel = useDataModel(layout, model);
const viewService = useViewService();
const dataModel = useDataModel(layout, model, viewService);
const selections = useSelections() as ExtendedSelections;

useEffect(() => {
if (dataModel.hasData && rect?.width && rect?.height && constraints && selections) {
console.debug('render', { layout, selections, constraints, dataModel, rect, model });
if (dataModel.hasData && rect?.width && rect?.height && constraints && selections && viewService) {
console.debug('render', { layout, selections, constraints, dataModel, rect, model, viewService });
render(element, {
rect,
constraints,
dataModel,
selections
selections,
viewService
});
}
}, [dataModel, rect?.width, rect?.height, constraints, selections]);
}, [dataModel, rect?.width, rect?.height, constraints, selections, viewService]);

useEffect(() => () => {
teardown(element);
Expand Down
Loading

0 comments on commit 877dfec

Please sign in to comment.