diff --git a/dev/svelte/SwitchPositions.svelte b/dev/svelte/SwitchPositions.svelte
index 22d1431e0..a5db8d477 100644
--- a/dev/svelte/SwitchPositions.svelte
+++ b/dev/svelte/SwitchPositions.svelte
@@ -1,23 +1,88 @@
-
\ No newline at end of file
+
diff --git a/src/Deeptable.ts b/src/Deeptable.ts
index 9d07536d5..4d7134a22 100644
--- a/src/Deeptable.ts
+++ b/src/Deeptable.ts
@@ -24,6 +24,7 @@ import {
Int32,
Int8,
tableToIPC,
+ Struct,
} from 'apache-arrow';
import { Scatterplot } from './scatterplot';
import { wrapArrowTable } from './wrap_arrow';
@@ -34,6 +35,8 @@ import type {
IdSelectParams,
} from './selection';
import { DataSelection } from './selection';
+import { Some, TupleMap } from './utilityFunctions';
+import { getNestedVector } from './regl_rendering';
type TransformationStatus = 'queued' | 'in progress' | 'complete' | 'failed';
@@ -70,7 +73,8 @@ export class Deeptable {
...defaultTransformations,
};
public _plot: Scatterplot | null;
- private extents: Record = {};
+ private extents: TupleMap =
+ new TupleMap();
// A 3d identifier for the tile. Usually [z, x, y]
private _extent?: Rectangle;
public _ix_seed = 0;
@@ -134,6 +138,9 @@ export class Deeptable {
this.root_tile = new Tile(defaultManifest, null, this);
const preProcessRootTile = this.root_tile.preprocessRootTileInfo();
+ // At instantiation, the deeptable isn't ready; only once this
+ // async stuff is done can the deeptable be used.
+ // TODO: Add an async static method as the preferred initialization method.
this.promise = preProcessRootTile.then(async () => {
const batch = await this.root_tile.get_arrow(null);
const schema = batch.schema;
@@ -346,13 +353,24 @@ export class Deeptable {
domain(
columnName: string,
+ subfield?: string[],
): [T[1], T[1]] {
- if (this.extents[columnName]) {
- return this.extents[columnName];
+ const key = [columnName, ...(subfield || [])] as Some;
+ if (this.extents.get(key)) {
+ return this.extents.get(key);
+ }
+
+ // First -- look at the schema metadata.
+ let dim = this._schema?.fields.find((d) => d.name === columnName);
+ for (const sub in subfield) {
+ if (dim === undefined) {
+ continue;
+ }
+ console.log({ dim });
+ dim = (dim as Field>).type.children.find(
+ (d) => d.name === sub,
+ );
}
- const dim = this._schema?.fields.find(
- (d) => d.name === columnName,
- ) as Field;
if (dim !== undefined) {
let min: T[0] | undefined = undefined;
let max: T[0] | undefined = undefined;
@@ -375,24 +393,30 @@ export class Deeptable {
'Date field extents in metadata must be passed as strings',
);
}
- return (this.extents[columnName] = [new Date(min), new Date(max)]);
+ this.extents.set(key, [new Date(min), new Date(max)]);
+ return this.extents.get(key);
}
if (typeof max === 'string') {
throw new Error('Failed to parse min-max as numbers');
}
if (min !== undefined) {
- return (this.extents[columnName] = [min as T[1], max as T[1]] as
+ this.extents.set(key, [min as T[1], max as T[1]] as
| [number, number]
| [Date, Date]);
+ return this.extents.get(key);
}
}
+
const vectors: Vector[] = this.map((tile) => tile)
.filter((d) => d.hasLoadedColumn(columnName))
- .map((d) => d.record_batch.getChild(columnName) as Vector);
+ .map((d) => getNestedVector(d, [columnName, ...(subfield || [])]));
+
const extented = extent([...new Vector(vectors)]) as [T[1], T[1]] as
| [number, number]
| [Date, Date];
- return (this.extents[columnName] = extented);
+
+ this.extents.set(key, extented);
+ return this.extents.get(key);
}
*points(bbox: Rectangle | undefined, max_ix = 1e99) {
diff --git a/src/aesthetics/Aesthetic.ts b/src/aesthetics/Aesthetic.ts
index 9dd1ae15d..4c1ef3b49 100644
--- a/src/aesthetics/Aesthetic.ts
+++ b/src/aesthetics/Aesthetic.ts
@@ -1,10 +1,11 @@
import type { TextureSet } from './AestheticSet';
import { isConstantChannel } from '../typing';
-import { Type, Vector } from 'apache-arrow';
+import { Struct, Type, Vector } from 'apache-arrow';
import { StructRowProxy } from 'apache-arrow/row/struct';
import { isNumber } from 'lodash';
import type * as DS from '../types';
import { Scatterplot } from '../scatterplot';
+import { Some } from '../utilityFunctions';
/**
* An Aesthetic bundles all operations in mapping from user dataspace to webGL based aesthetics.
@@ -26,6 +27,7 @@ export abstract class Aesthetic<
public abstract default_range: [Output['rangeType'], Output['rangeType']];
public scatterplot: Scatterplot;
public field: string | null = null;
+ public subfield: string[] = [];
public _texture_buffer: Float32Array | Uint8Array | null = null;
protected abstract _func?: (d: Input['domainType']) => Output['rangeType'];
public aesthetic_map: TextureSet;
@@ -76,9 +78,25 @@ export abstract class Aesthetic<
this.field = null;
} else {
this.field = encoding.field;
+ if (encoding.subfield) {
+ this.subfield = Array.isArray(encoding.subfield)
+ ? encoding.subfield
+ : [encoding.subfield];
+ }
}
}
+ /**
+ * Returns the keys that are used to access the data in the record batch,
+ * including with any nesting.
+ */
+ get columnKeys(): null | Some {
+ if (this.field === null) {
+ return null;
+ }
+ return [this.field, ...this.subfield] as Some;
+ }
+
get deeptable() {
return this.scatterplot.deeptable;
}
@@ -100,10 +118,14 @@ export abstract class Aesthetic<
value_for(point: Datum): Input['domainType'] | null {
if (this.field && point[this.field]) {
- return point[this.field] as Input['domainType'];
+ let v = point[this.field] as Input['domainType'];
+ for (let i = 0; i < this.subfield.length; i++) {
+ v = v[this.subfield[i]] as Input['domainType'];
+ }
+ return v;
+ // Needs a default perhaps?
+ return null;
}
- // Needs a default perhaps?
- return null;
}
get map_position() {
@@ -136,9 +158,17 @@ export abstract class Aesthetic<
if (this.field === null || this.field === undefined) {
return (this.column = null);
}
- return (this.column = this.deeptable.root_tile.record_batch.getChild(
- this.field,
- ) as Vector);
+ let output: Vector | Vector | null = null;
+ for (const f of [this.field, ...this.subfield]) {
+ if (output === null) {
+ output = this.deeptable.root_tile.record_batch.getChild(f) as Vector<
+ Input['arrowType']
+ >;
+ } else {
+ output = (output as Vector).getChild(f) as Vector;
+ }
+ }
+ return (this.column = output as Vector);
}
is_dictionary(): boolean {
diff --git a/src/aesthetics/AestheticSet.ts b/src/aesthetics/AestheticSet.ts
index 04daf719d..20273a172 100644
--- a/src/aesthetics/AestheticSet.ts
+++ b/src/aesthetics/AestheticSet.ts
@@ -6,6 +6,7 @@ import type { Deeptable } from '../Deeptable';
import { StatefulAesthetic } from './StatefulAesthetic';
import type { Encoding } from '../types';
import type * as DS from '../types';
+import { TupleSet } from '../utilityFunctions';
type AesMap = {
[K in keyof typeof dimensions]: StatefulAesthetic<
@@ -83,6 +84,12 @@ export class AestheticSet {
}
}
+ _neededFields: TupleSet = new TupleSet();
+
+ get neededFields(): string[][] {
+ return [...this._neededFields.values()];
+ }
+
apply_encoding(encoding: Encoding) {
if (
encoding['jitter_radius'] &&
@@ -107,6 +114,17 @@ export class AestheticSet {
this.dim(k).update(encoding[k] as DS.ChannelType | null);
}
+ // Update the needed fields.
+ this._neededFields.clear();
+
+ for (const v of Object.values(this.store)) {
+ if (v instanceof StatefulAesthetic) {
+ for (const f of v.neededFields) {
+ this._neededFields.add(f);
+ }
+ }
+ }
+
// Apply settings that are not full-on aesthetics.
for (const setting of ['jitter_method'] as const) {
this.options[setting].last = this.options[setting].current;
diff --git a/src/aesthetics/ScaledAesthetic.ts b/src/aesthetics/ScaledAesthetic.ts
index 072d6ff68..af31072ec 100644
--- a/src/aesthetics/ScaledAesthetic.ts
+++ b/src/aesthetics/ScaledAesthetic.ts
@@ -203,7 +203,7 @@ export abstract class ScaledAesthetic<
Input['domainType'],
];
} else {
- return this.scatterplot.deeptable.domain(this.field);
+ return this.scatterplot.deeptable.domain(this.field, this.subfield);
}
}
diff --git a/src/aesthetics/StatefulAesthetic.ts b/src/aesthetics/StatefulAesthetic.ts
index c631acb32..c1138965c 100644
--- a/src/aesthetics/StatefulAesthetic.ts
+++ b/src/aesthetics/StatefulAesthetic.ts
@@ -55,6 +55,7 @@ export type ConcreteScaledAesthetic =
import type { Deeptable } from '../Deeptable';
import type { Regl } from 'regl';
import type { TextureSet } from './AestheticSet';
+import { Some } from '../utilityFunctions';
export class StatefulAesthetic {
/**
@@ -97,11 +98,12 @@ export class StatefulAesthetic {
] as [T, T];
}
- get neededFields(): string[] {
- return [this.current.field, this.last.field].filter(
+ get neededFields(): Some[] {
+ return [this.current.columnKeys, this.last.columnKeys].filter(
(f) => f !== null,
- ) as string[];
+ );
}
+
get current() {
return this.states[0];
}
diff --git a/src/regl_rendering.ts b/src/regl_rendering.ts
index 965580d57..85c020818 100644
--- a/src/regl_rendering.ts
+++ b/src/regl_rendering.ts
@@ -32,6 +32,7 @@ import { Scatterplot } from './scatterplot';
import {
Data,
Dictionary,
+ Struct,
StructRowProxy,
Type,
Utf8,
@@ -41,7 +42,7 @@ import { Color } from './aesthetics/ColorAesthetic';
import { StatefulAesthetic } from './aesthetics/StatefulAesthetic';
import { Filter, Foreground } from './aesthetics/BooleanAesthetic';
import { ZoomTransform } from 'd3-zoom';
-import { TupleMap } from './utilityFunctions';
+import { Some, TupleMap, TupleSet } from './utilityFunctions';
// eslint-disable-next-line import/prefer-default-export
export class ReglRenderer extends Renderer {
public regl: Regl;
@@ -65,6 +66,11 @@ export class ReglRenderer extends Renderer {
public tick_num?: number;
public reglframe?: REGL.Cancellable;
public bufferManager: BufferManager;
+
+ private aes_to_buffer_num?: Record;
+ private variable_to_buffer_num?: TupleMap;
+ private buffer_num_to_variable?: string[][];
+
// public _renderer : Renderer;
constructor(
@@ -159,7 +165,7 @@ export class ReglRenderer extends Renderer {
] as [number, number];
const props: DS.GlobalDrawProps = {
// Copy the aesthetic as a string.
- aes: { encoding: this.aes.encoding },
+ // aes: { encoding: this.aes.encoding },
colors_as_grid: 0,
corners: this.zoom.current_corners(),
zoom_balance: prefs.zoom_balance,
@@ -224,12 +230,7 @@ export class ReglRenderer extends Renderer {
props.background_draw_needed[0] || props.background_draw_needed[1];
for (const tile of this.visible_tiles()) {
// Do the binding operation; returns truthy if it's already done.
- if (
- !this.bufferManager.ready(
- tile,
- this.needeedFields.map((d) => [d]),
- )
- ) {
+ if (!this.bufferManager.ready(tile, this.aes.neededFields)) {
continue;
}
@@ -259,6 +260,7 @@ export class ReglRenderer extends Renderer {
a.tile_id
);
});
+ // console.log({ prop_list });
this._renderer(prop_list);
}
@@ -282,7 +284,7 @@ export class ReglRenderer extends Renderer {
this.zoom.current_corners(),
this.props.max_ix,
5,
- this.needeedFields,
+ this.aes.neededFields.map((x) => x[0]),
'high',
);
} else {
@@ -291,7 +293,7 @@ export class ReglRenderer extends Renderer {
undefined,
prefs.max_points,
5,
- this.needeedFields,
+ this.aes.neededFields.map((x) => x[0]),
'high',
);
}
@@ -886,7 +888,7 @@ export class ReglRenderer extends Renderer {
type BufferSummary = {
aesthetic: keyof typeof dimensions;
time: time;
- field: string;
+ field: [string, ...string[]];
};
const buffers: BufferSummary[] = [];
const priorities = [
@@ -910,7 +912,10 @@ export class ReglRenderer extends Renderer {
buffers.push({
aesthetic,
time,
- field: this.aes.dim(aesthetic)[time].field,
+ field: [
+ this.aes.dim(aesthetic)[time].field,
+ ...this.aes.dim(aesthetic)[time].subfield,
+ ],
});
}
} catch (error) {
@@ -935,20 +940,20 @@ export class ReglRenderer extends Renderer {
const aes_to_buffer_num: Record = {}; // eg 'x' => 3
// Pre-allocate the 'ix' buffer and the 'ix_in_tile' buffers.
- const variable_to_buffer_num: Record = {
- ix: 0,
- ix_in_tile: 1,
- }; // eg 'year' => 3
+ const variable_to_buffer_num: TupleMap = new TupleMap([
+ [['ix'], 0],
+ [['ix_in_tile'], 1],
+ ]); // eg 'year' => 3
let num = 1;
for (const { aesthetic, time, field } of buffers) {
const k = `${aesthetic}--${time}`;
- if (variable_to_buffer_num[field] !== undefined) {
- aes_to_buffer_num[k] = variable_to_buffer_num[field];
+ if (variable_to_buffer_num.get(field) !== undefined) {
+ aes_to_buffer_num[k] = variable_to_buffer_num.get(field);
continue;
}
if (num++ < 16) {
aes_to_buffer_num[k] = num;
- variable_to_buffer_num[field] = num;
+ variable_to_buffer_num.set(field, num);
continue;
} else {
// Don't use the last value, use the current value.
@@ -959,18 +964,12 @@ export class ReglRenderer extends Renderer {
}
}
- const buffer_num_to_variable = [
- ...Object.keys(variable_to_buffer_num).map((k) => [k]),
- ];
+ const buffer_num_to_variable = [...variable_to_buffer_num.keys()];
this.aes_to_buffer_num = aes_to_buffer_num;
this.variable_to_buffer_num = variable_to_buffer_num;
this.buffer_num_to_variable = buffer_num_to_variable;
}
- aes_to_buffer_num?: Record;
- variable_to_buffer_num?: Record;
- buffer_num_to_variable?: string[][];
-
get discard_share() {
// If jitter is temporal, e.g., or filters are in place,
// it may make sense to estimate the number of hidden points.
@@ -1029,7 +1028,7 @@ export class BufferManager {
return this._integer_buffer;
}
- get(k: (string | Tile)[]): DS.BufferLocation | null {
+ get(k: Some): DS.BufferLocation | null {
const a = this.bufferMap.get(this.arrayMap.get(k));
return a;
}
@@ -1041,17 +1040,22 @@ export class BufferManager {
*/
ready(tile: Tile, needed_dimensions: Iterable): boolean {
// We don't allocate buffers for dimensions until they're needed.
+ for (const keyset of [['ix'], ['ix_in_tile'], ...needed_dimensions] as Some<
+ string[]
+ >) {
+ const current = this.get([tile, ...keyset]);
- for (const key of [['ix'], ['ix_in_tile'], ...needed_dimensions]) {
- const current = this.get([tile, ...key]);
if (current === null || current === undefined) {
- if (tile.hasLoadedColumn(key[0])) {
- this.create_regl_buffer(tile, key);
+ if (tile.hasLoadedColumn(keyset[0])) {
+ this.create_regl_buffer(tile, keyset);
} else {
- // console.log('not ready because of', key);
- if (key[0] === 'ix_in_tile') {
- this.create_regl_buffer(tile, key);
+ if (keyset[0] === 'ix_in_tile') {
+ this.create_regl_buffer(tile, keyset);
} else {
+ if (tile.readyToUse) {
+ // tile.get_column(keyset[0]);
+ } else {
+ }
return false;
}
}
@@ -1145,11 +1149,10 @@ export class BufferManager {
create_regl_buffer(tile: Tile, keys: string[]): void {
const { renderer } = this;
- const key = [tile, ...keys];
+ const key = [tile, ...keys] as Some;
if (this.arrayMap.has(key)) {
return;
}
- // console.log({ keys });
if (keys[0] === 'ix_in_tile') {
this.arrayMap.set(key, this.integer_array);
if (!this.bufferMap.has(this.integer_array)) {
@@ -1170,6 +1173,7 @@ export class BufferManager {
const data_length = data.length;
const buffer_desc = renderer.buffers.allocate_block(data_length, item_size);
+
buffer_desc.buffer.subdata(data, buffer_desc.offset);
this.bufferMap.set(vector.data[0].values, buffer_desc);
@@ -1177,7 +1181,7 @@ export class BufferManager {
}
// TODO: Build this out in next PR.
-function getNestedVector(
+export function getNestedVector(
tile: Tile,
key: string[],
): Vector {
@@ -1203,9 +1207,8 @@ function getNestedVector(
}
}
- let column: Vector = tile.record_batch.getChild(
- key[0],
- );
+ let column: Vector | Vector =
+ tile.record_batch.getChild(key[0]);
for (const k of key.slice(1)) {
column = column.getChild(k);
}
@@ -1216,7 +1219,17 @@ function getNestedVector(
if (!column.type || !column.type.typeId) {
throw new Error(`Column ${key} has no type.`);
}
+ function assertNotStruct(
+ value: Vector | Vector,
+ ): asserts value is Vector {
+ if (column.type.typeId === Type.Struct) {
+ throw new Error(
+ 'Structs are not supported for buffer data on column ' + key.join('->'),
+ );
+ }
+ }
+ assertNotStruct(column);
return column;
}
@@ -1323,7 +1336,6 @@ class MultipurposeBufferSet {
byte_size: items * bytes_per_item,
};
- // add a listener for GC on the value.
this.pointer += items * bytes_per_item;
return value;
}
@@ -1334,16 +1346,24 @@ class MultipurposeBufferSet {
* @param prefs The preferences object to be used.
*
* @returns The fields that need to be allocated in the buffers for
- * a tile to be drawn.
+ * a tile to be drawn. Returns a map of columns and any subfields in them that are needed.
*/
-export function neededFieldsToPlot(prefs: DS.CompletePrefs): Set {
- const needed_keys: Set = new Set();
- if (!prefs.encoding) {
+export function neededFieldsToPlot(
+ encoding: DS.Encoding | undefined,
+): TupleSet {
+ const needed_keys = new TupleSet([['ix']]);
+ if (!encoding) {
return needed_keys;
}
- for (const [_, v] of Object.entries(prefs.encoding)) {
+ for (const [_, v] of Object.entries(encoding)) {
if (v && typeof v !== 'string' && v['field'] !== undefined) {
- needed_keys.add(v['field'] as string);
+ const needed_key: Some = [v['field']];
+ if (v['subfield'] !== undefined) {
+ const subfield = v['subfield'];
+ const asArray = Array.isArray(subfield) ? [...subfield] : [subfield];
+ needed_key.push(...asArray);
+ }
+ needed_keys.add(needed_key);
}
}
return needed_keys;
diff --git a/src/rendering.ts b/src/rendering.ts
index 39674ab61..10a0262c2 100644
--- a/src/rendering.ts
+++ b/src/rendering.ts
@@ -168,21 +168,6 @@ export class Renderer {
return this.render_props.alpha;
}
- get needeedFields(): string[] {
- const { aes } = this;
- const needed = new Set();
- if (aes) {
- for (const v of Object.values(aes.store)) {
- if (v instanceof StatefulAesthetic) {
- for (const f of v.neededFields) {
- needed.add(f);
- }
- }
- }
- }
- return [...needed, 'ix'];
- }
-
get optimal_alpha() {
// This extends a formula suggested by Ricky Reusser to include
// discard share.
diff --git a/src/scatterplot.ts b/src/scatterplot.ts
index 96fd2ed1d..b1a932f07 100644
--- a/src/scatterplot.ts
+++ b/src/scatterplot.ts
@@ -121,6 +121,7 @@ export class Scatterplot {
this.ready = new Promise((resolve) => {
this.mark_ready = resolve;
});
+
this.click_handler = new ClickFunction(this);
this.tooltip_handler = new TooltipHTML(this);
this.label_click_handler = new LabelClick(this);
@@ -389,7 +390,7 @@ export class Scatterplot {
async reinitialize() {
const { prefs } = this;
- await this.deeptable.ready;
+ await this.deeptable.promise;
await this.deeptable.root_tile.get_column('x');
this._renderer = new ReglRenderer(
'#container-for-webgl-canvas',
@@ -417,8 +418,11 @@ export class Scatterplot {
ctx.fillRect(0, 0, window.innerWidth * 2, window.innerHeight * 2);
void this._renderer.initialize();
- void this.deeptable.promise.then(() => this.mark_ready());
- return this.ready;
+ await this.deeptable.promise.then(() => {
+ this.mark_ready();
+ });
+ this.mark_ready();
+ return;
}
/*
@@ -632,11 +636,28 @@ export class Scatterplot {
}
await this.plot_queue;
+ // Ensure that the deeptable exists.
+ if (this._root === undefined) {
+ const { source_url, arrow_table, arrow_buffer } =
+ prefs as DS.InitialAPICall;
+ const dataSpec = { source_url, arrow_table, arrow_buffer } as DS.DataSpec;
+ if (Object.values(dataSpec).filter((x) => x !== undefined).length !== 1) {
+ throw new Error(
+ 'The initial API call specify exactly one of source_url, arrow_table, or arrow_buffer',
+ );
+ }
+ await this.load_deeptable(dataSpec);
+ }
+ this.update_prefs(prefs);
+ // Then ensure the renderer and interaction handlers exist.
+ if (this._zoom === undefined || this._renderer === undefined) {
+ await this.reinitialize();
+ }
if (prefs) {
await this.start_transformations(prefs);
}
+
this.plot_queue = this.unsafe_plotAPI(prefs);
- await this.plot_queue;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
for (const [_, hook] of Object.entries(this.hooks)) {
@@ -666,23 +687,23 @@ export class Scatterplot {
if (this.prefs.duration < delay) {
delay = this.prefs.duration;
}
- const needed_keys: Set = neededFieldsToPlot(this.prefs);
if (!prefs.encoding) {
resolve();
}
- if (this._renderer) {
- this.deeptable.root_tile.require_columns(needed_keys);
- // Immediately start loading what we can onto the GPUs, too.
- for (const tile of this.renderer.visible_tiles()) {
- this._renderer.bufferManager.ready(
- tile,
- [...needed_keys].map((k) => [k]),
- );
- }
- resolve();
- } else {
- resolve();
+ if (!this._renderer) {
+ throw new Error('No renderer has been initialized');
}
+ //
+ const needed_keys = neededFieldsToPlot(prefs.encoding);
+ this.deeptable.root_tile.require_columns(
+ [...needed_keys].map((k) => k[0]),
+ );
+ // Immediately start loading what we can onto the GPUs, too.
+ for (const tile of this.renderer.visible_tiles()) {
+ this._renderer.bufferManager.ready(tile, needed_keys);
+ }
+ // TODO: There should be a setTimeout here before the resolution
+ resolve();
});
}
/**
@@ -724,18 +745,6 @@ export class Scatterplot {
this.update_prefs(prefs);
- if (this._root === undefined) {
- const { source_url, arrow_table, arrow_buffer } =
- prefs as DS.InitialAPICall;
- const dataSpec = { source_url, arrow_table, arrow_buffer } as DS.DataSpec;
- if (Object.values(dataSpec).filter((x) => x !== undefined).length !== 1) {
- throw new Error(
- 'The initial API call specify exactly one of source_url, arrow_table, or arrow_buffer',
- );
- }
- await this.load_deeptable(dataSpec);
- }
-
if (prefs.transformations) {
for (const [k, func] of Object.entries(prefs.transformations)) {
if (!this.deeptable.transformations[k]) {
@@ -743,9 +752,7 @@ export class Scatterplot {
}
}
}
- if (this._zoom === undefined || this._renderer === undefined) {
- await this.reinitialize();
- }
+
const renderer = this._renderer;
const zoom = this._zoom;
diff --git a/src/selection.ts b/src/selection.ts
index 3abd9b888..970086b38 100644
--- a/src/selection.ts
+++ b/src/selection.ts
@@ -885,10 +885,14 @@ export class DataSelection {
}
}
-function bigintmatcher(field: string, matches: bigint[]) {
+function bigintmatcher(
+ field: string,
+ matches: bigint[],
+ subfield: string | string[] | null = null,
+) {
const matchings = new Set(matches);
return async function (tile: Tile) {
- const col = (await tile.get_column(field)).data[0];
+ const col = (await tile.get_column(field, subfield)).data[0];
const values = col.values as bigint[];
const bitmask = new Bitmask(tile.record_batch.numRows);
for (let i = 0; i < tile.record_batch.numRows; i++) {
@@ -920,7 +924,11 @@ function bigintmatcher(field: string, matches: bigint[]) {
* @param matches A list of strings to match in that column
* @returns
*/
-function stringmatcher(field: string, matches: string[]) {
+function stringmatcher(
+ field: string,
+ matches: string[],
+ subfield: string | string[] | null = null,
+) {
if (field === undefined) {
throw new Error('Field must be defined');
}
@@ -958,7 +966,8 @@ function stringmatcher(field: string, matches: string[]) {
* The Deepscatter transformation function.
*/
return async function (tile: Tile) {
- const col = ((await tile.get_column(field)) as Vector).data[0];
+ const col = ((await tile.get_column(field, subfield)) as Vector)
+ .data[0];
const bytes = col.values;
const offsets = col.valueOffsets;
diff --git a/src/tile.ts b/src/tile.ts
index 36b187d03..95e4c572d 100644
--- a/src/tile.ts
+++ b/src/tile.ts
@@ -64,6 +64,9 @@ export class Tile {
private _partialManifest: Partial | Partial;
private _manifest?: TileManifest | LazyTileManifest;
+ // Does the tile have a loaded manifest and other features sufficient to plot.
+ public readyToUse = false;
+
// A cache of fetchCalls for downloaded arrow tables, including any table schema metadata.
// Tables may contain more than a single column, so this prevents multiple dispatch.
//private _promiseOfChildren: Promise;
@@ -134,22 +137,38 @@ export class Tile {
*
*
* @param colname The name of the column to retrive.
+ * @param subfield If the column is a struct vector, the subfield to retrieve. When a string, retrieves a single
+ * subfield. When an array, treats it as a nesting order.
* @returns An Arrow Vector of the column.
*/
- async get_column(colname: string): Promise {
- const existing = this._batch?.getChild(colname);
- if (existing) {
- return existing;
+ async get_column(
+ colname: string,
+ subfield: string | string[] | null = null,
+ ): Promise {
+ const subfields =
+ subfield === null ? [] : Array.isArray(subfield) ? subfield : [subfield];
+ let existing = this._batch?.getChild(colname);
+
+ if (!existing) {
+ if (this.deeptable.transformations[colname]) {
+ await this.apply_transformation(colname);
+ existing = this.record_batch.getChild(colname);
+ if (existing === null) {
+ throw new Error(`Column ${colname} not found after transformation`);
+ }
+ }
}
- if (this.deeptable.transformations[colname]) {
- await this.apply_transformation(colname);
- const vector = this.record_batch.getChild(colname);
- if (vector === null) {
- throw new Error(`Column ${colname} not found after transformation`);
+
+ // If subfields are passed, use them.
+ for (let i = 0; i < subfields.length; i++) {
+ existing = existing.getChild(subfields[i]);
+ if (existing === null) {
+ throw new Error(
+ `Column ${colname} lacks subfield ${subfields.slice(0, i).join(' >> ')}`,
+ );
}
- return vector;
}
- throw new Error(`Column ${colname} not found`);
+ return existing;
}
/**
@@ -286,6 +305,7 @@ export class Tile {
});
this.highest_known_ix = manifest.max_ix;
this._manifest = manifest;
+ this.readyToUse = true;
}
set highest_known_ix(val) {
diff --git a/src/tixrixqid.ts b/src/tixrixqid.ts
index 1522b6343..402a160fd 100644
--- a/src/tixrixqid.ts
+++ b/src/tixrixqid.ts
@@ -1,4 +1,11 @@
-import type { Bool, Data, Field, Struct, StructRowProxy, Vector } from 'apache-arrow';
+import type {
+ Bool,
+ Data,
+ Field,
+ Struct,
+ StructRowProxy,
+ Vector,
+} from 'apache-arrow';
import type { Tile } from './deepscatter';
import { Bitmask, DataSelection, Deeptable } from './deepscatter';
@@ -102,7 +109,7 @@ export function tixToZxy(tix: Tix): [number, number, number] {
*/
export function getQidFromRow(
row: StructRowProxy,
- dataset: Deeptable
+ dataset: Deeptable,
): [number, number] {
const tile = getTileFromRow(row, dataset);
const rix = row[Symbol.for('rowIndex')] as number;
@@ -110,7 +117,6 @@ export function getQidFromRow(
}
export function getTileFromRow(row: StructRowProxy, dataset: Deeptable): Tile {
-
const parent = row[Symbol.for('parent')] as Data;
const parentsColumns = parent.children;
@@ -119,8 +125,8 @@ export function getTileFromRow(row: StructRowProxy, dataset: Deeptable): Tile {
// need to find the tile that matches the most columns, not assume
// that every column matches exactly.
let best_match: [Tile | null, number] = [null, 0];
- const parentNames : [string, Data][] = parent.type.children.map(
- (d: Field, i: number) => [d.name, parentsColumns[i]]
+ const parentNames: [string, Data][] = parent.type.children.map(
+ (d: Field, i: number) => [d.name, parentsColumns[i]],
);
dataset.map((t: Tile) => {
@@ -144,7 +150,7 @@ export function getTileFromRow(row: StructRowProxy, dataset: Deeptable): Tile {
});
if (best_match[0] === undefined) {
throw new Error(
- 'No tiles found for this row.' + JSON.stringify({ ...row })
+ 'No tiles found for this row.' + JSON.stringify({ ...row }),
);
}
return best_match[0];
diff --git a/src/types.ts b/src/types.ts
index 32ee18c02..a173d0811 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -21,6 +21,7 @@ import { Scatterplot } from './scatterplot';
import { ZoomTransform } from 'd3-zoom';
import type { Tile } from './tile';
import type { Rectangle } from './tile';
+import { TupleMap } from './utilityFunctions';
export type { Renderer, Deeptable, ConcreteAesthetic };
/**
@@ -294,6 +295,8 @@ export type NumericScaleChannel<
> = {
/** The name of a column in the data table to be encoded. */
field: string;
+ // If field is a struct, subfield indicates which child to extract.
+ subfield?: string | string[];
/**
* A transformation to apply on the field.
* 'literal' maps in the implied dataspace set by 'x', 'y', while
@@ -312,6 +315,8 @@ export type LambdaChannel<
> = {
lambda?: (v: DomainType) => RangeType;
field: string;
+ // If field is a struct, subfield indicates which child to extract.
+ subfield?: string | string[];
};
/**
@@ -342,6 +347,8 @@ type TwoArgumentOp = {
export type OpChannel = {
field: string;
+ // If field is a struct, subfield indicates which child to extract.
+ subfield?: string | string[];
} & (OneArgumentOp | TwoArgumentOp);
export type ConstantChannel = {
@@ -376,12 +383,16 @@ export type ChannelType =
export type CategoricalColorScale = {
field: string;
+ // If field is a struct, subfield indicates which child to extract.
+ subfield?: string | string[];
domain: string | [string, string, ...string[]];
range: Colorname[];
};
export type LinearColorScale = {
field: string;
+ // If field is a struct, subfield indicates which child to extract.
+ subfield?: string | string[];
domain: [number, number]; // TODO: | [number, number, number]
// TODO: implement some codegen for these values
range: 'viridis' | 'magma' | 'ylorrd';
@@ -605,7 +616,7 @@ export type RowFunction = (
// Props that are needed for all the draws in a single tick.
export type GlobalDrawProps = {
- aes: { encoding: Encoding };
+ // aes: { encoding: Encoding };
colors_as_grid: 0 | 1;
corners: Rectangle;
zoom_balance: number;
@@ -627,7 +638,7 @@ export type GlobalDrawProps = {
grid_mode: 1 | 0;
buffer_num_to_variable: string[][];
aes_to_buffer_num: Record;
- variable_to_buffer_num: Record;
+ variable_to_buffer_num: TupleMap;
color_picker_mode: 0 | 1 | 2 | 3;
zoom_matrix: [
number,