From 32cc29a294d571970ce74a740f48c5a9ef6f3309 Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Mon, 17 Feb 2025 15:03:19 -0300 Subject: [PATCH 1/5] data-lake: Add utility function for finding all variable IDs in a given string --- src/libs/utils-data-lake.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libs/utils-data-lake.ts b/src/libs/utils-data-lake.ts index 93accf8cf..610584f64 100644 --- a/src/libs/utils-data-lake.ts +++ b/src/libs/utils-data-lake.ts @@ -70,6 +70,16 @@ export const replaceDataLakeInputsInString = (input: string, replaceFunction?: ( return input.toString().replace(dataLakeInputRegex, (match) => replaceFunctionToUse(match)) } +/** + * Find all data lake variable ids in a string. + * @param {string} input The string to search for data lake variable ids + * @returns {string[]} An array of data lake variable ids + */ +export const findDataLakeVariablesIdsInString = (input: string): string[] => { + const inputs = findDataLakeInputsInString(input) + return inputs.map((i) => getDataLakeVariableIdFromInput(i)).filter((id) => id !== null) +} + export const replaceDataLakeInputsInJsonString = (jsonString: string): string => { let parsedJson = jsonString From f471f6ea05b3ccd7eda9ef7232e1c5a930739801 Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Mon, 24 Feb 2025 19:35:06 -0300 Subject: [PATCH 2/5] utils: Add function to create "machine-friendly" versions of strings --- src/libs/utils.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libs/utils.ts b/src/libs/utils.ts index a3b8d00bf..4db03b8ce 100644 --- a/src/libs/utils.ts +++ b/src/libs/utils.ts @@ -224,3 +224,17 @@ export const humanizeString = (str: string): string => { .trim() .replace(/\b\w/g, (match) => match.toUpperCase()) } + +/** + * Convert a string to a machine-friendly version of it + * @param {string} str The string to convert + * @returns {string} The machine-friendly string + */ +export const machinizeString = (str: string): string => { + return str + .toLowerCase() + .trim() + .replace(/[^a-zA-Z0-9-]/g, '-') + .replace(/-+/g, '-') + .replace(/^-+|-+$/g, '') +} From 1b5e42a954d744d7d03f2484b9d7b79bc481944c Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Mon, 17 Feb 2025 15:04:51 -0300 Subject: [PATCH 3/5] data-lake: Allow creation of transforming functions A transforming function is a new DataLake variable in which the value is calculated based on a JavsScript expression, that can include other DataLake variables in it. --- src/App.vue | 7 + src/libs/actions/data-lake-transformations.ts | 195 ++++++++++ src/stores/appInterface.ts | 1 + src/views/ConfigurationDataLakeView.vue | 339 ++++++++++++++++++ 4 files changed, 542 insertions(+) create mode 100644 src/libs/actions/data-lake-transformations.ts create mode 100644 src/views/ConfigurationDataLakeView.vue diff --git a/src/App.vue b/src/App.vue index d74fab760..c205b8ea7 100644 --- a/src/App.vue +++ b/src/App.vue @@ -357,6 +357,7 @@ import { useWidgetManagerStore } from './stores/widgetManager' import { SubMenuComponent } from './types/general' import ConfigurationActionsView from './views/ConfigurationActionsView.vue' import ConfigurationAlertsView from './views/ConfigurationAlertsView.vue' +import ConfigurationDataLakeView from './views/ConfigurationDataLakeView.vue' import ConfigurationDevelopmentView from './views/ConfigurationDevelopmentView.vue' import ConfigurationGeneralView from './views/ConfigurationGeneralView.vue' import ConfigurationJoystickView from './views/ConfigurationJoystickView.vue' @@ -439,6 +440,12 @@ const configMenu = [ componentName: SubMenuComponentName.SettingsActions, component: markRaw(ConfigurationActionsView) as SubMenuComponent, }, + { + icon: 'mdi-database-outline', + title: 'Data Lake', + componentName: SubMenuComponentName.SettingsDataLake, + component: markRaw(ConfigurationDataLakeView) as SubMenuComponent, + }, ] const toolsMenu = [ diff --git a/src/libs/actions/data-lake-transformations.ts b/src/libs/actions/data-lake-transformations.ts new file mode 100644 index 000000000..edc8e3727 --- /dev/null +++ b/src/libs/actions/data-lake-transformations.ts @@ -0,0 +1,195 @@ +import { + findDataLakeVariablesIdsInString, + getDataLakeVariableIdFromInput, + replaceDataLakeInputsInString, +} from '../utils-data-lake' +import { + createDataLakeVariable, + DataLakeVariable, + deleteDataLakeVariable, + listenDataLakeVariable, + setDataLakeVariableData, + unlistenDataLakeVariable, +} from './data-lake' + +const transformingFunctionsKey = 'cockpit-transforming-functions' + +let globalTransformingFunctions: TransformingFunction[] = [] + +const loadTransformingFunctions = (): void => { + const transformingFunctions = localStorage.getItem(transformingFunctionsKey) + if (!transformingFunctions) { + globalTransformingFunctions = [] + return + } + globalTransformingFunctions = JSON.parse(transformingFunctions) + updateTransformingFunctionListeners() +} + +const saveTransformingFunctions = (): void => { + localStorage.setItem(transformingFunctionsKey, JSON.stringify(globalTransformingFunctions)) + updateTransformingFunctionListeners() +} + +const getExpressionValue = (func: TransformingFunction): string | number | boolean => { + const expressionWithValues = replaceDataLakeInputsInString(func.expression) + if (func.expression.includes('return')) { + return eval(`(function() { ${expressionWithValues} })()`) + } + return eval(`(function() { return ${expressionWithValues} })()`) +} + +const variablesListeners: Record> = {} + +const nextDelayToEvaluateFaillingTransformingFunction: Record = {} +const lastTimeTriedToEvaluateFaillingTransformingFunction: Record = {} + +const setupTransformingFunctionsListeners = (): void => { + globalTransformingFunctions.forEach((func) => { + const dataLakeVariablesInExpression = getDataLakeVariableIdFromInput(func.expression) + if (dataLakeVariablesInExpression) { + const variableIds = findDataLakeVariablesIdsInString(func.expression) + variableIds.forEach((variableId) => { + const listenerId = listenDataLakeVariable(variableId, () => { + try { + // If the function is failing, we don't want to evaluate it too often + const currentDelay = nextDelayToEvaluateFaillingTransformingFunction[func.id] || 10 + const lastTimeTried = lastTimeTriedToEvaluateFaillingTransformingFunction[func.id] || 0 + if (currentDelay > 0 && lastTimeTried + currentDelay > Date.now()) { + return + } else { + const expressionValue = getExpressionValue(func) + setDataLakeVariableData(func.id, expressionValue) + } + } catch (error) { + lastTimeTriedToEvaluateFaillingTransformingFunction[func.id] = Date.now() + const currentDelay = nextDelayToEvaluateFaillingTransformingFunction[func.id] || 10 + const nextDelay = Math.min(2 * currentDelay, 10000) + nextDelayToEvaluateFaillingTransformingFunction[func.id] = nextDelay + const msg = `Error evaluating expression for transforming function '${func.id}'. Next evaluation in ${nextDelay} ms. Error: ${error}` + console.error(msg) + } + }) + if (!variablesListeners[func.id]) { + variablesListeners[func.id] = { [variableId]: [listenerId] } + } else if (!variablesListeners[func.id][variableId]) { + variablesListeners[func.id][variableId] = [listenerId] + } else { + variablesListeners[func.id][variableId].push(listenerId) + } + }) + } + }) +} + +const deleteAllTransformingFunctionsListeners = (): void => { + Object.keys(nextDelayToEvaluateFaillingTransformingFunction).forEach((funcId) => { + delete nextDelayToEvaluateFaillingTransformingFunction[funcId] + delete lastTimeTriedToEvaluateFaillingTransformingFunction[funcId] + }) + Object.keys(lastTimeTriedToEvaluateFaillingTransformingFunction).forEach((funcId) => { + delete lastTimeTriedToEvaluateFaillingTransformingFunction[funcId] + }) + Object.entries(variablesListeners).forEach(([funcId, variableListeners]) => { + Object.entries(variableListeners).forEach(([variableId, listenerIds]) => { + listenerIds.forEach((listenerId) => unlistenDataLakeVariable(variableId, listenerId)) + }) + delete variablesListeners[funcId] + }) +} + +const deleteAllTransformingFunctionsVariables = (): void => { + globalTransformingFunctions.forEach((func) => { + deleteDataLakeVariable(func.id) + }) +} + +const setupAllTransformingFunctionsVariables = (): void => { + globalTransformingFunctions.forEach((func) => { + try { + createDataLakeVariable(new DataLakeVariable(func.id, func.name, func.type, func.description)) + } catch (createError) { + const msg = `Could not create data lake variable info for transforming function ${func.id}. Error: ${createError}` + console.error(msg) + } + }) +} + +const updateTransformingFunctionListeners = (): void => { + deleteAllTransformingFunctionsListeners() + deleteAllTransformingFunctionsVariables() + setupAllTransformingFunctionsVariables() + setupTransformingFunctionsListeners() +} + +/** + * Interface for a transforming function that creates a new data lake variable + * based on an expression using other variables + */ +export interface TransformingFunction { + /** Name of the new variable */ + name: string + /** ID of the new variable */ + id: string + /** Type of the new variable */ + type: 'string' | 'number' | 'boolean' + /** Description of the new variable */ + description?: string + /** JavaScript expression that defines how to calculate the new variable */ + expression: string +} + +/** + * Creates a new transforming function that listens to its dependencies + * and updates its value when they change + * @param {string} id - ID of the new variable + * @param {string} name - Name of the new variable + * @param {'string' | 'number' | 'boolean'} type - Type of the new variable + * @param {string} expression - Expression to calculate the variable's value + * @param {string?} description - Description of the new variable + */ +export const createTransformingFunction = ( + id: string, + name: string, + type: 'string' | 'number' | 'boolean', + expression: string, + description?: string +): void => { + const transformingFunction: TransformingFunction = { name, id, type, expression, description } + globalTransformingFunctions.push(transformingFunction) + createDataLakeVariable(new DataLakeVariable(id, name, type, description)) + saveTransformingFunctions() +} + +/** + * Returns all transforming functions + * @returns {TransformingFunction[]} All transforming functions + */ +export const getAllTransformingFunctions = (): TransformingFunction[] => { + return globalTransformingFunctions +} + +/** + * Updates a transforming function + * @param {TransformingFunction} func - The function to update + */ +export const updateTransformingFunction = (func: TransformingFunction): void => { + const index = globalTransformingFunctions.findIndex((f) => f.id === func.id) + if (index !== -1) { + globalTransformingFunctions[index] = func + saveTransformingFunctions() + } +} + +/** + * Deletes a transforming function and cleans up its subscriptions + * @param {TransformingFunction} func - The function to delete + */ +export const deleteTransformingFunction = (func: TransformingFunction): void => { + // Remove the variable from the data lake + globalTransformingFunctions = globalTransformingFunctions.filter((f) => f.id !== func.id) + deleteDataLakeVariable(func.id) + saveTransformingFunctions() +} + +loadTransformingFunctions() diff --git a/src/stores/appInterface.ts b/src/stores/appInterface.ts index f330e555e..503e54a30 100644 --- a/src/stores/appInterface.ts +++ b/src/stores/appInterface.ts @@ -28,6 +28,7 @@ export enum SubMenuComponentName { SettingsDev = 'settings-dev', SettingsMission = 'settings-mission', SettingsActions = 'settings-actions', + SettingsDataLake = 'settings-datalake', ToolsMAVLink = 'tools-mavlink', ToolsDataLake = 'tools-datalake', } diff --git a/src/views/ConfigurationDataLakeView.vue b/src/views/ConfigurationDataLakeView.vue new file mode 100644 index 000000000..fec3ab077 --- /dev/null +++ b/src/views/ConfigurationDataLakeView.vue @@ -0,0 +1,339 @@ + + + + + From 17f998863c8993a745dda0e802cddf03219a98ac Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Mon, 24 Feb 2025 15:54:36 -0300 Subject: [PATCH 4/5] data-lake: Use Monaco editor in the transforming functions --- src/views/ConfigurationDataLakeView.vue | 241 ++++++++++++++++++------ 1 file changed, 185 insertions(+), 56 deletions(-) diff --git a/src/views/ConfigurationDataLakeView.vue b/src/views/ConfigurationDataLakeView.vue index fec3ab077..d5f5aa0d8 100644 --- a/src/views/ConfigurationDataLakeView.vue +++ b/src/views/ConfigurationDataLakeView.vue @@ -112,15 +112,18 @@ :items="['string', 'number', 'boolean']" :rules="[(v) => !!v || 'Type is required']" /> - +
+ +
+
+ Create complex transformations by combining existing Data Lake variables using JavaScript + expressions. Type '\{\{' to access available variables. Return is optional, but should be included + in complex expressions. Remember to set the type accordingly. +
+
-
- -
- - - - - - -
-
@@ -171,7 +148,12 @@ + + diff --git a/src/views/ConfigurationDataLakeView.vue b/src/views/ConfigurationDataLakeView.vue deleted file mode 100644 index d5f5aa0d8..000000000 --- a/src/views/ConfigurationDataLakeView.vue +++ /dev/null @@ -1,468 +0,0 @@ - - - - - diff --git a/src/views/ToolsDataLakeView.vue b/src/views/ToolsDataLakeView.vue index 365945d79..faccf5868 100644 --- a/src/views/ToolsDataLakeView.vue +++ b/src/views/ToolsDataLakeView.vue @@ -24,6 +24,10 @@ @click="searchQuery = ''" /> + + mdi-plus + Add compound variable + -
-

{{ item.name }}

-
- - -
-

{{ item.id }}

+
+

{{ item.name }}

+

@@ -86,11 +93,32 @@

+ +
+ + +
+ + +