diff --git a/spark-ui/src/reducers/Alerts/MemoryAlertsReducer.ts b/spark-ui/src/reducers/Alerts/MemoryAlertsReducer.ts index 6175c82..e3afd78 100644 --- a/spark-ui/src/reducers/Alerts/MemoryAlertsReducer.ts +++ b/spark-ui/src/reducers/Alerts/MemoryAlertsReducer.ts @@ -2,76 +2,118 @@ import { Alerts, ConfigStore, StatusStore, SparkExecutorStore } from "../../inte import { humanFileSizeSparkConfigFormat, humanFileSize, calculatePercentage } from "../../utils/FormatUtils"; import { EnvironmentInfo } from "../../interfaces/ApplicationInfo"; -const MAX_MEMORY_PERCENTAGE_TOO_HIGH_THRESHOLD = 95; -const MAX_MEMORY_PERCENTAGE_TOO_LOW_THRESHOLD = 70; -const MEMORY_INCREASE_RATIO = 0.2; -const MEMORY_DECREASE_SAFETY_BUFFER = 0.2; +const MAX_EXECUTOR_MEMORY_PERCENTAGE_TOO_HIGH_THRESHOLD = 95; +const MAX_EXECUTOR_MEMORY_PERCENTAGE_TOO_LOW_THRESHOLD = 70; +const EXECUTOR_MEMORY_INCREASE_RATIO = 0.2; +const EXECUTOR_MEMORY_DECREASE_SAFETY_BUFFER = 0.2; + +const MAX_DRIVER_MEMORY_PERCENTAGE_TOO_HIGH_THRESHOLD = 95; +const DRIVER_MEMORY_INCREASE_RATIO = 0.2; export function reduceMemoryAlerts( { executors: statusExecutors }: StatusStore, - { executorMemoryBytes, executorMemoryBytesSparkFormatString }: ConfigStore, + config: ConfigStore, environmentInfo: EnvironmentInfo | undefined, executors: SparkExecutorStore[], alerts: Alerts ) { - if (statusExecutors?.maxExecutorMemoryBytes) { - checkMemoryUsage( - statusExecutors.maxExecutorMemoryPercentage, - executorMemoryBytes, - executorMemoryBytesSparkFormatString, - "executor", - alerts - ); + if (statusExecutors?.maxExecutorMemoryBytes !== undefined && + statusExecutors.maxExecutorMemoryBytes !== 0) { + reduceExecutorMemoryAlerts(statusExecutors, config, alerts); } - if (environmentInfo?.driverXmxBytes) { - const driverExecutor = executors?.find((exec) => exec.id === "driver"); - const driverMaxMemory = environmentInfo?.driverXmxBytes ?? 1; - const driverMemoryUsage = driverExecutor?.HeapMemoryUsageBytes ?? 0; - const driverMemoryUsagePercentage = calculatePercentage(driverMemoryUsage, driverMaxMemory); - - if (driverMemoryUsage) { - checkMemoryUsage( - driverMemoryUsagePercentage, - driverMaxMemory, - humanFileSizeSparkConfigFormat(driverMaxMemory), - "driver", - alerts - ); - } + reduceDriverMemoryAlerts(executors, config, environmentInfo, alerts); } } -function checkMemoryUsage( - memoryPercentage: number, - maxMemoryBytes: number, - maxMemoryBytesString: string, - type: "executor" | "driver", +function reduceExecutorMemoryAlerts( + executors: StatusStore["executors"], + config: ConfigStore, alerts: Alerts ) { - const sourceMetric = type === "driver" ? "driverMemory" : "memory"; - const createAlert = (alertType: "High" | "Low", alertLevel: "error" | "warning") => { + const maxExecutorMemoryPercentage = executors!.maxExecutorMemoryPercentage; + const maxExecutorMemoryBytes = executors!.maxExecutorMemoryBytes; + const maxExecutorMemoryBytesString = executors!.maxExecutorMemoryBytesString; + + if (maxExecutorMemoryPercentage > MAX_EXECUTOR_MEMORY_PERCENTAGE_TOO_HIGH_THRESHOLD) { + const suggestedMemory = humanFileSizeSparkConfigFormat( + config.executorMemoryBytes * (1 + EXECUTOR_MEMORY_INCREASE_RATIO) + ); + alerts.push({ + id: `executorMemoryTooHigh_${maxExecutorMemoryPercentage.toFixed(2)}`, + name: "executorMemoryTooHigh", + title: "Executor Memory Under-Provisioned", + location: "In: Summery Page -> Memory Utilization", + message: `Max Executor Memory usage is ${maxExecutorMemoryPercentage.toFixed(2)}% which is too high, and can cause spills and OOMs`, + suggestion: ` + 1. Increase executor memory provisioning by changing "spark.executor.memory" to ${suggestedMemory} + (the current usage is ${maxExecutorMemoryPercentage.toFixed(2)}% but set to higher as it needs some buffer) + from current value "${config.executorMemoryBytesSparkFormatString}"`, + type: "error", + source: { + type: "status", + metric: "memory" + } + }); + } + else if (maxExecutorMemoryPercentage < MAX_EXECUTOR_MEMORY_PERCENTAGE_TOO_LOW_THRESHOLD) { const suggestedMemory = humanFileSizeSparkConfigFormat( - maxMemoryBytes * (1 + (alertType === "High" ? MEMORY_INCREASE_RATIO : MEMORY_DECREASE_SAFETY_BUFFER)) + maxExecutorMemoryBytes * (1 + EXECUTOR_MEMORY_DECREASE_SAFETY_BUFFER) ); alerts.push({ - id: `${type}MemoryToo${alertType}_${memoryPercentage.toFixed(2)}`, - name: `${type}MemoryToo${alertType}`, - title: `${type.charAt(0).toUpperCase() + type.slice(1)} Memory ${alertType === "High" ? "Under" : "Over"}-Provisioned`, - location: `In: Summary Page -> Memory Usage`, - message: `Max ${type} Memory usage is ${memoryPercentage.toFixed(2)}% which is ${alertType === "High" ? "too high, and can cause spills and OOMs" : "too low, which means you can provision less memory and save $$$"}`, + id: `executorMemoryTooLow_${maxExecutorMemoryPercentage.toFixed(2)}`, + name: "executorMemoryTooLow", + title: "Executor Memory Over-Provisioned", + location: "In: Summery Page -> Memory Utilization", + message: `Max executor memory usage is only ${maxExecutorMemoryPercentage.toFixed(2)}%, which means you can provision less memory and save $$$`, suggestion: ` - 1. ${alertType === "High" ? "Increase" : "Decrease"} ${type} memory provisioning by changing "spark.${type}.memory" to ${suggestedMemory} - (the current usage is ${memoryPercentage.toFixed(2)}% but set to ${alertType === "High" ? "higher" : "lower"} as it needs some buffer) - from current value "${maxMemoryBytesString}"`, - type: alertLevel, - source: { type: "status", metric: sourceMetric }, + 1. Decrease executor memory provisioning by changing "spark.executor.memory" to ${suggestedMemory} + (the current usage is ${maxExecutorMemoryPercentage.toFixed(2)}% but set to higher as it needs some buffer) + from current value "${config.executorMemoryBytesSparkFormatString}"`, + type: "warning", + source: { + type: "status", + metric: "memory" + } + }); - }; + } +} - if (memoryPercentage > MAX_MEMORY_PERCENTAGE_TOO_HIGH_THRESHOLD) { - createAlert("High", "error"); - } else if (type === "executor" && memoryPercentage < MAX_MEMORY_PERCENTAGE_TOO_LOW_THRESHOLD) { - createAlert("Low", "warning"); +function reduceDriverMemoryAlerts( + executors: SparkExecutorStore[], + config: ConfigStore, + environmentInfo: EnvironmentInfo, + alerts: Alerts +) { + const driverExecutor = executors?.find((exec) => exec.id === "driver"); + if (!driverExecutor?.HeapMemoryUsageBytes || !environmentInfo.driverXmxBytes) { + return; } -} \ No newline at end of file + + const driverMaxMemory = environmentInfo.driverXmxBytes; + const driverMemoryUsage = driverExecutor.HeapMemoryUsageBytes; + const driverMemoryUsagePercentage = calculatePercentage(driverMemoryUsage, driverMaxMemory); + + if (driverMemoryUsagePercentage > MAX_DRIVER_MEMORY_PERCENTAGE_TOO_HIGH_THRESHOLD) { + const suggestedMemory = humanFileSizeSparkConfigFormat( + driverMaxMemory * (1 + DRIVER_MEMORY_INCREASE_RATIO) + ); + alerts.push({ + id: `driverMemoryTooHigh_${driverMemoryUsagePercentage.toFixed(2)}`, + name: "driverMemoryTooHigh", + title: "Driver Memory Under-Provisioned", + location: "In: Summery Page -> Memory Utilization", + message: `Max Driver Memory usage is ${driverMemoryUsagePercentage.toFixed(2)}% which is too high, and can cause spills and OOMs`, + suggestion: ` + 1. Increase driver memory provisioning by changing "spark.driver.memory" to ${suggestedMemory} + (the current usage is ${driverMemoryUsagePercentage.toFixed(2)}% but set to higher as it needs some buffer) + from current value "${humanFileSizeSparkConfigFormat(driverMaxMemory)}"`, + type: "error", + source: { + type: "status", + metric: "driverMemory" + } + }); + } +}