Skip to content

Commit

Permalink
fix 0.25.8 oom bug
Browse files Browse the repository at this point in the history 
fix 0.25.8 oom bug
  • Loading branch information
@v1xingyue
v1xingyue authored Feb 28, 2025
1 parent c04f89a commit 486bd69
Showing 1 changed file with 77 additions and 27 deletions.
104 changes: 77 additions & 27 deletions packages/core/src/generation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -349,9 +349,9 @@ export async function generateText({
maxSteps = 1,
stop,
customSystemPrompt,
// verifiableInference = process.env.VERIFIABLE_INFERENCE_ENABLED === "true",
// verifiableInferenceOptions,
}: {
}: // verifiableInference = process.env.VERIFIABLE_INFERENCE_ENABLED === "true",
// verifiableInferenceOptions,
{
runtime: IAgentRuntime;
context: string;
modelClass: ModelClass;
Expand Down Expand Up @@ -504,8 +504,7 @@ export async function generateText({
const max_context_length =
modelConfiguration?.maxInputTokens || modelSettings.maxInputTokens;
const max_response_length =
modelConfiguration?.maxOutputTokens ||
modelSettings.maxOutputTokens;
modelConfiguration?.maxOutputTokens || modelSettings.maxOutputTokens;
const experimental_telemetry =
modelConfiguration?.experimental_telemetry ||
modelSettings.experimental_telemetry;
Expand Down Expand Up @@ -956,7 +955,10 @@ export async function generateText({
experimental_telemetry: experimental_telemetry,
});

response = ollamaResponse.replace(/<think>[\s\S]*?<\/think>\s*\n*/g, '');
response = ollamaResponse.replace(
/<think>[\s\S]*?<\/think>\s*\n*/g,
""
);
}
elizaLogger.debug("Received response from Ollama model.");
break;
Expand Down Expand Up @@ -1168,8 +1170,10 @@ export async function generateText({
// console.warn("veniceResponse:")
// console.warn(veniceResponse)
//rferrari: remove all text from <think> to </think>\n\n
response = veniceResponse
.replace(/<think>[\s\S]*?<\/think>\s*\n*/g, '');
response = veniceResponse.replace(
/<think>[\s\S]*?<\/think>\s*\n*/g,
""
);
// console.warn(response)

// response = veniceResponse;
Expand Down Expand Up @@ -1303,7 +1307,7 @@ export async function generateText({
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${apiKey}`,
}
},
});
const secretAi = secretAiProvider(model);

Expand All @@ -1320,7 +1324,7 @@ export async function generateText({
response = secretAiResponse;
}
break;

case ModelProviderName.BEDROCK: {
elizaLogger.debug("Initializing Bedrock model.");

Expand All @@ -1332,7 +1336,7 @@ export async function generateText({
frequencyPenalty: frequency_penalty,
presencePenalty: presence_penalty,
experimental_telemetry: experimental_telemetry,
prompt: context
prompt: context,
});

response = bedrockResponse;
Expand Down Expand Up @@ -1430,6 +1434,26 @@ export async function splitChunks(
): Promise<string[]> {
elizaLogger.debug(`[splitChunks] Starting text split`);

// Validate parameters
if (chunkSize <= 0) {
elizaLogger.warn(
`Invalid chunkSize (${chunkSize}), using default 1500`
);
chunkSize = 1500;
}

if (bleed >= chunkSize) {
elizaLogger.warn(
`Bleed (${bleed}) >= chunkSize (${chunkSize}), adjusting bleed to 1/4 of chunkSize`
);
bleed = Math.floor(chunkSize / 4);
}

if (bleed < 0) {
elizaLogger.warn(`Invalid bleed (${bleed}), using default 100`);
bleed = 100;
}

const chunks = splitText(content, chunkSize, bleed);

elizaLogger.debug(`[splitChunks] Split complete:`, {
Expand All @@ -1442,14 +1466,29 @@ export async function splitChunks(
return chunks;
}

export function splitText(content: string, chunkSize: number, bleed: number): string[] {
export function splitText(
content: string,
chunkSize: number,
bleed: number
): string[] {
const chunks: string[] = [];
let start = 0;

while (start < content.length) {
const end = Math.min(start + chunkSize, content.length);
chunks.push(content.substring(start, end));
start = end - bleed; // Apply overlap

// Ensure we're not creating empty or invalid chunks
if (end > start) {
chunks.push(content.substring(start, end));
}

// Ensure forward progress and prevent infinite loops
const nextStart = end - bleed;
if (nextStart <= start) {
start = end; // If no progress would be made, skip the bleed
} else {
start = nextStart;
}
}

return chunks;
Expand Down Expand Up @@ -1709,7 +1748,9 @@ export const generateImage = async (
}> => {
const modelSettings = getImageModelSettings(runtime.imageModelProvider);
if (!modelSettings) {
elizaLogger.warn("No model settings found for the image model provider.");
elizaLogger.warn(
"No model settings found for the image model provider."
);
return { success: false, error: "No model settings available" };
}
const model = modelSettings.name;
Expand Down Expand Up @@ -1739,11 +1780,21 @@ export const generateImage = async (
return runtime.getSetting("SECRET_AI_API_KEY");
case ModelProviderName.NEARAI:
try {
// Read auth config from ~/.nearai/config.json if it exists
const config = JSON.parse(fs.readFileSync(path.join(os.homedir(), '.nearai/config.json'), 'utf8'));
return JSON.stringify(config?.auth);
// Read auth config from ~/.nearai/config.json if it exists
const config = JSON.parse(
fs.readFileSync(
path.join(
os.homedir(),
".nearai/config.json"
),
"utf8"
)
);
return JSON.stringify(config?.auth);
} catch (e) {
elizaLogger.warn(`Error loading NEAR AI config. The environment variable NEARAI_API_KEY will be used. ${e}`);
elizaLogger.warn(
`Error loading NEAR AI config. The environment variable NEARAI_API_KEY will be used. ${e}`
);
}
return runtime.getSetting("NEARAI_API_KEY");
default:
Expand Down Expand Up @@ -2154,10 +2205,10 @@ export const generateObject = async ({
schemaDescription,
stop,
mode = "json",
// verifiableInference = false,
// verifiableInferenceAdapter,
// verifiableInferenceOptions,
}: GenerationOptions): Promise<GenerateObjectResult<unknown>> => {
}: // verifiableInference = false,
// verifiableInferenceAdapter,
// verifiableInferenceOptions,
GenerationOptions): Promise<GenerateObjectResult<unknown>> => {
if (!context) {
const errorMessage = "generateObject context is empty";
console.error(errorMessage);
Expand Down Expand Up @@ -2319,8 +2370,7 @@ async function handleOpenAI({
}: ProviderOptions): Promise<GenerateObjectResult<unknown>> {
const endpoint =
runtime.character.modelEndpointOverride || getEndpoint(provider);
const baseURL =
getCloudflareGatewayBaseURL(runtime, "openai") || endpoint;
const baseURL = getCloudflareGatewayBaseURL(runtime, "openai") || endpoint;
const openai = createOpenAI({ apiKey, baseURL });
return await aiGenerateObject({
model: openai.languageModel(model),
Expand Down Expand Up @@ -2439,7 +2489,7 @@ async function handleGoogle({
mode = "json",
modelOptions,
}: ProviderOptions): Promise<GenerateObjectResult<unknown>> {
const google = createGoogleGenerativeAI({apiKey});
const google = createGoogleGenerativeAI({ apiKey });
return await aiGenerateObject({
model: google(model),
schema,
Expand Down Expand Up @@ -2662,7 +2712,7 @@ async function handleSecretAi({
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${apiKey}`,
}
},
});
const secretAi = secretAiProvider(model);
return await aiGenerateObject({
Expand Down

0 comments on commit 486bd69

Please sign in to comment.