Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: fix 0.25.8 oom bug #3722

Merged
merged 2 commits into from
Feb 28, 2025
Merged

fix: fix 0.25.8 oom bug #3722

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
486bd69
fix 0.25.8 oom bug
v1xingyue Feb 28, 2025
6db000f
Merge branch 'develop' into patch-1
odilitime Feb 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 75 additions & 24 deletions packages/core/src/generation.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -366,9 +366,9 @@ export async function generateText({
maxSteps = 1,
stop,
customSystemPrompt,
// verifiableInference = process.env.VERIFIABLE_INFERENCE_ENABLED === "true",
// verifiableInferenceOptions,
}: {
}: // verifiableInference = process.env.VERIFIABLE_INFERENCE_ENABLED === "true",
// verifiableInferenceOptions,
{
runtime: IAgentRuntime;
context: string;
modelClass: ModelClass;
Expand Down Expand Up @@ -521,8 +521,7 @@ export async function generateText({
const max_context_length =
modelConfiguration?.maxInputTokens || modelSettings.maxInputTokens;
const max_response_length =
modelConfiguration?.maxOutputTokens ||
modelSettings.maxOutputTokens;
modelConfiguration?.maxOutputTokens || modelSettings.maxOutputTokens;
const experimental_telemetry =
modelConfiguration?.experimental_telemetry ||
modelSettings.experimental_telemetry;
Expand Down Expand Up @@ -973,7 +972,10 @@ export async function generateText({
experimental_telemetry: experimental_telemetry,
});

response = ollamaResponse.replace(/<think>[\s\S]*?<\/think>\s*\n*/g, '');
response = ollamaResponse.replace(
/<think>[\s\S]*?<\/think>\s*\n*/g,
""
);
}
elizaLogger.debug("Received response from Ollama model.");
break;
Expand Down Expand Up @@ -1185,8 +1187,10 @@ export async function generateText({
// console.warn("veniceResponse:")
// console.warn(veniceResponse)
//rferrari: remove all text from <think> to </think>\n\n
response = veniceResponse
.replace(/<think>[\s\S]*?<\/think>\s*\n*/g, '');
response = veniceResponse.replace(
/<think>[\s\S]*?<\/think>\s*\n*/g,
""
);
// console.warn(response)

// response = veniceResponse;
Expand Down Expand Up @@ -1320,7 +1324,7 @@ export async function generateText({
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${apiKey}`,
}
},
});
const secretAi = secretAiProvider(model);

Expand Down Expand Up @@ -1349,7 +1353,7 @@ export async function generateText({
frequencyPenalty: frequency_penalty,
presencePenalty: presence_penalty,
experimental_telemetry: experimental_telemetry,
prompt: context
prompt: context,
});

response = bedrockResponse;
Expand Down Expand Up @@ -1447,6 +1451,26 @@ export async function splitChunks(
): Promise<string[]> {
elizaLogger.debug(`[splitChunks] Starting text split`);

// Validate parameters
if (chunkSize <= 0) {
elizaLogger.warn(
`Invalid chunkSize (${chunkSize}), using default 1500`
);
chunkSize = 1500;
}

if (bleed >= chunkSize) {
elizaLogger.warn(
`Bleed (${bleed}) >= chunkSize (${chunkSize}), adjusting bleed to 1/4 of chunkSize`
);
bleed = Math.floor(chunkSize / 4);
}

if (bleed < 0) {
elizaLogger.warn(`Invalid bleed (${bleed}), using default 100`);
bleed = 100;
}

const chunks = splitText(content, chunkSize, bleed);

elizaLogger.debug(`[splitChunks] Split complete:`, {
Expand All @@ -1459,14 +1483,29 @@ export async function splitChunks(
return chunks;
}

export function splitText(content: string, chunkSize: number, bleed: number): string[] {
export function splitText(
content: string,
chunkSize: number,
bleed: number
): string[] {
const chunks: string[] = [];
let start = 0;

while (start < content.length) {
const end = Math.min(start + chunkSize, content.length);
chunks.push(content.substring(start, end));
start = end - bleed; // Apply overlap

// Ensure we're not creating empty or invalid chunks
if (end > start) {
chunks.push(content.substring(start, end));
}

// Ensure forward progress and prevent infinite loops
const nextStart = end - bleed;
if (nextStart <= start) {
start = end; // If no progress would be made, skip the bleed
} else {
start = nextStart;
}
}

return chunks;
Expand Down Expand Up @@ -1726,7 +1765,9 @@ export const generateImage = async (
}> => {
const modelSettings = getImageModelSettings(runtime.imageModelProvider);
if (!modelSettings) {
elizaLogger.warn("No model settings found for the image model provider.");
elizaLogger.warn(
"No model settings found for the image model provider."
);
return { success: false, error: "No model settings available" };
}
const model = modelSettings.name;
Expand Down Expand Up @@ -1756,11 +1797,21 @@ export const generateImage = async (
return runtime.getSetting("SECRET_AI_API_KEY");
case ModelProviderName.NEARAI:
try {
// Read auth config from ~/.nearai/config.json if it exists
const config = JSON.parse(fs.readFileSync(path.join(os.homedir(), '.nearai/config.json'), 'utf8'));
return JSON.stringify(config?.auth);
// Read auth config from ~/.nearai/config.json if it exists
const config = JSON.parse(
fs.readFileSync(
path.join(
os.homedir(),
".nearai/config.json"
),
"utf8"
)
);
return JSON.stringify(config?.auth);
} catch (e) {
elizaLogger.warn(`Error loading NEAR AI config. The environment variable NEARAI_API_KEY will be used. ${e}`);
elizaLogger.warn(
`Error loading NEAR AI config. The environment variable NEARAI_API_KEY will be used. ${e}`
);
}
return runtime.getSetting("NEARAI_API_KEY");
default:
Expand Down Expand Up @@ -2171,10 +2222,10 @@ export const generateObject = async ({
schemaDescription,
stop,
mode = "json",
// verifiableInference = false,
// verifiableInferenceAdapter,
// verifiableInferenceOptions,
}: GenerationOptions): Promise<GenerateObjectResult<unknown>> => {
}: // verifiableInference = false,
// verifiableInferenceAdapter,
// verifiableInferenceOptions,
GenerationOptions): Promise<GenerateObjectResult<unknown>> => {
if (!context) {
const errorMessage = "generateObject context is empty";
console.error(errorMessage);
Expand Down Expand Up @@ -2314,7 +2365,7 @@ async function handleOpenAI({
modelOptions,
provider,
runtime,
}: ProviderOptions): Promise<GenerationResult> {
}: ProviderOptions): Promise<GenerateObjectResult<unknown>> {
const endpoint = runtime.character.modelEndpointOverride || getEndpoint(provider);
const baseURL = getCloudflareGatewayBaseURL(runtime, "openai") || endpoint;
const openai = createOpenAI({
Expand Down Expand Up @@ -2452,7 +2503,7 @@ async function handleGoogle({
mode = "json",
modelOptions,
runtime,
}: ProviderOptions): Promise<GenerationResult> {
}: ProviderOptions): Promise<GenerateObjectResult<unknown>> {
const google = createGoogleGenerativeAI({
apiKey,
fetch: runtime.fetch
Expand Down
Loading