diff --git a/plugin/constants.ts b/plugin/constants.ts new file mode 100644 index 00000000..38e6b4a9 --- /dev/null +++ b/plugin/constants.ts @@ -0,0 +1,39 @@ +import { Notice } from "obsidian"; + +export const VALID_IMAGE_EXTENSIONS = ["png", "jpg", "jpeg", "gif", "svg", "webp"]; + +export const VALID_AUDIO_EXTENSIONS = [ + "mp3", + "mp4", + "mpeg", + "mpga", + "m4a", + "wav", + "webm", +]; + +export const VALID_MEDIA_EXTENSIONS = [ + ...VALID_IMAGE_EXTENSIONS, + ...VALID_AUDIO_EXTENSIONS, +]; + +export const VALID_TEXT_EXTENSIONS = ["md", "txt"]; + +export const VALID_EXTENSIONS = [ + ...VALID_MEDIA_EXTENSIONS, + ...VALID_TEXT_EXTENSIONS, + "pdf", +]; + +/** + * Validates if a given file extension is supported by FileOrganizer + * @param extension - The file extension to validate (without the dot) + * @returns boolean indicating if the extension is supported + */ +export const isValidExtension = (extension: string): boolean => { + const isSupported = VALID_EXTENSIONS.includes(extension); + if (!isSupported) { + new Notice("Sorry, FileOrganizer does not support this file type."); + } + return isSupported; +}; \ No newline at end of file diff --git a/plugin/fileUtils.ts b/plugin/fileUtils.ts index 33e27a06..3cb41ac3 100644 --- a/plugin/fileUtils.ts +++ b/plugin/fileUtils.ts @@ -1,5 +1,4 @@ import { App, TFolder, TFile, normalizePath } from "obsidian"; -import { FileOrganizerSettings } from "./FileOrganizerSettings"; import { Notice } from "obsidian"; export async function ensureFolderExists(app: App, folderPath: string) { if (!(await app.vault.adapter.exists(folderPath))) { diff --git a/plugin/handlers/commandHandlers.ts b/plugin/handlers/commandHandlers.ts index b6fba939..37c87b70 100644 --- a/plugin/handlers/commandHandlers.ts +++ b/plugin/handlers/commandHandlers.ts @@ -1,7 +1,8 @@ import { WorkspaceLeaf } from "obsidian"; import FileOrganizer from "../index"; -import { ORGANIZER_VIEW_TYPE, AssistantViewWrapper } from "../views/organizer"; +import { ORGANIZER_VIEW_TYPE, AssistantViewWrapper } from "../views/organizer/view"; import { AIChatView, CHAT_VIEW_TYPE } from "../views/ai-chat/view"; +import { Inbox } from "../inbox"; export function initializeChat(plugin: FileOrganizer) { plugin.registerView( @@ -50,8 +51,9 @@ export function initializeFileOrganizationCommands(plugin: FileOrganizer) { name: "Put in inbox", callback: async () => { const activeFile = plugin.app.workspace.getActiveFile(); + // move to file to inbox if (activeFile) { - await plugin.processFileV2(activeFile); + await plugin.app.vault.rename(activeFile, `${plugin.settings.pathToWatch}/${activeFile.name}`); } }, }); diff --git a/plugin/handlers/eventHandlers.ts b/plugin/handlers/eventHandlers.ts index bb4e13f9..b5fb7f50 100644 --- a/plugin/handlers/eventHandlers.ts +++ b/plugin/handlers/eventHandlers.ts @@ -1,12 +1,17 @@ import { TFile } from "obsidian"; import FileOrganizer from ".."; +import { Inbox } from "../inbox"; export function registerEventHandlers(plugin: FileOrganizer) { plugin.registerEvent( - plugin.app.vault.on("create", (file) => { + plugin.app.vault.on("create", file => { if (!file.path.includes(plugin.settings.pathToWatch)) return; if (file instanceof TFile) { - plugin.processFileV2(file); + if (plugin.settings.useInbox) { + Inbox.getInstance().enqueueFiles([file]); + } else { + plugin.processFileV2(file); + } } }) ); @@ -15,8 +20,12 @@ export function registerEventHandlers(plugin: FileOrganizer) { plugin.app.vault.on("rename", (file, oldPath) => { if (!file.path.includes(plugin.settings.pathToWatch)) return; if (file instanceof TFile) { - plugin.processFileV2(file, oldPath); + if (plugin.settings.useInbox) { + Inbox.getInstance().enqueueFiles([file]); + } else { + plugin.processFileV2(file, oldPath); + } } }) ); -} \ No newline at end of file +} diff --git a/plugin/inbox/constants.ts b/plugin/inbox/constants.ts new file mode 100644 index 00000000..7cbf586e --- /dev/null +++ b/plugin/inbox/constants.ts @@ -0,0 +1,38 @@ +export const VALID_MEDIA_EXTENSIONS = [ + "png", + "jpg", + "jpeg", + "gif", + "bmp", + "svg", + "mp3", + "wav", + "mp4", + "mov", + "wmv", +]; + +export const CHUNK_SIZE = 1024 * 1024; // 1MB +export const MAX_CONCURRENT_TASKS = 100; +export const BATCH_DELAY = 100; // ms +export const MAX_BATCH_SIZE = 10; +export const CACHE_TTL = 1000 * 60 * 60 * 24; // 24 hours +export const MAX_LOG_SIZE = 100; +export const ERROR_FOLDER = "_FileOrganizer2000/Error"; + +export const NOTIFICATION_DURATIONS = { + CRITICAL: 10000, // 10 seconds + HIGH: 5000, // 5 seconds + MEDIUM: 3000, // 3 seconds + LOW: 2000, // 2 seconds +}; + +export const FILE_PRIORITIES = { + SMALL: 3, // Small files (<100KB) + MARKDOWN: 2, // Markdown files + DEFAULT: 1, // Default priority +}; + +export const SIZE_THRESHOLDS = { + SMALL: 1024 * 100, // 100KB +}; \ No newline at end of file diff --git a/plugin/inbox/index.ts b/plugin/inbox/index.ts new file mode 100644 index 00000000..3a3c7716 --- /dev/null +++ b/plugin/inbox/index.ts @@ -0,0 +1,445 @@ +import { TFile, moment, Notice, Vault } from "obsidian"; +import { VALID_MEDIA_EXTENSIONS } from "../constants"; +import FileOrganizer from "../index"; +import { validateFile } from "../utils"; +import { Queue } from "./services/queue"; +import { RecordManager } from "./services/record-manager"; +import { FileRecord, QueueStatus } from "./types"; +import { logMessage } from "../../utils"; +import { IdService } from "./services/id-service"; + +export interface FolderSuggestion { + isNewFolder: boolean; + score: number; + folder: string; + reason: string; +} + +export interface LogEntry { + id: string; + fileName: string; + timestamp: string; + status: "queued" | "processing" | "completed" | "error"; + newPath?: string; + newName?: string; + classification?: string; + addedTags?: string[]; + errors?: string[]; + messages: string[]; +} + +interface EventRecord { + id: string; + fileRecordId: string; + timestamp: string; + message: string; + metadata?: Record; +} + +interface ProcessingContext { + file: TFile; + hash: string; + record: FileRecord; + content?: string; + newPath?: string; + newName?: string; + tags?: string[]; + containerFile?: TFile; + attachmentFile?: TFile; + plugin: FileOrganizer; + recordManager: RecordManager; + idService: IdService; + queue: Queue; +} + +export class Inbox { + protected static instance: Inbox; + private plugin: FileOrganizer; + private readonly MAX_CONCURRENT_TASKS: number = 100; + + private queue: Queue; + private recordManager: RecordManager; + private idService: IdService; + + private constructor(plugin: FileOrganizer) { + this.plugin = plugin; + this.recordManager = RecordManager.getInstance(); + this.idService = IdService.getInstance(); + this.initializeQueue(); + } + + public static initialize(plugin: FileOrganizer): Inbox { + if (!Inbox.instance) { + Inbox.instance = new Inbox(plugin); + } + return Inbox.instance; + } + + public static getInstance(): Inbox { + if (!Inbox.instance) { + throw new Error("Inbox not initialized. Call initialize() first."); + } + return Inbox.instance; + } + + public static cleanup(): void { + if (Inbox.instance) { + Inbox.instance.queue.clear(); + // @ts-ignore - We know what we're doing here + Inbox.instance = null; + } + } + + public enqueueFile(file: TFile): void { + this.enqueueFiles([file]); + } + + public enqueueFiles(files: TFile[]): void { + logMessage(`Enqueuing ${files.length} files`); + for (const file of files) { + const hash = this.idService.generateFileHash(file); + const record = this.recordManager.createOrUpdateFileRecord(file); + + this.recordManager.updateFileStatus( + record, + "queued", + "File enqueued for processing" + ); + + this.queue.add(file, { + metadata: { hash }, + }); + } + logMessage(`Enqueued ${this.getQueueStats().queued} files`); + } + + private initializeQueue(): void { + this.queue = new Queue({ + concurrency: this.MAX_CONCURRENT_TASKS, + timeout: 30000, + onProcess: async (file: TFile, metadata?: Record) => { + await this.processInboxFile(file); + }, + onComplete: (file: TFile, metadata?: Record) => {}, + onError: (error: Error, file: TFile, metadata?: Record) => { + console.error("Queue processing error:", error); + }, + }); + } + + public getFileStatus(filePath: string): FileRecord | undefined { + return this.recordManager.getRecordByPath(filePath); + } + + public getFileEvents(fileId: string): EventRecord[] { + return this.recordManager.getFileEvents(fileId); + } + + public getAllFiles(): FileRecord[] { + return this.recordManager.getAllRecords(); + } + + public getQueueStats(): QueueStatus { + return this.queue.getStats(); + } + + // Refactored method using a pipeline-style processing + private async processInboxFile(file: TFile): Promise { + const hash = this.idService.generateFileHash(file); + const record = this.recordManager.getRecordByHash(hash); + if (!record) return; + + const context: ProcessingContext = { + file, + hash, + record, + plugin: this.plugin, + recordManager: this.recordManager, + idService: this.idService, + queue: this.queue, + }; + + try { + await startProcessing(context) + .then(validateFileStep) + .then(processContentStep) + .then(determineDestinationStep) + .then(async ctx => { + if (this.shouldCreateMarkdownContainer(ctx.file)) { + return await processMediaFileStep(ctx); + } else { + return await processNonMediaFileStep(ctx); + } + }) + .then(completeProcessing); + this.queue.remove(context.hash); + } catch (error) { + console.error("Error processing inbox file:", error); + await handleError(error, context); + } + } + + private shouldCreateMarkdownContainer(file: TFile): boolean { + return ( + VALID_MEDIA_EXTENSIONS.includes(file.extension) || + file.extension === "pdf" + ); + } +} + +// Pipeline processing steps + +async function startProcessing( + context: ProcessingContext +): Promise { + context.recordManager.recordProcessingStart(context.record); + return context; +} + +async function validateFileStep( + context: ProcessingContext +): Promise { + if (!validateFile(context.file)) { + context.queue.bypass(context.hash); + await moveFileToBypassedFolder(context); + context.recordManager.recordProcessingBypassed( + context.record, + "File validation failed" + ); + throw new Error("File validation failed"); + } + return context; +} + +async function processContentStep( + context: ProcessingContext +): Promise { + const result = await processContent(context.plugin, context.file); + if (!result) { + context.recordManager.recordProcessingBypassed( + context.record, + "No content to process" + ); + throw new Error("No content to process"); + } + context.content = result.text; + return context; +} + +async function determineDestinationStep( + context: ProcessingContext +): Promise { + // Get AI classified folder and update record + context.newPath = await context.plugin.getAIClassifiedFolder( + context.content!, + context.file.path + ); + context.recordManager.updateFileStatus( + context.record, + "processing", + `Determined destination folder: ${context.newPath}` + ); + + // Generate AI name and update record + context.newName = await context.plugin.generateNameFromContent( + context.content!, + context.file.name + ); + context.recordManager.updateFileStatus( + context.record, + "processing", + `Generated new name: ${context.newName}` + ); + + return context; +} + +async function processMediaFileStep( + context: ProcessingContext +): Promise { + // Create container file + const containerPath = `${context.newPath}/${context.newName}.md`; + context.containerFile = await createFile( + context, + containerPath, + context.content! + ); + + // Move attachment + const attachmentFolderPath = `${context.newPath}/attachments`; + await moveFile( + context, + context.file, + `${attachmentFolderPath}/${context.file.name}` + ); + context.attachmentFile = context.plugin.app.vault.getAbstractFileByPath( + `${attachmentFolderPath}/${context.file.name}` + ) as TFile; + + // Update container with attachment reference + await context.plugin.app.vault.modify( + context.containerFile, + `${context.content}\n\n![[${context.attachmentFile.path}]]` + ); + + // Generate tags + const existingTags = await context.plugin.getAllVaultTags(); + context.tags = ( + await context.plugin.guessRelevantTags( + context.content!, + context.file.path, + existingTags + ) + ).map(tag => tag.tag); + + return context; +} + +async function processNonMediaFileStep( + context: ProcessingContext +): Promise { + const existingTags = await context.plugin.getAllVaultTags(); + context.tags = ( + await context.plugin.guessRelevantTags( + context.content!, + context.file.path, + existingTags + ) + ).map(tag => tag.tag); + + const finalPath = `${context.newPath}/${context.newName}.md`; + await moveFile(context, context.file, finalPath); + + return context; +} + +async function completeProcessing( + context: ProcessingContext +): Promise { + context.recordManager.recordProcessingComplete(context.record, { + newPath: context.newPath!, + newName: context.newName!, + tags: context.tags, + }); + return context; +} + +// Error handling + +async function handleError( + error: any, + context: ProcessingContext +): Promise { + context.recordManager.recordError(context.record, error); + await moveFileToErrorFolder(context); + context.recordManager.updateFileStatus( + context.record, + "error", + `Processing error: ${error.message}`, + { + errorDetails: { + message: error.message, + stack: error.stack, + fileName: context.file.name, + newLocation: context.plugin.settings.errorFilePath, + }, + } + ); +} + +// Helper functions for file operations + +async function moveFileToBypassedFolder( + context: ProcessingContext +): Promise { + const bypassedFolderPath = context.plugin.settings.bypassedFilePath; + const newPath = `${bypassedFolderPath}/${context.file.name}`; + await moveFile(context, context.file, newPath); +} + +async function moveFileToErrorFolder( + context: ProcessingContext +): Promise { + const errorFolderPath = context.plugin.settings.errorFilePath; + const newPath = `${errorFolderPath}/${context.file.name}`; + await moveFile(context, context.file, newPath); +} + +async function processContent( + plugin: FileOrganizer, + file: TFile +): Promise<{ text: string } | null> { + try { + const text = await plugin.getTextFromFile(file); + return { text }; + } catch (error) { + console.error("Error in processContent:", error); + throw new Error("Error in processContent"); + } +} + +async function moveFile( + context: ProcessingContext, + file: TFile, + newPath: string +): Promise { + try { + await ensureFolder(context, newPath); + + const exists = await context.plugin.app.vault.adapter.exists(newPath); + if (exists) { + const timestamp = moment().format("YYYY-MM-DD-HHmmss"); + const parts = newPath.split("."); + const ext = parts.pop(); + newPath = `${parts.join(".")}-${timestamp}.${ext}`; + } + + await context.plugin.app.vault.rename(file, newPath); + } catch (error) { + console.error(`Failed to move file ${file.path} to ${newPath}:`, error); + throw new Error(`Failed to move file: ${error.message}`); + } +} + +async function createFile( + context: ProcessingContext, + path: string, + content: string +): Promise { + try { + await ensureFolder(context, path); + return await context.plugin.app.vault.create(path, content); + } catch (error) { + console.error(`Failed to create file at ${path}:`, error); + throw new Error(`Failed to create file: ${error.message}`); + } +} + +async function ensureFolder( + context: ProcessingContext, + path: string +): Promise { + const folderPath = path.split("/").slice(0, -1).join("/"); + try { + await context.plugin.app.vault.createFolder(folderPath); + } catch (error) { + if (!error.message.includes("already exists")) { + console.error("Error creating folder:", error); + throw error; + } + } +} + +// Helper functions for initialization and usage +export function initializeInboxQueue(plugin: FileOrganizer): void { + Inbox.cleanup(); + Inbox.initialize(plugin); +} + +export function enqueueFiles(files: TFile[]): void { + Inbox.getInstance().enqueueFiles(files); +} + +export function getInboxStatus(): QueueStatus { + return Inbox.getInstance().getQueueStats(); +} diff --git a/plugin/inbox/services/error-service.ts b/plugin/inbox/services/error-service.ts new file mode 100644 index 00000000..3ee90dd8 --- /dev/null +++ b/plugin/inbox/services/error-service.ts @@ -0,0 +1,150 @@ +import { Notice } from "obsidian"; +import { logMessage } from "../../utils"; + +export enum ErrorSeverity { + LOW = "low", + MEDIUM = "medium", + HIGH = "high", + CRITICAL = "critical" +} + +export interface ErrorDetails { + message: string; + severity: ErrorSeverity; + context?: Record; + error?: Error; + shouldNotify?: boolean; +} + +export class ErrorService { + private static instance: ErrorService; + private errorLog: ErrorDetails[] = []; + private readonly MAX_LOG_SIZE = 100; + private isDebugEnabled = false; + + private constructor() {} + + public static getInstance(): ErrorService { + if (!ErrorService.instance) { + ErrorService.instance = new ErrorService(); + } + return ErrorService.instance; + } + + public handleError(details: ErrorDetails): void { + // Log the error + this.logError(details); + + // Show notification if needed + if (details.shouldNotify) { + this.showNotification(details); + } + + // Log to console in debug mode + if (this.isDebugEnabled) { + console.error("[FileOrganizer Error]", { + ...details, + timestamp: new Date().toISOString() + }); + } + } + + private logError(details: ErrorDetails): void { + this.errorLog.unshift({ + ...details, + context: { + ...details.context, + timestamp: new Date().toISOString() + } + }); + + // Trim log if it gets too large + if (this.errorLog.length > this.MAX_LOG_SIZE) { + this.errorLog = this.errorLog.slice(0, this.MAX_LOG_SIZE); + } + } + + private showNotification(details: ErrorDetails): void { + const duration = this.getNotificationDuration(details.severity); + new Notice( + `FileOrganizer: ${details.message}`, + duration + ); + } + + private getNotificationDuration(severity: ErrorSeverity): number { + switch (severity) { + case ErrorSeverity.CRITICAL: + return 10000; // 10 seconds + case ErrorSeverity.HIGH: + return 5000; // 5 seconds + case ErrorSeverity.MEDIUM: + return 3000; // 3 seconds + default: + return 2000; // 2 seconds + } + } + + public getRecentErrors(): ErrorDetails[] { + return [...this.errorLog]; + } + + public clearErrors(): void { + this.errorLog = []; + } + + public enableDebug(): void { + this.isDebugEnabled = true; + } + + public disableDebug(): void { + this.isDebugEnabled = false; + } + + // Helper methods for common error scenarios + public handleFileOperationError(operation: string, path: string, error: Error): void { + this.handleError({ + message: `Failed to ${operation} file: ${path}`, + severity: ErrorSeverity.HIGH, + context: { operation, path }, + error, + shouldNotify: true + }); + } + + public handleAPIError(endpoint: string, error: Error): void { + this.handleError({ + message: `API request failed: ${endpoint}`, + severity: ErrorSeverity.MEDIUM, + context: { endpoint }, + error, + shouldNotify: false + }); + } + + public handleProcessingError(fileName: string, error: Error): void { + this.handleError({ + message: `Error processing file: ${fileName}`, + severity: ErrorSeverity.HIGH, + context: { fileName }, + error, + shouldNotify: true + }); + } +} + +// Helper function to wrap async operations with error handling +export async function withErrorHandling( + operation: () => Promise, + errorDetails: Omit +): Promise { + try { + return await operation(); + } catch (error) { + ErrorService.getInstance().handleError({ + ...errorDetails, + error: error as Error + }); + return null; + } +} \ No newline at end of file diff --git a/plugin/inbox/services/id-service.ts b/plugin/inbox/services/id-service.ts new file mode 100644 index 00000000..34815069 --- /dev/null +++ b/plugin/inbox/services/id-service.ts @@ -0,0 +1,31 @@ +import { TFile } from "obsidian"; +import { createHash } from "crypto"; + +export class IdService { + private static instance: IdService; + + public static getInstance(): IdService { + if (!IdService.instance) { + IdService.instance = new IdService(); + } + return IdService.instance; + } + + public generateFileHash(file: TFile): string { + // Create a unique hash based on file path and last modified time + const content = `${file.path}-${file.stat.mtime}`; + return createHash('sha256').update(content).digest('hex').slice(0, 12); + } + + public generateEventId(fileHash: string, timestamp: number): string { + return `evt-${fileHash}-${timestamp}`; + } + + public generateStepId(fileHash: string, type: string): string { + return `step-${fileHash}-${type}`; + } + + public validateHash(hash: string): boolean { + return typeof hash === 'string' && hash.length === 12; + } +} \ No newline at end of file diff --git a/plugin/inbox/services/queue.ts b/plugin/inbox/services/queue.ts new file mode 100644 index 00000000..18e4ca50 --- /dev/null +++ b/plugin/inbox/services/queue.ts @@ -0,0 +1,203 @@ +import { EventEmitter } from 'events'; +import { ErrorService, ErrorSeverity } from './error-service'; +import { MAX_CONCURRENT_TASKS } from '../constants'; +import { IdService } from './id-service'; +import { TFile } from "obsidian"; + +interface QueueItem { + hash: string; + data: T; + metadata?: Record; + addedAt: number; +} + +interface QueueOptions { + concurrency?: number; + timeout?: number; + onProcess: (item: T, metadata?: Record) => Promise; + onComplete?: (item: T, metadata?: Record) => void; + onError?: (error: Error, item: T, metadata?: Record) => void; +} + +interface QueueStatus { + queued: number; + processing: number; + completed: number; + errors: number; + bypassed: number; + total: number; +} + +export class Queue extends EventEmitter { + private items: Map> = new Map(); + private processing: Set = new Set(); + private options: Required>; + private idService: IdService; + + private completedItems: Set = new Set(); + private errorItems: Set = new Set(); + private bypassedItems: Set = new Set(); + + private queue: string[] = []; + + constructor(options: QueueOptions) { + super(); + this.options = { + concurrency: MAX_CONCURRENT_TASKS, + timeout: 30000, + onComplete: options.onComplete || ((item: T, metadata?: Record) => {}), + onError: options.onError || ((error: Error, item: T, metadata?: Record) => {}), + ...options + }; + this.idService = IdService.getInstance(); + } + + public add(file: TFile, { metadata = {} } = {}): string { + const hash = this.idService.generateFileHash(file); + const item: QueueItem = { + hash, + data: file as unknown as T, + metadata: { ...metadata, hash }, + addedAt: Date.now() + }; + + this.items.set(hash, item); + this.queue.push(hash); + + if (this.processing.size < this.options.concurrency) { + this.processNext(); + } + + return hash; + } + + private async processNext(): Promise { + if (this.processing.size >= this.options.concurrency || this.queue.length === 0) { + return; + } + + const hash = this.queue.shift(); + if (!hash) return; + + const item = this.items.get(hash); + if (!item) return; + + this.processing.add(hash); + + try { + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error('Processing timeout')), this.options.timeout); + }); + + await Promise.race([ + this.options.onProcess(item.data, item.metadata), + timeoutPromise + ]); + + this.completedItems.add(hash); + this.items.delete(hash); + this.options.onComplete?.(item.data, item.metadata); + } catch (error) { + this.errorItems.add(hash); + this.items.delete(hash); + ErrorService.getInstance().handleError({ + message: 'Queue processing error', + severity: ErrorSeverity.HIGH, + error: error as Error, + context: { + itemId: hash, + metadata: item.metadata + } + }); + this.options.onError?.(error as Error, item.data, item.metadata); + } finally { + this.processing.delete(hash); + + this.emit('statsUpdated', this.getStats()); + + if (this.queue.length > 0) { + this.processNext(); + } else if (this.processing.size === 0) { + this.emit('drain'); + } + } + } + + public remove(hash: string): boolean { + if (!this.idService.validateHash(hash) || this.processing.has(hash)) { + return false; + } + + const index = this.queue.indexOf(hash); + if (index !== -1) { + this.queue.splice(index, 1); + this.items.delete(hash); + return true; + } + + return false; + } + + public clear(): void { + this.queue = []; + this.processing.clear(); + this.completedItems.clear(); + this.errorItems.clear(); + this.bypassedItems.clear(); + } + + public pause(): void { + this.emit('pause'); + } + + public resume(): void { + this.emit('resume'); + while (this.processing.size < this.options.concurrency && this.queue.length > 0) { + this.processNext(); + } + } + + public get size(): number { + return this.items.size; + } + + public get activeCount(): number { + return this.processing.size; + } + + public getItem(hash: string): QueueItem | undefined { + return this.items.get(hash); + } + + public getStats(): QueueStatus { + return { + queued: this.queue.length, + processing: this.processing.size, + completed: this.completedItems.size, + errors: this.errorItems.size, + bypassed: this.bypassedItems.size, + total: this.items.size + this.completedItems.size + this.errorItems.size + this.bypassedItems.size + }; + } + + public bypass(hash: string): boolean { + if (!this.idService.validateHash(hash) || this.processing.has(hash)) { + return false; + } + + const index = this.queue.indexOf(hash); + if (index !== -1) { + this.queue.splice(index, 1); + const item = this.items.get(hash); + if (item) { + this.bypassedItems.add(hash); + this.items.delete(hash); + this.emit('bypass', item); + this.emit('statsUpdated', this.getStats()); + return true; + } + } + + return false; + } +} \ No newline at end of file diff --git a/plugin/inbox/services/record-manager.ts b/plugin/inbox/services/record-manager.ts new file mode 100644 index 00000000..85a1ff8e --- /dev/null +++ b/plugin/inbox/services/record-manager.ts @@ -0,0 +1,221 @@ +import { TFile } from "obsidian"; +import { FileRecord, FileStatus, FileMetadata, EventRecord } from "../types"; +import { IdService } from "./id-service"; +import { ErrorService, ErrorSeverity } from "./error-service"; +import { isMediaFile } from "../utils/file"; +import moment from "moment"; + +export class RecordManager { + private static instance: RecordManager; + private fileRecords: Map = new Map(); + private eventRecords: Map = new Map(); + private idService: IdService; + + private constructor() { + this.idService = IdService.getInstance(); + } + + public static getInstance(): RecordManager { + if (!RecordManager.instance) { + RecordManager.instance = new RecordManager(); + } + return RecordManager.instance; + } + + public createOrUpdateFileRecord( + file: TFile, + updates?: Partial + ): FileRecord { + try { + const hash = this.idService.generateFileHash(file); + const existingRecord = this.getRecordByHash(hash); + + if (existingRecord) { + if (updates) { + return this.updateRecord(hash, updates); + } + return existingRecord; + } + + const now = moment().format(); + const metadata: FileMetadata = { + size: file.stat.size, + extension: file.extension, + createdTime: file.stat.ctime, + modifiedTime: file.stat.mtime, + isMediaFile: isMediaFile(file), + }; + + const newRecord: FileRecord = { + id: hash, + filePath: file.path, + fileName: file.basename, + previousName: file.basename, + status: "queued" as FileStatus, + createdAt: now, + updatedAt: now, + metadata, + errors: [], + ...updates, + }; + + this.fileRecords.set(hash, newRecord); + this.addEvent(hash, "File record initialized", { metadata }); + + return newRecord; + } catch (error) { + ErrorService.getInstance().handleError({ + message: "Failed to create/update file record", + severity: ErrorSeverity.HIGH, + error: error as Error, + context: { filePath: file.path }, + }); + throw error; + } + } + + private updateRecord(hash: string, updates: Partial): FileRecord { + const record = this.getRecordByHash(hash); + if (!record) { + throw new Error(`Record not found for hash: ${hash}`); + } + + const updatedRecord = { + ...record, + ...updates, + updatedAt: moment().format(), + }; + + this.fileRecords.set(hash, updatedRecord); + return updatedRecord; + } + + public updateFileStatus( + record: FileRecord, + status: FileStatus, + message?: string, + metadata?: Record + ): void { + const hash = record.id; + + // Update the record with new status + this.updateRecord(hash, { + status, + updatedAt: moment().format(), + }); + + // Add event with status change + const statusMessage = message || `File status changed to ${status}`; + this.addEvent(hash, statusMessage, { + status, + previousStatus: record.status, + ...metadata, + }); + } + + public recordProcessingStart(record: FileRecord): void { + this.updateFileStatus(record, "processing", "Started processing file"); + } + + public recordProcessingComplete( + record: FileRecord, + metadata?: { + newPath?: string; + newName?: string; + tags?: string[]; + } + ): void { + const updates: Partial = { + status: "completed", + updatedAt: moment().format(), + }; + + if (metadata) { + // Update file paths + if (metadata.newPath) updates.newPath = metadata.newPath; + if (metadata.newName) updates.newName = metadata.newName; + + // Update tags + if (metadata.tags) updates.tags = metadata.tags; + + // Update processing information + } + + this.updateRecord(record.id, updates); + this.addEvent(record.id, "File processing completed", metadata); + } + + public recordProcessingBypassed(record: FileRecord, reason?: string): void { + this.updateFileStatus( + record, + "bypassed", + reason || "File processing bypassed" + ); + } + + public recordError(record: FileRecord, error: Error): void { + const hash = record.id; + const errorRecord = { + timestamp: moment().format(), + message: error.message, + stack: error.stack, + }; + + this.updateRecord(hash, { + errors: [...(record.errors || []), errorRecord], + }); + this.addEvent(hash, `Error: ${error.message}`, { error: errorRecord }); + } + + public updateDestination( + record: FileRecord, + newName: string, + newPath: string + ): void { + const hash = record.id; + this.updateRecord(hash, { newName, newPath }); + this.addEvent(hash, `Updated destination: ${newPath}/${newName}`); + } + + public addTags(record: FileRecord, tags: string[]): void { + const hash = record.id; + this.updateRecord(hash, { tags }); + this.addEvent(hash, `Added tags: ${tags.join(", ")}`); + } + + public getRecordByHash(hash: string): FileRecord | undefined { + return this.fileRecords.get(hash); + } + + public getRecordByPath(path: string): FileRecord | undefined { + return Array.from(this.fileRecords.values()).find( + record => record.filePath === path + ); + } + + public getAllRecords(): FileRecord[] { + return Array.from(this.fileRecords.values()); + } + + public getFileEvents(fileId: string): EventRecord[] { + return this.eventRecords.get(fileId) || []; + } + + private addEvent( + hash: string, + message: string, + metadata?: Record + ): void { + const event: EventRecord = { + id: this.idService.generateEventId(hash, Date.now()), + fileRecordId: hash, + timestamp: moment().format(), + message, + metadata, + }; + + const events = this.eventRecords.get(hash) || []; + events.push(event); + this.eventRecords.set(hash, events); + } +} diff --git a/plugin/inbox/types.ts b/plugin/inbox/types.ts new file mode 100644 index 00000000..a069e367 --- /dev/null +++ b/plugin/inbox/types.ts @@ -0,0 +1,80 @@ +import { TFile } from "obsidian"; + +export type FileStatus = "queued" | "processing" | "completed" | "error" | "bypassed"; + +export interface FileMetadata { + size: number; + extension: string; + createdTime: number; + modifiedTime: number; + isMediaFile: boolean; +} + +export interface FileRecord { + id: string; + filePath: string; + fileName: string; + previousName: string; + status: FileStatus; + createdAt: string; + updatedAt: string; + metadata: FileMetadata; + errors: Array<{ + timestamp: string; + message: string; + stack?: string; + }>; + newPath?: string; + newName?: string; + tags?: string[]; +} + +export interface EventRecord { + id: string; + fileRecordId: string; + timestamp: string; + message: string; + metadata?: Record; +} + +export interface QueueStatus { + queued: number; + processing: number; + completed: number; + errors: number; + bypassed: number; + total: number; +} + +export interface ProcessingResult { + text: string; + classification?: string; + formattedText: string; + tags?: string[]; +} + +export interface FileOperation { + type: 'move' | 'create' | 'modify'; + file: TFile; + newPath?: string; + content?: string; +} + +export interface BatchRequest { + content: string; + resolve: (value: T) => void; + reject: (error: Error) => void; +} + +export interface ClassificationResult { + classification: string; + formattedText: string; + confidence?: number; + metadata?: Record; +} + +export interface CacheEntry { + value: T; + timestamp: number; + hash: string; +} \ No newline at end of file diff --git a/plugin/inbox/utils/file.ts b/plugin/inbox/utils/file.ts new file mode 100644 index 00000000..be868515 --- /dev/null +++ b/plugin/inbox/utils/file.ts @@ -0,0 +1,48 @@ +import { TFile, App } from "obsidian"; +import { VALID_MEDIA_EXTENSIONS } from "../constants"; + +export function isMediaFile(file: TFile): boolean { + return ( + VALID_MEDIA_EXTENSIONS.includes(file.extension) || + file.extension === "pdf" + ); +} + +export function getFilePath(file: TFile): string { + return file.path; +} + +export function getFileName(file: TFile): string { + return file.basename; +} + +export function getFileExtension(file: TFile): string { + return file.extension; +} + +export async function ensureFolderExists(app: App, path: string): Promise { + const folderExists = await app.vault.adapter.exists(path); + if (!folderExists) { + await app.vault.createFolder(path); + } +} + +export function generateId(prefix: string = ''): string { + return `${prefix}${Date.now()}-${Math.random().toString(36).slice(2)}`; +} + +export function getParentFolder(path: string): string { + return path.split('/').slice(0, -1).join('/'); +} + +export function joinPaths(...parts: string[]): string { + return parts.filter(Boolean).join('/'); +} + +export function getAttachmentPath(basePath: string, fileName: string): string { + return joinPaths(basePath, 'attachments', fileName); +} + +export function getContainerPath(basePath: string, fileName: string): string { + return joinPaths(basePath, `${fileName}.md`); +} \ No newline at end of file diff --git a/plugin/index.ts b/plugin/index.ts index c40c6457..6ddb5b24 100644 --- a/plugin/index.ts +++ b/plugin/index.ts @@ -10,10 +10,11 @@ import { normalizePath, loadPdfJs, requestUrl, + arrayBufferToBase64, } from "obsidian"; import { logMessage, formatToSafeName, sanitizeTag } from "../utils"; import { FileOrganizerSettingTab } from "./views/settings/view"; -import { ORGANIZER_VIEW_TYPE } from "./views/organizer"; +import { ORGANIZER_VIEW_TYPE } from "./views/organizer/view"; import { CHAT_VIEW_TYPE } from "./views/ai-chat/view"; import Jimp from "jimp/es/index"; @@ -30,37 +31,23 @@ import { checkAndCreateFolders, checkAndCreateTemplates, moveFile, - getAllFolders, } from "./fileUtils"; + import { checkLicenseKey } from "./apiUtils"; import { makeApiRequest } from "./apiUtils"; +import { + VALID_IMAGE_EXTENSIONS, + VALID_AUDIO_EXTENSIONS, + VALID_MEDIA_EXTENSIONS, +} from "./constants"; +import { initializeInboxQueue, Inbox } from "./inbox"; +import { validateFile } from "./utils"; + type TagCounts = { [key: string]: number; }; -const validImageExtensions = ["png", "jpg", "jpeg", "gif", "svg", "webp"]; -const validAudioExtensions = [ - "mp3", - "mp4", - "mpeg", - "mpga", - "m4a", - "wav", - "webm", -]; -export const validMediaExtensions = [ - ...validImageExtensions, - ...validAudioExtensions, -]; -const validTextExtensions = ["md", "txt"]; - -const validExtensions = [ - ...validMediaExtensions, - ...validTextExtensions, - "pdf", -]; - export interface FolderSuggestion { isNewFolder: boolean; score: number; @@ -68,13 +55,6 @@ export interface FolderSuggestion { reason: string; } -const isValidExtension = (extension: string) => { - if (!validExtensions.includes(extension)) { - new Notice("Sorry, FileOrganizer does not support this file type."); - return false; - } - return true; -}; // determine sever url interface ProcessingResult { text: string; @@ -107,6 +87,7 @@ interface TitleSuggestion { } export default class FileOrganizer extends Plugin { + public inbox: Inbox; settings: FileOrganizerSettings; async loadSettings() { @@ -160,7 +141,7 @@ export default class FileOrganizer extends Plugin { // Step 1-3: Initialize and validate await this.initializeProcessing(logFilePath, processedFileName); - if (!this.validateFile(originalFile)) { + if (!validateFile(originalFile)) { await this.log( logFilePath, `2. Unsupported file type. Skipping ${processedFileName}` @@ -229,15 +210,7 @@ export default class FileOrganizer extends Plugin { await this.log(logFilePath, `3. Verified necessary folders exist`); } - private validateFile(file: TFile): boolean { - if (!file.extension || !isValidExtension(file.extension)) { - new Notice("Unsupported file type. Skipping.", 3000); - return false; - } - return true; - } - - private async processContent( + async processContent( file: TFile, logFilePath: string ): Promise { @@ -319,7 +292,7 @@ export default class FileOrganizer extends Plugin { `8c. Moved container to: [[${newPath}/${newName}]]` ); - await this.addMetadata(containerFile, content, newName); + await this.generateAndAppendSimilarTags(containerFile, content, newName); } private async processNonMediaFile( @@ -339,7 +312,7 @@ export default class FileOrganizer extends Plugin { ); } - await this.addMetadata(file, content, newName); + await this.generateAndAppendSimilarTags(file, content, newName); await this.moveFile(file, newName, newPath); await this.log( logFilePath, @@ -347,18 +320,7 @@ export default class FileOrganizer extends Plugin { ); } - private async addMetadata( - file: TFile, - content: string, - newName: string - ): Promise { - if (this.settings.useSimilarTags) { - await this.generateAndAppendSimilarTags(file, content, newName); - } - - } - - private async createMediaContainer(content: string): Promise { + async createMediaContainer(content: string): Promise { const containerContent = `${content}`; const containerFilePath = `${ this.settings.defaultDestinationPath @@ -467,7 +429,8 @@ export default class FileOrganizer extends Plugin { shouldCreateMarkdownContainer(file: TFile): boolean { return ( - validMediaExtensions.includes(file.extension) || file.extension === "pdf" + VALID_MEDIA_EXTENSIONS.includes(file.extension) || + file.extension === "pdf" ); } @@ -686,7 +649,7 @@ export default class FileOrganizer extends Plugin { ); } - async identifyConcepts(content: string): Promise { + async _experimentalIdentifyConcepts(content: string): Promise { try { const response = await makeApiRequest(() => requestUrl({ @@ -889,10 +852,7 @@ export default class FileOrganizer extends Plugin { async generateTranscriptFromAudio( file: TFile ): Promise> { - new Notice( - `Generating transcription for ${file.basename}. This may take a few minutes.`, - 8000 - ); + try { const audioBuffer = await this.app.vault.readBinary(file); const response = await this.transcribeAudio(audioBuffer, file.extension); @@ -911,7 +871,6 @@ export default class FileOrganizer extends Plugin { } } - new Notice(`Transcription started for ${file.basename}`, 5000); return generateTranscript(); } catch (e) { console.error("Error generating transcript", e); @@ -939,29 +898,24 @@ export default class FileOrganizer extends Plugin { classifications: string[] ): Promise { const serverUrl = this.getServerUrl(); - try { - const response = await fetch(`${serverUrl}/api/classify1`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${this.settings.API_KEY}`, - }, - body: JSON.stringify({ - content, - templateNames: classifications, - }), - }); - - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } + const response = await fetch(`${serverUrl}/api/classify1`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.settings.API_KEY}`, + }, + body: JSON.stringify({ + content, + templateNames: classifications, + }), + }); - const { documentType } = await response.json(); - return documentType; - } catch (error) { - console.error("Error in classifyContentV2:", error); - throw error; + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); } + + const { documentType } = await response.json(); + return documentType; } async organizeFile(file: TFile, content: string) { @@ -1015,9 +969,9 @@ export default class FileOrganizer extends Plugin { const pdfContent = await this.extractTextFromPDF(file); return pdfContent; } - case validImageExtensions.includes(file.extension): + case VALID_IMAGE_EXTENSIONS.includes(file.extension): return await this.generateImageAnnotation(file); - case validAudioExtensions.includes(file.extension): { + case VALID_AUDIO_EXTENSIONS.includes(file.extension): { // Change this part to consume the iterator const transcriptIterator = await this.generateTranscriptFromAudio(file); let transcriptText = ""; @@ -1103,8 +1057,9 @@ export default class FileOrganizer extends Plugin { this.settings.templatePaths, this.settings.fabricPaths, this.settings.pathToWatch, - '_FileOrganizer2000', - '/' + this.settings.errorFilePath, + "_FileOrganizer2000", + "/", ]; logMessage("ignoredFolders", ignoredFolders); // remove empty strings @@ -1123,14 +1078,18 @@ export default class FileOrganizer extends Plugin { return allFoldersPaths.filter(folder => { // Check if the folder is not in the ignored folders list - return !ignoredFolders.includes(folder) && - !ignoredFolders.some(ignoredFolder => - folder.startsWith(ignoredFolder + "/") - ); + return ( + !ignoredFolders.includes(folder) && + !ignoredFolders.some(ignoredFolder => + folder.startsWith(ignoredFolder + "/") + ) + ); }); } - async getSimilarFiles(fileToCheck: TFile): Promise { + async _experimentalGenerateSimilarFiles( + fileToCheck: TFile + ): Promise { if (!fileToCheck) { return []; } @@ -1156,7 +1115,7 @@ export default class FileOrganizer extends Plugin { name: file.path, })); - const similarFiles = await this.generateRelationships( + const similarFiles = await this._experimentalGenerateRelationships( activeFileContent, fileContents ); @@ -1168,7 +1127,7 @@ export default class FileOrganizer extends Plugin { ); } - async generateRelationships( + async _experimentalGenerateRelationships( activeFileContent: string, files: { name: string }[] ): Promise { @@ -1219,6 +1178,7 @@ export default class FileOrganizer extends Plugin { return formatToSafeName(name); } + // should be deprecated to use v2 api routes async generateDocumentTitle( content: string, currentName: string, @@ -1280,10 +1240,7 @@ export default class FileOrganizer extends Plugin { } async generateImageAnnotation(file: TFile) { - new Notice( - `Generating annotation for ${file.basename} this can take up to a minute`, - 8000 - ); + const arrayBuffer = await this.app.vault.readBinary(file); const fileContent = Buffer.from(arrayBuffer); @@ -1310,16 +1267,16 @@ export default class FileOrganizer extends Plugin { } async extractTextFromImage(image: ArrayBuffer): Promise { - const base64Image = this.arrayBufferToBase64(image); - + const base64Image = arrayBufferToBase64(image); + const response = await makeApiRequest(() => requestUrl({ url: `${this.getServerUrl()}/api/vision`, method: "POST", contentType: "application/json", - body: JSON.stringify({ + body: JSON.stringify({ image: base64Image, - instructions: this.settings.imageInstructions + instructions: this.settings.imageInstructions, }), throw: false, headers: { @@ -1331,15 +1288,6 @@ export default class FileOrganizer extends Plugin { return text; } - arrayBufferToBase64(buffer: ArrayBuffer): string { - let binary = ""; - const bytes = new Uint8Array(buffer); - const len = bytes.byteLength; - for (let i = 0; i < len; i++) { - binary += String.fromCharCode(bytes[i]); - } - return window.btoa(binary); - } async getBacklog() { const allFiles = this.app.vault.getFiles(); const pendingFiles = allFiles.filter(file => @@ -1349,6 +1297,13 @@ export default class FileOrganizer extends Plugin { } async processBacklog() { const pendingFiles = await this.getBacklog(); + if (this.settings.useInbox) { + logMessage("Enqueuing files from backlog V3"); + Inbox.getInstance().enqueueFiles(pendingFiles); + return; + } + logMessage("Processing files from backlog V2"); + for (const file of pendingFiles) { await this.processFileV2(file); } @@ -1376,6 +1331,7 @@ export default class FileOrganizer extends Plugin { return file instanceof TFolder; } + // should be reprecatd and only use guessRelevantFolders async getAIClassifiedFolder( content: string, filePath: string @@ -1450,31 +1406,6 @@ export default class FileOrganizer extends Plugin { return suggestedFolders; } - async createNewFolder( - content: string, - fileName: string, - existingFolders: string[] - ): Promise { - const response = await makeApiRequest(() => - requestUrl({ - url: `${this.getServerUrl()}/api/create-folder`, - method: "POST", - contentType: "application/json", - body: JSON.stringify({ - content, - fileName, - existingFolders, - }), - throw: false, - headers: { - Authorization: `Bearer ${this.settings.API_KEY}`, - }, - }) - ); - const { folderName } = await response.json; - return folderName; - } - async appendTag(file: TFile, tag: string) { // Ensure the tag starts with a hash symbol const formattedTag = sanitizeTag(tag); @@ -1485,26 +1416,14 @@ export default class FileOrganizer extends Plugin { } await this.app.vault.append(file, `\n${formattedTag}`); } - d; - - validateAPIKey() { - if (!this.settings.usePro) { - // atm we assume no api auth for self hosted - return true; - } - - if (!this.settings.API_KEY) { - throw new Error( - "Please enter your API Key in the settings of the FileOrganizer plugin." - ); - } - } - async onload() { + this.inbox = Inbox.initialize(this); await this.initializePlugin(); await this.saveSettings(); + initializeInboxQueue(this); + // Initialize different features initializeChat(this); initializeOrganizer(this); @@ -1606,113 +1525,10 @@ export default class FileOrganizer extends Plugin { return templateFiles.map(file => file.basename); } - async getExistingFolders( + async guessTitles( content: string, filePath: string - ): Promise { - if (this.settings.ignoreFolders.includes("*")) { - return [this.settings.defaultDestinationPath]; - } - const currentFolder = - this.app.vault.getAbstractFileByPath(filePath)?.parent?.path || ""; - const filteredFolders = this.getAllUserFolders() - .filter(folder => folder !== currentFolder) - - // if this.settings.ignoreFolders has one or more folder specified, filter them out including subfolders - .filter(folder => { - const hasIgnoreFolders = - this.settings.ignoreFolders.length > 0 && - this.settings.ignoreFolders[0] !== ""; - if (!hasIgnoreFolders) return true; - const isFolderIgnored = this.settings.ignoreFolders.some(ignoreFolder => - folder.startsWith(ignoreFolder) - ); - return !isFolderIgnored; - }) - .filter(folder => folder !== "/"); - - try { - const apiEndpoint = this.settings.useFolderEmbeddings - ? "/api/folders/embeddings" - : "/api/folders/existing"; - - const response = await makeApiRequest(() => - requestUrl({ - url: `${this.getServerUrl()}${apiEndpoint}`, - method: "POST", - contentType: "application/json", - body: JSON.stringify({ - content, - fileName: filePath, - folders: filteredFolders, - }), - throw: false, - headers: { - Authorization: `Bearer ${this.settings.API_KEY}`, - }, - }) - ); - const { folders: guessedFolders } = await response.json; - return guessedFolders; - } catch (error) { - console.error("Error in getExistingFolders:", error); - return [this.settings.defaultDestinationPath]; - } - } - async getNewFolders(content: string, filePath: string): Promise { - const uniqueFolders = await this.getAllUserFolders(); - if (this.settings.ignoreFolders.includes("*")) { - return [this.settings.defaultDestinationPath]; - } - const currentFolder = - this.app.vault.getAbstractFileByPath(filePath)?.parent?.path || ""; - const filteredFolders = uniqueFolders - .filter(folder => folder !== currentFolder) - .filter(folder => folder !== filePath) - .filter(folder => folder !== this.settings.defaultDestinationPath) - .filter(folder => folder !== this.settings.attachmentsPath) - .filter(folder => folder !== this.settings.logFolderPath) - .filter(folder => folder !== this.settings.pathToWatch) - .filter(folder => folder !== this.settings.templatePaths) - .filter(folder => !folder.includes("_FileOrganizer2000")) - // if this.settings.ignoreFolders has one or more folder specified, filter them out including subfolders - .filter(folder => { - const hasIgnoreFolders = - this.settings.ignoreFolders.length > 0 && - this.settings.ignoreFolders[0] !== ""; - if (!hasIgnoreFolders) return true; - const isFolderIgnored = this.settings.ignoreFolders.some(ignoreFolder => - folder.startsWith(ignoreFolder) - ); - return !isFolderIgnored; - }) - .filter(folder => folder !== "/"); - try { - const response = await makeApiRequest(() => - requestUrl({ - url: `${this.getServerUrl()}/api/folders/new`, - method: "POST", - contentType: "application/json", - body: JSON.stringify({ - content, - fileName: filePath, - folders: filteredFolders, - }), - throw: false, - headers: { - Authorization: `Bearer ${this.settings.API_KEY}`, - }, - }) - ); - const { folders: guessedFolders } = await response.json; - return guessedFolders; - } catch (error) { - console.error("Error in getNewFolders:", error); - return [this.settings.defaultDestinationPath]; - } - } - - async guessTitles(content: string, filePath: string): Promise { + ): Promise { const customInstructions = this.settings.enableFileRenaming ? this.settings.renameInstructions : undefined; diff --git a/plugin/settings.ts b/plugin/settings.ts index 0514963b..d083f442 100644 --- a/plugin/settings.ts +++ b/plugin/settings.ts @@ -9,6 +9,10 @@ export class FileOrganizerSettings { backupFolderPath = "_FileOrganizer2000/Backups"; templatePaths = "_FileOrganizer2000/Templates"; fabricPaths = "_FileOrganizer2000/Fabric"; + bypassedFilePath = "_FileOrganizer2000/Bypassed"; + errorFilePath = "_FileOrganizer2000/Errors"; + + useSimilarTags = true; renameInstructions = "Create a concise, descriptive name for the document based on its key content. Prioritize clarity and searchability, using specific terms that will make the document easy to find later. Avoid generic words and focus on unique, identifying elements."; usePro = true; @@ -31,6 +35,7 @@ export class FileOrganizerSettings { selectedModel: "gpt-4o" | "llama3.2" = "gpt-4o"; tagScoreThreshold = 70; formatBehavior: "override" | "newFile" = "override"; + useInbox = false; imageInstructions = "Analyze the image and provide a clear, detailed description focusing on the main elements, context, and any text visible in the image. Include relevant details that would be useful for searching and organizing the image later."; } diff --git a/plugin/utils.ts b/plugin/utils.ts new file mode 100644 index 00000000..6d37fd08 --- /dev/null +++ b/plugin/utils.ts @@ -0,0 +1,22 @@ +import { TFile } from "obsidian"; + +import { Notice } from "obsidian"; +import { isValidExtension } from "./constants"; + +export function arrayBufferToBase64(buffer: ArrayBuffer): string { + let binary = ""; + const bytes = new Uint8Array(buffer); + const len = bytes.byteLength; + for (let i = 0; i < len; i++) { + binary += String.fromCharCode(bytes[i]); + } + return window.btoa(binary); + } + +export function validateFile(file: TFile): boolean { + if (!file.extension || !isValidExtension(file.extension)) { + new Notice("Unsupported file type. Skipping.", 3000); + return false; + } + return true; +} diff --git a/plugin/views/ai-chat/chat.tsx b/plugin/views/ai-chat/chat.tsx index b062c362..518dd8a7 100644 --- a/plugin/views/ai-chat/chat.tsx +++ b/plugin/views/ai-chat/chat.tsx @@ -302,7 +302,7 @@ export const ChatComponent: React.FC = ({ const tags = await plugin.getAllVaultTags(); setAllTags(tags); - const folders = plugin.getAllNonFo2kFolders(); + const folders = plugin.getAllUserFolders(); setAllFolders(folders); }; diff --git a/plugin/views/organizer/components/inbox-logs.tsx b/plugin/views/organizer/components/inbox-logs.tsx new file mode 100644 index 00000000..c03d6b34 --- /dev/null +++ b/plugin/views/organizer/components/inbox-logs.tsx @@ -0,0 +1,281 @@ +import * as React from "react"; +import FileOrganizer from "../../../index"; +import { FileRecord, FileStatus } from "../../../inbox/types"; +import moment from "moment"; +import { ChevronDown, Search, ExternalLink } from "lucide-react"; +import { App, TFile } from "obsidian"; +import { motion, AnimatePresence } from "framer-motion"; + +interface InboxLogsProps { + plugin: FileOrganizer & { + inbox: { + getAllFiles: () => FileRecord[]; + getQueueStats: () => { + queued: number; + processing: number; + completed: number; + bypassed: number; + errors: number; + }; + }; + app: App; + }; +} + +function calculateStatsFromRecords(records: FileRecord[]) { + return records.reduce( + (acc, record) => { + acc[record.status]++; + return acc; + }, + { + queued: 0, + processing: 0, + completed: 0, + bypassed: 0, + error: 0, + } + ); +} + +export const InboxLogs: React.FC = ({ plugin }) => { + const [files, setFiles] = React.useState([]); + const [sortConfig, setSortConfig] = React.useState<{ + key: keyof FileRecord; + direction: "asc" | "desc"; + }>({ + key: "updatedAt", + direction: "desc", + }); + const [searchTerm, setSearchTerm] = React.useState(""); + const [statusFilter, setStatusFilter] = React.useState("all"); + + const filteredAndSortedFiles = React.useMemo(() => { + return files + .filter(file => { + const matchesSearch = + file.fileName.toLowerCase().includes(searchTerm.toLowerCase()) || + file.status.toLowerCase().includes(searchTerm.toLowerCase()); + + const matchesStatus = + statusFilter === "all" || file.status === statusFilter; + + return matchesSearch && matchesStatus; + }) + .sort((a, b) => { + const aValue = a[sortConfig.key]; + const bValue = b[sortConfig.key]; + + if (!aValue || !bValue) return 0; + + if (aValue < bValue) return sortConfig.direction === "asc" ? -1 : 1; + if (aValue > bValue) return sortConfig.direction === "asc" ? 1 : -1; + return 0; + }); + }, [files, sortConfig, searchTerm, statusFilter]); + + React.useEffect(() => { + const updateFiles = () => { + const allFiles = plugin.inbox.getAllFiles(); + setFiles(prevFiles => { + if (JSON.stringify(prevFiles) !== JSON.stringify(allFiles)) { + return allFiles; + } + return prevFiles; + }); + }; + + updateFiles(); + const interval = setInterval(updateFiles, 100); + return () => clearInterval(interval); + }, [plugin]); + React.useEffect(() => { + console.log("files", files); + }, [files]); + + const getStatusBadgeClass = (status: FileStatus) => { + const baseClasses = + "px-2 py-1 rounded-full text-sm font-medium whitespace-nowrap"; + const statusColors = { + queued: + "bg-opacity-20 text-[--text-warning] border border-[--text-warning]", + processing: + "bg-opacity-20 text-[--text-accent] border border-[--text-accent]", + completed: + "bg-opacity-20 text-[--text-success] border border-[--text-success]", + error: "bg-opacity-20 text-[--text-error] border border-[--text-error]", + bypassed: + "bg-opacity-20 text-[--text-muted] border border-[--text-muted]", + } as const; + + return `${baseClasses} ${statusColors[status] || ""}`; + }; + + const stats = React.useMemo(() => { + return calculateStatsFromRecords(files); + }, [files]); + + const nameTransitionVariants = { + initial: { + opacity: 1, + }, + renamed: { + opacity: 0.8, + textDecoration: "line-through", + }, + }; + + return ( +
+ {/* Header with stats - numbers are now derived from records */} +
+
+
+
+ {stats.queued} +
+
Queued
+
+
+
+ {stats.processing} +
+
Processing
+
+
+
+ {stats.completed} +
+
Completed
+
+
+
+ {stats.bypassed} +
+
Bypassed
+
+
+
+ {stats.error} +
+
Errors
+
+
+
+ + {/* Search and filters */} +
+
+ + setSearchTerm(e.target.value)} + className="w-full pl-10 pr-4 py-2 rounded bg-[--background-modifier-form-field] border border-[--background-modifier-border] text-[--text-normal]" + /> +
+ +
+ + {/* File cards */} +
+ + {filteredAndSortedFiles.map(file => ( + +
+ + +
+
+
+ Project folder: +
+
+ {file.newPath || "—"} +
+
+ +
+ {file.status} +
+
+
+ + ))} + +
+ + {filteredAndSortedFiles.length === 0 && ( +
+ Drag files in the inbox to automatically organize them. +
+ )} +
+ ); +}; diff --git a/plugin/views/organizer/files.tsx b/plugin/views/organizer/files.tsx index 576cb32d..eeab18d7 100644 --- a/plugin/views/organizer/files.tsx +++ b/plugin/views/organizer/files.tsx @@ -15,7 +15,7 @@ export const SimilarFilesBox: React.FC = ({ plugin, file } if (!file) return; setLoading(true); try { - const similarFiles = await plugin.getSimilarFiles(file); + const similarFiles = await plugin._experimentalGenerateSimilarFiles(file); setFilePaths(similarFiles); } catch (error) { console.error("Error fetching similar files:", error); diff --git a/plugin/views/organizer/index.tsx b/plugin/views/organizer/index.tsx deleted file mode 100644 index 3db43d71..00000000 --- a/plugin/views/organizer/index.tsx +++ /dev/null @@ -1,48 +0,0 @@ -import { ItemView, WorkspaceLeaf } from "obsidian"; -import * as React from "react"; - -import { Root, createRoot } from "react-dom/client"; -import { AssistantView } from "./view"; -import FileOrganizer from "../.."; - - -export const ORGANIZER_VIEW_TYPE = "fo2k.assistant.sidebar2"; - -export class AssistantViewWrapper extends ItemView { - root: Root | null = null; - plugin: FileOrganizer; - - constructor(leaf: WorkspaceLeaf, plugin: FileOrganizer) { - super(leaf); - this.plugin = plugin; - } - - getViewType(): string { - return ORGANIZER_VIEW_TYPE; - } - - getDisplayText(): string { - return "Fo2k Assistant"; - } - - getIcon(): string { - return "sparkle"; - } - - async onOpen(): Promise { - this.root = createRoot(this.containerEl.children[1]); - this.render(); - } - - render(): void { - this.root?.render( - - - - ); - } - - async onClose(): Promise { - this.root?.unmount(); - } -} \ No newline at end of file diff --git a/plugin/views/organizer/organizer.tsx b/plugin/views/organizer/organizer.tsx new file mode 100644 index 00000000..eb56e2db --- /dev/null +++ b/plugin/views/organizer/organizer.tsx @@ -0,0 +1,269 @@ +import * as React from "react"; +import { TFile, WorkspaceLeaf, Notice } from "obsidian"; +import FileOrganizer from "../../index"; +import { debounce } from "lodash"; + +import { SectionHeader } from "./components/section-header"; +import { SimilarTags } from "./tags"; +import { DocumentChunks } from "./chunks"; +import { RenameSuggestion } from "./titles/box"; +import { SimilarFolderBox } from "./folders/box"; +import { RefreshButton } from "./components/refresh-button"; +import { ClassificationContainer } from "./ai-format/templates"; +import { TranscriptionButton } from "./transcript"; +import { SimilarFilesBox } from "./files"; +import { EmptyState } from "./components/empty-state"; +import { logMessage } from "../../../utils"; +import { LicenseValidator } from "./components/license-validator"; +import { VALID_MEDIA_EXTENSIONS } from "../../constants"; + +interface AssistantViewProps { + plugin: FileOrganizer; + leaf: WorkspaceLeaf; +} + +const checkIfIsMediaFile = (file: TFile | null): boolean => { + if (!file) return false; + return VALID_MEDIA_EXTENSIONS.includes(file.extension); +}; + +export const AssistantView: React.FC = ({ + plugin, + leaf, +}) => { + const [activeFile, setActiveFile] = React.useState(null); + const [noteContent, setNoteContent] = React.useState(""); + const [refreshKey, setRefreshKey] = React.useState(0); + const [error, setError] = React.useState(null); + const [isLicenseValid, setIsLicenseValid] = React.useState(false); + + const isMediaFile = React.useMemo( + () => checkIfIsMediaFile(activeFile), + [activeFile] + ); + + const isInIgnoredPatterns = React.useMemo( + () => + plugin + .getAllIgnoredFolders() + .some(folder => activeFile?.path.startsWith(folder)), + [activeFile, plugin.getAllIgnoredFolders] + ); + + const updateActiveFile = React.useCallback(async () => { + logMessage("updating active file"); + // Check if the Assistant view is visible before processing + const isVisible = + leaf.view.containerEl.isShown() && + !plugin.app.workspace.rightSplit.collapsed; + if (!isVisible) return; + + try { + const file = plugin.app.workspace.getActiveFile(); + if (file && !isMediaFile) { + const content = await plugin.app.vault.read(file); + setNoteContent(content); + } + setActiveFile(file); + } catch (err) { + console.error("Error updating active file:", err); + setError("Failed to load file content"); + } + }, [ + plugin.app.workspace, + plugin.app.vault, + leaf.view.containerEl, + plugin.app.workspace.rightSplit.collapsed, + leaf.view.containerEl.isShown, + isMediaFile, + ]); + + React.useEffect(() => { + updateActiveFile(); + const debouncedUpdate = debounce(updateActiveFile, 300); + + // Attach event listeners + plugin.app.workspace.on("file-open", debouncedUpdate); + plugin.app.workspace.on("active-leaf-change", debouncedUpdate); + + // Cleanup function to remove event listeners + return () => { + plugin.app.workspace.off("file-open", debouncedUpdate); + plugin.app.workspace.off("active-leaf-change", debouncedUpdate); + debouncedUpdate.cancel(); + }; + }, [updateActiveFile, plugin.app.workspace]); + + const refreshContext = React.useCallback(() => { + setRefreshKey(prevKey => prevKey + 1); + setError(null); + updateActiveFile(); + }, [updateActiveFile]); + + const renderSection = React.useCallback( + (component: React.ReactNode, errorMessage: string) => { + try { + return component; + } catch (err) { + console.error(errorMessage, err); + return
{errorMessage}
; + } + }, + [] + ); + + const handleDelete = React.useCallback(async () => { + if (!activeFile) return; + + try { + await plugin.app.vault.delete(activeFile); + new Notice("File deleted successfully"); + } catch (err) { + console.error("Error deleting file:", err); + setError("Failed to delete file"); + } + }, [activeFile, plugin.app.vault]); + + if (!isLicenseValid) { + return ( + setIsLicenseValid(true)} + plugin={plugin} + /> + ); + } + + if (error) { + return ( + + ); + } + + if (!activeFile) { + return ; + } + if (isInIgnoredPatterns) { + return ( + + ); + } + + if (isMediaFile) { + return ( + + ); + } + if (!noteContent.trim()) { + return ( + + ); + } + + return ( +
+
+ +
+
+ {activeFile.basename} +
+
+
+ + {renderSection( + , + "Error loading classification" + )} + + + {renderSection( + , + "Error loading tags" + )} + + + {renderSection( + , + "Error loading title suggestions" + )} + + + {renderSection( + , + "Error loading folder suggestions" + )} + + {plugin.settings.enableSimilarFiles && ( + <> + + {renderSection( + , + "Error loading similar files" + )} + + )} + + {plugin.settings.enableAtomicNotes && ( + <> + + {renderSection( + , + "Error loading atomic notes" + )} + + )} + + {hasAudioEmbed(noteContent) && ( + <> + + {renderSection( + , + "Error loading transcription button" + )} + + )} + + +
+ ); +}; + +const hasAudioEmbed = (content: string): boolean => { + const audioRegex = /!\[\[(.*\.(mp3|wav|m4a|ogg|webm))]]/i; + return audioRegex.test(content); +}; diff --git a/plugin/views/organizer/view.tsx b/plugin/views/organizer/view.tsx index 5fdc89b8..419877f7 100644 --- a/plugin/views/organizer/view.tsx +++ b/plugin/views/organizer/view.tsx @@ -1,266 +1,133 @@ +import { ItemView, WorkspaceLeaf } from "obsidian"; import * as React from "react"; -import { TFile, WorkspaceLeaf, Notice } from "obsidian"; -import FileOrganizer, { validMediaExtensions } from "../../index"; -import { debounce } from "lodash"; - +import { Root, createRoot } from "react-dom/client"; +import { AssistantView } from "./organizer"; +import FileOrganizer from "../.."; +import { InboxLogs } from "./components/inbox-logs"; import { SectionHeader } from "./components/section-header"; -import { SimilarTags } from "./tags"; -import { DocumentChunks } from "./chunks"; -import { RenameSuggestion } from "./titles/box"; -import { SimilarFolderBox } from "./folders/box"; -import { RefreshButton } from "./components/refresh-button"; -import { ClassificationContainer } from "./ai-format/templates"; -import { TranscriptionButton } from "./transcript"; -import { SimilarFilesBox } from "./files"; -import { EmptyState } from "./components/empty-state"; -import { logMessage } from "../../../utils"; -import { LicenseValidator } from "./components/license-validator"; -interface AssistantViewProps { - plugin: FileOrganizer; - leaf: WorkspaceLeaf; +export const ORGANIZER_VIEW_TYPE = "fo2k.assistant.sidebar2"; + +type Tab = 'organizer' | 'inbox'; + +function TabContent({ + activeTab, + plugin, + leaf +}: { + activeTab: Tab, + plugin: FileOrganizer, + leaf: WorkspaceLeaf +}) { + if (activeTab === 'organizer') { + return ; + } + + if (activeTab === 'inbox') { + return ( + <> + + + + ); + } + + return null; } -const checkIfIsMediaFile = (file: TFile | null): boolean => { - if (!file) return false; - return validMediaExtensions.includes(file.extension); -}; - -export const AssistantView: React.FC = ({ - plugin, - leaf, -}) => { - const [activeFile, setActiveFile] = React.useState(null); - const [noteContent, setNoteContent] = React.useState(""); - const [refreshKey, setRefreshKey] = React.useState(0); - const [error, setError] = React.useState(null); - const [isLicenseValid, setIsLicenseValid] = React.useState(false); - - const isMediaFile = React.useMemo( - () => checkIfIsMediaFile(activeFile), - [activeFile] - ); - - const isInIgnoredPatterns = React.useMemo( - () => - plugin - .getAllIgnoredFolders() - .some(folder => activeFile?.path.startsWith(folder)), - [activeFile, plugin.getAllIgnoredFolders] +function TabButton({ + isActive, + onClick, + children +}: { + isActive: boolean, + onClick: () => void, + children: React.ReactNode +}) { + return ( + ); +} - const updateActiveFile = React.useCallback(async () => { - logMessage("updating active file"); - // Check if the Assistant view is visible before processing - const isVisible = - leaf.view.containerEl.isShown() && - !plugin.app.workspace.rightSplit.collapsed; - if (!isVisible) return; - - try { - const file = plugin.app.workspace.getActiveFile(); - if (file && !isMediaFile) { - const content = await plugin.app.vault.read(file); - setNoteContent(content); - } - setActiveFile(file); - } catch (err) { - console.error("Error updating active file:", err); - setError("Failed to load file content"); - } - }, [ - plugin.app.workspace, - plugin.app.vault, - leaf.view.containerEl, - plugin.app.workspace.rightSplit.collapsed, - leaf.view.containerEl.isShown, - isMediaFile, - ]); - - React.useEffect(() => { - updateActiveFile(); - const debouncedUpdate = debounce(updateActiveFile, 300); - - // Attach event listeners - plugin.app.workspace.on("file-open", debouncedUpdate); - plugin.app.workspace.on("active-leaf-change", debouncedUpdate); - - // Cleanup function to remove event listeners - return () => { - plugin.app.workspace.off("file-open", debouncedUpdate); - plugin.app.workspace.off("active-leaf-change", debouncedUpdate); - debouncedUpdate.cancel(); - }; - }, [updateActiveFile, plugin.app.workspace]); - - const refreshContext = React.useCallback(() => { - setRefreshKey(prevKey => prevKey + 1); - setError(null); - updateActiveFile(); - }, [updateActiveFile]); +function OrganizerContent({ plugin, leaf }: { plugin: FileOrganizer, leaf: WorkspaceLeaf }) { + const [activeTab, setActiveTab] = React.useState('organizer'); - const renderSection = React.useCallback( - (component: React.ReactNode, errorMessage: string) => { - try { - return component; - } catch (err) { - console.error(errorMessage, err); - return
{errorMessage}
; - } - }, - [] + return ( +
+
+ setActiveTab('organizer')} + > + Assistant + + {plugin.settings.useInbox && ( + setActiveTab('inbox')} + > + Inbox + + )} +
+ + +
); +} - const handleDelete = React.useCallback(async () => { - if (!activeFile) return; - - try { - await plugin.app.vault.delete(activeFile); - new Notice("File deleted successfully"); - } catch (err) { - console.error("Error deleting file:", err); - setError("Failed to delete file"); - } - }, [activeFile, plugin.app.vault]); +export class AssistantViewWrapper extends ItemView { + root: Root | null = null; + plugin: FileOrganizer; - if (!isLicenseValid) { - return ( - setIsLicenseValid(true)} - plugin={plugin} - /> - ); + constructor(leaf: WorkspaceLeaf, plugin: FileOrganizer) { + super(leaf); + this.plugin = plugin; } - if (error) { - return ( - - ); + getViewType(): string { + return ORGANIZER_VIEW_TYPE; } - if (!activeFile) { - return ; - } - if (isInIgnoredPatterns) { - return ( - - ); + getDisplayText(): string { + return "Fo2k Assistant"; } - if (isMediaFile) { - return ( - - ); + getIcon(): string { + return "sparkle"; } - if (!noteContent.trim()) { - return ( - - ); + + async onOpen(): Promise { + this.root = createRoot(this.containerEl.children[1]); + this.render(); } - return ( -
-
- -
-
- {activeFile.basename} -
+ render(): void { + this.root?.render( + +
+
-
- - {renderSection( - , - "Error loading classification" - )} - - - {renderSection( - , - "Error loading tags" - )} - - - {renderSection( - , - "Error loading title suggestions" - )} - - - {renderSection( - , - "Error loading folder suggestions" - )} - - {plugin.settings.enableSimilarFiles && ( - <> - - {renderSection( - , - "Error loading similar files" - )} - - )} - - {plugin.settings.enableAtomicNotes && ( - <> - - {renderSection( - , - "Error loading atomic notes" - )} - - )} - - {hasAudioEmbed(noteContent) && ( - <> - - {renderSection( - , - "Error loading transcription button" - )} - - )} -
- ); -}; + + ); + } -const hasAudioEmbed = (content: string): boolean => { - const audioRegex = /!\[\[(.*\.(mp3|wav|m4a|ogg|webm))]]/i; - return audioRegex.test(content); -}; + async onClose(): Promise { + this.root?.unmount(); + } +} \ No newline at end of file diff --git a/plugin/views/settings/experiment-tab.tsx b/plugin/views/settings/experiment-tab.tsx index f9a421a3..74358db3 100644 --- a/plugin/views/settings/experiment-tab.tsx +++ b/plugin/views/settings/experiment-tab.tsx @@ -11,6 +11,7 @@ export const ExperimentTab: React.FC = ({ plugin }) => { const [enableAtomicNotes, setEnableAtomicNotes] = useState(plugin.settings.enableAtomicNotes); const [enableScreenpipe, setEnableScreenpipe] = useState(plugin.settings.enableScreenpipe); const [enableFabric, setEnableFabric] = useState(plugin.settings.enableFabric); + const [useInbox, setUseInbox] = useState(plugin.settings.useInbox); const handleToggleChange = async (value: boolean, setter: React.Dispatch>, settingKey: keyof typeof plugin.settings) => { setter(value); @@ -53,6 +54,25 @@ export const ExperimentTab: React.FC = ({ plugin }) => { value={enableAtomicNotes} onChange={(value) => handleToggleChange(value, setEnableAtomicNotes, 'enableAtomicNotes')} /> + +

Enable the new inbox system designed for processing large volumes of files efficiently.

+
+

✨ New Features:

+
    +
  • Real-time processing status in sidebar
    • +
  • Improved batch file handling
    • +
  • Progress logging and monitoring
    • +
+

Note: Currently does not support classifications and tagging

+
+
+ } + value={useInbox} + onChange={(value) => handleToggleChange(value, setUseInbox, 'useInbox')} + /> @@ -94,7 +114,7 @@ export const ExperimentTab: React.FC = ({ plugin }) => { interface ToggleSettingProps { name: string; - description: string; + description: string | JSX.Element; value: boolean; onChange: (value: boolean) => void; } @@ -103,14 +123,16 @@ const ToggleSetting: React.FC = ({ name, description, value,
{name}
-
{description}
+
+ {description} +
onChange(e.target.checked)} - className="form-checkbox h-5 w-5 text-[--interactive-accent] rounded border-[--background-modifier-border]" + className="text-[--interactive-accent] rounded border-[--background-modifier-border]" />
diff --git a/web/app/api/(newai)/organize-all/route.ts b/web/app/api/(newai)/organize-all/route.ts new file mode 100644 index 00000000..848b1a81 --- /dev/null +++ b/web/app/api/(newai)/organize-all/route.ts @@ -0,0 +1,92 @@ +import { NextRequest, NextResponse } from "next/server"; +import { handleAuthorization } from "@/lib/handleAuthorization"; +import { incrementAndLogTokenUsage } from "@/lib/incrementAndLogTokenUsage"; +import { getModel } from "@/lib/models"; +import { z } from "zod"; +import { generateObject } from "ai"; + +const sanitizeFileName = (fileName: string) => { + return fileName.replace(/[^a-zA-Z0-9\s]/g, "_"); +}; + +export async function POST(request: NextRequest) { + try { + const { userId } = await handleAuthorization(request); + const { content, fileName, folders, existingTags, customInstructions } = await request.json(); + const model = getModel(process.env.MODEL_NAME); + + const response = await generateObject({ + model, + schema: z.object({ + classification: z.object({ + documentType: z.string(), + confidence: z.number().min(0).max(100), + reasoning: z.string(), + }), + folders: z.array( + z.object({ + score: z.number().min(0).max(100), + isNewFolder: z.boolean(), + folder: z.string(), + reason: z.string(), + }) + ), + titles: z.array( + z.object({ + score: z.number().min(0).max(100), + title: z.string(), + reason: z.string(), + }) + ), + tags: z.array( + z.object({ + score: z.number().min(0).max(100), + isNew: z.boolean(), + tag: z.string(), + reason: z.string(), + }) + ), + }), + system: `You are an expert document organizer. Analyze the given content and: +1. Classify the document type +2. Suggest relevant folders (using existing folders: ${folders.join(", ")}) +3. Generate clear, concise titles (avoid special characters) +4. Suggest relevant tags (existing tags: ${existingTags?.join(", ") || "none"}) +${customInstructions ? `Additional instructions: ${customInstructions}` : ""} + +Consider the relationships between all suggestions to maintain consistency. +Current filename: "${fileName}"`, + prompt: `Content to analyze: "${content}"`, + }); + + // Sanitize titles + const safeTitles = response.object.titles.map(title => ({ + ...title, + title: sanitizeFileName(title.title), + })); + + // Sort all arrays by score + const sortByScore = (arr: T[]) => + [...arr].sort((a, b) => b.score - a.score); + + const result = { + classification: response.object.classification, + folders: sortByScore(response.object.folders), + titles: sortByScore(safeTitles), + tags: sortByScore(response.object.tags), + }; + + // Log and increment token usage + const tokens = response.usage.totalTokens; + console.log("incrementing token usage organize-all", userId, tokens); + await incrementAndLogTokenUsage(userId, tokens); + + return NextResponse.json(result); + } catch (error) { + console.error("Error in organize-all:", error); + return NextResponse.json( + { error: error.message }, + { status: error?.status || 500 } + ); + } +} \ No newline at end of file