speed up get_datasite_states

[abyesilyurt]

Preliminary work on identifying bottlenecks in syncing on client and server side.

benchmark hash_dir

from pathlib import Path

from syftbox.lib.hash import hash_dir
from syftbox.lib.profiling import pyspy

path = Path("/Users/azizwork/SyftBox/datasites")


@pyspy()
def profile_hash_dir():
    for _ in range(1):
        hash_dir(path, root_dir=path)


profile_hash_dir()

benchmark get_datasite_states

from collections import defaultdict
from fastapi import Request
import httpx
from loguru import logger
from syftbox.client.server_client import SyftBoxClient

import sqlite3
import traceback
from pathlib import Path
from typing import Dict, List

from syftbox.lib.profiling import pyspy
from syftbox.server.api.v1.sync_router import dir_state, get_db_connection, get_file_store
from syftbox.server.db.db import get_all_datasites
from syftbox.server.db.file_store import FileStore
from syftbox.server.models.sync_models import FileMetadata, RelativePath
from syftbox.server.settings import ServerSettings, get_server_settings


def get_datasite_states(
    conn: sqlite3.Connection,
    file_store: FileStore,
    server_settings: ServerSettings,
    email: str,
) -> Dict[str, List[FileMetadata]]:
    # for datasite in all_datasites:
    #     try:
    file_metadata = file_store.list_for_user(None, email)
    # except Exception as e:
    #     logger.error(f"Failed to get dir state for {datasite}: {e} {traceback.format_exc()}")
    #     continue
    # datasite_states[datasite] = datasite_state

    # dict of datasite -> list of files
    datasite_states = defaultdict(list)
    for metadata in file_metadata:
        user_email = metadata.path.root
        datasite_states[user_email].append(metadata)

    datasite_states = dict(datasite_states)
    return datasite_states


@pyspy()
def main_loop():
    for i in range(1000):
        main()


def main():
    # Setup dependencies
    request = Request(scope={"type": "http", "method": "POST", "path": "/sync/datasite_states"})
    request.state.server_settings = ServerSettings(data_folder=Path("/Users/azizwork/Workspace/syft/server_data"))
    email = "aziz@openmined.org"  # Replace with actual email

    # Run function
    conn = get_db_connection(request).__next__()
    file_store = get_file_store(request).__next__()
    server_settings = get_server_settings(request)
    states = get_datasite_states(conn, file_store, server_settings, email)


if __name__ == "__main__":
    main_loop()

[abyesilyurt]

some changes needs to be reverted before merge.