suppl_fig1_profile_memory.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/_settings.py:63: UserWarning: Since v1.0.0, scvi-tools no longer uses a random seed by default. Run `scvi.settings.seed = 0` to reproduce results from previous versions.\n",
      "  self.seed = seed\n",
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/_settings.py:70: UserWarning: Setting `dl_pin_memory_gpu_training` is deprecated in v1.0 and will be removed in v1.1. Please pass in `pin_memory` to the data loaders instead.\n",
      "  self.dl_pin_memory_gpu_training = (\n"
     ]
    }
   ],
   "source": [
    "import scanpy as sc\n",
    "import scvi\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from memory_profiler import profile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scvi.hub import HubModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def minified_hub_model(a):\n",
    "    hubmodel = HubModel.pull_from_huggingface_hub(\"scvi-tools/human-lung-cell-atlas\")\n",
    "    adata = hubmodel.adata\n",
    "    model = hubmodel.model\n",
    "    return None\n",
    "    #model.get_normalized_expression(adata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3991725/3303627859.py:1: UserWarning: No revision was passed, so the default (latest) revision will be used.\n",
      "  hubmodel = HubModel.pull_from_huggingface_hub(\"scvi-tools/human-lung-cell-atlas\", local_dir='hlca')\n",
      "Fetching 3 files: 100%|██████████| 3/3 [00:00<00:00, 4274.09it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "hubmodel = HubModel.pull_from_huggingface_hub(\"scvi-tools/human-lung-cell-atlas\", local_dir='hlca')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = hubmodel.large_training_adata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mINFO    \u001b[0m Loading model\u001b[33m...\u001b[0m                                                                                          \n",
      "\u001b[34mINFO    \u001b[0m File \u001b[35m/home/cane/Documents/yoseflab/can/scvi-hub/scvi-hub-reproducibility/hlca/\u001b[0m\u001b[95mmodel.pt\u001b[0m already downloaded \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/anndata/__init__.py:51: FutureWarning: `anndata.read` is deprecated, use `anndata.read_h5ad` instead. `ad.read` will be removed in mid 2024.\n",
      "  warnings.warn(\n",
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/abc.py:119: FutureWarning: SparseDataset is deprecated and will be removed in late 2024. It has been replaced by the public classes CSRDataset and CSCDataset.\n",
      "\n",
      "For instance checks, use `isinstance(X, (anndata.experimental.CSRDataset, anndata.experimental.CSCDataset))` instead.\n",
      "\n",
      "For creation, use `anndata.experimental.sparse_dataset(X)` instead.\n",
      "\n",
      "  return _abc_instancecheck(cls, instance)\n",
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/data/fields/_dataframe_field.py:227: UserWarning: Category 18 in adata.obs['_scvi_labels'] has fewer than 3 cells. Models may not train properly.\n",
      "  new_mapping = _make_column_categorical(\n",
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/data/fields/_scanvi.py:58: UserWarning: Category 18 in adata.obs['_scvi_labels'] has fewer than 3 cells. Models may not train properly.\n",
      "  mapping = _make_column_categorical(\n"
     ]
    }
   ],
   "source": [
    "model = hubmodel.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AnnData object with n_obs × n_vars = 584944 × 2000\n",
       "    obs: 'suspension_type', 'donor_id', 'is_primary_data', 'assay_ontology_term_id', 'cell_type_ontology_term_id', 'development_stage_ontology_term_id', 'disease_ontology_term_id', 'self_reported_ethnicity_ontology_term_id', 'tissue_ontology_term_id', 'organism_ontology_term_id', 'sex_ontology_term_id', 'BMI', 'age_or_mean_of_age_range', 'age_range', 'anatomical_region_ccf_score', 'ann_coarse_for_GWAS_and_modeling', 'ann_finest_level', 'ann_level_1', 'ann_level_2', 'ann_level_3', 'ann_level_4', 'ann_level_5', 'cause_of_death', 'dataset', 'entropy_dataset_leiden_3', 'entropy_original_ann_level_1_leiden_3', 'entropy_original_ann_level_2_clean_leiden_3', 'entropy_original_ann_level_3_clean_leiden_3', 'entropy_subject_ID_leiden_3', 'fresh_or_frozen', 'leiden_1', 'leiden_2', 'leiden_3', 'leiden_4', 'leiden_5', 'log10_total_counts', 'lung_condition', 'mixed_ancestry', 'n_genes_detected', 'original_ann_highest_res', 'original_ann_level_1', 'original_ann_level_2', 'original_ann_level_3', 'original_ann_level_4', 'original_ann_level_5', 'original_ann_nonharmonized', 'reannotation_type', 'reference_genome', 'sample', 'scanvi_label', 'sequencing_platform', 'size_factors', 'smoking_status', 'study', 'subject_type', 'tissue_dissociation_protocol', 'tissue_level_2', 'tissue_level_3', 'tissue_sampling_method', 'cell_type', 'assay', 'disease', 'organism', 'sex', 'tissue', 'self_reported_ethnicity', 'development_stage', '_scvi_batch', '_scvi_labels'\n",
       "    var: 'feature_is_filtered', 'feature_name', 'feature_reference', 'feature_biotype'\n",
       "    obsm: 'X_scanvi_emb', 'X_umap'\n",
       "    obsp: 'connectivities', 'distances'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ad.write('hlca/large_training_adata_subset.h5ad')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3991725/846078938.py:1: UserWarning: No revision was passed, so the default (latest) revision will be used.\n",
      "  hubmodel = HubModel.pull_from_huggingface_hub(\"scvi-tools/human-lung-cell-atlas\", local_dir='hlca')\n",
      "Fetching 3 files: 100%|██████████| 3/3 [00:00<00:00, 59074.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mINFO    \u001b[0m Reading adata\u001b[33m...\u001b[0m                                                                                          \n",
      "\u001b[34mINFO    \u001b[0m Loading model\u001b[33m...\u001b[0m                                                                                          \n",
      "\u001b[34mINFO    \u001b[0m File \u001b[35m/home/cane/Documents/yoseflab/can/scvi-hub/scvi-hub-reproducibility/hlca/\u001b[0m\u001b[95mmodel.pt\u001b[0m already downloaded \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/anndata/__init__.py:51: FutureWarning: `anndata.read` is deprecated, use `anndata.read_h5ad` instead. `ad.read` will be removed in mid 2024.\n",
      "  warnings.warn(\n",
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/data/fields/_dataframe_field.py:227: UserWarning: Category 18 in adata.obs['_scvi_labels'] has fewer than 3 cells. Models may not train properly.\n",
      "  new_mapping = _make_column_categorical(\n",
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/data/fields/_scanvi.py:58: UserWarning: Category 18 in adata.obs['_scvi_labels'] has fewer than 3 cells. Models may not train properly.\n",
      "  mapping = _make_column_categorical(\n"
     ]
    }
   ],
   "source": [
    "hubmodel = HubModel.pull_from_huggingface_hub(\"scvi-tools/human-lung-cell-atlas\", local_dir='hlca')\n",
    "adata = hubmodel.adata\n",
    "model = hubmodel.model\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "adata.obsm['scvi'] = model.get_latent_representation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mINFO    \u001b[0m File hlca/model.pt already downloaded                                                                     \n",
      "\u001b[34mINFO    \u001b[0m Found \u001b[1;36m100.0\u001b[0m% reference vars in query data.                                                                \n",
      "\u001b[34mINFO    \u001b[0m File hlca/model.pt already downloaded                                                                     \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/model/base/_base_model.py:676: UserWarning: `use_gpu` is deprecated in v1.0 and will be removed in v1.1. Please use `accelerator` and `devices` instead.\n",
      "  _, _, device = parse_device_args(\n",
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/data/fields/_base_field.py:64: UserWarning: adata.X does not contain unnormalized count data. Are you sure this is what you want?\n",
      "  self.validate_field(adata)\n",
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/data/fields/_dataframe_field.py:227: UserWarning: Category 18 in adata.obs['_scvi_labels'] has fewer than 3 cells. Models may not train properly.\n",
      "  new_mapping = _make_column_categorical(\n",
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/data/fields/_scanvi.py:58: UserWarning: Category 18 in adata.obs['_scvi_labels'] has fewer than 3 cells. Models may not train properly.\n",
      "  mapping = _make_column_categorical(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">ScanVI Model with the following params: \n",
       "unlabeled_category: unlabeled, n_hidden: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">128</span>, n_latent: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">30</span>, n_layers: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, dropout_rate: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1</span>, dispersion: gene, \n",
       "gene_likelihood: nb\n",
       "Training status: Trained\n",
       "Model's adata is minified?: <span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "ScanVI Model with the following params: \n",
       "unlabeled_category: unlabeled, n_hidden: \u001b[1;36m128\u001b[0m, n_latent: \u001b[1;36m30\u001b[0m, n_layers: \u001b[1;36m2\u001b[0m, dropout_rate: \u001b[1;36m0.1\u001b[0m, dispersion: gene, \n",
       "gene_likelihood: nb\n",
       "Training status: Trained\n",
       "Model's adata is minified?: \u001b[3;91mFalse\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scvi.model.SCANVI.prepare_query_anndata(ad, \"hlca\")\n",
    "scvi.model.SCANVI.load(\"hlca\", adata=ad, use_gpu=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.adata = ad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "adata_url = \"https://cellxgene.cziscience.com/e/066943a2-fdac-4b29-b348-40cede398e4e.cxg/\"\n",
    "b = scvi.data.cellxgene(adata_url, filename=\"large_training_adata.h5ad\", save_path=\"./hlca\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The memory_profiler extension is already loaded. To reload it, use:\n",
      "  %reload_ext memory_profiler\n"
     ]
    }
   ],
   "source": [
    "%load_ext memory_profiler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython.core\n",
    "IPython.core.page = print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3930981/3741317311.py:2: UserWarning: No revision was passed, so the default (latest) revision will be used.\n",
      "  hubmodel = HubModel.pull_from_huggingface_hub(\"scvi-tools/human-lung-cell-atlas\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR: Could not find file /tmp/ipykernel_3930981/3741317311.py\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching 3 files: 100%|██████████| 3/3 [00:00<00:00, 4341.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mINFO    \u001b[0m Reading adata\u001b[33m...\u001b[0m                                                                                          \n",
      "\u001b[34mINFO    \u001b[0m Loading model\u001b[33m...\u001b[0m                                                                                          \n",
      "\u001b[34mINFO    \u001b[0m File                                                                                                      \n",
      "         \u001b[35m/home/cane/.cache/huggingface/hub/models--scvi-tools--human-lung-cell-atlas/snapshots/7af8f610b2c46b715e62\u001b[0m\n",
      "         \u001b[35mcf46bc14163d8a3e679e/\u001b[0m\u001b[95mmodel.pt\u001b[0m already downloaded                                                          \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/anndata/__init__.py:51: FutureWarning: `anndata.read` is deprecated, use `anndata.read_h5ad` instead. `ad.read` will be removed in mid 2024.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/data/fields/_dataframe_field.py:227: UserWarning: Category 18 in adata.obs['_scvi_labels'] has fewer than 3 cells. Models may not train properly.\n",
      "  new_mapping = _make_column_categorical(\n",
      "/home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/data/fields/_scanvi.py:58: UserWarning: Category 18 in adata.obs['_scvi_labels'] has fewer than 3 cells. Models may not train properly.\n",
      "  mapping = _make_column_categorical(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": []
    }
   ],
   "source": [
    "%mprun -f minified_hub_model minified_hub_model(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<string>:1: UserWarning: No revision was passed, so the default (latest) revision will be used.\n",
      "Fetching 3 files: 100%|██████████| 3/3 [00:00<00:00, 4557.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Filename: /home/cane/miniconda3/envs/hub_paper/lib/python3.10/site-packages/scvi/hub/hub_model.py\n",
      "\n",
      "Line #    Mem usage    Increment  Occurrences   Line Contents\n",
      "=============================================================\n",
      "   148  12759.0 MiB  12759.0 MiB           1       @classmethod\n",
      "   149                                             def pull_from_huggingface_hub(\n",
      "   150                                                 cls,\n",
      "   151                                                 repo_name: str,\n",
      "   152                                                 cache_dir: Optional[str] = None,\n",
      "   153                                                 revision: Optional[str] = None,\n",
      "   154                                                 **kwargs,\n",
      "   155                                             ):\n",
      "   156                                                 \"\"\"Download the given model repo from huggingface.\n",
      "   157                                         \n",
      "   158                                                 The model, its card, data, metadata are downloaded to a cached location on disk\n",
      "   159                                                 selected by huggingface and an instance of this class is created with that info\n",
      "   160                                                 and returned.\n",
      "   161                                         \n",
      "   162                                                 Parameters\n",
      "   163                                                 ----------\n",
      "   164                                                 repo_name\n",
      "   165                                                     ID of the huggingface repo where this model needs to be uploaded\n",
      "   166                                                 cache_dir\n",
      "   167                                                     The directory where the downloaded model artifacts will be cached\n",
      "   168                                                 revision\n",
      "   169                                                     The revision to pull from the repo. This can be a branch name, a tag, or a full-length commit hash.\n",
      "   170                                                     If None, the default (latest) revision is pulled.\n",
      "   171                                                 kwargs\n",
      "   172                                                     Additional keyword arguments to pass to :meth:`~huggingface_hub.snapshot_download`.\n",
      "   173                                                 \"\"\"\n",
      "   174  12759.0 MiB      0.0 MiB           1           if revision is None:\n",
      "   175  12759.0 MiB      0.0 MiB           2               warnings.warn(\n",
      "   176  12759.0 MiB      0.0 MiB           1                   \"No revision was passed, so the default (latest) revision will be used.\",\n",
      "   177  12759.0 MiB      0.0 MiB           1                   UserWarning,\n",
      "   178  12759.0 MiB      0.0 MiB           1                   stacklevel=settings.warnings_stacklevel,\n",
      "   179                                                     )\n",
      "   180  12759.0 MiB      0.0 MiB           3           snapshot_folder = snapshot_download(\n",
      "   181  12759.0 MiB      0.0 MiB           1               repo_id=repo_name,\n",
      "   182  12759.0 MiB      0.0 MiB           1               allow_patterns=[\"model.pt\", \"adata.h5ad\", _SCVI_HUB.METADATA_FILE_NAME],\n",
      "   183  12759.0 MiB      0.0 MiB           1               cache_dir=cache_dir,\n",
      "   184  12759.0 MiB      0.0 MiB           1               revision=revision,\n",
      "   185  12759.0 MiB      0.0 MiB           1               **kwargs,\n",
      "   186                                                 )\n",
      "   187  12759.0 MiB      0.0 MiB           1           model_card = ModelCard.load(repo_name)\n",
      "   188  12759.0 MiB      0.0 MiB           1           return cls(snapshot_folder, model_card=model_card)"
     ]
    }
   ],
   "source": [
    "mprof_results = %mprun -f scvi.hub.HubModel.pull_from_huggingface_hub scvi.hub.HubModel.pull_from_huggingface_hub(\"scvi-tools/human-lung-cell-atlas\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hub_paper",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}