Cleanup datapipes (#1332)

pytorch · Oct 17, 2024 · c64801f · c64801f
1 parent 769c570
commit c64801f
Show file tree

Hide file tree

Showing 115 changed files with 247 additions and 15,710 deletions.
diff --git a/.github/scripts/check_complete_doc.py b/.github/scripts/check_complete_doc.py
@@ -38,14 +38,13 @@ def compare_sets(set_a, set_b, ignore_set=None):
 
 
 def main():
-    datapipes_folder = os.path.join("torchdata", "datapipes")
     init_file = "__init__.py"
     docs_source_folder = os.path.join("docs", "source")
     exit_code = 0
 
-    for target, ignore_set in zip(["iter", "map", "utils"], [{"IterDataPipe", "Extractor"}, {"MapDataPipe"}, {}]):
-        init_path = os.path.join(datapipes_folder, target, init_file)
-        rst_path = os.path.join(docs_source_folder, "torchdata.datapipes." + target + ".rst")
+    for target, ignore_set in [("stateful_dataloader", {})]:
+        init_path = os.path.join("torchdata", target, init_file)
+        rst_path = os.path.join(docs_source_folder, "torchdata." + target + ".rst")
 
         init_set = collect_init_dps(init_path)
         rst_set = collect_rst_dps(rst_path)

diff --git a/.github/workflows/aistore_ci.yml b/.github/workflows/aistore_ci.yml
diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml
@@ -35,7 +35,7 @@ jobs:
         include:
           - repository: pytorch/data
             pre-script: packaging/pre_build_script_linux.sh
-            post-script: packaging/post_build_script_linux.sh
+            post-script: ""
             smoke-test-script: test/smoke_test/smoke_test.py
             package-name: torchdata
     name: ${{ matrix.repository }}

diff --git a/.gitmodules b/.gitmodules
diff --git a/CMakeLists.txt b/CMakeLists.txt
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -112,21 +112,10 @@
     "<class 'torch.utils.data.sampler.SequentialSampler'>": "SequentialSampler",
     "torch.utils.data.datapipes.iter.combining.T_co": "T_co",
     "torch.utils.data.datapipes.iter.combinatorics.T_co": "T_co",
-    "torchdata.datapipes.iter.transform.bucketbatcher.T_co": "T_co",
     "torch.utils.data.datapipes.map.grouping.T": "T",
     "torch.utils.data.datapipes.map.combining.T_co": "T_co",
     "torch.utils.data.datapipes.map.combinatorics.T_co": "T_co",
-    "torchdata.datapipes.iter.util.cycler.T_co": "T_co",
-    "torchdata.datapipes.iter.util.paragraphaggregator.T_co": "T_co",
-    "torchdata.datapipes.map.util.cacheholder.T_co": "T_co",
-    "Sequence[torchdata.datapipes.map.util.unzipper.T]": "Sequence[T]",
-    "torchdata.datapipes.iter.util.samplemultiplexer.T_co": "T_co",
-    "torchdata.datapipes.iter.util.indexadder.K": "K",
-    "torchdata.datapipes.iter.util.unzipper.T": "T",
     "torch.utils.data.datapipes.iter.grouping.T_co": "T_co",
-    "torchdata.datapipes.iter.util.dataframemaker.T_co": "T_co",
-    "torchdata.datapipes.iter.util.cacheholder.T_co": "T_co",
-    "torchdata.datapipes.iter.util.header.T_co": "T_co",
     "<class 'torch.utils.data.datapipes.datapipe.DataChunk'>": "List",
     "typing.": "",
     "Union[IterDataPipe, MapDataPipe]": "DataPipe",

diff --git a/docs/source/dataloader2.rst b/docs/source/dataloader2.rst
diff --git a/docs/source/dlv2_tutorial.rst b/docs/source/dlv2_tutorial.rst