diff --git a/test/test_local_io.py b/test/test_local_io.py index 9fcef2b74..43c310059 100644 --- a/test/test_local_io.py +++ b/test/test_local_io.py @@ -22,7 +22,7 @@ FileOpener, HashChecker, IoPathFileLister, - IoPathFileLoader, + IoPathFileOpener, IoPathSaver, IterableWrapper, JsonParser, @@ -539,7 +539,7 @@ def test_io_path_file_lister_iterdatapipe(self): @skipIfNoIoPath def test_io_path_file_loader_iterdatapipe(self): datapipe1 = IoPathFileLister(root=self.temp_sub_dir.name) - datapipe2 = IoPathFileLoader(datapipe1) + datapipe2 = IoPathFileOpener(datapipe1) # check contents of file match for _, f in datapipe2: @@ -548,7 +548,7 @@ def test_io_path_file_loader_iterdatapipe(self): # Reset Test: Ensure the resulting streams are still readable after the DataPipe is reset/exhausted self._write_text_files() lister_dp = FileLister(self.temp_dir.name, "*.text") - iopath_file_loader_dp = IoPathFileLoader(lister_dp, mode="rb") + iopath_file_loader_dp = IoPathFileOpener(lister_dp, mode="rb") n_elements_before_reset = 2 res_before_reset, res_after_reset = reset_after_n_next_calls(iopath_file_loader_dp, n_elements_before_reset) diff --git a/torchdata/datapipes/iter/__init__.py b/torchdata/datapipes/iter/__init__.py index 90c6095a2..61fae45e5 100644 --- a/torchdata/datapipes/iter/__init__.py +++ b/torchdata/datapipes/iter/__init__.py @@ -33,6 +33,7 @@ from torchdata.datapipes.iter.load.iopath import ( IoPathFileListerIterDataPipe as IoPathFileLister, IoPathFileLoaderIterDataPipe as IoPathFileLoader, + IoPathFileOpenerIterDataPipe as IoPathFileOpener, IoPathSaverIterDataPipe as IoPathSaver, ) @@ -107,6 +108,7 @@ "IndexAdder", "IoPathFileLister", "IoPathFileLoader", + "IoPathFileOpener", "IoPathSaver", "IterDataPipe", "IterKeyZipper", diff --git a/torchdata/datapipes/iter/load/iopath.py b/torchdata/datapipes/iter/load/iopath.py index 70500d335..3d5982fd3 100644 --- a/torchdata/datapipes/iter/load/iopath.py +++ b/torchdata/datapipes/iter/load/iopath.py @@ -1,5 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. import os +import warnings from typing import Any, Callable, Iterator, List, Optional, Tuple, Union @@ -82,7 +83,18 @@ def __iter__(self) -> Iterator[str]: @functional_datapipe("load_file_by_iopath") class IoPathFileLoaderIterDataPipe(IterDataPipe[Tuple[str, StreamWrapper]]): - r""":class:`IoPathFileLoaderIterDataPipe`. + def __new__(cls, source_datapipe: IterDataPipe[str], mode: str = "r", pathmgr=None): + warnings.warn( + "IoPathFileLoaderIterDataPipe and its functional API has been renamed and will be removed " + "from this package. Please use 'IoPathFileOpenerIterDataPipe' instead.", + DeprecationWarning, + ) + return IoPathFileOpenerIterDataPipe(source_datapipe=source_datapipe, mode=mode, pathmgr=pathmgr) + + +@functional_datapipe("open_file_by_iopath") +class IoPathFileOpenerIterDataPipe(IterDataPipe[Tuple[str, StreamWrapper]]): + r""":class:`IoPathFileOpenerIterDataPipe`. Iterable DataPipe to open files from input datapipe which contains pathnames or URLs, and yields a tuple of pathname and opened file stream.