diff --git a/torchvision/prototype/datasets/_folder.py b/torchvision/prototype/datasets/_folder.py index efffaa80f99..fbca8b07b1a 100644 --- a/torchvision/prototype/datasets/_folder.py +++ b/torchvision/prototype/datasets/_folder.py @@ -7,8 +7,7 @@ from typing import Union, Tuple, List, Dict, Any import torch -from torch.utils.data import IterDataPipe -from torch.utils.data.datapipes.iter import FileLister, FileLoader, Mapper, Shuffler, Filter +from torchdata.datapipes.iter import IterDataPipe, FileLister, FileOpener, Mapper, Shuffler, Filter from torchvision.prototype.datasets.decoder import pil from torchvision.prototype.datasets.utils._internal import INFINITE_BUFFER_SIZE, hint_sharding @@ -54,7 +53,7 @@ def from_data_folder( dp: IterDataPipe = Filter(dp, functools.partial(_is_not_top_level_file, root=root)) dp = hint_sharding(dp) dp = Shuffler(dp, buffer_size=INFINITE_BUFFER_SIZE) - dp = FileLoader(dp) + dp = FileOpener(dp, mode="rb") return ( Mapper(dp, functools.partial(_collate_and_decode_data, root=root, categories=categories, decoder=decoder)), categories, diff --git a/torchvision/prototype/datasets/utils/_internal.py b/torchvision/prototype/datasets/utils/_internal.py index 824594dd28e..d0071206dd8 100644 --- a/torchvision/prototype/datasets/utils/_internal.py +++ b/torchvision/prototype/datasets/utils/_internal.py @@ -30,7 +30,7 @@ import torch import torch.distributed as dist import torch.utils.data -from torchdata.datapipes.iter import IoPathFileLister, IoPathFileLoader, IterDataPipe, ShardingFilter, Shuffler +from torchdata.datapipes.iter import IoPathFileLister, IoPathFileOpener, IterDataPipe, ShardingFilter, Shuffler from torchdata.datapipes.utils import StreamWrapper @@ -254,7 +254,7 @@ def _make_sharded_datapipe(root: str, dataset_size: int) -> IterDataPipe[Dict[st dp = IoPathFileLister(root=root) dp = SharderDataPipe(dp) dp = dp.shuffle(buffer_size=INFINITE_BUFFER_SIZE) - dp = IoPathFileLoader(dp, mode="rb") + dp = IoPathFileOpener(dp, mode="rb") dp = PicklerDataPipe(dp) # dp = dp.cycle(2) dp = TakerDataPipe(dp, dataset_size) diff --git a/torchvision/prototype/datasets/utils/_resource.py b/torchvision/prototype/datasets/utils/_resource.py index e7aa5731b1b..2fc6bbca49e 100644 --- a/torchvision/prototype/datasets/utils/_resource.py +++ b/torchvision/prototype/datasets/utils/_resource.py @@ -9,7 +9,7 @@ from torchdata.datapipes.iter import ( IterableWrapper, FileLister, - FileLoader, + FileOpener, IterDataPipe, ZipArchiveReader, TarArchiveReader, @@ -62,9 +62,9 @@ def _decompress(file: pathlib.Path) -> pathlib.Path: def _default_loader(self, path: pathlib.Path) -> IterDataPipe[Tuple[str, IO]]: if path.is_dir(): - return FileLoader(FileLister(str(path), recursive=True)) + return FileOpener(FileLister(str(path), recursive=True), mode="rb") - dp = FileLoader(IterableWrapper((str(path),))) + dp = FileOpener(IterableWrapper((str(path),)), mode="rb") archive_loader = self._guess_archive_loader(path) if archive_loader: