Skip to content

*FileLoader -> *FileOpener #5128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jan 6, 2022
Merged

*FileLoader -> *FileOpener #5128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1b6405f
*FileLoader -> *FileOpener
pmeier Dec 27, 2021
786bdf0
Merge branch 'main' into file-loader-opener
NivekT Jan 4, 2022
9eeda59
fix opening mode
pmeier Jan 5, 2022
0108611
Merge branch 'main' into file-loader-opener
pmeier Jan 5, 2022
6c11358
Merge branch 'main' into file-loader-opener
NivekT Jan 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions torchvision/prototype/datasets/_folder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
from typing import Union, Tuple, List, Dict, Any

import torch
from torch.utils.data import IterDataPipe
from torch.utils.data.datapipes.iter import FileLister, FileLoader, Mapper, Shuffler, Filter
from torchdata.datapipes.iter import IterDataPipe, FileLister, FileOpener, Mapper, Shuffler, Filter
from torchvision.prototype.datasets.decoder import pil
from torchvision.prototype.datasets.utils._internal import INFINITE_BUFFER_SIZE, hint_sharding

Expand Down Expand Up @@ -54,7 +53,7 @@ def from_data_folder(
dp: IterDataPipe = Filter(dp, functools.partial(_is_not_top_level_file, root=root))
dp = hint_sharding(dp)
dp = Shuffler(dp, buffer_size=INFINITE_BUFFER_SIZE)
dp = FileLoader(dp)
dp = FileOpener(dp, mode="rb")
return (
Mapper(dp, functools.partial(_collate_and_decode_data, root=root, categories=categories, decoder=decoder)),
categories,
Expand Down
4 changes: 2 additions & 2 deletions torchvision/prototype/datasets/utils/_internal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import torch
import torch.distributed as dist
import torch.utils.data
from torchdata.datapipes.iter import IoPathFileLister, IoPathFileLoader, IterDataPipe, ShardingFilter, Shuffler
from torchdata.datapipes.iter import IoPathFileLister, IoPathFileOpener, IterDataPipe, ShardingFilter, Shuffler
from torchdata.datapipes.utils import StreamWrapper


Expand Down Expand Up @@ -254,7 +254,7 @@ def _make_sharded_datapipe(root: str, dataset_size: int) -> IterDataPipe[Dict[st
dp = IoPathFileLister(root=root)
dp = SharderDataPipe(dp)
dp = dp.shuffle(buffer_size=INFINITE_BUFFER_SIZE)
dp = IoPathFileLoader(dp, mode="rb")
dp = IoPathFileOpener(dp, mode="rb")
dp = PicklerDataPipe(dp)
# dp = dp.cycle(2)
dp = TakerDataPipe(dp, dataset_size)
Expand Down
6 changes: 3 additions & 3 deletions torchvision/prototype/datasets/utils/_resource.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from torchdata.datapipes.iter import (
IterableWrapper,
FileLister,
FileLoader,
FileOpener,
IterDataPipe,
ZipArchiveReader,
TarArchiveReader,
Expand Down Expand Up @@ -62,9 +62,9 @@ def _decompress(file: pathlib.Path) -> pathlib.Path:

def _default_loader(self, path: pathlib.Path) -> IterDataPipe[Tuple[str, IO]]:
if path.is_dir():
return FileLoader(FileLister(str(path), recursive=True))
return FileOpener(FileLister(str(path), recursive=True), mode="rb")

dp = FileLoader(IterableWrapper((str(path),)))
dp = FileOpener(IterableWrapper((str(path),)), mode="rb")

archive_loader = self._guess_archive_loader(path)
if archive_loader:
Expand Down