Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Implement approximate mode #440

Draft
wants to merge 13 commits into
base: main
Choose a base branch
from
+520 −259
Draft

[WIP] Implement approximate mode #440

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
16d698f
Start implementation of approximate mode
scotts Dec 16, 2024
9a5abce
Merge branch 'main' of github.com:pytorch/torchcodec into approx
scotts Dec 18, 2024
d95b128
Initial seek mode implementation in VideoDecoder.
scotts Dec 19, 2024
97ac764
Merge branch 'main' of github.com:pytorch/torchcodec into approx
scotts Dec 19, 2024
35f2e59
Added Python side support, extended tests.
scotts Dec 20, 2024
8c9aeac
Apply lints
scotts Dec 20, 2024
921b822
Default C++ tests to approximate mode
scotts Dec 20, 2024
b349282
Apply lints
scotts Dec 20, 2024
081a5bb
Updated metadata; all tests pass.
scotts Dec 20, 2024
802b881
Removed commened out code.
scotts Dec 20, 2024
911a3bc
Consolidated logic for timestamp batch. Big perf win.
scotts Dec 20, 2024
7267b5a
Consolidated logic for timestamp range.
scotts Dec 21, 2024
ae44f78
More mode consolidation.
scotts Dec 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 35 additions & 19 deletions benchmarks/decoders/benchmark_decoders_library.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
get_frames_by_pts,
get_json_metadata,
get_next_frame,
scan_all_streams_to_update_metadata,
seek_to_pts,
)

Expand Down Expand Up @@ -154,8 +153,7 @@ def __init__(self, num_threads=None, color_conversion_library=None, device="cpu"
self._device = device

def decode_frames(self, video_file, pts_list):
decoder = create_from_file(video_file)
scan_all_streams_to_update_metadata(decoder)
decoder = create_from_file(video_file, seek_mode="exact")
_add_video_stream(
decoder,
num_threads=self._num_threads,
Expand All @@ -170,7 +168,7 @@ def decode_frames(self, video_file, pts_list):
return frames

def decode_first_n_frames(self, video_file, n):
decoder = create_from_file(video_file)
decoder = create_from_file(video_file, seek_mode="approximate")
_add_video_stream(
decoder,
num_threads=self._num_threads,
Expand All @@ -197,7 +195,7 @@ def __init__(self, num_threads=None, color_conversion_library=None, device="cpu"
self.transforms_v2 = transforms_v2

def decode_frames(self, video_file, pts_list):
decoder = create_from_file(video_file)
decoder = create_from_file(video_file, seek_mode="approximate")
num_threads = int(self._num_threads) if self._num_threads else 0
_add_video_stream(
decoder,
Expand All @@ -216,7 +214,7 @@ def decode_frames(self, video_file, pts_list):

def decode_first_n_frames(self, video_file, n):
num_threads = int(self._num_threads) if self._num_threads else 0
decoder = create_from_file(video_file)
decoder = create_from_file(video_file, seek_mode="approximate")
_add_video_stream(
decoder,
num_threads=num_threads,
Expand All @@ -233,7 +231,7 @@ def decode_first_n_frames(self, video_file, n):

def decode_and_resize(self, video_file, pts_list, height, width, device):
num_threads = int(self._num_threads) if self._num_threads else 1
decoder = create_from_file(video_file)
decoder = create_from_file(video_file, seek_mode="approximate")
_add_video_stream(
decoder,
num_threads=num_threads,
Expand Down Expand Up @@ -263,8 +261,7 @@ def __init__(self, num_threads=None, color_conversion_library=None, device="cpu"
self._device = device

def decode_frames(self, video_file, pts_list):
decoder = create_from_file(video_file)
scan_all_streams_to_update_metadata(decoder)
decoder = create_from_file(video_file, seek_mode="exact")
_add_video_stream(
decoder,
num_threads=self._num_threads,
Expand All @@ -279,8 +276,7 @@ def decode_frames(self, video_file, pts_list):
return frames

def decode_first_n_frames(self, video_file, n):
decoder = create_from_file(video_file)
scan_all_streams_to_update_metadata(decoder)
decoder = create_from_file(video_file, seek_mode="exact")
_add_video_stream(
decoder,
num_threads=self._num_threads,
Expand All @@ -297,9 +293,10 @@ def decode_first_n_frames(self, video_file, n):


class TorchCodecPublic(AbstractDecoder):
def __init__(self, num_ffmpeg_threads=None, device="cpu"):
def __init__(self, num_ffmpeg_threads=None, device="cpu", seek_mode="exact"):
self._num_ffmpeg_threads = num_ffmpeg_threads
self._device = device
self._seek_mode = seek_mode

from torchvision.transforms import v2 as transforms_v2

Expand All @@ -310,7 +307,10 @@ def decode_frames(self, video_file, pts_list):
int(self._num_ffmpeg_threads) if self._num_ffmpeg_threads else 0
)
decoder = VideoDecoder(
video_file, num_ffmpeg_threads=num_ffmpeg_threads, device=self._device
video_file,
num_ffmpeg_threads=num_ffmpeg_threads,
device=self._device,
seek_mode=self._seek_mode,
)
return decoder.get_frames_played_at(pts_list)

Expand All @@ -319,7 +319,10 @@ def decode_first_n_frames(self, video_file, n):
int(self._num_ffmpeg_threads) if self._num_ffmpeg_threads else 0
)
decoder = VideoDecoder(
video_file, num_ffmpeg_threads=num_ffmpeg_threads, device=self._device
video_file,
num_ffmpeg_threads=num_ffmpeg_threads,
device=self._device,
seek_mode=self._seek_mode,
)
frames = []
count = 0
Expand All @@ -335,17 +338,21 @@ def decode_and_resize(self, video_file, pts_list, height, width, device):
int(self._num_ffmpeg_threads) if self._num_ffmpeg_threads else 1
)
decoder = VideoDecoder(
video_file, num_ffmpeg_threads=num_ffmpeg_threads, device=self._device
video_file,
num_ffmpeg_threads=num_ffmpeg_threads,
device=self._device,
seek_mode=self._seek_mode,
)
frames = decoder.get_frames_played_at(pts_list)
frames = self.transforms_v2.functional.resize(frames.data, (height, width))
return frames


class TorchCodecPublicNonBatch(AbstractDecoder):
def __init__(self, num_ffmpeg_threads=None, device="cpu"):
def __init__(self, num_ffmpeg_threads=None, device="cpu", seek_mode="exact"):
self._num_ffmpeg_threads = num_ffmpeg_threads
self._device = device
self._seek_mode = seek_mode

from torchvision.transforms import v2 as transforms_v2

Expand All @@ -356,7 +363,10 @@ def decode_frames(self, video_file, pts_list):
int(self._num_ffmpeg_threads) if self._num_ffmpeg_threads else 0
)
decoder = VideoDecoder(
video_file, num_ffmpeg_threads=num_ffmpeg_threads, device=self._device
video_file,
num_ffmpeg_threads=num_ffmpeg_threads,
device=self._device,
seek_mode=self._seek_mode,
)

frames = []
Expand All @@ -370,7 +380,10 @@ def decode_first_n_frames(self, video_file, n):
int(self._num_ffmpeg_threads) if self._num_ffmpeg_threads else 0
)
decoder = VideoDecoder(
video_file, num_ffmpeg_threads=num_ffmpeg_threads, device=self._device
video_file,
num_ffmpeg_threads=num_ffmpeg_threads,
device=self._device,
seek_mode=self._seek_mode,
)
frames = []
count = 0
Expand All @@ -386,7 +399,10 @@ def decode_and_resize(self, video_file, pts_list, height, width, device):
int(self._num_ffmpeg_threads) if self._num_ffmpeg_threads else 1
)
decoder = VideoDecoder(
video_file, num_ffmpeg_threads=num_ffmpeg_threads, device=self._device
video_file,
num_ffmpeg_threads=num_ffmpeg_threads,
device=self._device,
seek_mode=self._seek_mode,
)

frames = []
Expand Down
Loading
Loading