diff --git a/tensorflow_io/core/kernels/ffmpeg_kernels.cc b/tensorflow_io/core/kernels/ffmpeg_kernels.cc
index 32f63db53..1ab37f830 100644
--- a/tensorflow_io/core/kernels/ffmpeg_kernels.cc
+++ b/tensorflow_io/core/kernels/ffmpeg_kernels.cc
@@ -669,8 +669,7 @@ class FFmpegVideoStream : public FFmpegStream {
     char* base = ((char*)(value->flat<uint8>().data()));
     int64 datasize = height_ * width_ * channels_;
     for (size_t i = 0; i < frames_.size(); i++) {
-      memcpy(base, reinterpret_cast<char*>(frames_buffer_.front().get()),
-             datasize);
+      memcpy(base, reinterpret_cast<char*>(frames_buffer_[i].get()), datasize);
       base += datasize;
     }
     frames_.clear();
diff --git a/tests/test_ffmpeg.py b/tests/test_ffmpeg.py
index 28988c72f..2353f1c5b 100644
--- a/tests/test_ffmpeg.py
+++ b/tests/test_ffmpeg.py
@@ -84,6 +84,7 @@ def test_ffmpeg_decode_video(video_path):
     video = tfio.experimental.ffmpeg.decode_video(content, 0)
     assert video.shape == [166, 320, 560, 3]
     assert video.dtype == tf.uint8
+    assert np.abs(video[0] - video[-1]).sum() > 0
 
 
 def test_ffmpeg_decode_video_invalid_content():