From 7f3b73677eff68e41c0d3697e718e89e01f2646a Mon Sep 17 00:00:00 2001 From: Dramalf Date: Tue, 21 Jan 2025 00:48:24 -0800 Subject: [PATCH 1/7] fix: resolve size mismatch issue in MediaStreamClip during tab recording --- packages/av-cliper/src/clips/iclip.ts | 2 +- .../av-cliper/src/clips/media-stream-clip.ts | 44 +++++++++++++------ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/packages/av-cliper/src/clips/iclip.ts b/packages/av-cliper/src/clips/iclip.ts index 2c82e880..b6390990 100644 --- a/packages/av-cliper/src/clips/iclip.ts +++ b/packages/av-cliper/src/clips/iclip.ts @@ -1,4 +1,4 @@ -interface IClipMeta { +export interface IClipMeta { width: number; height: number; duration: number; diff --git a/packages/av-cliper/src/clips/media-stream-clip.ts b/packages/av-cliper/src/clips/media-stream-clip.ts index 33e3cf5c..26cd6f5a 100644 --- a/packages/av-cliper/src/clips/media-stream-clip.ts +++ b/packages/av-cliper/src/clips/media-stream-clip.ts @@ -1,5 +1,5 @@ import { autoReadStream } from '@webav/internal-utils'; -import { IClip } from './iclip'; +import { IClip, IClipMeta } from './iclip'; /** * 包装实时音视频流，仅用于 [AVCanvas](../../av-canvas/classes/AVCanvas.html) @@ -44,24 +44,40 @@ export class MediaStreamClip implements IClip { #ms: MediaStream; constructor(ms: MediaStream) { this.#ms = ms; + this.audioTrack = ms.getAudioTracks()[0] ?? null; + this.#meta.duration = Infinity; const videoTrack = ms.getVideoTracks()[0]; if (videoTrack != null) { - const { width, height } = videoTrack.getSettings(); videoTrack.contentHint = 'motion'; - this.#meta.width = width ?? 0; - this.#meta.height = height ?? 0; - - this.#cvs = new OffscreenCanvas(width ?? 0, height ?? 0); - this.#stopRenderCvs = renderVideoTrackToCvs( - this.#cvs.getContext('2d')!, - videoTrack, + // The videoTrack.getSettings() is unstable in some cases if the videoTrack is not attached to a video element + // The width and height will change after the first call of getSettings() (e.g. 3452x2240 -> 3452x1940) + // This is something to do with the stream initialization, there is a default size for the videoTrack which may not match the real size + // Create a video element to initialize the videoTrack to ensure the correct size + const video_for_initialization = document.createElement('video'); + video_for_initialization.style.display = 'none'; + document.body.appendChild(video_for_initialization); + const loadedPromise: Promise = new Promise( + (resolve, reject) => { + video_for_initialization.onerror = reject; + video_for_initialization.onloadedmetadata = () => { + const { width, height } = videoTrack.getSettings(); + this.#cvs = new OffscreenCanvas(width ?? 0, height ?? 0); + this.#stopRenderCvs = renderVideoTrackToCvs( + this.#cvs.getContext('2d')!, + videoTrack, + ); + this.#meta.width = width ?? 0; + this.#meta.height = height ?? 0; + video_for_initialization.remove(); + resolve(this.meta); + }; + }, ); + this.ready = loadedPromise; + video_for_initialization.srcObject = ms; + } else { + this.ready = Promise.resolve(this.meta); } - - this.audioTrack = ms.getAudioTracks()[0] ?? null; - - this.#meta.duration = Infinity; - this.ready = Promise.resolve(this.meta); } async tick(): Promise<{ From 9e48e49b6f6db4fc20c49495ffed1e4412f86d1d Mon Sep 17 00:00:00 2001 From: Dramalf Date: Tue, 21 Jan 2025 01:07:43 -0800 Subject: [PATCH 2/7] fix(av-cliper): resolve size mismatch issue in MediaStreamClip during tab recording --- .changeset/shiny-onions-retire.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/shiny-onions-retire.md diff --git a/.changeset/shiny-onions-retire.md b/.changeset/shiny-onions-retire.md new file mode 100644 index 00000000..6b1c10dc --- /dev/null +++ b/.changeset/shiny-onions-retire.md @@ -0,0 +1,5 @@ +--- +'@webav/av-cliper': major +--- + +resolve size mismatch issue in MediaStreamClip during tab recording From 1055cac94c3b8c98e5f5b2eb795082bd81bd4e1a Mon Sep 17 00:00:00 2001 From: Dramalf Date: Tue, 21 Jan 2025 01:10:46 -0800 Subject: [PATCH 3/7] fix(av-cliper): resolve size mismatch issue in MediaStreamClip during tab recording --- .changeset/thin-cows-hug.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/thin-cows-hug.md diff --git a/.changeset/thin-cows-hug.md b/.changeset/thin-cows-hug.md new file mode 100644 index 00000000..edf227e9 --- /dev/null +++ b/.changeset/thin-cows-hug.md @@ -0,0 +1,5 @@ +--- +'@webav/av-cliper': patch +--- + +resolve size mismatch issue in MediaStreamClip during tab recording From e34d60b1f76b8c135ac7c6e2eaf4746f2e42fc14 Mon Sep 17 00:00:00 2001 From: Dramalf Date: Tue, 21 Jan 2025 01:11:20 -0800 Subject: [PATCH 4/7] fix(av-cliper): resolve size mismatch issue in MediaStreamClip during tab recording --- .changeset/shiny-onions-retire.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/shiny-onions-retire.md diff --git a/.changeset/shiny-onions-retire.md b/.changeset/shiny-onions-retire.md deleted file mode 100644 index 6b1c10dc..00000000 --- a/.changeset/shiny-onions-retire.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@webav/av-cliper': major ---- - -resolve size mismatch issue in MediaStreamClip during tab recording From 63fee170f24bb62e9f3f6822e78ee9d93eec8763 Mon Sep 17 00:00:00 2001 From: Dramalf Date: Tue, 21 Jan 2025 01:12:27 -0800 Subject: [PATCH 5/7] fix(av-cliper): resolve size mismatch issue in MediaStreamClip during tab recording #343 --- .changeset/thin-cows-hug.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/thin-cows-hug.md b/.changeset/thin-cows-hug.md index edf227e9..99c8a620 100644 --- a/.changeset/thin-cows-hug.md +++ b/.changeset/thin-cows-hug.md @@ -2,4 +2,4 @@ '@webav/av-cliper': patch --- -resolve size mismatch issue in MediaStreamClip during tab recording +resolve size mismatch issue in MediaStreamClip during tab recording #343 From aa6e3feac1c8f0bd8604bfea420290210a545928 Mon Sep 17 00:00:00 2001 From: Dramalf Date: Wed, 22 Jan 2025 12:24:49 -0800 Subject: [PATCH 6/7] fix: Initialize MediaStreamClip's width, height and cvs in the first call of onChunk with the firstFrame information #343 --- .changeset/seven-olives-worry.md | 5 ++ .vscode/settings.json | 3 ++ .../av-cliper/src/clips/media-stream-clip.ts | 47 ++++++++----------- 3 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 .changeset/seven-olives-worry.md diff --git a/.changeset/seven-olives-worry.md b/.changeset/seven-olives-worry.md new file mode 100644 index 00000000..d505331a --- /dev/null +++ b/.changeset/seven-olives-worry.md @@ -0,0 +1,5 @@ +--- +'@webav/av-cliper': patch +--- + +fix: Initialize MediaStreamClip's width, height and cvs in the first call of onChunk with the firstFrame information #343 diff --git a/.vscode/settings.json b/.vscode/settings.json index d48d8a08..97b5d675 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,5 +11,8 @@ }, "[jsonc]": { "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[typescript]": { + "editor.defaultFormatter": "vscode.typescript-language-features" } } diff --git a/packages/av-cliper/src/clips/media-stream-clip.ts b/packages/av-cliper/src/clips/media-stream-clip.ts index 26cd6f5a..84385e5b 100644 --- a/packages/av-cliper/src/clips/media-stream-clip.ts +++ b/packages/av-cliper/src/clips/media-stream-clip.ts @@ -49,32 +49,14 @@ export class MediaStreamClip implements IClip { const videoTrack = ms.getVideoTracks()[0]; if (videoTrack != null) { videoTrack.contentHint = 'motion'; - // The videoTrack.getSettings() is unstable in some cases if the videoTrack is not attached to a video element - // The width and height will change after the first call of getSettings() (e.g. 3452x2240 -> 3452x1940) - // This is something to do with the stream initialization, there is a default size for the videoTrack which may not match the real size - // Create a video element to initialize the videoTrack to ensure the correct size - const video_for_initialization = document.createElement('video'); - video_for_initialization.style.display = 'none'; - document.body.appendChild(video_for_initialization); - const loadedPromise: Promise = new Promise( - (resolve, reject) => { - video_for_initialization.onerror = reject; - video_for_initialization.onloadedmetadata = () => { - const { width, height } = videoTrack.getSettings(); - this.#cvs = new OffscreenCanvas(width ?? 0, height ?? 0); - this.#stopRenderCvs = renderVideoTrackToCvs( - this.#cvs.getContext('2d')!, - videoTrack, - ); - this.#meta.width = width ?? 0; - this.#meta.height = height ?? 0; - video_for_initialization.remove(); - resolve(this.meta); - }; - }, - ); - this.ready = loadedPromise; - video_for_initialization.srcObject = ms; + this.ready = new Promise((resolve) => { + this.#stopRenderCvs = renderVideoTrackToCvs(videoTrack, (cvs) => { + this.#meta.width = cvs.width; + this.#meta.height = cvs.height; + this.#cvs = cvs; + resolve(this.meta); + }); + }); } else { this.ready = Promise.resolve(this.meta); } @@ -107,15 +89,26 @@ export class MediaStreamClip implements IClip { } function renderVideoTrackToCvs( - cvsCtx: OffscreenCanvasRenderingContext2D, track: MediaStreamVideoTrack, + onOffscreenCanvasReady: (cvs: OffscreenCanvas) => void, ) { + let emitFF = false; + let cvsCtx: OffscreenCanvasRenderingContext2D; return autoReadStream( new MediaStreamTrackProcessor({ track, }).readable, { onChunk: async (frame) => { + if (!emitFF) { + const { displayHeight, displayWidth } = frame; + const width = displayWidth ?? 0; + const height = displayHeight ?? 0; + const cvs = new OffscreenCanvas(width, height); + cvsCtx = cvs.getContext('2d')!; + onOffscreenCanvasReady(cvs); + emitFF = true; + } cvsCtx.drawImage(frame, 0, 0); frame.close(); }, From a6b088dfd2cd0b83e92f7a37b8b7b2571bdd97d9 Mon Sep 17 00:00:00 2001 From: Dramalf Date: Wed, 22 Jan 2025 15:26:02 -0800 Subject: [PATCH 7/7] fix(av-cliper): remove unnecessary import #343 --- packages/av-cliper/src/clips/iclip.ts | 2 +- packages/av-cliper/src/clips/media-stream-clip.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/av-cliper/src/clips/iclip.ts b/packages/av-cliper/src/clips/iclip.ts index b6390990..2c82e880 100644 --- a/packages/av-cliper/src/clips/iclip.ts +++ b/packages/av-cliper/src/clips/iclip.ts @@ -1,4 +1,4 @@ -export interface IClipMeta { +interface IClipMeta { width: number; height: number; duration: number; diff --git a/packages/av-cliper/src/clips/media-stream-clip.ts b/packages/av-cliper/src/clips/media-stream-clip.ts index 84385e5b..311e5dc5 100644 --- a/packages/av-cliper/src/clips/media-stream-clip.ts +++ b/packages/av-cliper/src/clips/media-stream-clip.ts @@ -1,5 +1,5 @@ import { autoReadStream } from '@webav/internal-utils'; -import { IClip, IClipMeta } from './iclip'; +import { IClip } from './iclip'; /** * 包装实时音视频流，仅用于 [AVCanvas](../../av-canvas/classes/AVCanvas.html)