Merge pull request #283 from bilibili/feat/sprite-playback-rate

Feat/sprite playback rate
bilibili · Sep 18, 2024 · b2d1f49 · b2d1f49
2 parents d661a0a + e2a1387
commit b2d1f49
Show file tree

Hide file tree

Showing 10 changed files with 106 additions and 11 deletions.
diff --git a/.changeset/strange-sheep-rule.md b/.changeset/strange-sheep-rule.md
@@ -0,0 +1,5 @@
+---
+'@webav/av-cliper': minor
+---
+
+feat: support change playbackRate for Sprite
diff --git a/packages/av-canvas/CHANGELOG.md b/packages/av-canvas/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 ## 0.15.5
 
+## 0.14.15
+
 ### Patch Changes
 
 - Updated dependencies [e3b9a74]

diff --git a/packages/av-cliper/src/__tests__/av-utils.test.ts b/packages/av-cliper/src/__tests__/av-utils.test.ts
@@ -1,6 +1,11 @@
 import { test, expect } from 'vitest';
 // import './mock';
-import { concatFloat32Array, concatPCMFragments, mixinPCM } from '../av-utils';
+import {
+  changePCMPlaybackRate,
+  concatFloat32Array,
+  concatPCMFragments,
+  mixinPCM,
+} from '../av-utils';
 
 test('concatArrayBuffer', () => {
   expect(
@@ -46,3 +51,11 @@ test('concatFragmentPCM', () => {
     new Float32Array([...chan1, ...chan1]),
   ]);
 });
+
+test('changePCMPlaybackRate', () => {
+  const pcm = new Float32Array([1, 2, 3, 4, 5]);
+  expect(changePCMPlaybackRate(pcm, 0.5)).toEqual(
+    new Float32Array([1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5]),
+  );
+  expect(changePCMPlaybackRate(pcm, 2)).toEqual(new Float32Array([1, 3]));
+});
diff --git a/packages/av-cliper/src/av-utils.ts b/packages/av-cliper/src/av-utils.ts
@@ -436,3 +436,33 @@ export function createGoPVideoDecoder(conf: VideoDecoderConfig) {
     },
   };
 }
+
+/**
+ * 改变 PCM 数据的播放速率，1 表示正常播放，0.5 表示播放速率减半，2 表示播放速率加倍
+ */
+export function changePCMPlaybackRate(
+  pcmData: Float32Array,
+  playbackRate: number,
+) {
+  // 计算新的采样率
+  const newLength = Math.floor(pcmData.length / playbackRate);
+  const newPcmData = new Float32Array(newLength);
+
+  // 线性插值
+  for (let i = 0; i < newLength; i++) {
+    // 原始数据中的位置
+    const originalIndex = i * playbackRate;
+    const intIndex = Math.floor(originalIndex);
+    const frac = originalIndex - intIndex;
+
+    // 边界检查
+    if (intIndex + 1 < pcmData.length) {
+      newPcmData[i] =
+        pcmData[intIndex] * (1 - frac) + pcmData[intIndex + 1] * frac;
+    } else {
+      newPcmData[i] = pcmData[intIndex]; // 最后一个样本
+    }
+  }
+
+  return newPcmData;
+}
diff --git a/packages/av-cliper/src/combinator.ts b/packages/av-cliper/src/combinator.ts
@@ -449,7 +449,7 @@ export function createAudioTrackBuf(adFrames: number) {
   let audioTs = 0;
   const adDuration = (adFrames / DEFAULT_AUDIO_CONF.sampleRate) * 1e6;
 
-  // 缺少音频数据是占位
+  // 缺少音频数据时占位
   const placeholderData = new Float32Array(adDataSize);
 
   const getAudioData = (ts: number) => {

diff --git a/packages/av-cliper/src/sprite/__tests__/visible-sprite.test.ts b/packages/av-cliper/src/sprite/__tests__/visible-sprite.test.ts
@@ -1,6 +1,7 @@
 import { expect, test, vi } from 'vitest';
 import { VisibleSprite } from '../visible-sprite';
 import { AudioClip } from '../../clips';
+import { sleep } from '../../av-utils';
 
 function createSprite() {
   return new VisibleSprite(new AudioClip([new Float32Array()]));
@@ -26,3 +27,20 @@ test('rect change event', async () => {
   expect(handler).toBeCalledWith({ rect: { y: 10 } });
   expect(handler).toBeCalledTimes(2);
 });
+
+test('sprite playbackRate', async () => {
+  const spr = createSprite();
+  const clip = spr.getClip();
+  const spyTick = vi.spyOn(clip, 'tick');
+  spr.time.playbackRate = 2;
+  const cvs = new OffscreenCanvas(100, 100);
+  const ctx = cvs.getContext('2d')!;
+  spr.render(ctx, 1e6);
+  expect(spyTick).toBeCalledWith(2e6);
+
+  await sleep(100);
+
+  spr.time.playbackRate = 0.5;
+  spr.render(ctx, 2e6);
+  expect(spyTick).toHaveBeenLastCalledWith(1e6);
+});
diff --git a/packages/av-cliper/src/sprite/base-sprite.ts b/packages/av-cliper/src/sprite/base-sprite.ts
@@ -28,14 +28,26 @@ export abstract class BaseSprite {
   rect = new Rect();
 
   /**
-   * 控制素材在视频中的时间偏移与时长，常用于剪辑场景时间轴（轨道）模块
-   *
+   * 控制素材在的时间偏移、时长、播放速率，常用于剪辑场景时间轴（轨道）模块
    * duration 不能大于引用 {@link IClip} 的时长，单位 微秒
+   *
+   * playbackRate 控制当前素材的播放速率，1 表示正常播放；
+   * **注意**
+   *    1. 设置 playbackRate 时需要主动修正 duration
+   *    2. 音频使用最简单的插值算法来改变速率，所以改变速率后音调会产生变化，自定义算法请使用 {@link MP4Clip.tickInterceptor} 配合实现
+   *
    */
-  time = {
+  #time = {
     offset: 0,
     duration: 0,
+    playbackRate: 1,
   };
+  get time(): { offset: number; duration: number; playbackRate: number } {
+    return this.#time;
+  }
+  set time(v: { offset: number; duration: number; playbackRate?: number }) {
+    Object.assign(this.#time, v);
+  }
 
   /**
    * 元素是否可见，用于不想删除，期望临时隐藏 Sprite 的场景

diff --git a/packages/av-cliper/src/sprite/offscreen-sprite.ts b/packages/av-cliper/src/sprite/offscreen-sprite.ts
@@ -1,6 +1,7 @@
 import { BaseSprite } from './base-sprite';
 import { IClip } from '../clips';
 import { Log } from '../log';
+import { changePCMPlaybackRate } from '../av-utils';
 
 /**
  * 包装 {@link IClip} 给素材扩展坐标、层级、透明度等信息，用于 {@link Combinator} 在后台合成视频
@@ -47,13 +48,21 @@ export class OffscreenSprite extends BaseSprite {
     audio: Float32Array[];
     done: boolean;
   }> {
-    this.animate(time);
+    const ts = time * this.time.playbackRate;
+    this.animate(ts);
     super._render(ctx);
     const { w, h } = this.rect;
-    const { video, audio, state } = await this.#clip.tick(time);
+    const { video, audio, state } = await this.#clip.tick(ts);
+    let outAudio = audio ?? [];
+    if (audio != null && this.time.playbackRate !== 1) {
+      outAudio = audio.map((pcm) =>
+        changePCMPlaybackRate(pcm, this.time.playbackRate),
+      );
+    }
+
     if (state === 'done') {
       return {
-        audio: audio ?? [],
+        audio: outAudio,
         done: true,
       };
     }
@@ -69,7 +78,7 @@ export class OffscreenSprite extends BaseSprite {
     }
 
     return {
-      audio: audio ?? [],
+      audio: outAudio,
       done: false,
     };
   }

diff --git a/packages/av-cliper/src/sprite/visible-sprite.ts b/packages/av-cliper/src/sprite/visible-sprite.ts
@@ -1,6 +1,7 @@
 import { BaseSprite } from './base-sprite';
 import { IClip } from '../clips';
 import { Log } from '../log';
+import { changePCMPlaybackRate } from '../av-utils';
 
 /**
  * 包装 {@link IClip} 给素材扩展坐标、层级、透明度等信息，用于 {@link [AVCanvas](../../av-canvas/classes/AVCanvas.html)} 响应用户交互
@@ -43,13 +44,18 @@ export class VisibleSprite extends BaseSprite {
     if (this.#ticking) return;
     this.#ticking = true;
     this.#clip
-      .tick(time)
+      .tick(time * this.time.playbackRate)
       .then(({ video, audio }) => {
         if (video != null) {
           this.#lastVf?.close();
           this.#lastVf = video ?? null;
         }
         this.#lastAudio = audio ?? [];
+        if (audio != null && this.time.playbackRate !== 1) {
+          this.#lastAudio = audio.map((pcm) =>
+            changePCMPlaybackRate(pcm, this.time.playbackRate),
+          );
+        }
       })
       .finally(() => {
         this.#ticking = false;

diff --git a/scripts/verify-commit.js b/scripts/verify-commit.js
@@ -6,7 +6,7 @@ const msgPath = resolve('.git/COMMIT_EDITMSG');
 const msg = readFileSync(msgPath, 'utf-8').trim();
 
 const commitRE =
-  /^(Release v)|(Merge branch)|((revert: )?(feat|fix|docs|dx|style|refactor|perf|test|workflow|build|ci|chore|types|wip|release|version)(\(.+\))?: .+)/;
+  /^(Release v)|(Merge .* branch)|((revert: )?(feat|fix|docs|dx|style|refactor|perf|test|workflow|build|ci|chore|types|wip|release|version)(\(.+\))?: .+)/;
 
 if (!commitRE.test(msg)) {
   console.error(