diff --git a/package.json b/package.json index d64617d85..15e0e75b8 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "fast-sha256": "^1.3.0", "libsodium": "^0.7.9", "libsodium-wrappers": "^0.7.9", + "pcm-player": "^0.0.11", "ts-proto": "^1.101.0", "wasm-feature-detect": "^1.2.11", "zstddec": "^0.0.2" diff --git a/src/codec.js b/src/codec.js index e35f72d85..43f992533 100644 --- a/src/codec.js +++ b/src/codec.js @@ -42,11 +42,11 @@ export async function loadVp9(callback) { ); } -export function loadOpus(callback) { +export function loadOpus(callback, channels, rate) { window.OGVLoader.loadClass( "OGVDecoderAudioOpusW", (audioCodecClass) => { - audioCodecClass({ audioFormat: {} }).then((decoder) => { + audioCodecClass({ audioFormat: { channels, rate } }).then((decoder) => { decoder.init(() => { callback(decoder); }) diff --git a/src/connection.ts b/src/connection.ts index 85249f45e..712c1ced4 100644 --- a/src/connection.ts +++ b/src/connection.ts @@ -25,6 +25,7 @@ export default class Connection { _firstFrame: Boolean | undefined; _videoDecoder: any; _audioDecoder: any; + _audioPlayer: any; _password: Uint8Array | undefined; _options: any; _videoTestSpeed: number[]; @@ -58,7 +59,6 @@ export default class Connection { } }, 1); this.loadVideoDecoder(); - this.loadAudioDecoder(); const uri = getDefaultUri(); const ws = new Websock(uri, true); this._ws = ws; @@ -242,7 +242,13 @@ export default class Connection { } else if (msg?.misc) { this.handleMisc(msg?.misc); } else if (msg?.audio_frame) { - // + const dec = this._audioDecoder; + dec.processAudio(msg?.audio_frame.data.slice(0).buffer, (res: any) => { + // ogv.js bug here, audioBuffer always empty + // and ogv.js vpx version is very old (2015), to-do: discard ogv + // let samples = dec.audioBuffer; + // console.log(res, dec); + }); } } } @@ -261,7 +267,6 @@ export default class Connection { clearInterval(this._interval); this._ws?.close(); this._videoDecoder?.close(); - this._audioDecoder?.close(); } refresh() { @@ -423,7 +428,9 @@ export default class Connection { handleMisc(misc: message.Misc) { if (misc.audio_format) { - // + this._audioDecoder?.destroy(); + // this._audioPlayer = globals.newAudioPlayer(misc.audio_format.channels, misc.audio_format.sample_rate); + this.loadAudioDecoder(misc.audio_format.channels, misc.audio_format.sample_rate); } else if (misc.chat_message) { globals.pushEvent("chat", misc.chat_message.text); } else if (misc.permission_info) { @@ -648,12 +655,12 @@ export default class Connection { }); } - loadAudioDecoder() { + loadAudioDecoder(channels: number, sample_rate: number) { this._audioDecoder?.close(); loadOpus((decoder: any) => { this._audioDecoder = decoder; console.log("opus loaded"); - }); + }, channels, sample_rate); } } diff --git a/src/globals.js b/src/globals.js index a7ceaf2b3..a404cedf5 100644 --- a/src/globals.js +++ b/src/globals.js @@ -4,6 +4,7 @@ import { CursorData } from "./message"; import { loadOpus, loadVp9 } from "./codec"; import { checkIfRetry, version } from "./gen_js_from_hbb"; import { initZstd, translate } from "./common"; +import PCMPlayer from "pcm-player"; var currentFrame = undefined; var events = []; @@ -301,3 +302,12 @@ export function getPeers() { return {}; } } + +export function newAudioPlayer(channels, sampleRate) { + return new PCMPlayer({ + encoding: '16bitInt', + channels, + sampleRate, + flushingTime: 2000 + }); +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index af7ff0b7d..cf4caafe5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -249,6 +249,11 @@ path-parse@^1.0.7: resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== +pcm-player@^0.0.11: + version "0.0.11" + resolved "https://registry.yarnpkg.com/pcm-player/-/pcm-player-0.0.11.tgz#1dd0e37d8e1238c10b153079939b1fc88958a9ff" + integrity sha512-+FmX62jiqZa7wDCqSRQ1g3DuU6JNgpymgOLCWhmiE/Lj/M+rOUNqgNwVQX509LdA9dtBtVD3EQQUSp9JqU6upw== + picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c"