hongshaorou/rustdesk
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

videotoolbox/mediacodec support changing bitrate dynamically (#10117)

Browse Source 
Signed-off-by: 21pages <sunboeasy@gmail.com>
This commit is contained in:
21pages 2024-11-30 11:44:51 +08:00 committed by GitHub
parent e0ed6ee986
commit d60b5a6ca0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 352 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/bridge.yml vendored
View File

@ -73,7 +73,7 @@ jobs:
- name: Install flutter rust bridge deps
shell: bash
run: |
cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid"
cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid" --locked
pushd flutter && sed -i -e 's/extended_text: 14.0.0/extended_text: 13.0.0/g' pubspec.yaml && flutter pub get && popd
- name: Run flutter rust bridge

2
.github/workflows/flutter-build.yml vendored
View File

@ -1032,7 +1032,7 @@ jobs:
ANDROID_NDK_ROOT: ${{ steps.setup-ndk.outputs.ndk-path }}
run: |
rustup target add ${{ matrix.job.target }}
cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }}
cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }} --locked
case ${{ matrix.job.target }} in
aarch64-linux-android)
./flutter/ndk_arm64.sh

6
.github/workflows/playground.yml vendored
View File

@ -149,7 +149,7 @@ jobs:
shell: bash
run: |
sed -i '' 's/3.1.0/2.17.0/g' flutter/pubspec.yaml;
cargo install flutter_rust_bridge_codegen --version ${{ matrix.job.bridge }} --features "uuid"
cargo install flutter_rust_bridge_codegen --version ${{ matrix.job.bridge }} --features "uuid" --locked
# below works for mac to make buildable on 3.13.9
# pushd flutter/lib; find . -name "*.dart" | xargs -I{} sed -i '' 's/textScaler: TextScaler.linear(\(.*\)),/textScaleFactor: \1,/g' {}; popd;
pushd flutter && flutter pub get && popd
@ -302,7 +302,7 @@ jobs:
- name: Install flutter rust bridge deps
run: |
git config --global core.longpaths true
cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid"
cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid" --locked
sed -i 's/uni_links_desktop/#uni_links_desktop/g' flutter/pubspec.yaml
pushd flutter/lib; find . | grep dart | xargs sed -i 's/textScaler: TextScaler.linear(\(.*\)),/textScaleFactor: \1,/g'; popd;
pushd flutter ; flutter pub get ; popd
@ -347,7 +347,7 @@ jobs:
ANDROID_NDK_ROOT: ${{ steps.setup-ndk.outputs.ndk-path }}
run: |
rustup target add ${{ matrix.job.target }}
cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }}
cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }} --locked
case ${{ matrix.job.target }} in
aarch64-linux-android)
./flutter/ndk_arm64.sh

8
libs/scrap/src/common/hwcodec.rs
View File

@ -193,15 +193,11 @@ impl EncoderApi for HwRamEncoder {
}
fn support_abr(&self) -> bool {
["qsv", "vaapi", "mediacodec", "videotoolbox"]
.iter()
.all(|&x| !self.config.name.contains(x))
["qsv", "vaapi"].iter().all(|&x| !self.config.name.contains(x))
}
fn support_changing_quality(&self) -> bool {
["vaapi", "mediacodec", "videotoolbox"]
.iter()
.all(|&x| !self.config.name.contains(x))
["vaapi"].iter().all(|&x| !self.config.name.contains(x))
}
fn latency_free(&self) -> bool {

84
res/vcpkg/ffmpeg/patch/0004-videotoolbox-changing-bitrate.patch Normal file
View File

@ -0,0 +1,84 @@
From 7f12898fe8fd12c1042c98b34825ab2eda89e54d Mon Sep 17 00:00:00 2001
From: 21pages <sunboeasy@gmail.com>
Date: Sun, 24 Nov 2024 12:58:39 +0800
Subject: [PATCH 1/2] videotoolbox changing bitrate
Signed-off-by: 21pages <sunboeasy@gmail.com>
---
libavcodec/videotoolboxenc.c | 39 ++++++++++++++++++++++++++++++++++++
1 file changed, 39 insertions(+)
diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
index 5ea9afee22..89c927cdcc 100644
--- a/libavcodec/videotoolboxenc.c
+++ b/libavcodec/videotoolboxenc.c
@@ -278,6 +278,8 @@ typedef struct VTEncContext {
int max_slice_bytes;
int power_efficient;
int max_ref_frames;
+
+ int last_bit_rate;
} VTEncContext;
static int vt_dump_encoder(AVCodecContext *avctx)
@@ -1174,6 +1176,7 @@ static int vtenc_create_encoder(AVCodecContext *avctx,
int64_t one_second_value = 0;
void *nums[2];
+ vtctx->last_bit_rate = bit_rate;
int status = VTCompressionSessionCreate(kCFAllocatorDefault,
avctx->width,
avctx->height,
@@ -2618,6 +2621,41 @@ static int vtenc_send_frame(AVCodecContext *avctx,
return 0;
}
+static void update_config(AVCodecContext *avctx)
+{
+ VTEncContext *vtctx = avctx->priv_data;
+
+ if (avctx->codec_id != AV_CODEC_ID_PRORES) {
+ if (avctx->bit_rate != vtctx->last_bit_rate) {
+ av_log(avctx, AV_LOG_INFO, "Setting bit rate to %d\n", avctx->bit_rate);
+ vtctx->last_bit_rate = avctx->bit_rate;
+ SInt32 bit_rate = avctx->bit_rate;
+ CFNumberRef bit_rate_num = CFNumberCreate(kCFAllocatorDefault,
+ kCFNumberSInt32Type,
+ &bit_rate);
+ if (!bit_rate_num) return;
+
+ if (vtctx->constant_bit_rate) {
+ int status = VTSessionSetProperty(vtctx->session,
+ compat_keys.kVTCompressionPropertyKey_ConstantBitRate,
+ bit_rate_num);
+ if (status == kVTPropertyNotSupportedErr) {
+ av_log(avctx, AV_LOG_ERROR, "Error: -constant_bit_rate true is not supported by the encoder.\n");
+ }
+ } else {
+ int status = VTSessionSetProperty(vtctx->session,
+ kVTCompressionPropertyKey_AverageBitRate,
+ bit_rate_num);
+ if (!status) {
+ av_log(avctx, AV_LOG_ERROR, "Error: cannot set average bit rate: %d\n", status);
+ }
+ }
+
+ CFRelease(bit_rate_num);
+ }
+ }
+}
+
static av_cold int vtenc_frame(
AVCodecContext *avctx,
AVPacket *pkt,
@@ -2630,6 +2668,7 @@ static av_cold int vtenc_frame(
CMSampleBufferRef buf = NULL;
ExtraSEI *sei = NULL;
+ update_config(avctx);
if (frame) {
status = vtenc_send_frame(avctx, vtctx, frame);
--
2.43.0.windows.1

259
res/vcpkg/ffmpeg/patch/0005-mediacodec-changing-bitrate.patch Normal file
View File

@ -0,0 +1,259 @@
From fb5cc7909a9b288f6bd13c75992b66ed257ab019 Mon Sep 17 00:00:00 2001
From: 21pages <sunboeasy@gmail.com>
Date: Sun, 24 Nov 2024 14:17:39 +0800
Subject: [PATCH 2/2] mediacodec changing bitrate
Signed-off-by: 21pages <sunboeasy@gmail.com>
---
libavcodec/mediacodec_wrapper.c | 96 +++++++++++++++++++++++++++++++++
libavcodec/mediacodec_wrapper.h | 7 +++
libavcodec/mediacodecenc.c | 18 +++++++
3 files changed, 121 insertions(+)
diff --git a/libavcodec/mediacodec_wrapper.c b/libavcodec/mediacodec_wrapper.c
index 306359071e..44fdd71869 100644
--- a/libavcodec/mediacodec_wrapper.c
+++ b/libavcodec/mediacodec_wrapper.c
@@ -35,6 +35,8 @@
#include "ffjni.h"
#include "mediacodec_wrapper.h"
+#define PARAMETER_KEY_VIDEO_BITRATE "video-bitrate"
+
struct JNIAMediaCodecListFields {
jclass mediacodec_list_class;
@@ -195,6 +197,8 @@ struct JNIAMediaCodecFields {
jmethodID set_input_surface_id;
jmethodID signal_end_of_input_stream_id;
+ jmethodID set_parameters_id;
+
jclass mediainfo_class;
jmethodID init_id;
@@ -248,6 +252,8 @@ static const struct FFJniField jni_amediacodec_mapping[] = {
{ "android/media/MediaCodec", "setInputSurface", "(Landroid/view/Surface;)V", FF_JNI_METHOD, OFFSET(set_input_surface_id), 0 },
{ "android/media/MediaCodec", "signalEndOfInputStream", "()V", FF_JNI_METHOD, OFFSET(signal_end_of_input_stream_id), 0 },
+ { "android/media/MediaCodec", "setParameters", "(Landroid/os/Bundle;)V", FF_JNI_METHOD, OFFSET(set_parameters_id), 0 },
+
{ "android/media/MediaCodec$BufferInfo", NULL, NULL, FF_JNI_CLASS, OFFSET(mediainfo_class), 1 },
{ "android/media/MediaCodec.BufferInfo", "<init>", "()V", FF_JNI_METHOD, OFFSET(init_id), 1 },
@@ -292,6 +298,24 @@ typedef struct FFAMediaCodecJni {
static const FFAMediaCodec media_codec_jni;
+struct JNIABundleFields
+{
+ jclass bundle_class;
+ jmethodID init_id;
+ jmethodID put_int_id;
+};
+
+#define OFFSET(x) offsetof(struct JNIABundleFields, x)
+static const struct FFJniField jni_abundle_mapping[] = {
+ { "android/os/Bundle", NULL, NULL, FF_JNI_CLASS, OFFSET(bundle_class), 1 },
+
+ { "android/os/Bundle", "<init>", "()V", FF_JNI_METHOD, OFFSET(init_id), 1 },
+ { "android/os/Bundle", "putInt", "(Ljava/lang/String;I)V", FF_JNI_METHOD, OFFSET(put_int_id), 1 },
+
+ { NULL }
+};
+#undef OFFSET
+
#define JNI_GET_ENV_OR_RETURN(env, log_ctx, ret) do { \
(env) = ff_jni_get_env(log_ctx); \
if (!(env)) { \
@@ -1761,6 +1785,64 @@ static int mediacodec_jni_signalEndOfInputStream(FFAMediaCodec *ctx)
return 0;
}
+static int mediacodec_jni_setParameter(FFAMediaCodec *ctx, const char* name, int value)
+{
+ JNIEnv *env = NULL;
+ struct JNIABundleFields jfields = { 0 };
+ jobject object = NULL;
+ jstring key = NULL;
+ FFAMediaCodecJni *codec = (FFAMediaCodecJni *)ctx;
+ void *log_ctx = codec;
+ int ret = -1;
+
+ JNI_GET_ENV_OR_RETURN(env, codec, AVERROR_EXTERNAL);
+
+ if (ff_jni_init_jfields(env, &jfields, jni_abundle_mapping, 0, log_ctx) < 0) {
+ av_log(log_ctx, AV_LOG_ERROR, "Failed to init jfields\n");
+ goto fail;
+ }
+
+ object = (*env)->NewObject(env, jfields.bundle_class, jfields.init_id);
+ if (!object) {
+ av_log(log_ctx, AV_LOG_ERROR, "Failed to create bundle object\n");
+ goto fail;
+ }
+
+ key = ff_jni_utf_chars_to_jstring(env, name, log_ctx);
+ if (!key) {
+ av_log(log_ctx, AV_LOG_ERROR, "Failed to convert key to jstring\n");
+ goto fail;
+ }
+
+ (*env)->CallVoidMethod(env, object, jfields.put_int_id, key, value);
+ if (ff_jni_exception_check(env, 1, log_ctx) < 0) {
+ goto fail;
+ }
+
+ (*env)->CallVoidMethod(env, codec->object, codec->jfields.set_parameters_id, object);
+ if (ff_jni_exception_check(env, 1, log_ctx) < 0) {
+ goto fail;
+ }
+
+ ret = 0;
+
+fail:
+ if (key) {
+ (*env)->DeleteLocalRef(env, key);
+ }
+ if (object) {
+ (*env)->DeleteLocalRef(env, object);
+ }
+ ff_jni_reset_jfields(env, &jfields, jni_abundle_mapping, 0, log_ctx);
+
+ return ret;
+}
+
+static int mediacodec_jni_setDynamicBitrate(FFAMediaCodec *ctx, int bitrate)
+{
+ return mediacodec_jni_setParameter(ctx, PARAMETER_KEY_VIDEO_BITRATE, bitrate);
+}
+
static const FFAMediaFormat media_format_jni = {
.class = &amediaformat_class,
@@ -1820,6 +1902,8 @@ static const FFAMediaCodec media_codec_jni = {
.getConfigureFlagEncode = mediacodec_jni_getConfigureFlagEncode,
.cleanOutputBuffers = mediacodec_jni_cleanOutputBuffers,
.signalEndOfInputStream = mediacodec_jni_signalEndOfInputStream,
+
+ .setDynamicBitrate = mediacodec_jni_setDynamicBitrate,
};
typedef struct FFAMediaFormatNdk {
@@ -1893,6 +1977,8 @@ typedef struct FFAMediaCodecNdk {
// Available since API level 26.
media_status_t (*setInputSurface)(AMediaCodec*, ANativeWindow *);
media_status_t (*signalEndOfInputStream)(AMediaCodec *);
+
+ media_status_t (*setParameters)(AMediaCodec *, const AMediaFormat *format);
} FFAMediaCodecNdk;
static const FFAMediaFormat media_format_ndk;
@@ -2154,6 +2240,8 @@ static inline FFAMediaCodec *ndk_codec_create(int method, const char *arg) {
GET_SYMBOL(setInputSurface, 0)
GET_SYMBOL(signalEndOfInputStream, 0)
+ GET_SYMBOL(setParameters, 0)
+
#undef GET_SYMBOL
switch (method) {
@@ -2428,6 +2516,12 @@ static int mediacodec_ndk_signalEndOfInputStream(FFAMediaCodec *ctx)
return 0;
}
+static int mediacodec_ndk_setDynamicBitrate(FFAMediaCodec *ctx, int bitrate)
+{
+ av_log(ctx, AV_LOG_ERROR, "ndk setDynamicBitrate unavailable\n");
+ return -1;
+}
+
static const FFAMediaFormat media_format_ndk = {
.class = &amediaformat_ndk_class,
@@ -2489,6 +2583,8 @@ static const FFAMediaCodec media_codec_ndk = {
.getConfigureFlagEncode = mediacodec_ndk_getConfigureFlagEncode,
.cleanOutputBuffers = mediacodec_ndk_cleanOutputBuffers,
.signalEndOfInputStream = mediacodec_ndk_signalEndOfInputStream,
+
+ .setDynamicBitrate = mediacodec_ndk_setDynamicBitrate,
};
FFAMediaFormat *ff_AMediaFormat_new(int ndk)
diff --git a/libavcodec/mediacodec_wrapper.h b/libavcodec/mediacodec_wrapper.h
index 11a4260497..86c64556ad 100644
--- a/libavcodec/mediacodec_wrapper.h
+++ b/libavcodec/mediacodec_wrapper.h
@@ -219,6 +219,8 @@ struct FFAMediaCodec {
// For encoder with FFANativeWindow as input.
int (*signalEndOfInputStream)(FFAMediaCodec *);
+
+ int (*setDynamicBitrate)(FFAMediaCodec *codec, int bitrate);
};
static inline char *ff_AMediaCodec_getName(FFAMediaCodec *codec)
@@ -343,6 +345,11 @@ static inline int ff_AMediaCodec_signalEndOfInputStream(FFAMediaCodec *codec)
return codec->signalEndOfInputStream(codec);
}
+static inline int ff_AMediaCodec_setDynamicBitrate(FFAMediaCodec *codec, int bitrate)
+{
+ return codec->setDynamicBitrate(codec, bitrate);
+}
+
int ff_Build_SDK_INT(AVCodecContext *avctx);
enum FFAMediaFormatColorRange {
diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c
index d3bf27cb7f..621529d686 100644
--- a/libavcodec/mediacodecenc.c
+++ b/libavcodec/mediacodecenc.c
@@ -73,6 +73,8 @@ typedef struct MediaCodecEncContext {
int bitrate_mode;
int level;
int pts_as_dts;
+
+ int last_bit_rate;
} MediaCodecEncContext;
enum {
@@ -155,6 +157,8 @@ static av_cold int mediacodec_init(AVCodecContext *avctx)
int ret;
int gop;
+ s->last_bit_rate = avctx->bit_rate;
+
if (s->use_ndk_codec < 0)
s->use_ndk_codec = !av_jni_get_java_vm(avctx);
@@ -515,12 +519,26 @@ static int mediacodec_send(AVCodecContext *avctx,
return 0;
}
+static void update_config(AVCodecContext *avctx)
+{
+ MediaCodecEncContext *s = avctx->priv_data;
+ if (avctx->bit_rate != s->last_bit_rate) {
+ s->last_bit_rate = avctx->bit_rate;
+ if (0 != ff_AMediaCodec_setDynamicBitrate(s->codec, avctx->bit_rate)) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to set bitrate to %d\n", avctx->bit_rate);
+ } else {
+ av_log(avctx, AV_LOG_INFO, "Set bitrate to %d\n", avctx->bit_rate);
+ }
+ }
+}
+
static int mediacodec_encode(AVCodecContext *avctx, AVPacket *pkt)
{
MediaCodecEncContext *s = avctx->priv_data;
int ret;
int got_packet = 0;
+ update_config(avctx);
// Return on three case:
// 1. Serious error
// 2. Got a packet success
--
2.43.0.windows.1

2
res/vcpkg/ffmpeg/portfile.cmake
View File

@ -13,6 +13,8 @@ vcpkg_from_github(
patch/0001-avcodec-amfenc-add-query_timeout-option-for-h264-hev.patch
patch/0002-libavcodec-amfenc-reconfig-when-bitrate-change.patch
patch/0003-amf-colorspace.patch
patch/0004-videotoolbox-changing-bitrate.patch
patch/0005-mediacodec-changing-bitrate.patch
)
if(SOURCE_PATH MATCHES " ")