videotoolbox/mediacodec support changing bitrate dynamically (#10117)

Signed-off-by: 21pages <sunboeasy@gmail.com>

.github/workflows/bridge.yml

@@ -73,7 +73,7 @@ jobs:
       - name: Install flutter rust bridge deps
         shell: bash
         run: |
-          cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid"
+          cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid" --locked
           pushd flutter && sed -i -e 's/extended_text: 14.0.0/extended_text: 13.0.0/g' pubspec.yaml && flutter pub get && popd
 
       - name: Run flutter rust bridge

.github/workflows/flutter-build.yml

@@ -1032,7 +1032,7 @@ jobs:
           ANDROID_NDK_ROOT: ${{ steps.setup-ndk.outputs.ndk-path }}
         run: |
           rustup target add ${{ matrix.job.target }}
-          cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }}
+          cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }} --locked
           case ${{ matrix.job.target }} in
             aarch64-linux-android)
               ./flutter/ndk_arm64.sh

.github/workflows/playground.yml

@@ -149,7 +149,7 @@ jobs:
         shell: bash
         run: |
           sed -i '' 's/3.1.0/2.17.0/g' flutter/pubspec.yaml;
-          cargo install flutter_rust_bridge_codegen --version ${{ matrix.job.bridge }} --features "uuid"
+          cargo install flutter_rust_bridge_codegen --version ${{ matrix.job.bridge }} --features "uuid" --locked
           # below works for mac to make buildable on 3.13.9
           # pushd flutter/lib; find . -name "*.dart" | xargs -I{} sed -i '' 's/textScaler: TextScaler.linear(\(.*\)),/textScaleFactor: \1,/g' {}; popd;
           pushd flutter && flutter pub get && popd
@@ -302,7 +302,7 @@ jobs:
       - name: Install flutter rust bridge deps
         run: |
           git config --global core.longpaths true
-          cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid"
+          cargo install flutter_rust_bridge_codegen --version ${{ env.FLUTTER_RUST_BRIDGE_VERSION }} --features "uuid" --locked
           sed -i 's/uni_links_desktop/#uni_links_desktop/g' flutter/pubspec.yaml
           pushd flutter/lib;  find . | grep dart | xargs sed -i 's/textScaler: TextScaler.linear(\(.*\)),/textScaleFactor: \1,/g'; popd;
           pushd flutter ; flutter pub get ; popd
@@ -347,7 +347,7 @@ jobs:
           ANDROID_NDK_ROOT: ${{ steps.setup-ndk.outputs.ndk-path }}
         run: |
           rustup target add ${{ matrix.job.target }}
-          cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }}
+          cargo install cargo-ndk --version ${{ env.CARGO_NDK_VERSION }} --locked
           case ${{ matrix.job.target }} in
             aarch64-linux-android)
               ./flutter/ndk_arm64.sh

libs/scrap/src/common/hwcodec.rs

@@ -193,15 +193,11 @@ impl EncoderApi for HwRamEncoder {
     }
 
     fn support_abr(&self) -> bool {
-        ["qsv", "vaapi", "mediacodec", "videotoolbox"]
+        ["qsv", "vaapi"].iter().all(|&x| !self.config.name.contains(x))
-            .iter()
-            .all(|&x| !self.config.name.contains(x))
     }
 
     fn support_changing_quality(&self) -> bool {
-        ["vaapi", "mediacodec", "videotoolbox"]
+        ["vaapi"].iter().all(|&x| !self.config.name.contains(x))
-            .iter()
-            .all(|&x| !self.config.name.contains(x))
     }
 
     fn latency_free(&self) -> bool {

res/vcpkg/ffmpeg/patch/0004-videotoolbox-changing-bitrate.patch

@@ -0,0 +1,84 @@
+From 7f12898fe8fd12c1042c98b34825ab2eda89e54d Mon Sep 17 00:00:00 2001
+From: 21pages <sunboeasy@gmail.com>
+Date: Sun, 24 Nov 2024 12:58:39 +0800
+Subject: [PATCH 1/2] videotoolbox changing bitrate
+
+Signed-off-by: 21pages <sunboeasy@gmail.com>
+---
+ libavcodec/videotoolboxenc.c | 39 ++++++++++++++++++++++++++++++++++++
+ 1 file changed, 39 insertions(+)
+
+diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
+index 5ea9afee22..89c927cdcc 100644
+--- a/libavcodec/videotoolboxenc.c
++++ b/libavcodec/videotoolboxenc.c
+@@ -278,6 +278,8 @@ typedef struct VTEncContext {
+     int max_slice_bytes;
+     int power_efficient;
+     int max_ref_frames;
++
++    int last_bit_rate;
+ } VTEncContext;
+ 
+ static int vt_dump_encoder(AVCodecContext *avctx)
+@@ -1174,6 +1176,7 @@ static int vtenc_create_encoder(AVCodecContext   *avctx,
+     int64_t      one_second_value = 0;
+     void         *nums[2];
+ 
++    vtctx->last_bit_rate = bit_rate;
+     int status = VTCompressionSessionCreate(kCFAllocatorDefault,
+                                             avctx->width,
+                                             avctx->height,
+@@ -2618,6 +2621,41 @@ static int vtenc_send_frame(AVCodecContext *avctx,
+     return 0;
+ }
+ 
++static void update_config(AVCodecContext *avctx)
++{
++    VTEncContext *vtctx = avctx->priv_data;
++    
++    if (avctx->codec_id != AV_CODEC_ID_PRORES) {
++        if (avctx->bit_rate != vtctx->last_bit_rate) {
++            av_log(avctx, AV_LOG_INFO, "Setting bit rate to %d\n", avctx->bit_rate);
++            vtctx->last_bit_rate = avctx->bit_rate;
++            SInt32 bit_rate = avctx->bit_rate;
++            CFNumberRef bit_rate_num = CFNumberCreate(kCFAllocatorDefault,
++                                      kCFNumberSInt32Type,
++                                      &bit_rate);
++            if (!bit_rate_num) return;
++
++            if (vtctx->constant_bit_rate) {
++                int status = VTSessionSetProperty(vtctx->session,
++                                            compat_keys.kVTCompressionPropertyKey_ConstantBitRate,
++                                            bit_rate_num);
++                if (status == kVTPropertyNotSupportedErr) {
++                    av_log(avctx, AV_LOG_ERROR, "Error: -constant_bit_rate true is not supported by the encoder.\n");
++                }
++            } else {
++                int status = VTSessionSetProperty(vtctx->session,
++                                            kVTCompressionPropertyKey_AverageBitRate,
++                                            bit_rate_num);
++                if (!status) {
++                    av_log(avctx, AV_LOG_ERROR, "Error: cannot set average bit rate: %d\n", status);
++                }
++            }
++
++            CFRelease(bit_rate_num);
++        }
++    }
++}
++
+ static av_cold int vtenc_frame(
+     AVCodecContext *avctx,
+     AVPacket       *pkt,
+@@ -2630,6 +2668,7 @@ static av_cold int vtenc_frame(
+     CMSampleBufferRef buf = NULL;
+     ExtraSEI *sei = NULL;
+ 
++    update_config(avctx);
+     if (frame) {
+         status = vtenc_send_frame(avctx, vtctx, frame);
+ 
+-- 
+2.43.0.windows.1
+

res/vcpkg/ffmpeg/patch/0005-mediacodec-changing-bitrate.patch

@@ -0,0 +1,259 @@
+From fb5cc7909a9b288f6bd13c75992b66ed257ab019 Mon Sep 17 00:00:00 2001
+From: 21pages <sunboeasy@gmail.com>
+Date: Sun, 24 Nov 2024 14:17:39 +0800
+Subject: [PATCH 2/2] mediacodec changing bitrate
+
+Signed-off-by: 21pages <sunboeasy@gmail.com>
+---
+ libavcodec/mediacodec_wrapper.c | 96 +++++++++++++++++++++++++++++++++
+ libavcodec/mediacodec_wrapper.h |  7 +++
+ libavcodec/mediacodecenc.c      | 18 +++++++
+ 3 files changed, 121 insertions(+)
+
+diff --git a/libavcodec/mediacodec_wrapper.c b/libavcodec/mediacodec_wrapper.c
+index 306359071e..44fdd71869 100644
+--- a/libavcodec/mediacodec_wrapper.c
++++ b/libavcodec/mediacodec_wrapper.c
+@@ -35,6 +35,8 @@
+ #include "ffjni.h"
+ #include "mediacodec_wrapper.h"
+ 
++#define PARAMETER_KEY_VIDEO_BITRATE "video-bitrate"
++
+ struct JNIAMediaCodecListFields {
+ 
+     jclass mediacodec_list_class;
+@@ -195,6 +197,8 @@ struct JNIAMediaCodecFields {
+     jmethodID set_input_surface_id;
+     jmethodID signal_end_of_input_stream_id;
+ 
++    jmethodID set_parameters_id;
++
+     jclass mediainfo_class;
+ 
+     jmethodID init_id;
+@@ -248,6 +252,8 @@ static const struct FFJniField jni_amediacodec_mapping[] = {
+         { "android/media/MediaCodec", "setInputSurface", "(Landroid/view/Surface;)V", FF_JNI_METHOD, OFFSET(set_input_surface_id), 0 },
+         { "android/media/MediaCodec", "signalEndOfInputStream", "()V", FF_JNI_METHOD, OFFSET(signal_end_of_input_stream_id), 0 },
+ 
++        { "android/media/MediaCodec", "setParameters", "(Landroid/os/Bundle;)V", FF_JNI_METHOD, OFFSET(set_parameters_id), 0 },
++
+     { "android/media/MediaCodec$BufferInfo", NULL, NULL, FF_JNI_CLASS, OFFSET(mediainfo_class), 1 },
+ 
+         { "android/media/MediaCodec.BufferInfo", "<init>", "()V", FF_JNI_METHOD, OFFSET(init_id), 1 },
+@@ -292,6 +298,24 @@ typedef struct FFAMediaCodecJni {
+ 
+ static const FFAMediaCodec media_codec_jni;
+ 
++struct JNIABundleFields
++{
++  jclass bundle_class;
++  jmethodID init_id;
++  jmethodID put_int_id;
++};
++
++#define OFFSET(x) offsetof(struct JNIABundleFields, x)
++static const struct FFJniField jni_abundle_mapping[] = {
++    { "android/os/Bundle", NULL, NULL, FF_JNI_CLASS, OFFSET(bundle_class), 1 },
++
++        { "android/os/Bundle", "<init>", "()V", FF_JNI_METHOD, OFFSET(init_id), 1 },
++        { "android/os/Bundle", "putInt", "(Ljava/lang/String;I)V", FF_JNI_METHOD, OFFSET(put_int_id), 1 },
++
++    { NULL }
++};
++#undef OFFSET
++
+ #define JNI_GET_ENV_OR_RETURN(env, log_ctx, ret) do {              \
+     (env) = ff_jni_get_env(log_ctx);                               \
+     if (!(env)) {                                                  \
+@@ -1761,6 +1785,64 @@ static int mediacodec_jni_signalEndOfInputStream(FFAMediaCodec *ctx)
+     return 0;
+ }
+ 
++static int mediacodec_jni_setParameter(FFAMediaCodec *ctx, const char* name, int value)
++{
++    JNIEnv *env = NULL;
++    struct JNIABundleFields jfields = { 0 };
++    jobject object = NULL;
++    jstring key = NULL;
++    FFAMediaCodecJni *codec = (FFAMediaCodecJni *)ctx;
++    void *log_ctx = codec;
++    int ret = -1;
++
++    JNI_GET_ENV_OR_RETURN(env, codec, AVERROR_EXTERNAL);
++
++    if (ff_jni_init_jfields(env, &jfields, jni_abundle_mapping, 0, log_ctx) < 0) {
++        av_log(log_ctx, AV_LOG_ERROR, "Failed to init jfields\n");
++        goto fail;
++    }
++
++    object = (*env)->NewObject(env, jfields.bundle_class, jfields.init_id);
++    if (!object) {
++        av_log(log_ctx, AV_LOG_ERROR, "Failed to create bundle object\n");
++        goto fail;
++    }
++
++    key = ff_jni_utf_chars_to_jstring(env, name, log_ctx);
++    if (!key) {
++        av_log(log_ctx, AV_LOG_ERROR, "Failed to convert key to jstring\n");
++        goto fail;
++    }
++
++    (*env)->CallVoidMethod(env, object, jfields.put_int_id, key, value);
++    if (ff_jni_exception_check(env, 1, log_ctx) < 0) {
++        goto fail;
++    }
++
++    (*env)->CallVoidMethod(env, codec->object, codec->jfields.set_parameters_id, object);
++    if (ff_jni_exception_check(env, 1, log_ctx) < 0) {
++        goto fail;
++    }
++
++    ret = 0;
++
++fail:
++    if (key) {
++        (*env)->DeleteLocalRef(env, key);
++    }
++    if (object) {
++        (*env)->DeleteLocalRef(env, object);
++    }
++    ff_jni_reset_jfields(env, &jfields, jni_abundle_mapping, 0, log_ctx);
++
++    return ret;
++}
++
++static int mediacodec_jni_setDynamicBitrate(FFAMediaCodec *ctx, int bitrate)
++{
++    return mediacodec_jni_setParameter(ctx, PARAMETER_KEY_VIDEO_BITRATE, bitrate);
++}
++
+ static const FFAMediaFormat media_format_jni = {
+     .class = &amediaformat_class,
+ 
+@@ -1820,6 +1902,8 @@ static const FFAMediaCodec media_codec_jni = {
+     .getConfigureFlagEncode = mediacodec_jni_getConfigureFlagEncode,
+     .cleanOutputBuffers = mediacodec_jni_cleanOutputBuffers,
+     .signalEndOfInputStream = mediacodec_jni_signalEndOfInputStream,
++
++    .setDynamicBitrate = mediacodec_jni_setDynamicBitrate,
+ };
+ 
+ typedef struct FFAMediaFormatNdk {
+@@ -1893,6 +1977,8 @@ typedef struct FFAMediaCodecNdk {
+     // Available since API level 26.
+     media_status_t (*setInputSurface)(AMediaCodec*, ANativeWindow *);
+     media_status_t (*signalEndOfInputStream)(AMediaCodec *);
++
++    media_status_t (*setParameters)(AMediaCodec *, const AMediaFormat *format);
+ } FFAMediaCodecNdk;
+ 
+ static const FFAMediaFormat media_format_ndk;
+@@ -2154,6 +2240,8 @@ static inline FFAMediaCodec *ndk_codec_create(int method, const char *arg) {
+     GET_SYMBOL(setInputSurface, 0)
+     GET_SYMBOL(signalEndOfInputStream, 0)
+ 
++    GET_SYMBOL(setParameters, 0)
++
+ #undef GET_SYMBOL
+ 
+     switch (method) {
+@@ -2428,6 +2516,12 @@ static int mediacodec_ndk_signalEndOfInputStream(FFAMediaCodec *ctx)
+     return 0;
+ }
+ 
++static int mediacodec_ndk_setDynamicBitrate(FFAMediaCodec *ctx, int bitrate)
++{
++    av_log(ctx, AV_LOG_ERROR, "ndk setDynamicBitrate unavailable\n");
++    return -1;
++}
++
+ static const FFAMediaFormat media_format_ndk = {
+     .class = &amediaformat_ndk_class,
+ 
+@@ -2489,6 +2583,8 @@ static const FFAMediaCodec media_codec_ndk = {
+     .getConfigureFlagEncode = mediacodec_ndk_getConfigureFlagEncode,
+     .cleanOutputBuffers = mediacodec_ndk_cleanOutputBuffers,
+     .signalEndOfInputStream = mediacodec_ndk_signalEndOfInputStream,
++
++    .setDynamicBitrate = mediacodec_ndk_setDynamicBitrate,
+ };
+ 
+ FFAMediaFormat *ff_AMediaFormat_new(int ndk)
+diff --git a/libavcodec/mediacodec_wrapper.h b/libavcodec/mediacodec_wrapper.h
+index 11a4260497..86c64556ad 100644
+--- a/libavcodec/mediacodec_wrapper.h
++++ b/libavcodec/mediacodec_wrapper.h
+@@ -219,6 +219,8 @@ struct FFAMediaCodec {
+ 
+     // For encoder with FFANativeWindow as input.
+     int (*signalEndOfInputStream)(FFAMediaCodec *);
++
++    int (*setDynamicBitrate)(FFAMediaCodec *codec, int bitrate);
+ };
+ 
+ static inline char *ff_AMediaCodec_getName(FFAMediaCodec *codec)
+@@ -343,6 +345,11 @@ static inline int ff_AMediaCodec_signalEndOfInputStream(FFAMediaCodec *codec)
+     return codec->signalEndOfInputStream(codec);
+ }
+ 
++static inline int ff_AMediaCodec_setDynamicBitrate(FFAMediaCodec *codec, int bitrate)
++{
++    return codec->setDynamicBitrate(codec, bitrate);
++}
++
+ int ff_Build_SDK_INT(AVCodecContext *avctx);
+ 
+ enum FFAMediaFormatColorRange {
+diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c
+index d3bf27cb7f..621529d686 100644
+--- a/libavcodec/mediacodecenc.c
++++ b/libavcodec/mediacodecenc.c
+@@ -73,6 +73,8 @@ typedef struct MediaCodecEncContext {
+     int bitrate_mode;
+     int level;
+     int pts_as_dts;
++
++    int last_bit_rate;
+ } MediaCodecEncContext;
+ 
+ enum {
+@@ -155,6 +157,8 @@ static av_cold int mediacodec_init(AVCodecContext *avctx)
+     int ret;
+     int gop;
+ 
++    s->last_bit_rate = avctx->bit_rate;
++
+     if (s->use_ndk_codec < 0)
+         s->use_ndk_codec = !av_jni_get_java_vm(avctx);
+ 
+@@ -515,12 +519,26 @@ static int mediacodec_send(AVCodecContext *avctx,
+     return 0;
+ }
+ 
++static void update_config(AVCodecContext *avctx)
++{
++    MediaCodecEncContext *s = avctx->priv_data;
++    if (avctx->bit_rate != s->last_bit_rate) {
++        s->last_bit_rate = avctx->bit_rate;
++        if (0 != ff_AMediaCodec_setDynamicBitrate(s->codec, avctx->bit_rate)) {
++            av_log(avctx, AV_LOG_ERROR, "Failed to set bitrate to %d\n", avctx->bit_rate);
++        } else {
++            av_log(avctx, AV_LOG_INFO, "Set bitrate to %d\n", avctx->bit_rate);
++        }
++    }
++}
++
+ static int mediacodec_encode(AVCodecContext *avctx, AVPacket *pkt)
+ {
+     MediaCodecEncContext *s = avctx->priv_data;
+     int ret;
+     int got_packet = 0;
+ 
++    update_config(avctx);
+     // Return on three case:
+     // 1. Serious error
+     // 2. Got a packet success
+-- 
+2.43.0.windows.1
+

res/vcpkg/ffmpeg/portfile.cmake

@@ -13,6 +13,8 @@ vcpkg_from_github(
     patch/0001-avcodec-amfenc-add-query_timeout-option-for-h264-hev.patch
     patch/0002-libavcodec-amfenc-reconfig-when-bitrate-change.patch
     patch/0003-amf-colorspace.patch
+    patch/0004-videotoolbox-changing-bitrate.patch
+    patch/0005-mediacodec-changing-bitrate.patch
 )
 
 if(SOURCE_PATH MATCHES " ")