video service 0.1

This commit is contained in:
csf 2022-01-20 15:57:54 +08:00
parent eeb30aa0d1
commit 668b34c228
8 changed files with 542 additions and 16 deletions

View File

@ -33,7 +33,7 @@ apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"
android {
compileSdkVersion 30
// ndkVersion '22.1.7171670' // * 使 NDK无法自动选择 使NDK版本 [CSF]
sourceSets {
main.java.srcDirs += 'src/main/kotlin'
}

View File

@ -28,17 +28,24 @@
android:resource="@drawable/launch_background"
/>
<intent-filter>
<action android:name="android.intent.action.MAIN"/>
<category android:name="android.intent.category.LAUNCHER"/>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
<service
android:name=".MainService"
android:enabled="true"
android:foregroundServiceType="mediaProjection"/>
<!-- Don't delete the meta-data below.
This is used by the Flutter tool to generate GeneratedPluginRegistrant.java -->
<meta-data
android:name="flutterEmbedding"
android:value="2" />
</application>
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
</application>
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
</manifest>

View File

@ -1,6 +1,106 @@
package com.carriez.flutter_hbb
import android.app.Activity
import android.content.Context
import android.content.Intent
import android.media.projection.MediaProjectionManager
import android.os.Build
import android.os.Bundle
import android.os.PersistableBundle
import android.util.Log
import androidx.annotation.RequiresApi
import io.flutter.embedding.android.FlutterActivity
import io.flutter.embedding.engine.FlutterEngine
import io.flutter.plugin.common.MethodCall
import io.flutter.plugin.common.MethodChannel
import java.nio.ByteBuffer
import kotlin.concurrent.thread
class MainActivity: FlutterActivity() {
class MainActivity : FlutterActivity() {
private val channelTag = "mChannel"
private var mediaProjectionResultIntent: Intent? = null
private val requestCode = 1
private val buf = ByteBuffer.allocate(16)
init {
System.loadLibrary("rustdesk")
}
override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
super.configureFlutterEngine(flutterEngine) // 必要 否则无法正确初始化flutter
MethodChannel(
flutterEngine.dartExecutor.binaryMessenger,
channelTag
).setMethodCallHandler { call, result ->
when (call.method) {
"getPer" -> {
Log.d(channelTag, "event from flutter,getPer")
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
getMediaProjection()
}
result.success(true)
}
"startSer" -> {
mStarService()
result.success(true)
}
"stopSer" -> {
mStopService()
result.success(true)
}
else -> {}
}
}
}
@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
private fun getMediaProjection() {
val mMediaProjectionManager =
getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
val mIntent = mMediaProjectionManager.createScreenCaptureIntent()
startActivityForResult(mIntent, requestCode)
}
private fun mStarService() {
if (mediaProjectionResultIntent == null) {
Log.w(channelTag, "mediaProjectionResultIntent is null")
return
}
Log.d(channelTag, "Start a service")
val serviceIntent = Intent(this, MainService::class.java)
serviceIntent.action = START_SERVICE
serviceIntent.putExtra(EXTRA_MP_DATA, mediaProjectionResultIntent)
// TEST api < O
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
startForegroundService(serviceIntent)
} else {
startService(serviceIntent)
}
}
private fun mStopService() {
Log.d(channelTag, "Stop service")
val serviceIntent = Intent(this, MainService::class.java)
serviceIntent.action = STOP_SERVICE
// TEST api < O
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
startForegroundService(serviceIntent)
} else {
startService(serviceIntent)
}
}
override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
super.onActivityResult(requestCode, resultCode, data)
if (resultCode == Activity.RESULT_OK && data != null) {
Log.d(channelTag, "got mediaProjectionResultIntent ok")
mediaProjectionResultIntent = data
}
}
}

View File

@ -0,0 +1,248 @@
package com.carriez.flutter_hbb
import android.annotation.SuppressLint
import android.app.*
import android.content.Context
import android.content.Intent
import android.graphics.Color
import android.graphics.PixelFormat
import android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC
import android.media.*
import android.media.projection.MediaProjection
import android.media.projection.MediaProjectionManager
import android.os.Build
import android.os.IBinder
import android.util.Log
import android.view.Surface
import android.widget.Toast
import androidx.annotation.RequiresApi
import androidx.core.app.NotificationCompat
import androidx.core.app.NotificationCompat.PRIORITY_MIN
import java.nio.ByteBuffer
import java.util.concurrent.Executors
const val EXTRA_MP_DATA = "mp_intent"
const val START_SERVICE = "start_service"
const val STOP_SERVICE = "stop_service"
const val MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_VP9
// 获取手机尺寸 建立连接时发送尺寸和基础信息
const val FIXED_WIDTH = 500 // 编码器有上限
const val FIXED_HEIGHT = 1000
const val M_KEY_BIT_RATE = 1024_000
const val M_KEY_FRAME_RATE = 30
class MainService : Service() {
fun rustGetRaw():ByteArray{
return rawByteArray!!
}
external fun init(ctx:Context)
init {
System.loadLibrary("rustdesk")
}
private val logTag = "LOG_SERVICE"
private var mMediaProjection: MediaProjection? = null
private var surface: Surface? = null
private val singleThread = Executors.newSingleThreadExecutor()
private var mEncoder: MediaCodec? = null
private var rawByteArray :ByteArray? = null
override fun onBind(intent: Intent): IBinder? {
return null
}
@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
Log.d("whichService", "this service:${Thread.currentThread()}")
init(this) // 注册到rust
if (intent?.action == START_SERVICE) {
Log.d(logTag, "service starting:${startId}:${Thread.currentThread()}")
createNotification()
val mMediaProjectionManager =
getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
mMediaProjection = intent.getParcelableExtra<Intent>(EXTRA_MP_DATA)?.let {
mMediaProjectionManager.getMediaProjection(Activity.RESULT_OK, it)
}
Log.d(logTag, "获取mMediaProjection成功$mMediaProjection")
if (testSupport()) {
startRecorder()
} else {
Toast.makeText(this, "此设备不支持:$MIME_TYPE", Toast.LENGTH_SHORT).show()
stopSelf(startId)
}
} else if (intent?.action == STOP_SERVICE) {
mEncoder?.let {
try {
Log.d(logTag, "正在释放encoder")
it.signalEndOfInputStream()
it.stop()
it.release()
} catch (e: Exception) {
null
}
}
stopSelf()
}
return super.onStartCommand(intent, flags, startId)
}
lateinit var mImageReader:ImageReader // * 注意 这里要成为成员变量,防止被回收 https://www.cnblogs.com/yongdaimi/p/11004560.html
@SuppressLint("WrongConstant")
@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
private fun startRecorder() {
Log.d(logTag, "startRecorder")
mMediaProjection?.let { mp ->
// 使用原始数据
mImageReader =
ImageReader.newInstance(FIXED_WIDTH, FIXED_HEIGHT, PixelFormat.RGBA_8888, 2) // 至少是2
mImageReader.setOnImageAvailableListener({ imageReader: ImageReader ->
Log.d(logTag, "on image")
try {
imageReader.acquireLatestImage().use { image ->
if (image == null) return@setOnImageAvailableListener
val planes = image.planes
val buffer = planes[0].buffer
buffer.rewind()
// 这里注意 处理不当会引发OOM
if (rawByteArray == null){
rawByteArray = ByteArray(buffer.capacity())
buffer.get(rawByteArray!!)
}else{
buffer.get(rawByteArray!!)
}
}
} catch (ignored: java.lang.Exception) {
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
imageReader.discardFreeBuffers()
}
}, null)
mp.createVirtualDisplay(
"rustdesk test",
FIXED_WIDTH, FIXED_HEIGHT, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC,
mImageReader.surface, null, null
)
// 使用内置编码器
// createMediaCodec()
// mEncoder?.let {
// surface = it.createInputSurface()
// if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
// surface!!.setFrameRate(1F, FRAME_RATE_COMPATIBILITY_DEFAULT)
// }
// it.setCallback(cb)
// it.start()
// mp.createVirtualDisplay(
// "rustdesk test",
// FIXED_WIDTH, FIXED_HEIGHT, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC,
// surface, null, null
// )
// }
} ?: let {
Log.d(logTag, "startRecorder fail,mMediaProjection is null")
}
}
private val cb: MediaCodec.Callback = @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
object : MediaCodec.Callback() {
override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {}
override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {}
override fun onOutputBufferAvailable(
codec: MediaCodec,
index: Int,
info: MediaCodec.BufferInfo
) {
codec.getOutputBuffer(index)?.let { buf ->
singleThread.execute {
// TODO 优化内存使用方式
val byteArray = ByteArray(buf.limit())
buf.get(byteArray)
sendVp9(byteArray)
codec.releaseOutputBuffer(index, false)
}
}
}
override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
Log.e(logTag, "MediaCodec.Callback error:$e")
}
}
external fun sendRaw(buf: ByteBuffer)
external fun sendVp9(data: ByteArray)
@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
private fun testSupport(): Boolean {
val res = MediaCodecList(MediaCodecList.ALL_CODECS)
.findEncoderForFormat(
MediaFormat.createVideoFormat(
MediaFormat.MIMETYPE_VIDEO_VP9,
FIXED_WIDTH,
FIXED_HEIGHT
)
)
return res?.let {
true
} ?: let {
false
}
}
private fun createMediaCodec() {
Log.d(logTag, "MediaFormat.MIMETYPE_VIDEO_VP9 :$MIME_TYPE")
mEncoder = MediaCodec.createEncoderByType(MIME_TYPE)
val mFormat = MediaFormat.createVideoFormat(MIME_TYPE, FIXED_WIDTH, FIXED_HEIGHT)
mFormat.setInteger(MediaFormat.KEY_BIT_RATE, M_KEY_BIT_RATE)
mFormat.setInteger(MediaFormat.KEY_FRAME_RATE, M_KEY_FRAME_RATE) // codec的帧率设置无效
mFormat.setInteger(
MediaFormat.KEY_COLOR_FORMAT,
MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible
)
mFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 5)
try {
mEncoder!!.configure(mFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
} catch (e: Exception) {
Log.e(logTag, "mEncoder.configure fail!")
}
}
private fun createNotification() {
val channelId =
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
createNotificationChannel("my_service", "My Background Service")
} else {
""
}
val notification: Notification = NotificationCompat.Builder(this, channelId)
.setOngoing(true)
.setContentTitle("Hello")
.setPriority(PRIORITY_MIN)
.setContentText("TEST TEST")
.build()
startForeground(11, notification)
}
@RequiresApi(Build.VERSION_CODES.O)
private fun createNotificationChannel(channelId: String, channelName: String): String {
val chan = NotificationChannel(
channelId,
channelName, NotificationManager.IMPORTANCE_NONE
)
chan.lightColor = Color.BLUE
chan.lockscreenVisibility = Notification.VISIBILITY_PRIVATE
val service = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
service.createNotificationChannel(chan)
return channelId
}
override fun onDestroy() {
Log.d(logTag, "service stop:${Thread.currentThread()}")
Toast.makeText(this, "service done", Toast.LENGTH_SHORT).show()
}
}

151
android_doc.md Normal file
View File

@ -0,0 +1,151 @@
# RustDesk 安卓端被控文档记录
### 1.获取屏幕录像
##### 原理 流程
MediaProjectionManager -> MediaProjection
-> VirtualDisplay -> Surface -> MediaCodec
- 获取mediaProjectionResultIntent
- **必须activity**
- activity获取mediaProjectionResultIntent
- 会提示用户 “获取屏幕录制权限”
- 获取MediaProjection
- **必须service**
- 将mediaProjectionResultIntent 传递到后台服务
- 通过后台服务获取MediaProjection
- 创建Surface(理解为一个buf)和Surface消费者
- MediaCodec生成Surface传入VirtualDisplay的入参中
- 设定编码等各类参数
- 获取VirtualDisplay(Surface 生产者)
- 前台服务
- MediaProjection createVirtualDisplay方法创建VirtualDisplay
- 创建VirtualDisplay的入参之一是Surface
- 需要设定正确的VirtualDisplay尺寸
- 获取编码后的buf
- 通过MediaCodec回调获取到可用的数据
- 通过jni传入Rust服务
- 直接通过jni调用rust端的函数将数据传递给video_service中
- 安卓VP9兼容性待测试
- 目前测试2017年一台安卓7机器不支持vp9硬件加速
- **安卓内置的编解码器并不一定是硬件解码**
##### 权限注意
```
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
<service
...
android:foregroundServiceType="mediaProjection"/>
```
- API大于O(26)时需要startForegroundService且需要正确设置通知栏
新特性中使用ForegroundService不会被系统杀掉
##### 资料
- 关于 FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION 权限
https://zhuanlan.zhihu.com/p/360356420
- 关于Notification 和 NotificationNotification
https://stackoverflow.com/questions/47531742/startforeground-fail-after-upgrade-to-android-8-1
https://developer.android.com/reference/android/support/v4/app/NotificationCompat.Builder.html#NotificationCompat.Builder(android.content.Context)
// TODO 使用 NotificationCompat 的区别
<hr>
### 2.获取控制
暂时可行的方案是使用安卓无障碍服务 参考droidVNC项目但droidVNC的实现并不完善droidVNC没有实现连续触控。
#### 无障碍服务获取权限
- https://developer.android.com/guide/topics/ui/accessibility/service?hl=zh-cn#manifest
- 清单文件
```
<application>
<service android:name=".MyAccessibilityService"
android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE"
android:label="@string/accessibility_service_label">
<intent-filter>
<action android:name="android.accessibilityservice.AccessibilityService" />
</intent-filter>
</service>
</application>
```
- 创建一个单独的xml文件用于无障碍服务配置
```
// 首先清单文件中增加文件地址
<service android:name=".MyAccessibilityService">
...
<meta-data
android:name="android.accessibilityservice"
android:resource="@xml/accessibility_service_config" />
</service>
// 然后在此位置添加xml
// <project_dir>/res/xml/accessibility_service_config.xml
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
...
android:canPerformGestures="true" // 这里最关键
/>
```
- 连续手势 https://developer.android.com/guide/topics/ui/accessibility/service?hl=zh-cn#continued-gestures
<hr>
### 其他
- Kotlin 与 compose 版本设置问题
- https://stackoverflow.com/questions/67600344/jetpack-compose-on-kotlin-1-5-0
- 在根目录的gradle中 设置两个正确对应版本
### Rust JVM 互相调用
rust端 引入 jni crate
https://docs.rs/jni/0.19.0/jni/index.html
Kotlin端
类中通过init{} 引入lib的调用
```kotlin
class Main{
init{
System.loadLibrary("$libname")
}
}
```
Rust端
使用jni规则进行函数命名
```rust
pub unsafe extern "system" fn Java_com_carriez_flutter_1hbb_MainActivity_init(
env: JNIEnv,
class: JClass,
ctx:JObject,
){
}
```
- 注意原项目包名flutter_hbb 带有下划线,通过安卓的编译提示获得的命名方式为如上。
- 将安卓的对象实例Context在init的过程中传入rust端
context通过env.new_global_ref()变成全局引用
env.get_java_vm()获取到jvm
- 原理上 Rust端通过类找静态方法也可行但在kotlin端测试失败会遇到类名找不到类静态方法找不到等问题目前仅使用绑定具体context对象即可。
- 将jvm和context 固定到全局变量中等待需要时候引用
- 使用时需要确保jvm与当前的线程绑定
jvm.attach_current_thread_permanently()
- 然后通过jvm获得env
jvm.get_env()
- 通过env.call_method()方法传入context.as_obj()使用对象的方法
传递数据
Kotlin 中的 ByteArray 类 会在JVM中编译成为java的byte[]
byte[]通过jni传递到rust端时
通过jni.rs的方法
env.convert_byte_array()即可转化为Vec<u8>

View File

@ -2,3 +2,6 @@
$ANDROID_NDK/toolchains/aarch64-linux-android-4.9/prebuilt/darwin-x86_64/bin/aarch64-linux-android-strip android/app/src/main/jniLibs/arm64-v8a/*
flutter build apk --target-platform android-arm64 --release --obfuscate --split-debug-info ./split-debug-info
flutter build appbundle --target-platform android-arm64 --release --obfuscate --split-debug-info ./split-debug-info
# build in linux
# $ANDROID_NDK/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/aarch64-linux-android-strip android/app/src/main/jniLibs/arm64-v8a/*

View File

@ -1,4 +1,5 @@
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:provider/provider.dart';
import 'package:tuple/tuple.dart';
import 'package:package_info/package_info.dart';
@ -21,6 +22,7 @@ class HomePage extends StatefulWidget {
class _HomePageState extends State<HomePage> {
final _idController = TextEditingController();
var _updateUrl = '';
static const toAndroidChannel = MethodChannel("mChannel");
@override
void initState() {
@ -96,9 +98,24 @@ class _HomePageState extends State<HomePage> {
fontWeight: FontWeight.bold)))),
getSearchBarUI(),
getPeers(),
ElevatedButton(onPressed:_toAndroidGetPer, child: Text("获取权限事件")),
ElevatedButton(onPressed:_toAndroidStartSer, child: Text("开启录屏服务")),
ElevatedButton(onPressed:_toAndroidStopSer, child: Text("停止录屏服务"))
]),
));
}
Future<Null> _toAndroidGetPer() async{
bool res = await toAndroidChannel.invokeMethod("getPer");
debugPrint("_toAndroidGetPer:$res");
}
Future<Null> _toAndroidStartSer() async{
bool res = await toAndroidChannel.invokeMethod("startSer");
debugPrint("_toAndroidStartSer:$res");
}
Future<Null> _toAndroidStopSer() async{
bool res = await toAndroidChannel.invokeMethod("stopSer");
debugPrint("_toAndroidStopSer:$res");
}
void onConnect() {
var id = _idController.text.trim();

View File

@ -21,7 +21,7 @@ packages:
name: async
url: "https://pub.dartlang.org"
source: hosted
version: "2.6.1"
version: "2.8.2"
boolean_selector:
dependency: transitive
description:
@ -35,14 +35,14 @@ packages:
name: characters
url: "https://pub.dartlang.org"
source: hosted
version: "1.1.0"
version: "1.2.0"
charcode:
dependency: transitive
description:
name: charcode
url: "https://pub.dartlang.org"
source: hosted
version: "1.2.0"
version: "1.3.1"
clock:
dependency: transitive
description:
@ -227,14 +227,14 @@ packages:
name: matcher
url: "https://pub.dartlang.org"
source: hosted
version: "0.12.10"
version: "0.12.11"
meta:
dependency: transitive
description:
name: meta
url: "https://pub.dartlang.org"
source: hosted
version: "1.3.0"
version: "1.7.0"
nested:
dependency: transitive
description:
@ -428,7 +428,7 @@ packages:
name: test_api
url: "https://pub.dartlang.org"
source: hosted
version: "0.3.0"
version: "0.4.3"
tuple:
dependency: "direct main"
description:
@ -491,7 +491,7 @@ packages:
name: vector_math
url: "https://pub.dartlang.org"
source: hosted
version: "2.1.0"
version: "2.1.1"
wakelock:
dependency: "direct main"
description:
@ -556,5 +556,5 @@ packages:
source: hosted
version: "3.1.0"
sdks:
dart: ">=2.13.0 <3.0.0"
dart: ">=2.14.0 <3.0.0"
flutter: ">=2.0.0"