diff --git a/Package.swift b/Package.swift index 4ca33f5e..e7d2903d 100644 --- a/Package.swift +++ b/Package.swift @@ -1,14 +1,14 @@ -// swift-tools-version:5.7 +// swift-tools-version:6.0 import PackageDescription let package = Package( name: "UIKit", - platforms: [.macOS(.v10_15)], + platforms: [.macOS(.v13)], products: [ .library(name: "UIKit", targets: ["UIKit"]) ], dependencies: [ - .package(path: "./swift-jni"), + .package(url: "https://github.com/SwiftAndroid/swift-jni", branch: "devel"), .package(path: "./SDL"), ], targets: [ @@ -23,5 +23,6 @@ let package = Package( exclude: ["Mac-Info.plist"] ), .target(name: "UIKit_C_API", path: "UIKit_C_API"), - ] + ], + swiftLanguageModes: [.v5] ) diff --git a/SDL b/SDL index 226ebcdb..617ec8c0 160000 --- a/SDL +++ b/SDL @@ -1 +1 @@ -Subproject commit 226ebcdb95ef2b9920b2dbbf03e655ccaa2669a3 +Subproject commit 617ec8c042a882060f133760eda4bf0cecde0bcc diff --git a/Sources/AVPlayer+Android.swift b/Sources/AVPlayer+Android.swift index 0cf20ac8..b980e064 100644 --- a/Sources/AVPlayer+Android.swift +++ b/Sources/AVPlayer+Android.swift @@ -1,15 +1,7 @@ #if os(Android) -// -// JNIVideo.swift -// UIKit -// -// Created by Chris on 13.09.17. -// Copyright © 2017 flowkey. All rights reserved. -// - import JNI -public class AVPlayer: JNIObject { +public final class AVPlayer: JNIObject { public override static var className: String { "org.uikit.AVPlayer" } public var onError: ((ExoPlaybackError) -> Void)? diff --git a/Sources/AVPlayerLayer+Android.swift b/Sources/AVPlayerLayer+Android.swift index b2250d00..7fcdc27e 100644 --- a/Sources/AVPlayerLayer+Android.swift +++ b/Sources/AVPlayerLayer+Android.swift @@ -1,12 +1,4 @@ #if os(Android) -// -// JNIVideo.swift -// UIKit -// -// Created by Chris on 13.09.17. -// Copyright © 2017 flowkey. All rights reserved. -// - import JNI public enum AVLayerVideoGravity: JavaInt { @@ -16,7 +8,55 @@ public enum AVLayerVideoGravity: JavaInt { } @MainActor -public class AVPlayerLayer: JNIObject { +final public class AVPlayerLayer: CALayer { + public var kotlinAVPlayerLayer: KotlinAVPlayerLayer? + + public convenience init(player: AVPlayer) { + self.init() + kotlinAVPlayerLayer = KotlinAVPlayerLayer(player: player) + } + + public var videoGravity: AVLayerVideoGravity = .resizeAspect { + didSet { kotlinAVPlayerLayer?.setVideoGravity(videoGravity) } + } + + override public var opacity: Float { + didSet { kotlinAVPlayerLayer?.setAlpha(opacity) } + } + + override public var isHidden: Bool { + didSet { kotlinAVPlayerLayer?.setIsHidden(isHidden) } + } + + override public func copy() -> AVPlayerLayer { + let copy = super.copy() + // Allow the presentation layer's frame to be animated: + copy.kotlinAVPlayerLayer = kotlinAVPlayerLayer + return copy + } + + override public var cornerRadius: CGFloat { + didSet { kotlinAVPlayerLayer?.setCornerRadius(Float(cornerRadius)) } + } + + override public var zPosition: CGFloat { + didSet { kotlinAVPlayerLayer?.setElevation(zPosition) } + } + + // [Frame Animations] + // `frame` is a computed property, so `position` and `bounds` are what actually gets animated + override public var bounds: CGRect { + didSet { kotlinAVPlayerLayer?.setFrame(frame) } + } + + override public var position: CGPoint { + didSet { kotlinAVPlayerLayer?.setFrame(frame) } + } + // [/Frame Animations] +} + +@MainActor +public final class KotlinAVPlayerLayer: JNIObject { override public static var className: String { "org.uikit.AVPlayerLayer" } public convenience init(player: AVPlayer) { @@ -24,24 +64,35 @@ public class AVPlayerLayer: JNIObject { try! self.init(arguments: parentView, player) } - public var videoGravity: AVLayerVideoGravity = .resizeAspect { - didSet { - try! call("setResizeMode", arguments: [videoGravity.rawValue]) - } + public func setVideoGravity(_ newValue: AVLayerVideoGravity) { + // Not implemented because we no longer user ExoPlayer's PlayerView } - public var frame: CGRect { - get { return .zero } // FIXME: This would require returning a JavaObject with the various params - set { - guard let scale = UIScreen.main?.scale else { return } - let scaledFrame = (newValue * scale) - try! call("setFrame", arguments: [ - JavaInt(scaledFrame.origin.x.rounded()), - JavaInt(scaledFrame.origin.y.rounded()), - JavaInt(scaledFrame.size.width.rounded()), - JavaInt(scaledFrame.size.height.rounded()) - ]) - } + public func setAlpha(_ newValue: Float) { + try! call("setAlpha", arguments: [newValue]) + } + + public func setFrame(_ newValue: CGRect) { + guard let scale = UIScreen.main?.scale else { return } + let scaledFrame = newValue * scale + try! call("setFrame", arguments: [ + JavaInt(scaledFrame.origin.x.rounded()), + JavaInt(scaledFrame.origin.y.rounded()), + JavaInt(scaledFrame.size.width.rounded()), + JavaInt(scaledFrame.size.height.rounded()) + ]) + } + + public func setCornerRadius(_ newValue: Float) { + try! call("setCornerRadius", arguments: [newValue]) + } + + public func setIsHidden(_ newValue: Bool) { + try! call("setIsHidden", arguments: [newValue]) + } + + public func setElevation(_ newValue: Double) { + try! call("setElevation", arguments: [Float(newValue)]) } deinit { diff --git a/Sources/AVPlayerLayer+Mac.swift b/Sources/AVPlayerLayer+Mac.swift index 565d3508..13a08b31 100644 --- a/Sources/AVPlayerLayer+Mac.swift +++ b/Sources/AVPlayerLayer+Mac.swift @@ -69,13 +69,12 @@ public final class AVPlayerLayer: CALayer { player?.currentItem?.remove(playerOutput) let aspectRatio = presentationSize.width / presentationSize.height - - let width = round(size.width) + + let width = (size.width * self.contentsScale).rounded() let widthAlignedTo4PixelPadding = (width.remainder(dividingBy: 8) == 0) ? width : // <-- no padding required - width + (8 - width.remainder(dividingBy: 8)) + width + (8 - width.remainder(dividingBy: 8).magnitude) - playerOutput = AVPlayerItemVideoOutput(pixelBufferAttributes: [ kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, kCVPixelBufferOpenGLCompatibilityKey as String: true, @@ -89,6 +88,7 @@ public final class AVPlayerLayer: CALayer { currentPlayerOutputSize = size } + @_optimize(speed) func updateVideoFrame() { updatePlayerOutput(size: frame.size) guard @@ -124,4 +124,4 @@ public final class AVPlayerLayer: CALayer { contents?.replacePixels(with: pixelBytes, bytesPerPixel: 4) } } -#endif +#endif // os(macOS) diff --git a/Sources/AVURLAsset+Android.swift b/Sources/AVURLAsset+Android.swift index 05610d3b..c190b84a 100644 --- a/Sources/AVURLAsset+Android.swift +++ b/Sources/AVURLAsset+Android.swift @@ -1,12 +1,4 @@ #if os(Android) -// -// AVPlayerItem+Android.swift -// UIKit -// -// Created by Geordie Jay on 24.05.17. -// Copyright © 2017 flowkey. All rights reserved. -// - import JNI public class AVPlayerItem { @@ -16,7 +8,7 @@ public class AVPlayerItem { } } -public class AVURLAsset: JNIObject { +public final class AVURLAsset: JNIObject { public override static var className: String { "org.uikit.AVURLAsset" } @MainActor diff --git a/Sources/CALayer+SDL.swift b/Sources/CALayer+SDL.swift index ce039aff..919cdf16 100644 --- a/Sources/CALayer+SDL.swift +++ b/Sources/CALayer+SDL.swift @@ -59,7 +59,7 @@ extension CALayer { // If a mask exists, take it into account when rendering by combining absoluteFrame with the mask's frame if let mask = mask { // XXX: we're probably not doing exactly what iOS does if there is a transform on here somewhere - let maskFrame = (mask._presentation ?? mask).frame + let maskFrame = (mask.presentation() ?? mask).frame let maskAbsoluteFrame = maskFrame.offsetBy(absoluteFrame.origin) // Don't intersect with previousClippingRect: in a case where both `masksToBounds` and `mask` are @@ -143,7 +143,7 @@ extension CALayer { transformAtSelfOrigin.setAsSDLgpuMatrix() for sublayer in sublayers { - (sublayer._presentation ?? sublayer).sdlRender(parentAbsoluteOpacity: opacity) + (sublayer.presentation() ?? sublayer).sdlRender(parentAbsoluteOpacity: opacity) } } diff --git a/Sources/CALayer+animations.swift b/Sources/CALayer+animations.swift index d11e5f03..02b5ac3c 100644 --- a/Sources/CALayer+animations.swift +++ b/Sources/CALayer+animations.swift @@ -13,7 +13,7 @@ extension CALayer { // animation.fromValue is optional, set it to currently visible state if nil if copy.fromValue == nil, let keyPath = copy.keyPath { - copy.fromValue = (_presentation ?? self).value(forKeyPath: keyPath) + copy.fromValue = (presentation() ?? self).value(forKeyPath: keyPath) } copy.animationGroup?.queuedAnimations += 1 diff --git a/Sources/CALayer.swift b/Sources/CALayer.swift index 86f858dc..3fbf579f 100644 --- a/Sources/CALayer.swift +++ b/Sources/CALayer.swift @@ -182,7 +182,10 @@ open class CALayer { public required init() {} public required init(layer: Any) { - guard let layer = layer as? CALayer else { fatalError() } + guard let layer = layer as? CALayer else { + fatalError("Copy of CALayer must be initialized from another CALayer") + } + bounds = layer.bounds delegate = layer.delegate transform = layer.transform @@ -208,8 +211,8 @@ open class CALayer { contentsGravity = layer.contentsGravity } - open func copy() -> Any { - return CALayer(layer: self) + open func copy() -> Self { + return Self(layer: self) } open func action(forKey event: String) -> CAAction? { @@ -221,7 +224,11 @@ open class CALayer { /// returns a non animating copy of the layer func createPresentation() -> CALayer { - let copy = CALayer(layer: self) + // XXX: Should we just return _presentation if it already exists?? + // This seems to break animations, but why? + // if let _presentation { return _presentation } + + let copy = self.copy() copy.isPresentationForAnotherLayer = true return copy } @@ -235,6 +242,14 @@ open class CALayer { didSet { onDidSetAnimations(wasEmpty: oldValue.isEmpty) } } + open func animationKeys() -> [String]? { + return animations.keys.isEmpty ? nil : animations.keys.map { $0 } + } + + open func animation(forKey key: String) -> CABasicAnimation? { + return animations[key] + } + /// We disable animation on parameters of views / layers that haven't been rendered yet. /// This is both a performance optimization (avoids lots of animations at the start) /// as well as a correctness fix (matches iOS behaviour). Maybe there's a better way though? diff --git a/Sources/CGImage.swift b/Sources/CGImage.swift index b04dfcaa..ce2a4e4f 100644 --- a/Sources/CGImage.swift +++ b/Sources/CGImage.swift @@ -43,12 +43,34 @@ public class CGImage { var data = sourceData guard let gpuImagePtr = data.withUnsafeMutableBytes({ buffer -> UnsafeMutablePointer? in - guard let ptr = buffer.baseAddress?.assumingMemoryBound(to: Int8.self) else { - return nil + var width: Int32 = 0 + var height: Int32 = 0 + var channels: Int32 = 4 + + #if os(Android) + // Android natively supports 2-channel textures. Use them to save 50% (GPU) RAM. + let data = stbi_load_from_memory(buffer.baseAddress, Int32(buffer.count), &width, &height, &channels, 0) + + let format: GPU_FormatEnum = switch channels { + case 1: GPU_FORMAT_ALPHA + case 2: GPU_FORMAT_LUMINANCE_ALPHA + case 3: GPU_FORMAT_RGB + case 4: GPU_FORMAT_RGBA + default: fatalError() } - - let rw = SDL_RWFromMem(ptr, Int32(buffer.count)) - return GPU_LoadImage_RW(rw, true) + #elseif os(macOS) + // OpenGL on macOS does not natively support 2-channel textures (`unit 0 GLD_TEXTURE_INDEX_2D is unloadable`). + // Instead, force `stb_image` to load all images as if they had 4 channels. + // This is more compatible, but requires more memory. + let data = stbi_load_from_memory(buffer.baseAddress, Int32(buffer.count), &width, &height, nil, channels) + let format = GPU_FORMAT_RGBA + #endif + + let img = GPU_CreateImage(UInt16(width), UInt16(height), format) + GPU_UpdateImageBytes(img, nil, data, width * channels) + data?.deallocate() + + return img }) else { return nil } self.init(gpuImagePtr, sourceData: data) diff --git a/Sources/UIApplicationDelegate.swift b/Sources/UIApplicationDelegate.swift index 462ebf1e..06c8e0de 100644 --- a/Sources/UIApplicationDelegate.swift +++ b/Sources/UIApplicationDelegate.swift @@ -29,18 +29,13 @@ public extension UIApplicationDelegate { func applicationWillResignActive(_ application: UIApplication) {} func applicationDidEnterBackground(_ application: UIApplication) {} - // Note: this is not used on Android, because there we have a library, so no `main` function will be called. @MainActor static func main() async throws { - #if os(macOS) - // On Mac (like on iOS), the main thread blocks here via RunLoop.current.run(). - defer { setupRenderAndRunLoop() } - #else - // Android is handled differently: we don't want to block the main thread because the system needs it. - // Instead, we call render periodically from Kotlin via the Android Choreographer API (see UIApplication). - // That said, this function won't even be called on platforms like Android where the app is built as a library, not an executable. - #endif - + #if !os(Android) // Unused on Android: we build a library, so no `main` function gets called. _ = UIApplicationMain(UIApplication.self, Self.self) + + // On Mac (like on iOS), the main thread blocks here via RunLoop.current.run(). + setupRenderAndRunLoop() + #endif // !os(Android) } } diff --git a/Sources/UIScrollView.swift b/Sources/UIScrollView.swift index 048c3f6a..2969d80f 100644 --- a/Sources/UIScrollView.swift +++ b/Sources/UIScrollView.swift @@ -52,7 +52,7 @@ open class UIScrollView: UIView { /// The contentOffset that is currently shown on the screen /// We won't need this once we implement animations via DisplayLink instead of with UIView.animate var visibleContentOffset: CGPoint { - return (layer._presentation ?? layer).bounds.origin + return (layer.presentation() ?? layer).bounds.origin } /// prevent `newContentOffset` being out of bounds diff --git a/Sources/UIView+SDL.swift b/Sources/UIView+SDL.swift index f041d35a..4b7523c5 100644 --- a/Sources/UIView+SDL.swift +++ b/Sources/UIView+SDL.swift @@ -10,7 +10,7 @@ internal import SDL extension UIView { final func sdlDrawAndLayoutTreeIfNeeded(parentAlpha: CGFloat = 1.0) { - let visibleLayer = (layer._presentation ?? layer) + let visibleLayer = (layer.presentation() ?? layer) let alpha = CGFloat(visibleLayer.opacity) * parentAlpha if visibleLayer.isHidden || alpha < 0.01 { return } diff --git a/Sources/UIView.swift b/Sources/UIView.swift index 972334d2..587128e8 100644 --- a/Sources/UIView.swift +++ b/Sources/UIView.swift @@ -351,7 +351,7 @@ open class UIView: UIResponder, CALayerDelegate, UIAccessibilityIdentification { let keyPath = AnimationKeyPath(stringLiteral: event) let beginFromCurrentState = prototype.animationGroup.options.contains(.beginFromCurrentState) - let state = beginFromCurrentState ? (layer._presentation ?? layer) : layer + let state = beginFromCurrentState ? (layer.presentation() ?? layer) : layer if let fromValue = state.value(forKeyPath: keyPath) { return prototype.createAnimation(keyPath: keyPath, fromValue: fromValue) diff --git a/Sources/androidNativeInit.swift b/Sources/androidNativeInit.swift index 6345d792..6f78b3a6 100644 --- a/Sources/androidNativeInit.swift +++ b/Sources/androidNativeInit.swift @@ -44,4 +44,10 @@ public func nativeDestroyScreen(env: UnsafeMutablePointer, view: JavaObj UIApplication.onWillEnterBackground() UIApplication.onDidEnterBackground() } + +@MainActor +@_cdecl("Java_org_libsdl_app_SDLActivity_onNativeShouldRelayout") +public func onNativeShouldRelayout(env: UnsafeMutablePointer, view: JavaObject) { + UIApplication.shared?.keyWindow?.setNeedsLayout() +} #endif diff --git a/UIKit_C_API/include/uikit.h b/UIKit_C_API/include/uikit.h index 751c963d..27485e1f 100644 --- a/UIKit_C_API/include/uikit.h +++ b/UIKit_C_API/include/uikit.h @@ -1,3 +1,11 @@ -#import "jni.h" +#import -JNIEXPORT jint JNICALL SDLJNI_OnLoad(JavaVM * _Nonnull vm, void * _Nullable reserved); \ No newline at end of file +#ifdef __cplusplus +extern "C" { +#endif + +JNIEXPORT jint JNICALL SDLJNI_OnLoad(JavaVM* _Nonnull vm, void* _Nullable reserved); + +#ifdef __cplusplus +} +#endif diff --git a/build.gradle b/build.gradle index 74bb6041..fa9f3e97 100644 --- a/build.gradle +++ b/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'com.android.library' apply plugin: 'kotlin-android' android { - compileSdkVersion 34 + compileSdkVersion 35 defaultConfig { minSdkVersion 21 diff --git a/samples/getting-started/android/app/build.gradle b/samples/getting-started/android/app/build.gradle index 25564f92..0b803a68 100644 --- a/samples/getting-started/android/app/build.gradle +++ b/samples/getting-started/android/app/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' android { - compileSdkVersion 34 + compileSdkVersion 35 defaultConfig { applicationId "com.example" minSdkVersion 24 diff --git a/samples/getting-started/android/build.gradle b/samples/getting-started/android/build.gradle index ddf93251..1d586224 100644 --- a/samples/getting-started/android/build.gradle +++ b/samples/getting-started/android/build.gradle @@ -1,13 +1,14 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. buildscript { - ext.kotlin_version = '1.9.22' + ext.kotlin_version = '2.2.20' + ext.agp_version = '8.13.0' repositories { google() mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:7.3.1' + classpath "com.android.tools.build:gradle:$agp_version" classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" // NOTE: Do not place your application dependencies here; they belong diff --git a/samples/getting-started/android/gradle/wrapper/gradle-wrapper.properties b/samples/getting-started/android/gradle/wrapper/gradle-wrapper.properties index 7bb2df6b..e6045a98 100644 --- a/samples/getting-started/android/gradle/wrapper/gradle-wrapper.properties +++ b/samples/getting-started/android/gradle/wrapper/gradle-wrapper.properties @@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip diff --git a/src/main/java/org/libsdl/app/SDLActivity.kt b/src/main/java/org/libsdl/app/SDLActivity.kt index 0a6460f4..5ffe4bc6 100644 --- a/src/main/java/org/libsdl/app/SDLActivity.kt +++ b/src/main/java/org/libsdl/app/SDLActivity.kt @@ -23,7 +23,7 @@ private const val TAG = "SDLActivity" open class SDLActivity internal constructor (context: Context?) : RelativeLayout(context), SDLOnKeyListener, SDLOnTouchListener, - SurfaceHolder.Callback, + TextureView.SurfaceTextureListener, Choreographer.FrameCallback, APKExtensionInputStreamOpener { @@ -39,8 +39,9 @@ open class SDLActivity internal constructor (context: Context?) : RelativeLayout internal var mSeparateMouseAndTouch = false } - private var mSurface: SurfaceView + private var mSurface: TextureView private var mIsSurfaceReady = false + private var renderingSurface: Surface? = null // cache a Surface from SurfaceTexture override var mHasFocus = false override var sessionStartTime: Long = SystemClock.uptimeMillis() @@ -52,6 +53,7 @@ open class SDLActivity internal constructor (context: Context?) : RelativeLayout private external fun onNativeResize(x: Int, y: Int, format: Int, rate: Float) private external fun onNativeSurfaceChanged() private external fun onNativeSurfaceDestroyed() + private external fun onNativeShouldRelayout() // SDLOnKeyListener conformance external override fun onNativeKeyDown(keycode: Int) @@ -78,20 +80,19 @@ open class SDLActivity internal constructor (context: Context?) : RelativeLayout Log.v(TAG, "Model: " + android.os.Build.MODEL) // Set up the surface - mSurface = SurfaceView(context) - mSurface.setZOrderMediaOverlay(true) // so we can cover the video (fixes Android 8 bug) + mSurface = TextureView(this.context).apply { + // For alpha blending with content behind: + isOpaque = false // == semi-transparent - // Enables the alpha value for colors on the SDLSurface - // which makes the VideoJNI SurfaceView behind it visible - mSurface.holder?.setFormat(PixelFormat.RGBA_8888) - mSurface.isFocusable = true - mSurface.isFocusableInTouchMode = true - mSurface.requestFocus() - - mSurface.holder?.addCallback(this) - mSurface.setOnTouchListener(this) + isFocusable = true + isFocusableInTouchMode = true + requestFocus() - this.addView(mSurface) + setSurfaceTextureListener(this@SDLActivity) + setOnTouchListener(this@SDLActivity) + } + addView(mSurface) + setBackgroundColor(Color.BLACK) } private fun getDeviceDensity(): Float = context.resources.displayMetrics.density @@ -106,13 +107,12 @@ open class SDLActivity internal constructor (context: Context?) : RelativeLayout val zeroRect = RectF(0f, 0f, 0f, 0f) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { - val activity = context as Activity val typeMask = - if (activity.window.navigationBarColor == Color.TRANSPARENT) WindowInsets.Type.displayCutout() or WindowInsets.Type.navigationBars() - else WindowInsets.Type.displayCutout() + WindowInsets.Type.displayCutout() or WindowInsets.Type.navigationBars() or WindowInsets.Type.statusBars() val insets = rootWindowInsets?.getInsets(typeMask) ?: return zeroRect val density = getDeviceDensity() + return RectF( insets.left.toFloat() / density, insets.top.toFloat() / density, @@ -228,7 +228,9 @@ open class SDLActivity internal constructor (context: Context?) : RelativeLayout /** Called by SDL using JNI. */ @Suppress("unused") - val nativeSurface: Surface get() = this.mSurface.holder.surface + val nativeSurface: Surface + get() = renderingSurface + ?: throw IllegalStateException("Surface not ready yet") // Input @@ -271,99 +273,65 @@ open class SDLActivity internal constructor (context: Context?) : RelativeLayout mSurface.setOnTouchListener(this) } - override fun surfaceCreated(holder: SurfaceHolder) { - Log.v(TAG, "surfaceCreated()") + override fun onSurfaceTextureAvailable(st: SurfaceTexture, width: Int, height: Int) { + Log.v(TAG, "onSurfaceTextureAvailable() $width x $height") + renderingSurface = Surface(st) mHasFocus = hasFocus() - + handleTextureSizeOrChange(width, height, isFirst = true) handleResume() } // Called when the surface is resized, e.g. orientation change or activity creation - override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) { - Log.v(TAG, "surfaceChanged()") - - var sdlFormat = 0x15151002 // SDL_PIXELFORMAT_RGB565 by default - when (format) { - PixelFormat.RGBA_8888 -> { - Log.v(TAG, "pixel format RGBA_8888") - sdlFormat = 0x16462004 // SDL_PIXELFORMAT_RGBA8888 - } - PixelFormat.RGBX_8888 -> { - Log.v(TAG, "pixel format RGBX_8888") - sdlFormat = 0x16261804 // SDL_PIXELFORMAT_RGBX8888 - } - PixelFormat.RGB_565 -> { - Log.v(TAG, "pixel format RGB_565") - sdlFormat = 0x15151002 // SDL_PIXELFORMAT_RGB565 - } - PixelFormat.RGB_888 -> { - Log.v(TAG, "pixel format RGB_888") - // Not sure this is right, maybe SDL_PIXELFORMAT_RGB24 instead? - sdlFormat = 0x16161804 // SDL_PIXELFORMAT_RGB888 - } - else -> Log.w("SDL", "pixel format unknown " + format) - } + override fun onSurfaceTextureSizeChanged(st: SurfaceTexture, width: Int, height: Int) { + Log.v(TAG, "onSurfaceTextureSizeChanged() $width x $height") + handleTextureSizeOrChange(width, height, isFirst = false) + } + private fun handleTextureSizeOrChange(width: Int, height: Int, isFirst: Boolean) { if (width == 0 || height == 0) { Log.v(TAG, "skipping due to invalid surface dimensions: $width x $height") return } - if (context is Activity) { - val activity = context as Activity - when (activity.requestedOrientation) { - ActivityInfo.SCREEN_ORIENTATION_SENSOR_LANDSCAPE, - ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE, - ActivityInfo.SCREEN_ORIENTATION_REVERSE_LANDSCAPE, - ActivityInfo.SCREEN_ORIENTATION_USER_LANDSCAPE -> { - if (width < height) { - Log.v(TAG, "skipping: orientation is landscape, but width < height") - return - } - } - ActivityInfo.SCREEN_ORIENTATION_PORTRAIT, - ActivityInfo.SCREEN_ORIENTATION_REVERSE_PORTRAIT, - ActivityInfo.SCREEN_ORIENTATION_SENSOR_PORTRAIT, - ActivityInfo.SCREEN_ORIENTATION_USER_PORTRAIT -> { - if (height < width) { - Log.v(TAG, "skipping: orientation is portrait, but height < width") - return - } - } - } - } - - if (mIsSurfaceReady && mWidth.toInt() == width && mHeight.toInt() == height) { - return - } - mWidth = width.toFloat() mHeight = height.toFloat() + // With TextureView you can’t read a holder format; assume 8888 which SDL expects on Android. + val sdlFormat = 0x16462004 // SDL_PIXELFORMAT_RGBA8888 this.onNativeResize(mWidth.toInt(), mHeight.toInt(), sdlFormat, display.refreshRate) - Log.v(TAG, "Window size: " + mWidth + "x" + mHeight) + Log.v(TAG, "Window size: ${mWidth}x${mHeight}") - // Set mIsSurfaceReady to 'true' *before* making a call to handleResume mIsSurfaceReady = true onNativeSurfaceChanged() - doNativeInitAndPostFrameCallbackIfNotRunning() + if (isFirst) { + doNativeInitAndPostFrameCallbackIfNotRunning() + } else if (!isRunning) { + postFrameCallbackIfNotRunning() + } if (mHasFocus) { handleSurfaceResume() } } + override fun onSurfaceTextureUpdated(p0: SurfaceTexture) {} + // Called when we lose the surface - override fun surfaceDestroyed(holder: SurfaceHolder) { - Log.v(TAG, "surfaceDestroyed()") + override fun onSurfaceTextureDestroyed(st: SurfaceTexture): Boolean { + Log.v(TAG, "onSurfaceTextureDestroyed()") mIsSurfaceReady = false onNativeSurfaceDestroyed() nativeDestroyScreen() removeFrameCallback() + nativeProcessEventsAndRender() - // renderer should now be destroyed but we need to process events once more to clean up - this.nativeProcessEventsAndRender() + // Release the Surface we created + renderingSurface?.release() + renderingSurface = null + + // Return true to let the system release the SurfaceTexture as well + return true } /** Called by SDL using JNI. */ @@ -371,8 +339,25 @@ open class SDLActivity internal constructor (context: Context?) : RelativeLayout fun removeCallbacks() { Log.v(TAG, "removeCallbacks()") mSurface.setOnTouchListener(null) - mSurface.holder?.removeCallback(this) // should only happen on SDL_Quit - nativeSurface.release() + mSurface.setSurfaceTextureListener(null) + } + + override fun onAttachedToWindow() { + super.onAttachedToWindow() + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return + setOnApplyWindowInsetsListener { v, insets -> + this.onNativeShouldRelayout() + insets + } + + requestApplyInsets() + } + + override fun onDetachedFromWindow() { + super.onDetachedFromWindow() + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return + setOnApplyWindowInsetsListener(null) } } diff --git a/src/main/java/org/uikit/VideoJNI.kt b/src/main/java/org/uikit/VideoJNI.kt index 1dbc8351..a9f5f817 100644 --- a/src/main/java/org/uikit/VideoJNI.kt +++ b/src/main/java/org/uikit/VideoJNI.kt @@ -1,15 +1,22 @@ package org.uikit import android.content.Context +import android.graphics.Matrix +import android.graphics.Outline import android.net.Uri import android.os.Handler import android.os.Looper -import android.widget.RelativeLayout import android.util.Log +import android.util.TypedValue +import android.view.TextureView +import android.view.View +import android.view.ViewOutlineProvider +import android.widget.RelativeLayout import androidx.media3.common.MediaItem import androidx.media3.common.PlaybackException import androidx.media3.common.PlaybackParameters import androidx.media3.common.Player +import androidx.media3.common.VideoSize import androidx.media3.database.ExoDatabaseProvider import androidx.media3.datasource.DataSource import androidx.media3.datasource.FileDataSource @@ -18,22 +25,36 @@ import androidx.media3.datasource.cache.CacheDataSource import androidx.media3.datasource.cache.LeastRecentlyUsedCacheEvictor import androidx.media3.datasource.cache.SimpleCache import androidx.media3.datasource.okhttp.OkHttpDataSource +import androidx.media3.exoplayer.DefaultRenderersFactory import androidx.media3.exoplayer.ExoPlayer import androidx.media3.exoplayer.SeekParameters -import androidx.media3.exoplayer.source.ProgressiveMediaSource -import androidx.media3.exoplayer.trackselection.DefaultTrackSelector -import androidx.media3.exoplayer.upstream.DefaultBandwidthMeter -import androidx.media3.ui.PlayerView -import org.libsdl.app.SDLActivity -import okhttp3.Cache as OkHttpCache import okhttp3.OkHttpClient import okhttp3.Protocol +import org.libsdl.app.SDLActivity import java.io.File +import java.util.concurrent.ThreadLocalRandom import kotlin.math.absoluteValue +import okhttp3.Cache as OkHttpCache @Suppress("unused") class AVPlayer(parent: SDLActivity, asset: AVURLAsset) { - internal val exoPlayer: ExoPlayer + private val renderersFactory = DefaultRenderersFactory(parent.context) + .setExtensionRendererMode(DefaultRenderersFactory.EXTENSION_RENDERER_MODE_OFF) + .setEnableDecoderFallback(true); + // .forceEnableMediaCodecAsynchronousQueueing(); Doesn't seem recommended on OS versions older than Android 12 + + private val mediaSourceFactory = + androidx.media3.exoplayer.source.DefaultMediaSourceFactory(parent.context) + .setDataSourceFactory(CacheDataSourceFactory(maxFileSize = 256L * 1024 * 1024)) + + internal val exoPlayer: ExoPlayer = ExoPlayer.Builder(parent.context) + .setRenderersFactory(renderersFactory) + .setMediaSourceFactory(mediaSourceFactory) + .build().apply { + setMediaItem(asset.mediaItem) + prepare() + } + private val listener: Player.Listener private var swiftAVPlayerInstancePtr: Long? = null @@ -46,24 +67,16 @@ class AVPlayer(parent: SDLActivity, asset: AVURLAsset) { external fun nativeOnVideoError(type: Int, message: String, swiftAVPlayerInstancePtr: Long) init { - val bandwidthMeter = DefaultBandwidthMeter.Builder(parent.context).build() - val trackSelector = DefaultTrackSelector(parent.context) - - exoPlayer = ExoPlayer.Builder(parent.context) - .setBandwidthMeter(bandwidthMeter) - .setTrackSelector(trackSelector) - .build().apply { - setMediaSource(asset.mediaSource) - prepare() - } - listener = object : Player.Listener { - override fun onPlayerStateChanged(playWhenReady: Boolean, playbackState: Int) { - this@AVPlayer.swiftAVPlayerInstancePtr?.let { userContext -> - when (playbackState) { - Player.STATE_READY -> nativeOnVideoReady(userContext) - Player.STATE_ENDED -> nativeOnVideoEnded(userContext) - Player.STATE_BUFFERING -> nativeOnVideoBuffering(userContext) + override fun onPlaybackStateChanged(state: Int) { + this@AVPlayer.swiftAVPlayerInstancePtr?.let { context -> + when (state) { + Player.STATE_READY -> { + nativeOnVideoReady(context) + resetRetryState() + } + Player.STATE_BUFFERING -> nativeOnVideoBuffering(context) + Player.STATE_ENDED -> nativeOnVideoEnded(context) else -> {} } } @@ -91,7 +104,20 @@ class AVPlayer(parent: SDLActivity, asset: AVURLAsset) { this@AVPlayer.swiftAVPlayerInstancePtr?.let { swiftAVPlayerInstancePtr -> Log.e("SDL", "ExoPlaybackException occurred") val message = error.message ?: "unknown" + Log.e("SDL", message) nativeOnVideoError(error.errorCode, message, swiftAVPlayerInstancePtr) + + val isCodecError = when (error.errorCode) { + PlaybackException.ERROR_CODE_DECODER_INIT_FAILED, + PlaybackException.ERROR_CODE_DECODER_QUERY_FAILED, + PlaybackException.ERROR_CODE_DECODING_FAILED -> true + else -> false + } + + if (isCodecError && retryAttempts < maxRetryAttempts) { + scheduleRetryWithBackoff() + return + } } } } @@ -118,7 +144,7 @@ class AVPlayer(parent: SDLActivity, asset: AVURLAsset) { fun getCurrentTimeInMilliseconds(): Long = exoPlayer.currentPosition fun getPlaybackRate(): Float = exoPlayer.playbackParameters.speed fun setPlaybackRate(rate: Float) { - exoPlayer.setPlaybackParameters(PlaybackParameters(rate, 1.0f)) + exoPlayer.playbackParameters = PlaybackParameters(rate, 1.0f) } private var isSeeking = false @@ -148,36 +174,130 @@ class AVPlayer(parent: SDLActivity, asset: AVURLAsset) { lastSeekedToTime = timeMs } + private var retryAttempts = 0 + private val maxRetryAttempts = 3 + private val baseDelayMs = 1_000L // 1s, then 2s, 4s, 8s... + private val maxDelayMs = 10_000L + private val jitterMs = 250L // adds 0..250ms to avoid thundering herds + private var pendingRetry: Runnable? = null + + private fun scheduleRetryWithBackoff() { + if (retryAttempts >= maxRetryAttempts) return + + val nextAttempt = retryAttempts + 1 + val backoff = (baseDelayMs shl (nextAttempt - 1)).coerceAtMost(maxDelayMs) + val delay = backoff + ThreadLocalRandom.current().nextLong(0, jitterMs + 1) + + // Only one scheduled retry at a time + pendingRetry?.let { mainHandler.removeCallbacks(it) } + + val task = Runnable { + retryAttempts = nextAttempt + + exoPlayer.prepare() + if (exoPlayer.playWhenReady) exoPlayer.play() + } + pendingRetry = task + mainHandler.postDelayed(task, delay) + } + + private fun resetRetryState() { + retryAttempts = 0 + pendingRetry?.let { mainHandler.removeCallbacks(it) } + pendingRetry = null + } + fun cleanup() { pendingSeek?.let { mainHandler.removeCallbacks(it) } exoPlayer.removeListener(listener) exoPlayer.release() + resetRetryState() } } @Suppress("unused") -class AVPlayerLayer(private val parent: SDLActivity, player: AVPlayer) { - private val exoPlayerView: PlayerView = PlayerView(parent.context).apply { - useController = false +class AVPlayerLayer constructor( + private val sdlView: SDLActivity, + private val player: AVPlayer +) : TextureView(sdlView.context, null, 0) { + init { tag = "ExoPlayer" - this.player = player.exoPlayer + player.exoPlayer.setVideoTextureView(this) + sdlView.addView(this, 0) + + player.exoPlayer.addListener(object : Player.Listener { + override fun onVideoSizeChanged(videoSize: VideoSize) { + this@AVPlayerLayer.setTransformMatrix() + } + }) } - init { - parent.addView(exoPlayerView, 0) + fun setCornerRadius(newValue: Float) { + val radiusPx = TypedValue.applyDimension( + TypedValue.COMPLEX_UNIT_DIP, newValue, resources.displayMetrics + ) + + clipToOutline = newValue > 0 + outlineProvider = object : ViewOutlineProvider() { + override fun getOutline(v: View, outline: Outline) { + // Ensure the outline matches the current size + outline.setRoundRect(0, 0, v.width, v.height, radiusPx) + } + } } fun setFrame(x: Int, y: Int, width: Int, height: Int) { - exoPlayerView.layoutParams = RelativeLayout.LayoutParams(width, height).also { + layoutParams = RelativeLayout.LayoutParams(width, height).also { it.setMargins(x, y, 0, 0) } + + this.setTransformMatrix() } - fun setResizeMode(resizeMode: Int) { - exoPlayerView.resizeMode = resizeMode + private fun setTransformMatrix() { + val viewHeight = this.layoutParams.height.toFloat() + val viewWidth = this.layoutParams.width.toFloat() + + val videoSize = this.player.exoPlayer.videoSize + val videoHeight = videoSize.height.toFloat() + val videoWidth = videoSize.width.toFloat() + + if (viewHeight <= 0 || viewWidth <= 0 || videoHeight <= 0 || videoWidth <= 0) { + return + } + + val sar = if (videoSize.pixelWidthHeightRatio > 0f) videoSize.pixelWidthHeightRatio else 1f + + val viewAspectRatio = viewWidth / viewHeight + val videoAspectRatio = (videoWidth * sar) / videoHeight // display aspect ratio + val diffAspectRatio = videoAspectRatio / viewAspectRatio // >1 => video "wider" than view + + // FILL (center-crop): ensure min(scaleX, scaleY) >= 1 and scaleX/scaleY == r + val (scaleX, scaleY) = if (diffAspectRatio > 1f) { + // wider: expand X, crop sides + diffAspectRatio to 1f + } else { + // taller/narrower: expand Y, crop top/bottom + 1f to (1f / diffAspectRatio) + } + + val matrix = Matrix() + matrix.setScale(maxOf(scaleX, 1f), maxOf(scaleY, 1f), viewWidth / 2f, viewHeight / 2f) + + if (matrix != this.matrix) { + this.setTransform(matrix) + this.invalidate() + } + } + + fun setIsHidden(newValue: Boolean) { + // `newValue` is the HIDDEN state, whereas we're setting the _VISIBILITY_ here: + visibility = if (!newValue) { View.VISIBLE } else { View.INVISIBLE } + // Note: there is another visibility state, `View.GONE`, which also removes the view + // from layout (similar to `display: none`), but we want to match iOS behaviour here. } - fun removeFromParent() = parent.removeViewInLayout(exoPlayerView) + fun removeFromParent() = sdlView.removeViewInLayout(this) } /** @@ -204,12 +324,8 @@ internal class CacheDataSourceFactory(private val maxFileSize: Long): DataSource */ object Media3Singleton { private var initialized = false - - lateinit var okHttpClient: OkHttpClient - private set - - lateinit var simpleCache: SimpleCache - private set + lateinit var okHttpClient: OkHttpClient private set + lateinit var simpleCache: SimpleCache private set /** * Initialize once with application context and cache sizes. @@ -235,26 +351,18 @@ object Media3Singleton { @Suppress("unused") class AVURLAsset(parent: SDLActivity, url: String) { - internal val mediaSource: ProgressiveMediaSource + internal val mediaItem: MediaItem private val context: Context = parent.context init { Media3Singleton.init( context = context, - - // the http cache holds HTTP responses/validators (ETags, headers, small bodies), - // so we get fast 304s and header compression. httpCacheSize = 20L * 1024 * 1024, - - // this cache actually holds the raw MP4 bytes mediaCacheSize = 512L * 1024 * 1024 ) - val mediaItem = MediaItem.fromUri(Uri.parse(url)) - val cacheFactory = CacheDataSourceFactory( - maxFileSize = 256L * 1024 * 1024 - ) - mediaSource = ProgressiveMediaSource.Factory(cacheFactory) - .createMediaSource(mediaItem) + mediaItem = MediaItem.Builder() + .setUri(Uri.parse(url)) + .build() } } diff --git a/swift-android-toolchain b/swift-android-toolchain index 9cf589df..0619ea8b 160000 --- a/swift-android-toolchain +++ b/swift-android-toolchain @@ -1 +1 @@ -Subproject commit 9cf589df3e0dd84dda7df39d4f1d916bf71c441f +Subproject commit 0619ea8b143b0517936220ed113fac2a89fdcb2e diff --git a/swift-jni b/swift-jni deleted file mode 160000 index 3d0a1c68..00000000 --- a/swift-jni +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3d0a1c686b602afaf5b9eb372de98b3232306a9a