dnrops
/
HaishinKit.swift
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adjust MultiCam CMSampleBuffer.

This commit is contained in:
shogo4405 2023-01-11 02:55:24 +09:00
parent d627d7f87b
commit 07562eda58
8 changed files with 147 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
HaishinKit.xcodeproj/project.pbxproj
View File

@ -416,6 +416,9 @@
BC94E52F264146540094C169 /* MP4ReaderConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC94E52C264146120094C169 /* MP4ReaderConvertible.swift */; };
BC94E530264146540094C169 /* MP4ReaderConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC94E52C264146120094C169 /* MP4ReaderConvertible.swift */; };
BC94E53A264192B00094C169 /* MP4FileReaderTests2.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCAD0C10263ED28500ADFB80 /* MP4FileReaderTests2.swift */; };
BC959EEF296EE4190067BA97 /* ImageTransform.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC959EEE296EE4190067BA97 /* ImageTransform.swift */; };
BC959EF0296EE4190067BA97 /* ImageTransform.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC959EEE296EE4190067BA97 /* ImageTransform.swift */; };
BC959EF1296EE4190067BA97 /* ImageTransform.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC959EEE296EE4190067BA97 /* ImageTransform.swift */; };
BC9CFA9323BDE8B700917EEF /* NetStreamDrawable.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC9CFA9223BDE8B700917EEF /* NetStreamDrawable.swift */; };
BC9CFA9423BDE8B700917EEF /* NetStreamDrawable.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC9CFA9223BDE8B700917EEF /* NetStreamDrawable.swift */; };
BC9CFA9523BDE8B700917EEF /* NetStreamDrawable.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC9CFA9223BDE8B700917EEF /* NetStreamDrawable.swift */; };
@ -895,6 +898,7 @@
BC94E505263FEA7F0094C169 /* MP4TrackFragmentBaseMediaDecodeTimeBox.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MP4TrackFragmentBaseMediaDecodeTimeBox.swift; sourceTree = "<group>"; };
BC94E509263FEBB60094C169 /* MP4TrackRunBox.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MP4TrackRunBox.swift; sourceTree = "<group>"; };
BC94E52C264146120094C169 /* MP4ReaderConvertible.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MP4ReaderConvertible.swift; sourceTree = "<group>"; };
BC959EEE296EE4190067BA97 /* ImageTransform.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageTransform.swift; sourceTree = "<group>"; };
BC9CFA9223BDE8B700917EEF /* NetStreamDrawable.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NetStreamDrawable.swift; sourceTree = "<group>"; };
BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Choreographer.swift; sourceTree = "<group>"; };
BCA2252B293CC5B600DD7CB2 /* IOScreenCaptureUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOScreenCaptureUnit.swift; sourceTree = "<group>"; };
@ -1397,6 +1401,7 @@
children = (
2941746A22D069B300A2944F /* AudioEffect.swift */,
BC9F9C7726F8C16600B01ED0 /* Choreographer.swift */,
BC959EEE296EE4190067BA97 /* ImageTransform.swift */,
29B876891CD70AFE00FC07DA /* IOAudioUnit.swift */,
BC1102492925147300D48035 /* IOCaptureUnit.swift */,
29B8768B1CD70AFE00FC07DA /* IOMixer.swift */,
@ -2241,6 +2246,7 @@
BC11024A2925147300D48035 /* IOCaptureUnit.swift in Sources */,
29B876921CD70AFE00FC07DA /* IOMixer.swift in Sources */,
29DC17B321D0CC0600E26CED /* Atomic.swift in Sources */,
BC959EEF296EE4190067BA97 /* ImageTransform.swift in Sources */,
BCC1A6DF264470F900661156 /* MP4DataEntryUrlBox.swift in Sources */,
2942424D1CF4C01300D65DCB /* MD5.swift in Sources */,
29EA87D31E799F360043A5F8 /* ExpressibleByIntegerLiteral+Extension.swift in Sources */,
@ -2396,6 +2402,7 @@
BCA97BFE263C54560027213C /* MP4SampleToChunkBox.swift in Sources */,
293B42EA2340B4840086F973 /* RTMPObjectEncoding.swift in Sources */,
29DC17B421D0CC0600E26CED /* Atomic.swift in Sources */,
BC959EF0296EE4190067BA97 /* ImageTransform.swift in Sources */,
BCC1A6B72643F41600661156 /* MP4WriterConvertible.swift in Sources */,
BCA97BE3263C095C0027213C /* MP4TimeToSampleBox.swift in Sources */,
BCAD0C22263EFCEF00ADFB80 /* MP4SegmentIndexBox.swift in Sources */,
@ -2626,6 +2633,7 @@
BC1102402917C35B00D48035 /* CVPixelBufferPool+Extension.swift in Sources */,
29EB3E161ED0588F001CAE8B /* MIME.swift in Sources */,
293B42EB2340B4840086F973 /* RTMPObjectEncoding.swift in Sources */,
BC959EF1296EE4190067BA97 /* ImageTransform.swift in Sources */,
295891301EEB8F4100CE51E1 /* FLVSoundType.swift in Sources */,
295891201EEB8E9600CE51E1 /* FLVSoundRate.swift in Sources */,
29EB3DF21ED05770001CAE8B /* DataConvertible.swift in Sources */,

2
Sources/Extension/CVPixelBuffer+Extension.swift
View File

@ -53,7 +53,7 @@ extension CVPixelBuffer {
}
@discardableResult
func split(_ pixelBuffer: CVPixelBuffer?, direction: vImage_Buffer.TransformDirection) -> Self {
func split(_ pixelBuffer: CVPixelBuffer?, direction: ImageTransform) -> Self {
guard var inputImageBuffer = try? pixelBuffer?.makevImage_Buffer(format: &Self.format) else {
return self
}

54
Sources/Extension/vImage_Buffer+Extension.swift
View File

@ -3,52 +3,6 @@ import CoreMedia
import Foundation
extension vImage_Buffer {
enum TransformDirection {
case north
case south
case east
case west
var opposite: TransformDirection {
switch self {
case .north:
return .south
case .south:
return .north
case .east:
return .west
case .west:
return .east
}
}
func tx(_ width: Double) -> Double {
switch self {
case .north:
return 0.0
case .south:
return Double.leastNonzeroMagnitude
case .east:
return width / 2
case .west:
return -(width / 2)
}
}
func ty(_ height: Double) -> Double {
switch self {
case .north:
return height / 2
case .south:
return -(height / 2)
case .east:
return Double.leastNonzeroMagnitude
case .west:
return 0.0
}
}
}
init?(height: vImagePixelCount, width: vImagePixelCount, pixelBits: UInt32, flags: vImage_Flags) {
self.init()
guard vImageBuffer_Init(
@ -116,13 +70,15 @@ extension vImage_Buffer {
}
@discardableResult
mutating func split(_ buffer: inout vImage_Buffer, direction: TransformDirection) -> Self {
mutating func split(_ buffer: inout vImage_Buffer, direction: ImageTransform) -> Self {
buffer.transform(direction.opposite)
var shape = ShapeFactory.shared.split(CGSize(width: CGFloat(width), height: CGFloat(height)), direction: direction.opposite)
vImageSelectChannels_ARGB8888(&shape, &buffer, &buffer, 0x8, vImage_Flags(kvImageNoFlags))
transform(direction)
guard vImageAlphaBlend_ARGB8888(
&self,
&buffer,
&self,
&self,
vImage_Flags(kvImageDoNotTile)
) == kvImageNoError else {
return self
@ -130,7 +86,7 @@ extension vImage_Buffer {
return self
}
private mutating func transform(_ direction: TransformDirection) {
private mutating func transform(_ direction: ImageTransform) {
let backgroundColor: [Pixel_8] = [0, 255, 255, 255]
var vImageTransform = vImage_CGAffineTransform(
a: 1,

4
Sources/Media/IOVideoUnit.swift
View File

@ -268,8 +268,8 @@ final class IOVideoUnit: NSObject, IOUnit {
regionOfInterest: multiCamCaptureSettings.regionOfInterest,
radius: multiCamCaptureSettings.cornerRadius
)
case .split(let direction):
buffer.split(multiCamPixelBuffer, direction: direction.transformDirection)
case .splitView(let direction):
buffer.split(multiCamPixelBuffer, direction: direction)
}
multiCamPixelBuffer.unlockBaseAddress()
}

65
Sources/Media/ImageTransform.swift Normal file
View File

@ -0,0 +1,65 @@
import Foundation
/// The type of image transform direction.
public enum ImageTransform {
/// The north direction.
case north
/// The south direction.
case south
/// The east direciton.
case east
/// The west direction.
case west
var opposite: ImageTransform {
switch self {
case .north:
return .south
case .south:
return .north
case .east:
return .west
case .west:
return .east
}
}
func tx(_ width: Double) -> Double {
switch self {
case .north:
return 0.0
case .south:
return Double.leastNonzeroMagnitude
case .east:
return width / 4.0
case .west:
return -(width / 4.0)
}
}
func ty(_ height: Double) -> Double {
switch self {
case .north:
return height / 4.0
case .south:
return -(height / 4.0)
case .east:
return Double.leastNonzeroMagnitude
case .west:
return 0.0
}
}
func makeRect(_ rect: CGRect) -> CGRect {
switch self {
case .north:
return .init(origin: .init(x: 0, y: 0), size: .init(width: rect.width, height: rect.height / 2))
case .south:
return .init(origin: .init(x: 0, y: rect.height / 2), size: .init(width: rect.width, height: rect.height / 2))
case .east:
return .init(origin: .init(x: rect.width / 2, y: 0), size: .init(width: rect.width / 2, height: rect.height))
case .west:
return .init(origin: .init(x: 0, y: 0), size: .init(width: rect.width / 2, height: rect.height))
}
}
}

27
Sources/Media/MultiCamCaptureSetting.swift
View File

@ -4,31 +4,15 @@ import Foundation
/// The MultiCamCaptureSetting represents the pip capture settings for the video capture.
public struct MultiCamCaptureSetting {
public enum TransformDirection {
case north
case south
case east
case west
var transformDirection: vImage_Buffer.TransformDirection {
switch self {
case .north:
return .north
case .south:
return .south
case .east:
return .east
case .west:
return .west
}
}
}
/// The type of image display mode.
public enum Mode {
/// The picture in picture mode means video stream playing within an inset window, freeing the rest of the screen for other tasks.
case pip
case split(direction: TransformDirection)
/// The split view means video stream playing within two individual windows.
case splitView(direction: ImageTransform)
}
/// The default setting for the stream.
public static let `default` = MultiCamCaptureSetting(
mode: .pip,
cornerRadius: 16.0,
@ -38,6 +22,7 @@ public struct MultiCamCaptureSetting {
)
)
/// The image display mode.
public let mode: Mode
/// The cornerRadius of the picture in picture image.
public let cornerRadius: CGFloat

42
Sources/Media/Shape.swift
View File

@ -1,3 +1,4 @@
import Accelerate
import Foundation
#if os(macOS)
@ -19,7 +20,7 @@ class RoundedSquareShape: Shape {
) else {
return nil
}
let path = CGPath.init(roundedRect: rect, cornerWidth: cornerRadius, cornerHeight: cornerRadius, transform: nil)
let path = CGPath(roundedRect: rect, cornerWidth: cornerRadius, cornerHeight: cornerRadius, transform: nil)
context.setFillColor(NSColor.white.cgColor)
context.addPath(path)
context.closePath()
@ -28,6 +29,28 @@ class RoundedSquareShape: Shape {
}
}
class HalfRectShape: Shape {
var rect: CGRect = .zero
var direction: ImageTransform = .east
func makeCGImage() -> CGImage? {
guard let context = CGContext(
data: nil,
width: Int(rect.width),
height: Int(rect.height),
bitsPerComponent: 8,
bytesPerRow: Int(rect.width) * 4,
space: CGColorSpaceCreateDeviceRGB(),
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedFirst.rawValue).rawValue
) else {
return nil
}
context.setFillColor(NSColor.white.cgColor)
context.addRect(direction.makeRect(rect))
context.fillPath()
return context.makeImage()
}
}
#else
import UIKit
@ -51,4 +74,21 @@ class RoundedSquareShape: Shape {
}
}
class HalfRectShape: Shape {
var rect: CGRect = .zero
var direction: ImageTransform = .east
func makeCGImage() -> CGImage? {
UIGraphicsBeginImageContext(rect.size)
guard let context = UIGraphicsGetCurrentContext() else {
return nil
}
context.setFillColor(UIColor.white.cgColor)
context.addRect(direction.makeRect(rect))
context.fillPath()
let image = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
return image?.cgImage
}
}
#endif

19
Sources/Media/ShapeFactory.swift
View File

@ -11,6 +11,7 @@ class ShapeFactory {
private var imageBuffers: [String: vImage_Buffer] = [:]
private var roundedSquareShape = RoundedSquareShape()
private var halfRectShape = HalfRectShape()
func cornerRadius(_ size: CGSize, cornerRadius: CGFloat) -> vImage_Buffer {
let key = "\(size.width):\(size.height):\(cornerRadius)"
@ -30,6 +31,24 @@ class ShapeFactory {
return imageBuffer
}
func split(_ size: CGSize, direction: ImageTransform) -> vImage_Buffer {
let key = "\(size.width):\(size.height):\(direction)"
if let buffer = imageBuffers[key] {
return buffer
}
var imageBuffer = vImage_Buffer()
halfRectShape.rect = .init(origin: .zero, size: size)
halfRectShape.direction = direction
guard
let image = halfRectShape.makeCGImage(),
var format = vImage_CGImageFormat(cgImage: image),
vImageBuffer_InitWithCGImage(&imageBuffer, &format, nil, image, vImage_Flags(kvImageNoFlags)) == kvImageNoError else {
return imageBuffer
}
imageBuffers[key] = imageBuffer
return imageBuffer
}
func removeAll() {
for buffer in imageBuffers.values {
buffer.free()