dnrops
/
HaishinKit.swift
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Advanced AudioCodec.

This commit is contained in:
shogo4405 2023-03-11 18:55:06 +09:00
parent 909823740f
commit 357a16c358
12 changed files with 94 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
HaishinKit.xcodeproj/project.pbxproj
View File

@ -338,9 +338,9 @@
BC34FA0B286CB90A00EFAF27 /* PiPHKView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC34FA0A286CB90A00EFAF27 /* PiPHKView.swift */; };
BC34FA0E286CBD6D00EFAF27 /* PiPHKView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC34FA0A286CB90A00EFAF27 /* PiPHKView.swift */; };
BC34FA0F286CBD6F00EFAF27 /* PiPHKView.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC34FA0A286CB90A00EFAF27 /* PiPHKView.swift */; };
BC44A1A923D31E92002D4297 /* AudioCodecBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC44A1A823D31E92002D4297 /* AudioCodecBuffer.swift */; };
BC44A1AA23D31E92002D4297 /* AudioCodecBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC44A1A823D31E92002D4297 /* AudioCodecBuffer.swift */; };
BC44A1AB23D31E92002D4297 /* AudioCodecBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC44A1A823D31E92002D4297 /* AudioCodecBuffer.swift */; };
BC44A1A923D31E92002D4297 /* AudioCodecRingBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC44A1A823D31E92002D4297 /* AudioCodecRingBuffer.swift */; };
BC44A1AA23D31E92002D4297 /* AudioCodecRingBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC44A1A823D31E92002D4297 /* AudioCodecRingBuffer.swift */; };
BC44A1AB23D31E92002D4297 /* AudioCodecRingBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC44A1A823D31E92002D4297 /* AudioCodecRingBuffer.swift */; };
BC4914A228DDD33D009E2DF6 /* VTSessionConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4914A128DDD33D009E2DF6 /* VTSessionConvertible.swift */; };
BC4914A328DDD33D009E2DF6 /* VTSessionConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4914A128DDD33D009E2DF6 /* VTSessionConvertible.swift */; };
BC4914A428DDD33D009E2DF6 /* VTSessionConvertible.swift in Sources */ = {isa = PBXBuildFile; fileRef = BC4914A128DDD33D009E2DF6 /* VTSessionConvertible.swift */; };
@ -743,7 +743,7 @@
BC3004F8296C351D00119932 /* RTMPPlaybackViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RTMPPlaybackViewController.swift; sourceTree = "<group>"; };
BC34DFD125EBB12C005F975A /* Logboard.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = Logboard.xcframework; path = Carthage/Build/Logboard.xcframework; sourceTree = "<group>"; };
BC34FA0A286CB90A00EFAF27 /* PiPHKView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PiPHKView.swift; sourceTree = "<group>"; };
BC44A1A823D31E92002D4297 /* AudioCodecBuffer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioCodecBuffer.swift; sourceTree = "<group>"; wrapsLines = 1; };
BC44A1A823D31E92002D4297 /* AudioCodecRingBuffer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioCodecRingBuffer.swift; sourceTree = "<group>"; wrapsLines = 1; };
BC4914A128DDD33D009E2DF6 /* VTSessionConvertible.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VTSessionConvertible.swift; sourceTree = "<group>"; };
BC4914A528DDD367009E2DF6 /* VTSessionOption.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VTSessionOption.swift; sourceTree = "<group>"; };
BC4914AD28DDF445009E2DF6 /* VTDecompressionSession+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VTDecompressionSession+Extension.swift"; sourceTree = "<group>"; };
@ -887,8 +887,8 @@
isa = PBXGroup;
children = (
29B876571CD70A7900FC07DA /* AudioCodec.swift */,
BC44A1A823D31E92002D4297 /* AudioCodecBuffer.swift */,
297E69112324E38800D418AB /* AudioCodecFormat.swift */,
BC44A1A823D31E92002D4297 /* AudioCodecRingBuffer.swift */,
BC7C56B6299E579F00C41A9B /* AudioCodecSettings.swift */,
29B876591CD70A7900FC07DA /* VideoCodec.swift */,
BC7C56BA299E595000C41A9B /* VideoCodecSettings.swift */,
@ -1887,7 +1887,7 @@
2958910E1EEB8D3C00CE51E1 /* FLVVideoCodec.swift in Sources */,
BC6FC9222961B3D800A746EE /* vImage_CGImageFormat+Extension.swift in Sources */,
299B13271D3B751400A1E8F5 /* HKView.swift in Sources */,
BC44A1A923D31E92002D4297 /* AudioCodecBuffer.swift in Sources */,
BC44A1A923D31E92002D4297 /* AudioCodecRingBuffer.swift in Sources */,
BC20DF38250377A3007BC608 /* IOUIScreenCaptureUnit.swift in Sources */,
29B876AF1CD70B2800FC07DA /* RTMPChunk.swift in Sources */,
29B876841CD70AE800FC07DA /* AVCConfigurationRecord.swift in Sources */,
@ -2073,7 +2073,7 @@
29B877051CD70D5A00FC07DA /* IOMixer.swift in Sources */,
294B2D3323785E3800CE7BDC /* RingBuffer.swift in Sources */,
2976A47F1D48FD6900B53EF2 /* IORecorder.swift in Sources */,
BC44A1AA23D31E92002D4297 /* AudioCodecBuffer.swift in Sources */,
BC44A1AA23D31E92002D4297 /* AudioCodecRingBuffer.swift in Sources */,
29B877071CD70D5A00FC07DA /* SoundTransform.swift in Sources */,
BC11023F2917C35B00D48035 /* CVPixelBufferPool+Extension.swift in Sources */,
29B877081CD70D5A00FC07DA /* IOVideoUnit.swift in Sources */,
@ -2197,7 +2197,7 @@
BCCBCE9929A90D880095B51C /* NALUnit.swift in Sources */,
29DC17B521D0CC0600E26CED /* Atomic.swift in Sources */,
BC7C56CF29A786AE00C41A9B /* ADTSHeader.swift in Sources */,
BC44A1AB23D31E92002D4297 /* AudioCodecBuffer.swift in Sources */,
BC44A1AB23D31E92002D4297 /* AudioCodecRingBuffer.swift in Sources */,
29EB3E261ED05A07001CAE8B /* RTMPStream.swift in Sources */,
29DF20642312A3DD004057C3 /* RTMPNWSocket.swift in Sources */,
29EB3E131ED05887001CAE8B /* SoundTransform.swift in Sources */,

52
Sources/Codec/AudioCodec.swift
View File

@ -41,37 +41,49 @@ public class AudioCodec {
guard var inSourceFormat, inSourceFormat != oldValue else {
return
}
audioBuffer = .init(&inSourceFormat)
ringBuffer = .init(&inSourceFormat)
audioConverter = makeAudioConvter(&inSourceFormat)
}
}
private var ringBuffer: AudioCodecRingBuffer?
private var audioConverter: AVAudioConverter?
private var audioBuffer: AudioCodecBuffer?
/// Append a CMSampleBuffer.
public func appendSampleBuffer(_ sampleBuffer: CMSampleBuffer, offset: Int = 0) {
guard CMSampleBufferDataIsReady(sampleBuffer), isRunning.value, let audioBuffer, let audioConverter, let buffer = makeOutputBuffer() else {
guard CMSampleBufferDataIsReady(sampleBuffer), isRunning.value else {
return
}
let numSamples = audioBuffer.appendSampleBuffer(sampleBuffer, offset: offset)
if audioBuffer.isReady {
for effect in effects {
effect.execute(audioBuffer.current, presentationTimeStamp: audioBuffer.presentationTimeStamp)
switch destination {
case .aac:
guard let audioConverter, let ringBuffer, let buffer = makeOutputBuffer() else {
return
}
var error: NSError?
audioConverter.convert(to: buffer, error: &error) { _, status in
status.pointee = .haveData
return audioBuffer.current
let numSamples = ringBuffer.appendSampleBuffer(sampleBuffer, offset: offset)
if ringBuffer.isReady {
for effect in effects {
effect.execute(ringBuffer.current, presentationTimeStamp: ringBuffer.presentationTimeStamp)
}
var error: NSError?
audioConverter.convert(to: buffer, error: &error) { _, status in
status.pointee = .haveData
return ringBuffer.current
}
if let error {
delegate?.audioCodec(self, errorOccurred: .faildToConvert(error: error))
} else {
delegate?.audioCodec(self, didOutput: buffer, presentationTimeStamp: ringBuffer.presentationTimeStamp)
}
ringBuffer.next()
}
if let error {
delegate?.audioCodec(self, errorOccurred: .faildToConvert(error: error))
} else {
delegate?.audioCodec(self, didOutput: buffer, presentationTimeStamp: audioBuffer.presentationTimeStamp)
if offset + numSamples < sampleBuffer.numSamples {
appendSampleBuffer(sampleBuffer, offset: offset + numSamples)
}
audioBuffer.next()
}
if offset + numSamples < sampleBuffer.numSamples {
appendSampleBuffer(sampleBuffer, offset: offset + numSamples)
case .pcm:
guard let buffer = makeInputBuffer() as? AVAudioCompressedBuffer else {
return
}
sampleBuffer.dataBuffer?.copyDataBytes(to: buffer.data)
appendAudioBuffer(buffer, presentationTimeStamp: sampleBuffer.presentationTimeStamp)
}
}
@ -137,7 +149,7 @@ extension AudioCodec: Running {
lockQueue.async {
self.inSourceFormat = nil
self.audioConverter = nil
self.audioBuffer = nil
self.ringBuffer = nil
self.isRunning.mutate { $0 = false }
}
}

8
Sources/Codec/AudioCodecBuffer.swift → Sources/Codec/AudioCodecRingBuffer.swift
View File

@ -1,7 +1,7 @@
import AVFoundation
import Foundation
final class AudioCodecBuffer {
final class AudioCodecRingBuffer {
enum Error: Swift.Error {
case isReady
case noBlockBuffer
@ -25,9 +25,9 @@ final class AudioCodecBuffer {
private var buffers: [AVAudioPCMBuffer] = []
private var cursor: Int = 0
private var workingBuffer: AVAudioPCMBuffer
private var maxBuffers: Int = AudioCodecBuffer.maxBuffers
private var maxBuffers: Int = AudioCodecRingBuffer.maxBuffers
init?(_ inSourceFormat: inout AudioStreamBasicDescription, numSamples: UInt32 = AudioCodecBuffer.numSamples) {
init?(_ inSourceFormat: inout AudioStreamBasicDescription, numSamples: UInt32 = AudioCodecRingBuffer.numSamples) {
guard
inSourceFormat.mFormatID == kAudioFormatLinearPCM,
let format = AVAudioFormat(streamDescription: &inSourceFormat),
@ -139,7 +139,7 @@ final class AudioCodecBuffer {
}
}
extension AudioCodecBuffer: CustomDebugStringConvertible {
extension AudioCodecRingBuffer: CustomDebugStringConvertible {
// MARK: CustomDebugStringConvertible
var debugDescription: String {
Mirror(reflecting: self).debugDescription

1
Sources/Codec/VideoCodec.swift
View File

@ -163,7 +163,6 @@ public class VideoCodec {
needsSync.mutate { $0 = false }
}
session?.inputBuffer(sampleBuffer) { [unowned self] status, _, imageBuffer, presentationTimeStamp, duration in
guard let imageBuffer, status == noErr else {
self.delegate?.videoCodec(self, errorOccurred: .failedToFlame(status: status))
return

5
Sources/Extension/CMBlockBuffer+Extension.swift
View File

@ -16,4 +16,9 @@ extension CMBlockBuffer {
}
return Data(bytes: buffer!, count: length)
}
@discardableResult
func copyDataBytes(to buffer: UnsafeMutableRawPointer) -> OSStatus {
return CMBlockBufferCopyDataBytes(self, atOffset: 0, dataLength: dataLength, destination: buffer)
}
}

7
Sources/Extension/CMFormatDescription+Extension.swift
View File

@ -2,6 +2,13 @@ import CoreMedia
import Foundation
extension CMFormatDescription {
@available(iOS, obsoleted: 13.0)
@available(tvOS, obsoleted: 13.0)
@available(macOS, obsoleted: 10.15)
var mediaType: CMMediaType {
CMFormatDescriptionGetMediaType(self)
}
func `extension`(by key: String) -> [String: AnyObject]? {
CMFormatDescriptionGetExtension(self, extensionKey: key as CFString) as? [String: AnyObject]
}

1
Sources/Extension/CMSampleBuffer+Extension.swift
View File

@ -1,4 +1,5 @@
import Accelerate
import AVFoundation
import CoreMedia
extension CMSampleBuffer {

8
Sources/MPEG/PacketizedElementaryStream.swift
View File

@ -68,7 +68,7 @@ struct PESOptionalHeader {
pesHeaderLength = UInt8(optionalFields.count)
}
func makeSampleTimingInfo(_ previousTimeStamp: CMTime) -> CMSampleTimingInfo? {
func makeSampleTimingInfo(_ previousPresentationTimeStamp: CMTime) -> CMSampleTimingInfo? {
var presentationTimeStamp: CMTime = .invalid
var decodeTimeStamp: CMTime = .invalid
if ptsDtsIndicator & 0x02 == 0x02 {
@ -80,7 +80,7 @@ struct PESOptionalHeader {
decodeTimeStamp = .init(value: dts, timescale: CMTimeScale(TSTimestamp.resolution))
}
return CMSampleTimingInfo(
duration: presentationTimeStamp - previousTimeStamp,
duration: presentationTimeStamp - previousPresentationTimeStamp,
presentationTimeStamp: presentationTimeStamp,
decodeTimeStamp: decodeTimeStamp
)
@ -326,7 +326,7 @@ struct PacketizedElementaryStream: PESPacketHeader {
return data.count
}
mutating func makeSampleBuffer(_ streamType: ESStreamType, previousTimeStamp: CMTime, formatDescription: CMFormatDescription?) -> CMSampleBuffer? {
mutating func makeSampleBuffer(_ streamType: ESStreamType, previousPresentationTimeStamp: CMTime, formatDescription: CMFormatDescription?) -> CMSampleBuffer? {
switch streamType {
case .h264:
_ = AVCFormatStream.toNALFileFormat(&data)
@ -336,7 +336,7 @@ struct PacketizedElementaryStream: PESPacketHeader {
let blockBuffer = data.makeBlockBuffer(advancedBy: streamType.headerSize)
var sampleBuffer: CMSampleBuffer?
var sampleSize: Int = blockBuffer?.dataLength ?? 0
var timing = optionalPESHeader?.makeSampleTimingInfo(previousTimeStamp) ?? .invalid
var timing = optionalPESHeader?.makeSampleTimingInfo(previousPresentationTimeStamp) ?? .invalid
guard let blockBuffer, CMSampleBufferCreate(
allocator: kCFAllocatorDefault,
dataBuffer: blockBuffer,

4
Sources/MPEG/TSReader.swift
View File

@ -75,9 +75,9 @@ public class TSReader {
pmt.removeAll()
programs.removeAll()
esSpecData.removeAll()
previousPresentationTimeStamps.removeAll()
formatDescriptions.removeAll()
packetizedElementaryStreams.removeAll()
previousPresentationTimeStamps.removeAll()
}
private func readPacketizedElementaryStream(_ packet: TSPacket) {
@ -126,7 +126,7 @@ public class TSReader {
}
let sampleBuffer = pes.makeSampleBuffer(
data.streamType,
previousTimeStamp: previousPresentationTimeStamps[id] ?? .invalid,
previousPresentationTimeStamp: previousPresentationTimeStamps[id] ?? .invalid,
formatDescription: formatDescriptions[id]
)
sampleBuffer?.isNotSync = isNotSync

38
Sources/Media/IOMixer.swift
View File

@ -27,8 +27,8 @@ public class IOMixer {
case passthrough
}
enum Mode {
case ready
enum ReadyState {
case standby
case encoding
case decoding
}
@ -76,7 +76,7 @@ public class IOMixer {
}
}
private var mode: Mode = .ready
private var readyState: ReadyState = .standby
/// The capture session instance.
public internal(set) lazy var session: AVCaptureSession = makeSession() {
@ -143,21 +143,21 @@ public class IOMixer {
private var videoTimeStamp = CMTime.zero
/// Append a CMSampleBuffer with media type.
public func appendSampleBuffer(_ sampleBuffer: CMSampleBuffer, with type: AVMediaType) {
switch mode {
public func appendSampleBuffer(_ sampleBuffer: CMSampleBuffer) {
switch readyState {
case .encoding:
break
case .decoding:
switch type {
case .audio:
switch sampleBuffer.formatDescription?.mediaType {
case kCMMediaType_Audio:
audioIO.codec.appendSampleBuffer(sampleBuffer)
case .video:
case kCMMediaType_Video:
videoIO.codec.formatDescription = sampleBuffer.formatDescription
mediaLink.enqueueVideo(sampleBuffer)
default:
break
}
case .ready:
case .standby:
break
}
}
@ -209,25 +209,34 @@ public class IOMixer {
extension IOMixer: IOUnitEncoding {
/// Starts encoding for video and audio data.
public func startEncoding(_ delegate: AVCodecDelegate) {
mode = .encoding
guard readyState == .standby else {
return
}
readyState = .encoding
videoIO.startEncoding(delegate)
audioIO.startEncoding(delegate)
}
/// Stop encoding.
public func stopEncoding() {
guard readyState == .encoding else {
return
}
videoTimeStamp = CMTime.zero
audioTimeStamp = CMTime.zero
videoIO.stopEncoding()
audioIO.stopEncoding()
mode = .ready
readyState = .standby
}
}
extension IOMixer: IOUnitDecoding {
/// Starts decoding for video and audio data.
public func startDecoding(_ audioEngine: AVAudioEngine) {
mode = .decoding
guard readyState == .standby else {
return
}
readyState = .decoding
mediaLink.startRunning()
audioIO.startDecoding(audioEngine)
videoIO.startDecoding(audioEngine)
@ -235,10 +244,13 @@ extension IOMixer: IOUnitDecoding {
/// Stop decoding.
public func stopDecoding() {
guard readyState == .decoding else {
return
}
mediaLink.stopRunning()
audioIO.stopDecoding()
videoIO.stopDecoding()
mode = .ready
readyState = .standby
}
}

4
Sources/Media/IOUIScreenCaptureUnit.swift
View File

@ -154,7 +154,7 @@ extension IOUIScreenCaptureUnit: Running {
self.colorSpace = CGColorSpaceCreateDeviceRGB()
self.displayLink = CADisplayLink(target: self, selector: #selector(onScreen))
self.displayLink.frameInterval = self.frameInterval
self.displayLink.add(to: .main, forMode: RunLoop.Mode.common)
self.displayLink.add(to: .main, forMode: .common)
}
}
@ -163,7 +163,7 @@ extension IOUIScreenCaptureUnit: Running {
guard self.isRunning.value else {
return
}
self.displayLink.remove(from: .main, forMode: RunLoop.Mode.common)
self.displayLink.remove(from: .main, forMode: .common)
self.displayLink.invalidate()
self.colorSpace = nil
self.displayLink = nil

8
Tests/Codec/AudioCodecBufferTests.swift
View File

@ -12,7 +12,7 @@ final class AudioCodecBufferTests: XCTestCase {
XCTFail()
return
}
let buffer = AudioCodecBuffer(&asbd, numSamples: 1024)
let buffer = AudioCodecRingBuffer(&asbd, numSamples: 1024)
for _ in 0..<1024/256 {
_ = buffer?.appendSampleBuffer(sampleBuffer, offset: 0)
}
@ -34,7 +34,7 @@ final class AudioCodecBufferTests: XCTestCase {
let sampleBuffer_1 = SinWaveUtil.createCMSampleBuffer(44100, numSamples: 920),
let sampleBuffer_2 = SinWaveUtil.createCMSampleBuffer(44100, numSamples: 921),
var asbd = sampleBuffer_1.formatDescription?.audioStreamBasicDescription,
let buffer = AudioCodecBuffer(&asbd, numSamples: 1024) else {
let buffer = AudioCodecRingBuffer(&asbd, numSamples: 1024) else {
XCTFail()
return
}
@ -75,7 +75,7 @@ final class AudioCodecBufferTests: XCTestCase {
let sampleBuffer_1 = SinWaveUtil.createCMSampleBuffer(44100, numSamples: 920),
let sampleBuffer_2 = SinWaveUtil.createCMSampleBuffer(44100, numSamples: 921),
var asbd = sampleBuffer_1.formatDescription?.audioStreamBasicDescription,
let buffer = AudioCodecBuffer(&asbd, numSamples: 1024) else {
let buffer = AudioCodecRingBuffer(&asbd, numSamples: 1024) else {
XCTFail()
return
}
@ -95,7 +95,7 @@ final class AudioCodecBufferTests: XCTestCase {
}
}
private func appendSampleBuffer(_ buffer: AudioCodecBuffer, sampleBuffer: CMSampleBuffer, offset: Int = 0) {
private func appendSampleBuffer(_ buffer: AudioCodecRingBuffer, sampleBuffer: CMSampleBuffer, offset: Int = 0) {
let numSamples = buffer.appendSampleBuffer(sampleBuffer, offset: offset)
if buffer.isReady {
buffer.next()