Improve transcript reliability with chunked multi-channel Apple Speech processing.

Keep system and microphone recordings separately, transcribe each channel in 30-second chunks with progress updates, and merge timeline segments while preserving compatibility for older mixed-only recordings.

Co-authored-by: Cursor <cursoragent@cursor.com>

meetings_app/Transcription/MeetingTranscriptionService.swift

@@ -0,0 +1,315 @@
+import Foundation
+import AVFoundation
+import Speech
+
+/// A single piece of transcript text attributed to a channel and a time range
+/// (offsets are in seconds from the start of the recording).
+struct TranscriptSegment: Codable, Hashable {
+    let speaker: String
+    let startOffset: TimeInterval
+    let endOffset: TimeInterval
+    let text: String
+}
+
+/// Logical speaker labels used when merging per-channel transcripts.
+enum TranscriptSpeaker: String {
+    case microphone = "You"
+    case system = "Meeting"
+}
+
+/// Progress snapshot for UI status updates.
+struct MeetingTranscriptionProgress: Sendable {
+    let totalChunks: Int
+    let completedChunks: Int
+}
+
+enum MeetingTranscriptionError: Error, LocalizedError {
+    case authorizationDenied
+    case authorizationRestricted
+    case recognizerUnavailable(locale: String)
+    case noAudioToTranscribe
+
+    var errorDescription: String? {
+        switch self {
+        case .authorizationDenied:
+            return "Speech recognition permission denied. Enable it in System Settings and try again."
+        case .authorizationRestricted:
+            return "Speech recognition is restricted on this Mac."
+        case .recognizerUnavailable(let locale):
+            return "Speech recognizer is unavailable for \(locale)."
+        case .noAudioToTranscribe:
+            return "No audio was available to transcribe."
+        }
+    }
+}
+
+/// Transcribes meeting audio by running Apple Speech on per-channel files
+/// in fixed-size chunks, falling back across a list of locales per chunk.
+final class MeetingTranscriptionService {
+    private struct ChunkPlan {
+        let index: Int
+        let startFrame: AVAudioFramePosition
+        let frameCount: AVAudioFrameCount
+        let startOffset: TimeInterval
+        let endOffset: TimeInterval
+    }
+
+    /// Shared progress counter used across concurrent channels.
+    private actor ProgressCounter {
+        private let total: Int
+        private var completed: Int = 0
+        private let onProgress: (@Sendable (MeetingTranscriptionProgress) -> Void)?
+
+        init(total: Int, onProgress: (@Sendable (MeetingTranscriptionProgress) -> Void)?) {
+            self.total = total
+            self.onProgress = onProgress
+        }
+
+        func emitInitial() {
+            onProgress?(MeetingTranscriptionProgress(totalChunks: total, completedChunks: 0))
+        }
+
+        func increment() {
+            completed += 1
+            onProgress?(MeetingTranscriptionProgress(totalChunks: total, completedChunks: completed))
+        }
+    }
+
+    func requestAuthorization() async throws {
+        switch SFSpeechRecognizer.authorizationStatus() {
+        case .authorized:
+            return
+        case .notDetermined:
+            let status: SFSpeechRecognizerAuthorizationStatus = await withCheckedContinuation { continuation in
+                SFSpeechRecognizer.requestAuthorization { continuation.resume(returning: $0) }
+            }
+            guard status == .authorized else { throw MeetingTranscriptionError.authorizationDenied }
+        case .denied:
+            throw MeetingTranscriptionError.authorizationDenied
+        case .restricted:
+            throw MeetingTranscriptionError.authorizationRestricted
+        @unknown default:
+            throw MeetingTranscriptionError.authorizationDenied
+        }
+    }
+
+    /// Transcribes the mic and system channel audio (either may be nil) and
+    /// returns a flat, time-ordered list of transcript segments labeled with
+    /// the speaker channel.
+    func transcribeMeeting(
+        micURL: URL?,
+        systemURL: URL?,
+        chunkSeconds: TimeInterval = 30,
+        overlapSeconds: TimeInterval = 0,
+        locales: [Locale] = [Locale(identifier: "en-US")],
+        onProgress: (@Sendable (MeetingTranscriptionProgress) -> Void)? = nil
+    ) async throws -> [TranscriptSegment] {
+        try await requestAuthorization()
+
+        let micPlan: (URL, [ChunkPlan])? = try micURL.flatMap { url -> (URL, [ChunkPlan])? in
+            guard FileManager.default.fileExists(atPath: url.path) else { return nil }
+            let chunks = try planChunks(for: url, chunkSeconds: chunkSeconds, overlapSeconds: overlapSeconds)
+            return chunks.isEmpty ? nil : (url, chunks)
+        }
+        let systemPlan: (URL, [ChunkPlan])? = try systemURL.flatMap { url -> (URL, [ChunkPlan])? in
+            guard FileManager.default.fileExists(atPath: url.path) else { return nil }
+            let chunks = try planChunks(for: url, chunkSeconds: chunkSeconds, overlapSeconds: overlapSeconds)
+            return chunks.isEmpty ? nil : (url, chunks)
+        }
+
+        let totalChunks = (micPlan?.1.count ?? 0) + (systemPlan?.1.count ?? 0)
+        guard totalChunks > 0 else {
+            throw MeetingTranscriptionError.noAudioToTranscribe
+        }
+
+        let counter = ProgressCounter(total: totalChunks, onProgress: onProgress)
+        await counter.emitInitial()
+
+        let effectiveLocales = locales.isEmpty ? [Locale(identifier: "en-US")] : locales
+
+        async let micSegments: [TranscriptSegment] = {
+            guard let plan = micPlan else { return [] }
+            return try await self.transcribeChannel(
+                url: plan.0,
+                chunks: plan.1,
+                speaker: .microphone,
+                locales: effectiveLocales,
+                counter: counter
+            )
+        }()
+        async let systemSegments: [TranscriptSegment] = {
+            guard let plan = systemPlan else { return [] }
+            return try await self.transcribeChannel(
+                url: plan.0,
+                chunks: plan.1,
+                speaker: .system,
+                locales: effectiveLocales,
+                counter: counter
+            )
+        }()
+
+        let combined = try await micSegments + systemSegments
+        return combined
+            .filter { $0.text.isEmpty == false }
+            .sorted { $0.startOffset < $1.startOffset }
+    }
+
+    // MARK: - Chunk planning
+
+    private func planChunks(for url: URL, chunkSeconds: TimeInterval, overlapSeconds: TimeInterval) throws -> [ChunkPlan] {
+        let audioFile = try AVAudioFile(forReading: url)
+        let sampleRate = audioFile.processingFormat.sampleRate
+        guard sampleRate > 0 else { return [] }
+        let totalFrames = audioFile.length
+        guard totalFrames > 0 else { return [] }
+
+        let chunkSamples = AVAudioFramePosition(max(1, chunkSeconds * sampleRate))
+        let overlapSamples = AVAudioFramePosition(max(0, overlapSeconds * sampleRate))
+        let step = max(AVAudioFramePosition(1), chunkSamples - overlapSamples)
+
+        var plans: [ChunkPlan] = []
+        var start: AVAudioFramePosition = 0
+        var index = 0
+        while start < totalFrames {
+            let end = min(start + chunkSamples, totalFrames)
+            let frameCount = AVAudioFrameCount(end - start)
+            let startOffset = Double(start) / sampleRate
+            let endOffset = Double(end) / sampleRate
+            plans.append(ChunkPlan(
+                index: index,
+                startFrame: start,
+                frameCount: frameCount,
+                startOffset: startOffset,
+                endOffset: endOffset
+            ))
+            index += 1
+            if end >= totalFrames { break }
+            start += step
+        }
+        return plans
+    }
+
+    // MARK: - Per-channel transcription
+
+    private func transcribeChannel(
+        url: URL,
+        chunks: [ChunkPlan],
+        speaker: TranscriptSpeaker,
+        locales: [Locale],
+        counter: ProgressCounter
+    ) async throws -> [TranscriptSegment] {
+        var segments: [TranscriptSegment] = []
+        segments.reserveCapacity(chunks.count)
+
+        for plan in chunks {
+            try Task.checkCancellation()
+            let buffer = try readChunkBuffer(url: url, startFrame: plan.startFrame, frameCount: plan.frameCount)
+            let text = await transcribeBufferWithLocaleFallback(buffer: buffer, locales: locales)
+            await counter.increment()
+            let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
+            if trimmed.isEmpty { continue }
+            segments.append(TranscriptSegment(
+                speaker: speaker.rawValue,
+                startOffset: plan.startOffset,
+                endOffset: plan.endOffset,
+                text: trimmed
+            ))
+        }
+        return segments
+    }
+
+    private func readChunkBuffer(url: URL, startFrame: AVAudioFramePosition, frameCount: AVAudioFrameCount) throws -> AVAudioPCMBuffer {
+        let audioFile = try AVAudioFile(forReading: url)
+        audioFile.framePosition = startFrame
+        guard let buffer = AVAudioPCMBuffer(pcmFormat: audioFile.processingFormat, frameCapacity: frameCount) else {
+            throw NSError(domain: "MeetingTranscriptionService", code: 1, userInfo: [NSLocalizedDescriptionKey: "Unable to allocate audio buffer."])
+        }
+        try audioFile.read(into: buffer, frameCount: frameCount)
+        return buffer
+    }
+
+    private func transcribeBufferWithLocaleFallback(buffer: AVAudioPCMBuffer, locales: [Locale]) async -> String {
+        for locale in locales {
+            guard let recognizer = SFSpeechRecognizer(locale: locale), recognizer.isAvailable else { continue }
+            do {
+                let text = try await transcribeBuffer(buffer: buffer, recognizer: recognizer)
+                let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
+                if trimmed.isEmpty == false { return trimmed }
+            } catch {
+                // One transient retry before moving on to the next locale.
+                try? await Task.sleep(nanoseconds: 500_000_000)
+                if let text = try? await transcribeBuffer(buffer: buffer, recognizer: recognizer) {
+                    let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
+                    if trimmed.isEmpty == false { return trimmed }
+                }
+                continue
+            }
+        }
+        return ""
+    }
+
+    private func transcribeBuffer(buffer: AVAudioPCMBuffer, recognizer: SFSpeechRecognizer) async throws -> String {
+        let request = SFSpeechAudioBufferRecognitionRequest()
+        request.shouldReportPartialResults = false
+        if #available(macOS 13.0, *) {
+            request.addsPunctuation = true
+        }
+
+        return try await withCheckedThrowingContinuation { continuation in
+            var hasResumed = false
+            let lock = NSLock()
+            func resumeOnce(with result: Result<String, Error>) {
+                lock.lock()
+                defer { lock.unlock() }
+                if hasResumed { return }
+                hasResumed = true
+                switch result {
+                case .success(let text):
+                    continuation.resume(returning: text)
+                case .failure(let error):
+                    continuation.resume(throwing: error)
+                }
+            }
+
+            let task = recognizer.recognitionTask(with: request) { result, error in
+                if let error {
+                    let nsError = error as NSError
+                    // "No speech detected" is a normal empty-chunk outcome (code 203 in kafAssistant domain).
+                    if nsError.domain == "kAFAssistantErrorDomain" && (nsError.code == 203 || nsError.code == 1110) {
+                        resumeOnce(with: .success(""))
+                        return
+                    }
+                    resumeOnce(with: .failure(error))
+                    return
+                }
+                if let result, result.isFinal {
+                    resumeOnce(with: .success(result.bestTranscription.formattedString))
+                }
+            }
+
+            request.append(buffer)
+            request.endAudio()
+            _ = task
+        }
+    }
+}
+
+extension Array where Element == TranscriptSegment {
+    /// Renders segments as a human-readable timeline like:
+    /// `[00:12] You: Hello everyone.`
+    func renderedTimelineText() -> String {
+        let formatter: (TimeInterval) -> String = { seconds in
+            let total = Int(seconds.rounded(.down))
+            let h = total / 3600
+            let m = (total % 3600) / 60
+            let s = total % 60
+            if h > 0 {
+                return String(format: "%02d:%02d:%02d", h, m, s)
+            }
+            return String(format: "%02d:%02d", m, s)
+        }
+        return self.map { segment in
+            "[\(formatter(segment.startOffset))] \(segment.speaker): \(segment.text)"
+        }.joined(separator: "\n")
+    }
+}

meetings_app/ViewController.swift

@@ -242,6 +242,7 @@ final class ViewController: NSViewController {
     private enum MeetingTranscriptSource: String, Codable {
         case meetApi
         case localAudioAppleSpeech
+        case localMultiChannelAppleSpeech
     }
 
     private enum PaywallFooterAction {
@@ -266,9 +267,12 @@ final class ViewController: NSViewController {
         let startedAt: Date
         let endedAt: Date
         let audioFilePath: String
+        var microphoneAudioFilePath: String?
+        var systemAudioFilePath: String?
         var transcriptStatusRaw: String?
         var transcriptSourceRaw: String?
         var transcriptText: String?
+        var transcriptSegmentsJSON: String?
         var transcriptErrorMessage: String?
     }
 
@@ -474,6 +478,8 @@ final class ViewController: NSViewController {
     private let nonPremiumJoinTrialConsumedDefaultsKey = "join.nonPremiumTrialConsumed"
     private let aiCompanionLocalRecordingsDefaultsKey = "aiCompanion.localRecordings"
     private let ratingEligibleUsageSeconds: TimeInterval = 30 * 60
+    private let meetingTranscriptionService = MeetingTranscriptionService()
+    private var aiCompanionTranscriptProgressByMeetingId: [String: String] = [:]
     private var darkModeEnabled: Bool {
         get {
             let hasValue = UserDefaults.standard.object(forKey: darkModeDefaultsKey) != nil
@@ -1392,6 +1398,9 @@ private extension ViewController {
         case .notRequested:
             return "Transcript not requested"
         case .processing:
+            if let progress = aiCompanionTranscriptProgressByMeetingId[recording.id], progress.isEmpty == false {
+                return progress
+            }
             return "Transcript processing..."
         case .ready:
             return "Transcript ready"
@@ -1604,7 +1613,7 @@ private extension ViewController {
         Task { [weak self] in
             guard let self else { return }
             if let stopSystemAudio { await stopSystemAudio() }
-            let finalURL = await self.finalizeMeetingAudioFile(
+            let finalized = await self.finalizeMeetingAudioFile(
                 systemURL: session.systemAudioFileURL,
                 microphoneURL: session.microphoneAudioFileURL,
                 recordingID: session.id
@@ -1616,10 +1625,13 @@ private extension ViewController {
                     meetURLString: session.meetURL.absoluteString,
                     startedAt: session.startedAt,
                     endedAt: Date(),
-                    audioFilePath: finalURL.path,
+                    audioFilePath: finalized.mixedURL.path,
+                    microphoneAudioFilePath: finalized.microphoneURL?.path,
+                    systemAudioFilePath: finalized.systemURL?.path,
                     transcriptStatusRaw: MeetingTranscriptStatus.notRequested.rawValue,
                     transcriptSourceRaw: nil,
                     transcriptText: nil,
+                    transcriptSegmentsJSON: nil,
                     transcriptErrorMessage: nil
                 )
                 self.aiCompanionLocalRecordings.insert(summary, at: 0)
@@ -1645,44 +1657,54 @@ private extension ViewController {
         return fileSize(at: url) >= minBytes
     }
 
-    private func finalizeMeetingAudioFile(systemURL: URL, microphoneURL: URL, recordingID: String) async -> URL {
+    private struct FinalizedMeetingAudio {
+        let mixedURL: URL
+        let microphoneURL: URL?
+        let systemURL: URL?
+    }
+
+    private func finalizeMeetingAudioFile(systemURL: URL, microphoneURL: URL, recordingID: String) async -> FinalizedMeetingAudio {
         let destinationURL = localRecordingDirectoryURL().appendingPathComponent("\(recordingID).m4a")
         let hasSystem = hasAudioPayload(at: systemURL)
         let hasMic = hasAudioPayload(at: microphoneURL)
 
+        if !hasSystem { try? FileManager.default.removeItem(at: systemURL) }
+        if !hasMic { try? FileManager.default.removeItem(at: microphoneURL) }
+
+        let savedMicURL: URL? = hasMic ? microphoneURL : nil
+        let savedSystemURL: URL? = hasSystem ? systemURL : nil
+
         if hasSystem && hasMic {
             do {
                 try await mixAudioFiles(systemURL: systemURL, microphoneURL: microphoneURL, destinationURL: destinationURL)
-                try? FileManager.default.removeItem(at: systemURL)
-                try? FileManager.default.removeItem(at: microphoneURL)
-                return destinationURL
+                return FinalizedMeetingAudio(mixedURL: destinationURL, microphoneURL: savedMicURL, systemURL: savedSystemURL)
             } catch {
-                // Fall back to best available single track.
+                // Fall back to best available single track for playback; per-channel files stay intact.
             }
         }
 
-        let chosenURL: URL
+        let chosenURL: URL?
         if hasSystem {
             chosenURL = systemURL
         } else if hasMic {
             chosenURL = microphoneURL
         } else {
-            let systemSize = fileSize(at: systemURL)
-            let micSize = fileSize(at: microphoneURL)
-            chosenURL = systemSize >= micSize ? systemURL : microphoneURL
+            chosenURL = nil
         }
 
-        if chosenURL.path != destinationURL.path {
-            try? FileManager.default.removeItem(at: destinationURL)
-            do {
-                try FileManager.default.copyItem(at: chosenURL, to: destinationURL)
-            } catch {
-                return chosenURL
+        if let chosenURL {
+            if chosenURL.path != destinationURL.path {
+                try? FileManager.default.removeItem(at: destinationURL)
+                do {
+                    try FileManager.default.copyItem(at: chosenURL, to: destinationURL)
+                } catch {
+                    return FinalizedMeetingAudio(mixedURL: chosenURL, microphoneURL: savedMicURL, systemURL: savedSystemURL)
+                }
             }
+            return FinalizedMeetingAudio(mixedURL: destinationURL, microphoneURL: savedMicURL, systemURL: savedSystemURL)
         }
-        try? FileManager.default.removeItem(at: systemURL)
-        try? FileManager.default.removeItem(at: microphoneURL)
-        return destinationURL
+
+        return FinalizedMeetingAudio(mixedURL: destinationURL, microphoneURL: nil, systemURL: nil)
     }
 
     private func mixAudioFiles(systemURL: URL, microphoneURL: URL, destinationURL: URL) async throws {
@@ -2990,10 +3012,12 @@ private extension ViewController {
                 )
                 await MainActor.run {
                     guard requestId == nil || self.aiCompanionTranscriptCurrentRequestId == requestId else { return }
+                    self.aiCompanionTranscriptProgressByMeetingId[meetingId] = nil
                     _ = self.aiCompanionUpdateRecording(meetingId: meetingId) { recording in
                         recording.transcriptStatusRaw = MeetingTranscriptStatus.ready.rawValue
                         recording.transcriptSourceRaw = result.source.rawValue
                         recording.transcriptText = result.text
+                        recording.transcriptSegmentsJSON = result.segmentsJSON
                         recording.transcriptErrorMessage = nil
                     }
                     self.aiCompanionTranscriptTextView?.string = result.text
@@ -3006,6 +3030,7 @@ private extension ViewController {
             } catch {
                 await MainActor.run {
                     guard requestId == nil || self.aiCompanionTranscriptCurrentRequestId == requestId else { return }
+                    self.aiCompanionTranscriptProgressByMeetingId[meetingId] = nil
                     let msg = error.localizedDescription.isEmpty ? "Failed to load transcript." : error.localizedDescription
                     _ = self.aiCompanionUpdateRecording(meetingId: meetingId) { recording in
                         recording.transcriptStatusRaw = MeetingTranscriptStatus.failed.rawValue
@@ -3027,23 +3052,71 @@ private extension ViewController {
         meetingId: String,
         interactiveAuth: Bool,
         presentingWindow: NSWindow?
-    ) async throws -> (text: String, source: MeetingTranscriptSource) {
+    ) async throws -> (text: String, segmentsJSON: String?, source: MeetingTranscriptSource) {
         guard let recording = aiCompanionLocalRecordings.first(where: { $0.id == meetingId }) else {
             throw NSError(domain: "AiCompanionTranscript", code: 10, userInfo: [NSLocalizedDescriptionKey: "Recording not found."])
         }
         _ = interactiveAuth
         _ = presentingWindow
 
-        let audioURL = URL(fileURLWithPath: recording.audioFilePath)
-        guard FileManager.default.fileExists(atPath: audioURL.path) else {
-            throw NSError(domain: "AiCompanionTranscript", code: 12, userInfo: [NSLocalizedDescriptionKey: "Local meeting audio is missing."])
+        let micURL: URL? = recording.microphoneAudioFilePath
+            .map { URL(fileURLWithPath: $0) }
+            .flatMap { FileManager.default.fileExists(atPath: $0.path) ? $0 : nil }
+        let systemURL: URL? = recording.systemAudioFilePath
+            .map { URL(fileURLWithPath: $0) }
+            .flatMap { FileManager.default.fileExists(atPath: $0.path) ? $0 : nil }
+
+        let hasPerChannel = micURL != nil || systemURL != nil
+
+        let mixedURL = URL(fileURLWithPath: recording.audioFilePath)
+        if hasPerChannel == false {
+            guard FileManager.default.fileExists(atPath: mixedURL.path) else {
+                throw NSError(domain: "AiCompanionTranscript", code: 12, userInfo: [NSLocalizedDescriptionKey: "Local meeting audio is missing."])
+            }
         }
-        let text = try await transcribeLocalAudioWithAppleSpeech(audioURL: audioURL)
-        let cleaned = text.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines)
-        guard cleaned.isEmpty == false else {
+
+        let progressHandler: @Sendable (MeetingTranscriptionProgress) -> Void = { [weak self] progress in
+            Task { @MainActor [weak self] in
+                guard let self else { return }
+                let text: String
+                if progress.totalChunks > 0 {
+                    text = "Transcribing \(progress.completedChunks)/\(progress.totalChunks) chunks..."
+                } else {
+                    text = "Transcript processing..."
+                }
+                self.aiCompanionTranscriptProgressByMeetingId[meetingId] = text
+                self.aiCompanionRefreshTranscriptStatusLabels(forMeetingID: meetingId)
+            }
+        }
+
+        let segments: [TranscriptSegment]
+        let source: MeetingTranscriptSource
+        if hasPerChannel {
+            segments = try await meetingTranscriptionService.transcribeMeeting(
+                micURL: micURL,
+                systemURL: systemURL,
+                onProgress: progressHandler
+            )
+            source = .localMultiChannelAppleSpeech
+        } else {
+            // Backward compatibility: old recordings only have a mixed file.
+            segments = try await meetingTranscriptionService.transcribeMeeting(
+                micURL: nil,
+                systemURL: mixedURL,
+                onProgress: progressHandler
+            )
+            source = .localAudioAppleSpeech
+        }
+
+        let renderedText = segments.renderedTimelineText().trimmingCharacters(in: .whitespacesAndNewlines)
+        guard renderedText.isEmpty == false else {
             throw NSError(domain: "AiCompanionTranscript", code: 14, userInfo: [NSLocalizedDescriptionKey: "Generated transcript was empty."])
         }
-        return (cleaned, .localAudioAppleSpeech)
+
+        let encoder = JSONEncoder()
+        let segmentsJSON = (try? encoder.encode(segments)).flatMap { String(data: $0, encoding: .utf8) }
+
+        return (renderedText, segmentsJSON, source)
     }
 
     @objc private func aiCompanionStopRecordingTapped(_ sender: NSButton) {