Ai companion: improve ended-meeting audio playback

Use an in-app AVPlayer with better status handling, plus a text-to-speech fallback when the audio URL can't produce audible output.

Made-with: Cursor

meetings_app/ViewController.swift

@@ -7,6 +7,8 @@
 
 import Cocoa
 import QuartzCore
+import AVFoundation
+import AVKit
 import WebKit
 import AuthenticationServices
 import StoreKit
@@ -267,6 +269,20 @@ final class ViewController: NSViewController {
     private var zoomJoinModeViews: [ZoomJoinMode: NSView] = [:]
     private var settingsActionByView = [ObjectIdentifier: SettingsAction]()
     private var aiCompanionAudioURLByView = [ObjectIdentifier: URL]()
+    private var aiCompanionAudioStatusLabelByView = [ObjectIdentifier: NSTextField]()
+    private var aiCompanionAudioPlayer: AVPlayer?
+    private var aiCompanionCurrentlyPlayingURL: URL?
+    private weak var aiCompanionCurrentlyPlayingButton: NSButton?
+    private var aiCompanionPlaybackEndObserver: NSObjectProtocol?
+    private var aiCompanionPlaybackFailedObserver: NSObjectProtocol?
+    private var aiCompanionTimeControlObserver: NSKeyValueObservation?
+    private var aiCompanionAudioRequestID = UUID()
+    private var aiCompanionNoProgressWorkItem: DispatchWorkItem?
+
+    private let aiCompanionSpeechSynthesizer = AVSpeechSynthesizer()
+    private var aiCompanionIsUsingSpeech = false
+    private var aiCompanionSpeechTextByView = [ObjectIdentifier: String]()
+    private var aiCompanionCurrentlySpeakingButtonId: ObjectIdentifier?
     private var paywallWindow: NSWindow?
     private weak var paywallOverlayView: NSView?
     private let paywallContentWidth: CGFloat = 520
@@ -450,6 +466,7 @@ final class ViewController: NSViewController {
 
     override func viewDidLoad() {
         super.viewDidLoad()
+        aiCompanionSpeechSynthesizer.delegate = self
         // Sync toggle + palette with current macOS appearance on launch.
         darkModeEnabled = systemPrefersDarkMode()
         palette = Palette(isDarkMode: darkModeEnabled)
@@ -1941,6 +1958,26 @@ private extension ViewController {
     }
 
     private func makeAiCompanionPageContent() -> NSView {
+        // Reset per-card mappings so stale buttons/labels from previous page builds don't linger.
+        aiCompanionAudioPlayer?.pause()
+        aiCompanionAudioPlayer = nil
+        aiCompanionCurrentlyPlayingURL = nil
+        aiCompanionCurrentlyPlayingButton = nil
+        aiCompanionTimeControlObserver = nil
+        aiCompanionNoProgressWorkItem?.cancel()
+        aiCompanionNoProgressWorkItem = nil
+
+        aiCompanionSpeechSynthesizer.stopSpeaking(at: .immediate)
+        aiCompanionIsUsingSpeech = false
+        aiCompanionCurrentlySpeakingButtonId = nil
+        if let token = aiCompanionPlaybackEndObserver { NotificationCenter.default.removeObserver(token) }
+        if let token = aiCompanionPlaybackFailedObserver { NotificationCenter.default.removeObserver(token) }
+        aiCompanionPlaybackEndObserver = nil
+        aiCompanionPlaybackFailedObserver = nil
+        aiCompanionAudioURLByView.removeAll()
+        aiCompanionAudioStatusLabelByView.removeAll()
+        aiCompanionSpeechTextByView.removeAll()
+
         let panel = NSView()
         panel.translatesAutoresizingMaskIntoConstraints = false
         panel.userInterfaceLayoutDirection = .leftToRight
@@ -2060,16 +2097,30 @@ private extension ViewController {
         if let audioURL = URL(string: audioLink) {
             aiCompanionAudioURLByView[ObjectIdentifier(audioButton)] = audioURL
         }
+        let trimmedSubtitle = meeting.subtitle?.trimmingCharacters(in: .whitespacesAndNewlines)
+        let speechText = {
+            let base = "Ended meeting: \(meeting.title)."
+            guard let trimmedSubtitle, trimmedSubtitle.isEmpty == false else { return base }
+            return "\(base) \(trimmedSubtitle)."
+        }()
+        aiCompanionSpeechTextByView[ObjectIdentifier(audioButton)] = speechText
 
         let audioURLLabel = textLabel(audioLink, font: typography.fieldLabel, color: palette.textMuted)
         audioURLLabel.alignment = .left
         audioURLLabel.maximumNumberOfLines = 2
         audioURLLabel.lineBreakMode = .byTruncatingTail
 
+        let audioStatusLabel = textLabel("Not playing", font: typography.fieldLabel, color: palette.textMuted)
+        audioStatusLabel.alignment = .left
+        audioStatusLabel.maximumNumberOfLines = 1
+        audioStatusLabel.lineBreakMode = .byTruncatingTail
+        aiCompanionAudioStatusLabelByView[ObjectIdentifier(audioButton)] = audioStatusLabel
+
         stack.addArrangedSubview(title)
         stack.addArrangedSubview(dateLabel)
         stack.addArrangedSubview(audioButton)
         stack.addArrangedSubview(audioURLLabel)
+        stack.addArrangedSubview(audioStatusLabel)
 
         card.addSubview(stack)
         NSLayoutConstraint.activate([
@@ -2083,8 +2134,268 @@ private extension ViewController {
     }
 
     @objc private func aiCompanionAudioTapped(_ sender: NSButton) {
-        guard let url = aiCompanionAudioURLByView[ObjectIdentifier(sender)] else { return }
-        NSWorkspace.shared.open(url)
+        let senderId = ObjectIdentifier(sender)
+        guard let url = aiCompanionAudioURLByView[senderId] else { return }
+
+        // Toggle play/pause if the same card is tapped.
+        if aiCompanionCurrentlyPlayingURL == url {
+            if aiCompanionIsUsingSpeech {
+                if aiCompanionSpeechSynthesizer.isSpeaking {
+                    aiCompanionSpeechSynthesizer.stopSpeaking(at: .immediate)
+                    aiCompanionIsUsingSpeech = false
+                    sender.title = "Play Audio"
+                    aiCompanionAudioStatusLabelByView[senderId]?.stringValue = "Stopped"
+                } else {
+                    aiCompanionStartSpeech(forButtonId: senderId)
+                    sender.title = "Pause Audio"
+                    aiCompanionAudioStatusLabelByView[senderId]?.stringValue = "Speaking..."
+                }
+                return
+            } else if let player = aiCompanionAudioPlayer {
+                if player.rate != 0 {
+                    player.pause()
+                    sender.title = "Play Audio"
+                    aiCompanionAudioStatusLabelByView[senderId]?.stringValue = "Paused"
+                } else {
+                    player.play()
+                    sender.title = "Pause Audio"
+                    aiCompanionAudioStatusLabelByView[senderId]?.stringValue = "Playing..."
+                }
+                return
+            }
+        }
+
+        // Stop any previous playback and remove observers.
+        aiCompanionAudioPlayer?.pause()
+        aiCompanionSpeechSynthesizer.stopSpeaking(at: .immediate)
+        aiCompanionIsUsingSpeech = false
+        aiCompanionNoProgressWorkItem?.cancel()
+        aiCompanionNoProgressWorkItem = nil
+        aiCompanionCurrentlySpeakingButtonId = nil
+        aiCompanionTimeControlObserver = nil
+        if let token = aiCompanionPlaybackEndObserver { NotificationCenter.default.removeObserver(token) }
+        if let token = aiCompanionPlaybackFailedObserver { NotificationCenter.default.removeObserver(token) }
+        aiCompanionPlaybackEndObserver = nil
+        aiCompanionPlaybackFailedObserver = nil
+
+        // Revert previous playing UI (if any).
+        if let previousButton = aiCompanionCurrentlyPlayingButton {
+            let previousId = ObjectIdentifier(previousButton)
+            previousButton.title = "Play Audio"
+            aiCompanionAudioStatusLabelByView[previousId]?.stringValue = "Not playing"
+        }
+
+        aiCompanionCurrentlyPlayingURL = url
+        aiCompanionCurrentlyPlayingButton = sender
+        sender.title = "Pause Audio"
+        aiCompanionAudioStatusLabelByView[senderId]?.stringValue = "Checking audio..."
+
+        aiCompanionAudioRequestID = UUID()
+        let requestID = aiCompanionAudioRequestID
+
+        let urlToCheck = url
+        let senderButton = sender
+
+        var request = URLRequest(url: urlToCheck)
+        request.setValue("bytes=0-0", forHTTPHeaderField: "Range") // lightweight probe
+        request.timeoutInterval = 10
+
+        URLSession.shared.dataTask(with: request) { [weak self] _, response, error in
+            guard let self else { return }
+
+            DispatchQueue.main.async {
+                guard self.aiCompanionAudioRequestID == requestID else { return } // stale tap
+
+                if let error {
+                    senderButton.title = "Play Audio"
+                    self.aiCompanionAudioTimeControlObserverResetForFailure()
+                    self.aiCompanionStartSpeech(forButtonId: senderId)
+                    return
+                }
+
+                guard let http = response as? HTTPURLResponse else {
+                    senderButton.title = "Play Audio"
+                    self.aiCompanionAudioTimeControlObserverResetForFailure()
+                    self.aiCompanionStartSpeech(forButtonId: senderId)
+                    return
+                }
+
+                let mime = http.mimeType?.lowercased() ?? ""
+                let okStatus = (200...299).contains(http.statusCode) || http.statusCode == 206
+                guard okStatus else {
+                    senderButton.title = "Play Audio"
+                    self.aiCompanionAudioTimeControlObserverResetForFailure()
+                    self.aiCompanionStartSpeech(forButtonId: senderId)
+                    return
+                }
+
+                if !mime.isEmpty && mime.hasPrefix("audio/") == false {
+                    senderButton.title = "Play Audio"
+                    self.aiCompanionAudioTimeControlObserverResetForFailure()
+                    self.aiCompanionStartSpeech(forButtonId: senderId)
+                    return
+                }
+
+                self.aiCompanionAudioStatusLabelByView[senderId]?.stringValue = "Loading..."
+
+                let player = AVPlayer(url: urlToCheck)
+                player.volume = 1.0
+                self.aiCompanionAudioPlayer = player
+
+                if let item = player.currentItem {
+                    self.aiCompanionPlaybackEndObserver = NotificationCenter.default.addObserver(
+                        forName: .AVPlayerItemDidPlayToEndTime,
+                        object: item,
+                        queue: .main
+                    ) { [weak self] _ in
+                        self?.aiCompanionAudioDidFinish()
+                    }
+
+                    self.aiCompanionPlaybackFailedObserver = NotificationCenter.default.addObserver(
+                        forName: .AVPlayerItemFailedToPlayToEndTime,
+                        object: item,
+                        queue: .main
+                    ) { [weak self] notification in
+                        self?.aiCompanionAudioDidFail(notification: notification)
+                    }
+                }
+
+                // Update the UI only when playback actually starts (helps diagnose "playing but no audio").
+                self.aiCompanionTimeControlObserver = player.observe(\.timeControlStatus, options: [.initial, .new]) { [weak self] p, _ in
+                    guard let self else { return }
+                    guard self.aiCompanionCurrentlyPlayingURL == urlToCheck else { return }
+
+                    switch p.timeControlStatus {
+                    case .playing:
+                        self.aiCompanionAudioStatusLabelByView[senderId]?.stringValue = "Playing..."
+                    case .waitingToPlayAtSpecifiedRate:
+                        self.aiCompanionAudioStatusLabelByView[senderId]?.stringValue = "Buffering..."
+                    case .paused:
+                        self.aiCompanionAudioStatusLabelByView[senderId]?.stringValue = "Paused"
+                    @unknown default:
+                        self.aiCompanionAudioStatusLabelByView[senderId]?.stringValue = "Playing..."
+                    }
+                }
+
+                // If the remote file is silent/unplayable, AVPlayer can still report "playing".
+                // After a short grace period, fall back to text-to-speech so you can always hear something.
+                let startSeconds = player.currentTime().seconds
+                player.play()
+                let playerRef = player
+                let urlRef = urlToCheck
+
+                self.aiCompanionNoProgressWorkItem?.cancel()
+                let workItem = DispatchWorkItem { [weak self] in
+                    guard let self else { return }
+                    guard self.aiCompanionAudioPlayer === playerRef else { return }
+                    guard self.aiCompanionCurrentlyPlayingURL == urlRef else { return }
+                    guard self.aiCompanionIsUsingSpeech == false else { return }
+
+                    let nowSeconds = playerRef.currentTime().seconds
+                    let progressed = startSeconds.isFinite && nowSeconds.isFinite && (nowSeconds - startSeconds) > 0.5
+                    let actuallyPlaying = playerRef.timeControlStatus == .playing
+
+                    if actuallyPlaying == false || progressed == false {
+                        self.aiCompanionStartSpeech(forButtonId: senderId)
+                    }
+                }
+                self.aiCompanionNoProgressWorkItem = workItem
+                DispatchQueue.main.asyncAfter(deadline: .now() + 3, execute: workItem)
+            }
+        }.resume()
+    }
+
+    private func aiCompanionAudioTimeControlObserverResetForFailure() {
+        aiCompanionAudioPlayer?.pause()
+        aiCompanionAudioPlayer = nil
+        aiCompanionTimeControlObserver = nil
+
+        if let token = aiCompanionPlaybackEndObserver { NotificationCenter.default.removeObserver(token) }
+        if let token = aiCompanionPlaybackFailedObserver { NotificationCenter.default.removeObserver(token) }
+        aiCompanionPlaybackEndObserver = nil
+        aiCompanionPlaybackFailedObserver = nil
+    }
+
+    private func aiCompanionStartSpeech(forButtonId buttonId: ObjectIdentifier) {
+        // Stop any remote audio playback first.
+        aiCompanionAudioPlayer?.pause()
+        aiCompanionAudioPlayer = nil
+        aiCompanionTimeControlObserver = nil
+        if let token = aiCompanionPlaybackEndObserver { NotificationCenter.default.removeObserver(token) }
+        if let token = aiCompanionPlaybackFailedObserver { NotificationCenter.default.removeObserver(token) }
+        aiCompanionPlaybackEndObserver = nil
+        aiCompanionPlaybackFailedObserver = nil
+
+        aiCompanionNoProgressWorkItem?.cancel()
+        aiCompanionNoProgressWorkItem = nil
+
+        aiCompanionIsUsingSpeech = true
+        aiCompanionCurrentlySpeakingButtonId = buttonId
+
+        if let button = aiCompanionCurrentlyPlayingButton, ObjectIdentifier(button) == buttonId {
+            button.title = "Pause Audio"
+        }
+
+        aiCompanionAudioStatusLabelByView[buttonId]?.stringValue = "Speaking..."
+
+        let text = aiCompanionSpeechTextByView[buttonId] ?? "Ended meeting."
+        let utterance = AVSpeechUtterance(string: text)
+        utterance.rate = 0.5
+        utterance.volume = 1.0
+
+        aiCompanionSpeechSynthesizer.stopSpeaking(at: .immediate)
+        aiCompanionSpeechSynthesizer.speak(utterance)
+    }
+
+    private func aiCompanionAudioDidFinish() {
+        guard let button = aiCompanionCurrentlyPlayingButton else { return }
+        let buttonId = ObjectIdentifier(button)
+
+        aiCompanionSpeechSynthesizer.stopSpeaking(at: .immediate)
+        aiCompanionIsUsingSpeech = false
+        aiCompanionCurrentlySpeakingButtonId = nil
+
+        button.title = "Play Audio"
+        aiCompanionAudioStatusLabelByView[buttonId]?.stringValue = "Finished"
+
+        aiCompanionAudioPlayer?.pause()
+        aiCompanionAudioPlayer = nil
+        aiCompanionCurrentlyPlayingURL = nil
+        aiCompanionCurrentlyPlayingButton = nil
+
+        if let token = aiCompanionPlaybackEndObserver { NotificationCenter.default.removeObserver(token) }
+        if let token = aiCompanionPlaybackFailedObserver { NotificationCenter.default.removeObserver(token) }
+        aiCompanionPlaybackEndObserver = nil
+        aiCompanionPlaybackFailedObserver = nil
+        aiCompanionTimeControlObserver = nil
+    }
+
+    private func aiCompanionAudioDidFail(notification: Notification) {
+        guard let button = aiCompanionCurrentlyPlayingButton else { return }
+        let buttonId = ObjectIdentifier(button)
+
+        aiCompanionSpeechSynthesizer.stopSpeaking(at: .immediate)
+        aiCompanionIsUsingSpeech = false
+        aiCompanionCurrentlySpeakingButtonId = nil
+
+        button.title = "Play Audio"
+
+        var message = "Failed to play audio"
+        if let item = notification.object as? AVPlayerItem, let error = item.error {
+            message = "Failed: \(error.localizedDescription)"
+        }
+        aiCompanionAudioStatusLabelByView[buttonId]?.stringValue = message
+
+        aiCompanionAudioPlayer?.pause()
+        aiCompanionAudioPlayer = nil
+        aiCompanionCurrentlyPlayingURL = nil
+        aiCompanionCurrentlyPlayingButton = nil
+
+        if let token = aiCompanionPlaybackEndObserver { NotificationCenter.default.removeObserver(token) }
+        if let token = aiCompanionPlaybackFailedObserver { NotificationCenter.default.removeObserver(token) }
+        aiCompanionPlaybackEndObserver = nil
+        aiCompanionPlaybackFailedObserver = nil
+        aiCompanionTimeControlObserver = nil
     }
 
     private func mockAudioURLString(for meeting: ScheduledMeeting) -> String {
@@ -4784,6 +5095,40 @@ extension ViewController: NSTextFieldDelegate {
     }
 }
 
+extension ViewController: AVSpeechSynthesizerDelegate {
+    func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) {
+        DispatchQueue.main.async { [weak self] in
+            guard let self else { return }
+            guard self.aiCompanionSpeechSynthesizer === synthesizer else { return }
+            guard let buttonId = self.aiCompanionCurrentlySpeakingButtonId else { return }
+
+            self.aiCompanionIsUsingSpeech = false
+            self.aiCompanionCurrentlySpeakingButtonId = nil
+
+            if let button = self.aiCompanionCurrentlyPlayingButton, ObjectIdentifier(button) == buttonId {
+                button.title = "Play Audio"
+            }
+            self.aiCompanionAudioStatusLabelByView[buttonId]?.stringValue = "Finished"
+        }
+    }
+
+    func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) {
+        DispatchQueue.main.async { [weak self] in
+            guard let self else { return }
+            guard self.aiCompanionSpeechSynthesizer === synthesizer else { return }
+            guard let buttonId = self.aiCompanionCurrentlySpeakingButtonId else { return }
+
+            self.aiCompanionIsUsingSpeech = false
+            self.aiCompanionCurrentlySpeakingButtonId = nil
+
+            if let button = self.aiCompanionCurrentlyPlayingButton, ObjectIdentifier(button) == buttonId {
+                button.title = "Play Audio"
+            }
+            self.aiCompanionAudioStatusLabelByView[buttonId]?.stringValue = "Stopped"
+        }
+    }
+}
+
 extension ViewController: NSWindowDelegate {
     func windowWillClose(_ notification: Notification) {
         guard let closingWindow = notification.object as? NSWindow else { return }