MeetingTranscriptionService.swift 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818
  1. import Foundation
  2. import AVFoundation
  3. import Speech
  4. /// A single piece of transcript text attributed to a channel and a time range
  5. /// (offsets are in seconds from the start of the recording).
  6. struct TranscriptSegment: Codable, Hashable {
  7. let speaker: String
  8. let startOffset: TimeInterval
  9. let endOffset: TimeInterval
  10. let text: String
  11. }
  12. /// Logical speaker labels used when merging per-channel transcripts.
  13. enum TranscriptSpeaker: String {
  14. case microphone = "You"
  15. case system = "Meeting"
  16. var localizedLabel: String {
  17. switch self {
  18. case .microphone:
  19. return "You".localized
  20. case .system:
  21. return "Meeting".localized
  22. }
  23. }
  24. }
  25. /// Progress snapshot for UI status updates.
  26. struct MeetingTranscriptionProgress: Sendable {
  27. let totalChunks: Int
  28. let completedChunks: Int
  29. }
  30. enum MeetingTranscriptionError: Error, LocalizedError {
  31. case authorizationDenied
  32. case authorizationRestricted
  33. case recognizerUnavailable(locale: String)
  34. case noAudioToTranscribe
  35. var errorDescription: String? {
  36. switch self {
  37. case .authorizationDenied:
  38. return "Speech recognition permission denied. Enable it in System Settings and try again.".localized
  39. case .authorizationRestricted:
  40. return "Speech recognition is restricted on this Mac.".localized
  41. case .recognizerUnavailable(let locale):
  42. return "Speech recognizer is unavailable for %@.".localized(with: locale)
  43. case .noAudioToTranscribe:
  44. return "No audio was available to transcribe.".localized
  45. }
  46. }
  47. }
  48. /// Transcribes meeting audio by running Apple Speech on per-channel files
  49. /// in fixed-size chunks, falling back across a list of locales per chunk.
  50. final class MeetingTranscriptionService {
  51. private struct ChunkPlan {
  52. let index: Int
  53. let startFrame: AVAudioFramePosition
  54. let frameCount: AVAudioFrameCount
  55. let startOffset: TimeInterval
  56. let endOffset: TimeInterval
  57. }
  58. /// Shared progress counter used across concurrent channels.
  59. private actor ProgressCounter {
  60. private let total: Int
  61. private var completed: Int = 0
  62. private let onProgress: (@Sendable (MeetingTranscriptionProgress) -> Void)?
  63. init(total: Int, onProgress: (@Sendable (MeetingTranscriptionProgress) -> Void)?) {
  64. self.total = total
  65. self.onProgress = onProgress
  66. }
  67. func emitInitial() {
  68. onProgress?(MeetingTranscriptionProgress(totalChunks: total, completedChunks: 0))
  69. }
  70. func increment() {
  71. completed += 1
  72. onProgress?(MeetingTranscriptionProgress(totalChunks: total, completedChunks: completed))
  73. }
  74. }
  75. func requestAuthorization() async throws {
  76. switch SFSpeechRecognizer.authorizationStatus() {
  77. case .authorized:
  78. return
  79. case .notDetermined:
  80. let status: SFSpeechRecognizerAuthorizationStatus = await withCheckedContinuation { continuation in
  81. SFSpeechRecognizer.requestAuthorization { continuation.resume(returning: $0) }
  82. }
  83. guard status == .authorized else { throw MeetingTranscriptionError.authorizationDenied }
  84. case .denied:
  85. throw MeetingTranscriptionError.authorizationDenied
  86. case .restricted:
  87. throw MeetingTranscriptionError.authorizationRestricted
  88. @unknown default:
  89. throw MeetingTranscriptionError.authorizationDenied
  90. }
  91. }
  92. /// Transcribes the mic and system channel audio (either may be nil) and
  93. /// returns a flat, time-ordered list of transcript segments labeled with
  94. /// the speaker channel.
  95. func transcribeMeeting(
  96. micURL: URL?,
  97. systemURL: URL?,
  98. chunkSeconds: TimeInterval = 7,
  99. overlapSeconds: TimeInterval = 1.2,
  100. locales: [Locale] = [],
  101. onProgress: (@Sendable (MeetingTranscriptionProgress) -> Void)? = nil
  102. ) async throws -> [TranscriptSegment] {
  103. try await requestAuthorization()
  104. let micPlan: (URL, [ChunkPlan])? = try micURL.flatMap { url -> (URL, [ChunkPlan])? in
  105. guard FileManager.default.fileExists(atPath: url.path) else { return nil }
  106. let chunks = try planChunks(for: url, chunkSeconds: chunkSeconds, overlapSeconds: overlapSeconds)
  107. return chunks.isEmpty ? nil : (url, chunks)
  108. }
  109. let systemPlan: (URL, [ChunkPlan])? = try systemURL.flatMap { url -> (URL, [ChunkPlan])? in
  110. guard FileManager.default.fileExists(atPath: url.path) else { return nil }
  111. let chunks = try planChunks(for: url, chunkSeconds: chunkSeconds, overlapSeconds: overlapSeconds)
  112. return chunks.isEmpty ? nil : (url, chunks)
  113. }
  114. let totalChunks = (micPlan?.1.count ?? 0) + (systemPlan?.1.count ?? 0)
  115. guard totalChunks > 0 else {
  116. throw MeetingTranscriptionError.noAudioToTranscribe
  117. }
  118. let counter = ProgressCounter(total: totalChunks, onProgress: onProgress)
  119. await counter.emitInitial()
  120. let effectiveLocales = locales.isEmpty ? preferredLocalesForGlobalRecognition() : locales
  121. async let micSegments: [TranscriptSegment] = {
  122. guard let plan = micPlan else { return [] }
  123. return try await self.transcribeChannel(
  124. url: plan.0,
  125. chunks: plan.1,
  126. speaker: .microphone,
  127. locales: effectiveLocales,
  128. counter: counter
  129. )
  130. }()
  131. async let systemSegments: [TranscriptSegment] = {
  132. guard let plan = systemPlan else { return [] }
  133. return try await self.transcribeChannel(
  134. url: plan.0,
  135. chunks: plan.1,
  136. speaker: .system,
  137. locales: effectiveLocales,
  138. counter: counter
  139. )
  140. }()
  141. let combined = try await micSegments + systemSegments
  142. return combined
  143. .filter { $0.text.isEmpty == false }
  144. .sorted { $0.startOffset < $1.startOffset }
  145. }
  146. // MARK: - Chunk planning
  147. private func planChunks(for url: URL, chunkSeconds: TimeInterval, overlapSeconds: TimeInterval) throws -> [ChunkPlan] {
  148. let audioFile = try AVAudioFile(forReading: url)
  149. let sampleRate = audioFile.processingFormat.sampleRate
  150. guard sampleRate > 0 else { return [] }
  151. let totalFrames = audioFile.length
  152. guard totalFrames > 0 else { return [] }
  153. let chunkSamples = AVAudioFramePosition(max(1, chunkSeconds * sampleRate))
  154. let overlapSamples = AVAudioFramePosition(max(0, overlapSeconds * sampleRate))
  155. let step = max(AVAudioFramePosition(1), chunkSamples - overlapSamples)
  156. var plans: [ChunkPlan] = []
  157. var start: AVAudioFramePosition = 0
  158. var index = 0
  159. while start < totalFrames {
  160. let end = min(start + chunkSamples, totalFrames)
  161. let frameCount = AVAudioFrameCount(end - start)
  162. let startOffset = Double(start) / sampleRate
  163. let endOffset = Double(end) / sampleRate
  164. plans.append(ChunkPlan(
  165. index: index,
  166. startFrame: start,
  167. frameCount: frameCount,
  168. startOffset: startOffset,
  169. endOffset: endOffset
  170. ))
  171. index += 1
  172. if end >= totalFrames { break }
  173. start += step
  174. }
  175. return plans
  176. }
  177. // MARK: - Per-channel transcription
  178. private func transcribeChannel(
  179. url: URL,
  180. chunks: [ChunkPlan],
  181. speaker: TranscriptSpeaker,
  182. locales: [Locale],
  183. counter: ProgressCounter
  184. ) async throws -> [TranscriptSegment] {
  185. var segments: [TranscriptSegment] = []
  186. segments.reserveCapacity(chunks.count)
  187. var preferredLocale: Locale?
  188. for plan in chunks {
  189. try Task.checkCancellation()
  190. let buffer = try readChunkBuffer(url: url, startFrame: plan.startFrame, frameCount: plan.frameCount)
  191. let result = await transcribeBufferWithLocaleFallback(
  192. buffer: buffer,
  193. locales: locales,
  194. preferredLocale: preferredLocale
  195. )
  196. await counter.increment()
  197. preferredLocale = result.locale ?? preferredLocale
  198. let trimmed = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
  199. if trimmed.isEmpty { continue }
  200. let textForSegment: String
  201. if let previous = segments.last {
  202. textForSegment = removeRepeatedPrefix(from: trimmed, previousText: previous.text)
  203. } else {
  204. textForSegment = trimmed
  205. }
  206. let normalized = textForSegment.trimmingCharacters(in: .whitespacesAndNewlines)
  207. if normalized.isEmpty { continue }
  208. segments.append(TranscriptSegment(
  209. speaker: speaker.localizedLabel,
  210. startOffset: plan.startOffset,
  211. endOffset: plan.endOffset,
  212. text: normalized
  213. ))
  214. }
  215. return segments
  216. }
  217. private func readChunkBuffer(url: URL, startFrame: AVAudioFramePosition, frameCount: AVAudioFrameCount) throws -> AVAudioPCMBuffer {
  218. let audioFile = try AVAudioFile(forReading: url)
  219. audioFile.framePosition = startFrame
  220. guard let buffer = AVAudioPCMBuffer(pcmFormat: audioFile.processingFormat, frameCapacity: frameCount) else {
  221. throw NSError(domain: "MeetingTranscriptionService", code: 1, userInfo: [NSLocalizedDescriptionKey: "Unable to allocate audio buffer.".localized])
  222. }
  223. try audioFile.read(into: buffer, frameCount: frameCount)
  224. return buffer
  225. }
  226. private func transcribeBufferWithLocaleFallback(
  227. buffer: AVAudioPCMBuffer,
  228. locales: [Locale],
  229. preferredLocale: Locale?
  230. ) async -> (text: String, locale: Locale?) {
  231. var orderedLocales: [Locale] = []
  232. if let preferredLocale {
  233. orderedLocales.append(preferredLocale)
  234. }
  235. for locale in locales where orderedLocales.contains(where: { $0.identifier == locale.identifier }) == false {
  236. orderedLocales.append(locale)
  237. }
  238. var bestCandidate: (text: String, locale: Locale, score: Double)?
  239. for locale in orderedLocales {
  240. guard let recognizer = SFSpeechRecognizer(locale: locale), recognizer.isAvailable else { continue }
  241. do {
  242. let text = try await transcribeBuffer(buffer: buffer, recognizer: recognizer)
  243. let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
  244. guard trimmed.isEmpty == false else { continue }
  245. let score = scoreTranscriptCandidate(trimmed, locale: locale, preferredLocale: preferredLocale)
  246. if let current = bestCandidate {
  247. if score > current.score {
  248. bestCandidate = (trimmed, locale, score)
  249. }
  250. } else {
  251. bestCandidate = (trimmed, locale, score)
  252. }
  253. } catch {
  254. // One transient retry before moving on to the next locale.
  255. try? await Task.sleep(nanoseconds: 500_000_000)
  256. if let text = try? await transcribeBuffer(buffer: buffer, recognizer: recognizer) {
  257. let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
  258. guard trimmed.isEmpty == false else { continue }
  259. let score = scoreTranscriptCandidate(trimmed, locale: locale, preferredLocale: preferredLocale)
  260. if let current = bestCandidate {
  261. if score > current.score {
  262. bestCandidate = (trimmed, locale, score)
  263. }
  264. } else {
  265. bestCandidate = (trimmed, locale, score)
  266. }
  267. }
  268. continue
  269. }
  270. }
  271. if let bestCandidate {
  272. return (bestCandidate.text, bestCandidate.locale)
  273. }
  274. return ("", nil)
  275. }
  276. private func preferredLocalesForGlobalRecognition() -> [Locale] {
  277. let supported = SFSpeechRecognizer.supportedLocales()
  278. guard supported.isEmpty == false else {
  279. return [Locale.current, Locale(identifier: "en-US")]
  280. }
  281. let normalizedSupportedByIdentifier: [String: Locale] = Dictionary(
  282. uniqueKeysWithValues: supported.map { locale in
  283. (normalizeLocaleIdentifier(locale.identifier), locale)
  284. }
  285. )
  286. var ordered: [Locale] = []
  287. var seen = Set<String>()
  288. func appendIfSupportedIdentifier(_ identifier: String) {
  289. let normalized = normalizeLocaleIdentifier(identifier)
  290. guard seen.contains(normalized) == false else { return }
  291. guard let supportedLocale = normalizedSupportedByIdentifier[normalized] else { return }
  292. seen.insert(normalized)
  293. ordered.append(supportedLocale)
  294. }
  295. appendIfSupportedIdentifier(Locale.current.identifier)
  296. for identifier in Locale.preferredLanguages {
  297. appendIfSupportedIdentifier(identifier)
  298. }
  299. for locale in supported.sorted(by: { $0.identifier < $1.identifier }) {
  300. appendIfSupportedIdentifier(locale.identifier)
  301. }
  302. if ordered.isEmpty {
  303. return [Locale(identifier: "en-US")]
  304. }
  305. return ordered
  306. }
  307. private func normalizeLocaleIdentifier(_ identifier: String) -> String {
  308. identifier.replacingOccurrences(of: "_", with: "-").lowercased()
  309. }
  310. private func scoreTranscriptCandidate(_ text: String, locale: Locale, preferredLocale: Locale?) -> Double {
  311. let normalizedWords = text
  312. .components(separatedBy: CharacterSet.letters.inverted)
  313. .filter { $0.isEmpty == false }
  314. guard normalizedWords.isEmpty == false else { return 0 }
  315. let totalChars = max(1, text.count)
  316. let letterCount = text.unicodeScalars.filter { CharacterSet.letters.contains($0) }.count
  317. let letterRatio = Double(letterCount) / Double(totalChars)
  318. var repeatedWordPenalty = 0.0
  319. if normalizedWords.count >= 3 {
  320. var longestRun = 1
  321. var run = 1
  322. for index in 1..<normalizedWords.count {
  323. if normalizedWords[index].caseInsensitiveCompare(normalizedWords[index - 1]) == .orderedSame {
  324. run += 1
  325. longestRun = max(longestRun, run)
  326. } else {
  327. run = 1
  328. }
  329. }
  330. if longestRun >= 3 {
  331. repeatedWordPenalty = Double(longestRun - 2) * 1.8
  332. }
  333. }
  334. // Bias toward continuity, but do not overpower quality score.
  335. let localeBias: Double
  336. if let preferredLocale,
  337. normalizeLocaleIdentifier(preferredLocale.identifier) == normalizeLocaleIdentifier(locale.identifier) {
  338. localeBias = 1.0
  339. } else {
  340. localeBias = 0.0
  341. }
  342. return Double(normalizedWords.count) + (letterRatio * 4.0) + localeBias - repeatedWordPenalty
  343. }
  344. /// Removes repeated leading words from a chunk that overlap with the
  345. /// trailing words of the previous chunk.
  346. private func removeRepeatedPrefix(from text: String, previousText: String) -> String {
  347. let separators = CharacterSet.whitespacesAndNewlines.union(.punctuationCharacters)
  348. let currentWords = text.components(separatedBy: separators).filter { $0.isEmpty == false }
  349. let previousWords = previousText.components(separatedBy: separators).filter { $0.isEmpty == false }
  350. guard currentWords.isEmpty == false, previousWords.isEmpty == false else { return text }
  351. let maxOverlap = min(12, currentWords.count, previousWords.count)
  352. guard maxOverlap > 0 else { return text }
  353. for overlap in stride(from: maxOverlap, through: 1, by: -1) {
  354. let previousSuffix = Array(previousWords.suffix(overlap)).map { $0.lowercased() }
  355. let currentPrefix = Array(currentWords.prefix(overlap)).map { $0.lowercased() }
  356. if previousSuffix == currentPrefix {
  357. let remainder = currentWords.dropFirst(overlap)
  358. return remainder.joined(separator: " ")
  359. }
  360. }
  361. return text
  362. }
  363. private func transcribeBuffer(buffer: AVAudioPCMBuffer, recognizer: SFSpeechRecognizer) async throws -> String {
  364. let request = SFSpeechAudioBufferRecognitionRequest()
  365. request.shouldReportPartialResults = false
  366. if #available(macOS 13.0, *) {
  367. request.addsPunctuation = true
  368. }
  369. return try await withCheckedThrowingContinuation { continuation in
  370. var hasResumed = false
  371. let lock = NSLock()
  372. func resumeOnce(with result: Result<String, Error>) {
  373. lock.lock()
  374. defer { lock.unlock() }
  375. if hasResumed { return }
  376. hasResumed = true
  377. switch result {
  378. case .success(let text):
  379. continuation.resume(returning: text)
  380. case .failure(let error):
  381. continuation.resume(throwing: error)
  382. }
  383. }
  384. let task = recognizer.recognitionTask(with: request) { result, error in
  385. if let error {
  386. let nsError = error as NSError
  387. // "No speech detected" is a normal empty-chunk outcome (code 203 in kafAssistant domain).
  388. if nsError.domain == "kAFAssistantErrorDomain" && (nsError.code == 203 || nsError.code == 1110) {
  389. resumeOnce(with: .success(""))
  390. return
  391. }
  392. resumeOnce(with: .failure(error))
  393. return
  394. }
  395. if let result, result.isFinal {
  396. resumeOnce(with: .success(result.bestTranscription.formattedString))
  397. }
  398. }
  399. request.append(buffer)
  400. request.endAudio()
  401. _ = task
  402. }
  403. }
  404. }
  405. extension Array where Element == TranscriptSegment {
  406. /// Renders segments as a human-readable timeline like:
  407. /// `[00:12] Hello everyone.`
  408. func renderedTimelineText() -> String {
  409. let formatter: (TimeInterval) -> String = { seconds in
  410. let total = Int(seconds.rounded(.down))
  411. let h = total / 3600
  412. let m = (total % 3600) / 60
  413. let s = total % 60
  414. if h > 0 {
  415. return String(format: "%02d:%02d:%02d", h, m, s)
  416. }
  417. return String(format: "%02d:%02d", m, s)
  418. }
  419. return self.map { segment in
  420. "[\(formatter(segment.startOffset))] \(segment.text)"
  421. }.joined(separator: "\n")
  422. }
  423. }
  424. enum MeetingNotesError: Error, LocalizedError {
  425. case missingAPIKey
  426. case invalidResponse
  427. case httpStatus(Int, String)
  428. case emptyNotes
  429. var errorDescription: String? {
  430. switch self {
  431. case .missingAPIKey:
  432. return "OpenAI API key is missing. Set OPENAI_API_KEY in the environment or app key vault.".localized
  433. case .invalidResponse:
  434. return "Notes generation returned an invalid response.".localized
  435. case let .httpStatus(code, body):
  436. return "Notes generation failed (%d): %@".localized(with: code, body)
  437. case .emptyNotes:
  438. return "Notes generation returned empty text.".localized
  439. }
  440. }
  441. }
  442. /// Generates concise meeting notes from transcript text.
  443. final class MeetingNotesService {
  444. private let session: URLSession
  445. private enum InAppKeyVault {
  446. private static let chunkOrder = [3, 9, 0, 7, 2, 10, 5, 1, 8, 4, 11, 6]
  447. private static let chunkSeeds: [UInt8] = [41, 77, 113, 149, 185, 221, 17, 53, 89, 125, 161, 197]
  448. private static let encryptedChunks: [[UInt8]] = [
  449. [75, 2, 114, 12, 39, 47, 198, 240, 254, 150, 129, 137, 150, 83],
  450. [117, 115, 43, 206, 197, 204, 221, 174, 164, 167, 158, 125, 44, 102],
  451. [2, 233, 190, 215, 195, 165, 182, 139, 154, 126, 54, 65, 115, 29],
  452. [193, 200, 230, 130, 237, 157, 171, 69, 73, 113, 102, 50, 14, 31],
  453. [192, 140, 136, 193, 169, 70, 105, 115, 46, 106, 91, 4, 221, 160],
  454. [130, 129, 174, 117, 116, 88, 112, 14, 11, 17, 208, 249, 221, 213],
  455. [69, 110, 30, 16, 30, 46, 29, 204, 210, 230, 238, 147, 234, 222],
  456. [102, 127, 49, 41, 74, 184, 171, 216, 210, 248, 136, 152, 84, 104],
  457. [9, 3, 35, 217, 247, 152, 220, 179, 175, 192, 83, 94, 71, 120],
  458. [23, 222, 174, 247, 153, 139, 173, 189, 104, 91, 20, 97, 56, 107],
  459. [238, 255, 240, 187, 150, 195, 79, 119, 106, 11, 40, 47, 44],
  460. [188, 130, 212, 186, 101, 120, 64, 122, 7, 22, 54, 236, 227, 229]
  461. ]
  462. static func resolveKey() -> String? {
  463. guard encryptedChunks.count == chunkOrder.count, chunkSeeds.count == chunkOrder.count else {
  464. return nil
  465. }
  466. var rebuiltByOriginalIndex = Array(repeating: [UInt8](), count: chunkOrder.count)
  467. for (scrambledIndex, encryptedChunk) in encryptedChunks.enumerated() {
  468. let seed = Int(chunkSeeds[scrambledIndex])
  469. let decrypted = encryptedChunk.enumerated().map { offset, byte in
  470. byte ^ UInt8((seed + (offset * 17)) & 0xFF)
  471. }
  472. let originalIndex = chunkOrder[scrambledIndex]
  473. guard originalIndex < rebuiltByOriginalIndex.count else { return nil }
  474. rebuiltByOriginalIndex[originalIndex] = decrypted
  475. }
  476. let joined = rebuiltByOriginalIndex.flatMap { $0 }
  477. guard joined.isEmpty == false else { return nil }
  478. return String(decoding: joined, as: UTF8.self)
  479. }
  480. }
  481. init(session: URLSession = .shared) {
  482. self.session = session
  483. }
  484. private enum APIKeySource: String {
  485. case argument
  486. case environment
  487. case inAppVault
  488. }
  489. func resolveAPIKey() -> String? {
  490. let env = normalizedAPIKey(from: ProcessInfo.processInfo.environment["OPENAI_API_KEY"])
  491. if let env, env.isEmpty == false { return env }
  492. let vault = normalizedAPIKey(from: InAppKeyVault.resolveKey())
  493. if let vault, vault.isEmpty == false { return vault }
  494. return nil
  495. }
  496. func generateNotes(
  497. from transcript: String,
  498. preferredLanguageIdentifier1: String? = nil,
  499. preferredLanguageIdentifier2: String? = nil,
  500. apiKey: String? = nil
  501. ) async throws -> String {
  502. let keys = resolveAPIKeyCandidates(apiKey: apiKey)
  503. guard keys.isEmpty == false else { throw MeetingNotesError.missingAPIKey }
  504. struct Message: Encodable {
  505. let role: String
  506. let content: String
  507. }
  508. struct Body: Encodable {
  509. let model: String
  510. let messages: [Message]
  511. let temperature: Double
  512. }
  513. var lastError: MeetingNotesError?
  514. for (index, candidate) in keys.enumerated() {
  515. do {
  516. let preference = preferredNotesLanguage(
  517. primary: preferredLanguageIdentifier1,
  518. secondary: preferredLanguageIdentifier2
  519. )
  520. let notesLanguage: DominantLanguage
  521. if let preference {
  522. notesLanguage = preference
  523. } else {
  524. notesLanguage = try await detectDominantLanguage(in: transcript, apiKey: candidate.key)
  525. }
  526. let notesPrompt = buildNotesPrompt(
  527. transcript: transcript,
  528. dominantLanguage: notesLanguage,
  529. preferredLanguage: preference
  530. )
  531. let body = Body(
  532. model: "gpt-4.1-mini",
  533. messages: [
  534. Message(role: "system", content: "You create practical meeting notes from imperfect transcripts."),
  535. Message(role: "user", content: notesPrompt)
  536. ],
  537. temperature: 0.2
  538. )
  539. return try await requestChatCompletionText(body: body, apiKey: candidate.key)
  540. } catch let error as MeetingNotesError {
  541. switch error {
  542. case .httpStatus(let code, _):
  543. lastError = error
  544. if code == 401, index < (keys.count - 1) {
  545. // Try next configured source when one key is stale/revoked.
  546. continue
  547. }
  548. throw error
  549. default:
  550. throw error
  551. }
  552. }
  553. }
  554. throw lastError ?? MeetingNotesError.invalidResponse
  555. }
  556. private func requestChatCompletionText(body: Encodable, apiKey: String) async throws -> String {
  557. var request = URLRequest(url: URL(string: "https://api.openai.com/v1/chat/completions")!)
  558. request.httpMethod = "POST"
  559. request.setValue("application/json", forHTTPHeaderField: "Content-Type")
  560. request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
  561. if let project = normalizedConfigValue(from: ProcessInfo.processInfo.environment["OPENAI_PROJECT_ID"])
  562. ?? normalizedConfigValue(from: Bundle.main.object(forInfoDictionaryKey: "OpenAIProjectID") as? String) {
  563. request.setValue(project, forHTTPHeaderField: "OpenAI-Project")
  564. }
  565. if let org = normalizedConfigValue(from: ProcessInfo.processInfo.environment["OPENAI_ORG_ID"])
  566. ?? normalizedConfigValue(from: Bundle.main.object(forInfoDictionaryKey: "OpenAIOrganizationID") as? String) {
  567. request.setValue(org, forHTTPHeaderField: "OpenAI-Organization")
  568. }
  569. request.httpBody = try JSONEncoder().encode(AnyEncodable(body))
  570. let (data, response) = try await session.data(for: request)
  571. guard let http = response as? HTTPURLResponse else { throw MeetingNotesError.invalidResponse }
  572. guard (200..<300).contains(http.statusCode) else {
  573. let bodyText = String(data: data, encoding: .utf8) ?? "<no body>"
  574. throw MeetingNotesError.httpStatus(http.statusCode, bodyText)
  575. }
  576. struct ChoiceMessage: Decodable {
  577. let content: String?
  578. }
  579. struct Choice: Decodable {
  580. let message: ChoiceMessage
  581. }
  582. struct ChatResponse: Decodable {
  583. let choices: [Choice]
  584. }
  585. let decoded = try JSONDecoder().decode(ChatResponse.self, from: data)
  586. let notes = decoded.choices.first?.message.content?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
  587. guard notes.isEmpty == false else { throw MeetingNotesError.emptyNotes }
  588. return notes
  589. }
  590. private struct DominantLanguage: Sendable {
  591. let tag: String
  592. let name: String
  593. }
  594. private func preferredNotesLanguage(primary: String?, secondary: String?) -> DominantLanguage? {
  595. // Notes should follow Preferred Language 1 when set (user intent),
  596. // because dominant-language detection can confuse similar scripts (e.g., Urdu vs Hindi).
  597. let cleaned = [
  598. primary?.trimmingCharacters(in: .whitespacesAndNewlines),
  599. secondary?.trimmingCharacters(in: .whitespacesAndNewlines)
  600. ]
  601. .compactMap { $0 }
  602. .filter { $0.isEmpty == false }
  603. guard let identifier = cleaned.first else { return nil }
  604. let tag = identifier.replacingOccurrences(of: "_", with: "-")
  605. let english = Locale(identifier: "en")
  606. let languageName = english.localizedString(forIdentifier: identifier)
  607. ?? english.localizedString(forIdentifier: tag)
  608. ?? identifier
  609. return DominantLanguage(tag: tag, name: languageName)
  610. }
  611. private func detectDominantLanguage(in transcript: String, apiKey: String) async throws -> DominantLanguage {
  612. struct DetectResponse: Decodable {
  613. let tag: String?
  614. let name: String?
  615. }
  616. let detectPrompt = """
  617. Identify the dominant language used in the transcript (the language used most overall).
  618. If multiple languages are present, still choose only ONE dominant language.
  619. Return ONLY valid JSON with this exact schema:
  620. {"tag":"<BCP-47 language tag>","name":"<language name in English>"}
  621. Transcript:
  622. \(transcript)
  623. """
  624. struct Message: Encodable {
  625. let role: String
  626. let content: String
  627. }
  628. struct Body: Encodable {
  629. let model: String
  630. let messages: [Message]
  631. let temperature: Double
  632. }
  633. let body = Body(
  634. model: "gpt-4.1-mini",
  635. messages: [
  636. Message(role: "system", content: "You detect languages and reply with strict JSON only."),
  637. Message(role: "user", content: detectPrompt)
  638. ],
  639. temperature: 0.0
  640. )
  641. let raw = try await requestChatCompletionText(body: body, apiKey: apiKey)
  642. let data = Data(raw.utf8)
  643. let decoded = try JSONDecoder().decode(DetectResponse.self, from: data)
  644. let tag = (decoded.tag ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
  645. let name = (decoded.name ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
  646. if tag.isEmpty == false, name.isEmpty == false {
  647. return DominantLanguage(tag: tag, name: name)
  648. }
  649. // Fallback: if decoding fails partially, still force non-English when possible.
  650. if tag.isEmpty == false {
  651. return DominantLanguage(tag: tag, name: tag)
  652. }
  653. return DominantLanguage(tag: "und", name: "the dominant language of the transcript")
  654. }
  655. private func buildNotesPrompt(
  656. transcript: String,
  657. dominantLanguage: DominantLanguage,
  658. preferredLanguage: DominantLanguage?
  659. ) -> String {
  660. let preferenceLine: String = {
  661. guard let preferredLanguage else { return "" }
  662. return "\nPreferred language selected by the user: \(preferredLanguage.name) (\(preferredLanguage.tag)). Follow this preference."
  663. }()
  664. return """
  665. You are a meeting assistant. Generate structured notes from the transcript.
  666. Important: the transcript may have missing words, dropped sentences, or minor recognition errors.
  667. Infer likely intent conservatively and produce useful notes without inventing specific facts.
  668. \(preferenceLine)
  669. Output language:
  670. - Write ALL notes (including headings) strictly in \(dominantLanguage.name) (\(dominantLanguage.tag)).
  671. - Do NOT respond in English unless \(dominantLanguage.name) is English.
  672. - Keep proper nouns, people names, product names, acronyms, URLs, and code tokens exactly as-is.
  673. Output sections (translate these section headings into \(dominantLanguage.name)):
  674. 1) Summary (3-5 bullets)
  675. 2) Decisions
  676. 3) Action Items (include owner if identifiable; otherwise write the equivalent of "Unassigned" in \(dominantLanguage.name))
  677. 4) Risks / Open Questions
  678. Transcript:
  679. \(transcript)
  680. """
  681. }
  682. private func resolveAPIKeyCandidates(apiKey: String?) -> [(source: APIKeySource, key: String)] {
  683. var candidates: [(APIKeySource, String)] = []
  684. if let value = normalizedAPIKey(from: apiKey) {
  685. candidates.append((.argument, value))
  686. }
  687. if let value = normalizedAPIKey(from: ProcessInfo.processInfo.environment["OPENAI_API_KEY"]) {
  688. candidates.append((.environment, value))
  689. }
  690. if let value = normalizedAPIKey(from: InAppKeyVault.resolveKey()) {
  691. candidates.append((.inAppVault, value))
  692. }
  693. var unique: [(APIKeySource, String)] = []
  694. var seen = Set<String>()
  695. for candidate in candidates where seen.contains(candidate.1) == false {
  696. seen.insert(candidate.1)
  697. unique.append(candidate)
  698. }
  699. return unique
  700. }
  701. private func normalizedConfigValue(from value: String?) -> String? {
  702. guard var cleaned = value?.trimmingCharacters(in: .whitespacesAndNewlines), cleaned.isEmpty == false else {
  703. return nil
  704. }
  705. cleaned = cleaned.trimmingCharacters(in: CharacterSet(charactersIn: "\"'`"))
  706. return cleaned.isEmpty ? nil : cleaned
  707. }
  708. private func normalizedAPIKey(from value: String?) -> String? {
  709. guard var cleaned = normalizedConfigValue(from: value) else { return nil }
  710. cleaned = cleaned.replacingOccurrences(of: "—", with: "-")
  711. cleaned = cleaned.replacingOccurrences(of: "–", with: "-")
  712. cleaned = cleaned.replacingOccurrences(of: " ", with: "")
  713. cleaned = cleaned.replacingOccurrences(of: "\u{200B}", with: "")
  714. cleaned = cleaned.replacingOccurrences(of: "\u{FEFF}", with: "")
  715. return cleaned.isEmpty ? nil : cleaned
  716. }
  717. }
  718. private struct AnyEncodable: Encodable {
  719. private let encodeBlock: (Encoder) throws -> Void
  720. init(_ wrapped: Encodable) {
  721. self.encodeBlock = wrapped.encode(to:)
  722. }
  723. func encode(to encoder: Encoder) throws {
  724. try encodeBlock(encoder)
  725. }
  726. }