indeed_ai_job_search/indeed_ai_job_search/Services/Indeed/IndeedWebKitPageFetcher.swift

//
//  IndeedWebKitPageFetcher.swift
//  indeed_ai_job_search
//
//  Loads Indeed search pages via WebKit (browser engine). Plain URLSession
//  is blocked by Cloudflare captcha; WKWebView matches real Safari enough
//  to receive job-card HTML for parsing.
//

import AppKit
import Foundation
import OSLog
import WebKit

@MainActor
final class IndeedWebKitPageFetcher: NSObject {
    static let shared = IndeedWebKitPageFetcher()

    private let log = Logger(
        subsystem: Bundle.main.bundleIdentifier ?? "indeed_ai_job_search",
        category: "IndeedFetch"
    )

    private var webView: WKWebView?
    /// Kept alive so WebKit completes loads off-screen (macOS often stalls without a window).
    private var hostWindow: NSWindow?
    private var continuation: CheckedContinuation<String, Error>?
    private var timeoutTask: Task<Void, Never>?

    private override init() {
        super.init()
    }

    /// Fetches fully rendered page HTML for `url`.
    func html(from url: URL, timeoutSeconds: TimeInterval = 35) async throws -> String {
        // Serialize loads — one WKWebView at a time.
        if continuation != nil {
            throw IndeedDataError.network("Another Indeed page load is already in progress.")
        }

        let configuration = WKWebViewConfiguration()
        // Persistent cookies help when Indeed has already verified this Mac in Safari/WebKit.
        configuration.websiteDataStore = .default()
        configuration.defaultWebpagePreferences.allowsContentJavaScript = true

        let view = WKWebView(frame: CGRect(x: 0, y: 0, width: 1280, height: 900), configuration: configuration)
        view.navigationDelegate = self
        self.webView = view

        let window = NSWindow(
            contentRect: NSRect(x: -10_000, y: -10_000, width: 1280, height: 900),
            styleMask: [.titled, .closable],
            backing: .buffered,
            defer: false
        )
        window.isReleasedWhenClosed = false
        window.contentView = view
        window.orderBack(nil)
        self.hostWindow = window

        log.info("Loading Indeed page \(url.absoluteString, privacy: .public)")

        return try await withCheckedThrowingContinuation { (cont: CheckedContinuation<String, Error>) in
            self.continuation = cont
            self.timeoutTask = Task { [weak self] in
                try? await Task.sleep(nanoseconds: UInt64(timeoutSeconds * 1_000_000_000))
                await MainActor.run {
                    guard let self, self.continuation != nil else { return }
                    self.fail(IndeedDataError.network("Timed out loading Indeed search results."))
                }
            }
            view.load(URLRequest(url: url, cachePolicy: .reloadIgnoringLocalCacheData, timeoutInterval: timeoutSeconds))
        }
    }

    private func tearDownWebView() {
        webView?.navigationDelegate = nil
        webView?.stopLoading()
        hostWindow?.contentView = nil
        hostWindow?.orderOut(nil)
        hostWindow = nil
        webView = nil
    }

    private func finish(_ html: String) {
        timeoutTask?.cancel()
        timeoutTask = nil
        guard let continuation else { return }
        self.continuation = nil
        tearDownWebView()
        continuation.resume(returning: html)
    }

    private func fail(_ error: Error) {
        timeoutTask?.cancel()
        timeoutTask = nil
        guard let continuation else { return }
        self.continuation = nil
        tearDownWebView()
        continuation.resume(throwing: error)
    }

    private func extractHTML(from webView: WKWebView) {
        let script = """
        (function() {
          if (window.INDEED_CLOUDFLARE_STATIC_PAGE) {
            return JSON.stringify({ blocked: true, html: document.documentElement.outerHTML });
          }
          return JSON.stringify({ blocked: false, html: document.documentElement.outerHTML });
        })();
        """
        webView.evaluateJavaScript(script) { [weak self] result, error in
            Task { @MainActor in
                guard let self else { return }
                if let error {
                    self.fail(IndeedDataError.network(error.localizedDescription))
                    return
                }
                guard let json = result as? String,
                      let data = json.data(using: .utf8),
                      let payload = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
                      let html = payload["html"] as? String else {
                    self.fail(IndeedDataError.invalidResponse("Could not read Indeed page HTML."))
                    return
                }
                if payload["blocked"] as? Bool == true || IndeedHTMLJobParser.isBlockedOrCaptcha(html) {
                    self.log.error("Indeed returned a security/captcha page")
                    self.fail(IndeedDataError.forbidden)
                    return
                }
                self.finish(html)
            }
        }
    }
}

extension IndeedWebKitPageFetcher: WKNavigationDelegate {
    func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) {
        // Give mosaic job-card scripts a moment to hydrate.
        Task { @MainActor in
            try? await Task.sleep(nanoseconds: 900_000_000)
            guard self.webView === webView, self.continuation != nil else { return }
            self.extractHTML(from: webView)
        }
    }

    func webView(
        _ webView: WKWebView,
        didFail navigation: WKNavigation!,
        withError error: Error
    ) {
        fail(IndeedDataError.network(error.localizedDescription))
    }

    func webView(
        _ webView: WKWebView,
        didFailProvisionalNavigation navigation: WKNavigation!,
        withError error: Error
    ) {
        fail(IndeedDataError.network(error.localizedDescription))
    }
}