indeed_ai_job_search/indeed_ai_job_search/Services/Search/JobSearchPipeline.swift

//
//  JobSearchPipeline.swift
//  indeed_ai_job_search
//
//  Phase 4 orchestrator: NL → criteria → Indeed fetch → filter → validate →
//  rank → dedupe (with optional in-memory metadata cache).
//

import Foundation
import OSLog

struct DefaultJobSearchPipeline: JobSearchPipeline {
    private static let log = Logger(
        subsystem: Bundle.main.bundleIdentifier ?? "indeed_ai_job_search",
        category: "SearchPipeline"
    )

    var interpreter: OpenAISearchService
    var dataSource: IndeedJobDataSource
    var rankingService: OpenAIRankingService
    var validationService: CompositeJobValidationService
    var cache: any JobListingCache
    var maxPages: Int

    init(
        interpreter: OpenAISearchService,
        dataSource: IndeedJobDataSource,
        rankingService: OpenAIRankingService,
        validationService: CompositeJobValidationService,
        cache: any JobListingCache = NullJobListingCache(),
        maxPages: Int = IndeedCachePolicy.maxPagesPerSearchUI
    ) {
        self.interpreter = interpreter
        self.dataSource = dataSource
        self.rankingService = rankingService
        self.validationService = validationService
        self.cache = cache
        self.maxPages = max(1, maxPages)
    }

    func search(_ request: JobSearchPipelineRequest) async throws -> JobSearchPipelineResult {
        let trimmed = request.naturalLanguageQuery.trimmingCharacters(in: .whitespacesAndNewlines)

        let interpreted: InterpretedSearch
        let criteria: SearchCriteria

        if request.reinterpret {
            interpreted = try await interpreter.interpret(query: trimmed)
            criteria = SearchCriteriaMerge.merge(
                interpreted: interpreted.criteria,
                userEdited: request.userCriteria
            )
        } else {
            criteria = request.userCriteria
            interpreted = InterpretedSearch(
                naturalLanguageQuery: trimmed,
                criteria: criteria,
                confidenceNote: "Using current filters without re-interpretation."
            )
        }

        Self.log.info(
            """
            Search start query=\(trimmed, privacy: .public) \
            country=\(criteria.country.isEmpty ? "(none)" : criteria.country, privacy: .public) \
            city=\(criteria.city.isEmpty ? "(none)" : criteria.city, privacy: .public) \
            title=\(criteria.jobTitle.isEmpty ? "(none)" : criteria.jobTitle, privacy: .public) \
            reinterpret=\(request.reinterpret, privacy: .public)
            """
        )

        if let cached = await cache.get(query: trimmed, criteria: criteria) {
            Self.log.debug("Cache hit — \(cached.listings.count, privacy: .public) listings")
            return JobSearchPipelineResult(
                interpreted: InterpretedSearch(
                    naturalLanguageQuery: trimmed,
                    criteria: criteria,
                    confidenceNote: interpreted.confidenceNote
                ),
                listings: cached.listings,
                totalEstimated: cached.totalEstimated,
                usedCache: true
            )
        }

        let fetched = try await fetchPages(criteria: criteria, query: trimmed)
        Self.log.info("Fetched \(fetched.listings.count, privacy: .public) raw listings")

        for listing in fetched.listings where listing.availability == .unavailable {
            Self.log.warning(
                "Source marked unavailable — key=\(listing.indeedJobKey, privacy: .public) title=\(listing.title, privacy: .public) url=\(listing.indeedURL.absoluteString, privacy: .public)"
            )
        }

        var listings = JobListingFilter.filter(fetched.listings, criteria: criteria, query: trimmed)
        listings = JobListingDeduper.dedupe(listings)

        if FeatureFlags.isListingValidationEnabled {
            let before = listings.count
            listings = await validationService.validateAll(listings)
            let dropped = listings.filter { $0.availability == .unavailable }
            for listing in dropped {
                Self.log.warning(
                    "Validation dropped unavailable — key=\(listing.indeedJobKey, privacy: .public) title=\(listing.title, privacy: .public) url=\(listing.indeedURL.absoluteString, privacy: .public)"
                )
            }
            listings = listings.filter { $0.availability != .unavailable }
            Self.log.debug(
                "Validation kept \(listings.count, privacy: .public)/\(before, privacy: .public) listings"
            )
        }

        listings = try await rankingService.rank(
            listings: listings,
            query: trimmed,
            criteria: criteria
        )
        // Final dedupe after ranking in case batches reintroduced keys.
        listings = JobListingDeduper.dedupe(listings)

        for listing in listings {
            let resolved = IndeedBrowseURLBuilder.openableURL(for: listing)
            if resolved.usedBrowseFallback, let reason = resolved.reason {
                Self.log.info(
                    "Open URL note key=\(listing.indeedJobKey, privacy: .public): \(reason, privacy: .public)"
                )
            }
        }

        let total = fetched.totalEstimated ?? listings.count
        await cache.set(
            CachedJobSearchPage(listings: listings, totalEstimated: total, storedAt: Date()),
            query: trimmed,
            criteria: criteria
        )

        Self.log.info("Search complete — \(listings.count, privacy: .public) listings (estimated \(total, privacy: .public))")

        return JobSearchPipelineResult(
            interpreted: InterpretedSearch(
                naturalLanguageQuery: trimmed,
                criteria: criteria,
                confidenceNote: interpreted.confidenceNote
            ),
            listings: listings,
            totalEstimated: total,
            usedCache: false
        )
    }

    private func fetchPages(
        criteria: SearchCriteria,
        query: String
    ) async throws -> (listings: [JobListing], totalEstimated: Int?) {
        var all: [JobListing] = []
        var totalEstimated: Int?
        var cursor: String?
        var pageIndex = 0

        while pageIndex < maxPages {
            let page = try await dataSource.searchPage(
                criteria: criteria,
                query: query,
                page: IndeedSearchPageRequest(
                    pageIndex: pageIndex,
                    pageSize: IndeedSearchPageRequest.defaultPageSize,
                    cursor: cursor
                )
            )

            all.append(contentsOf: page.listings)
            if totalEstimated == nil {
                totalEstimated = page.totalEstimated
            }

            guard page.hasMore else { break }
            cursor = page.nextCursor
            pageIndex += 1
        }

        return (all, totalEstimated)
    }
}

/// Adapts a plain `JobDataSource` (e.g. mock) into the paginated Indeed surface.
struct JobDataSourceIndeedAdapter: IndeedJobDataSource {
    let base: JobDataSource

    func searchPage(
        criteria: SearchCriteria,
        query: String,
        page: IndeedSearchPageRequest
    ) async throws -> IndeedSearchPage {
        // Non-paginated sources only serve page 0.
        if page.pageIndex > 0 {
            return IndeedSearchPage(
                listings: [],
                pageIndex: page.pageIndex,
                pageSize: page.pageSize,
                totalEstimated: 0,
                nextCursor: nil,
                hasMore: false
            )
        }

        let listings = try await base.search(criteria: criteria, query: query)
        return IndeedSearchPage(
            listings: listings,
            pageIndex: 0,
            pageSize: page.pageSize,
            totalEstimated: listings.count,
            nextCursor: nil,
            hasMore: false
        )
    }
}