"""API coverage audit for pypaperless.

Connects to the live PaperlessClient instance and:

  1. Fetches the OpenAPI schema (/api/schema/?format=json) as the authoritative
     source of field definitions for every endpoint.
  2. Also fetches a live object from each endpoint where possible.
  3. Compares both against the model_fields of the matching pypaperless model.

Status per model:
  OK            – model covers every field from the live response
  SCHEMA ✓      – no live data; model covers every field listed in the schema
  GAPS FOUND    – missing or extra fields detected
  NO DATA       – endpoint empty AND no schema component available (untyped)
  FETCH ERROR   – HTTP / parse error

The final summary also lists:
  • Schema paths not covered by any entry in const.EndpointPath (unimplemented)

Usage
-----
  # Activate your dev venv, then create .env from .env.example
  # (PYPAPERLESS_URL / PYPAPERLESS_TOKEN / PYPAPERLESS_TEST_DOC).
  python script/pngx_audit_coverage.py
"""

# ruff: noqa
# mypy: ignore-errors

# This file was fully automatically generated by GitHub Copilot (Claude Sonnet 4.6).

import argparse
import asyncio
import re
from dataclasses import dataclass, field
from enum import Enum
from typing import Any

import httpx

from pypaperless import PaperlessClient
from pypaperless.const import EndpointPath
from pypaperless.models import (
    Config,
    Correspondent,
    CustomField,
    Document,
    DocumentAISuggestions,
    DocumentHistory,
    DocumentMeta,
    DocumentNote,
    DocumentRoot,
    DocumentSuggestions,
    Group,
    MailAccount,
    MailRule,
    ProcessedMail,
    Profile,
    RemoteVersion,
    SavedView,
    SearchResult,
    ShareLink,
    ShareLinkBundle,
    Statistic,
    Status,
    StoragePath,
    Tag,
    Task,
    User,
    Workflow,
    WorkflowAction,
    WorkflowTrigger,
)
from pypaperless.models.config import (
    ArchiveFileGeneration,
    ColorConversionStrategy,
    OcrMode,
    OutputType,
    UnpaperClean,
)
from pypaperless.models.custom_fields import (
    CustomFieldType,
    CustomFieldValue,
)
from pypaperless.models.document_types import DocumentType
from pypaperless.models.mails.accounts import ImapSecurity, MailAccountType
from pypaperless.models.mails.rules import (
    MailRuleAction,
    MailRuleAttachmentType,
    MailRuleConsumptionScope,
    MailRuleCorrespondentSource,
    MailRulePdfLayout,
    MailRuleTitleSource,
)
from pypaperless.models.mixins.data_fields import MatchingAlgorithm
from pypaperless.models.saved_views import SavedViewDisplayMode, SavedViewFilterRule
from pypaperless.models.share_links import ShareLinkFileVersion
from pypaperless.models.status import (
    StatusDatabase,
    StatusDatabaseMigration,
    StatusStorage,
    StatusTasks,
)
from pypaperless.models.tasks import TaskStatus, TaskTriggerSource, TaskType
from pypaperless.models.workflows import (
    WorkflowActionEmail,
    WorkflowActionType,
    WorkflowActionWebhook,
    WorkflowTriggerScheduleDateField,
    WorkflowTriggerSource,
    WorkflowTriggerType,
)

from _dev_env import load_dev_env

# ── credentials ───────────────────────────────────────────────────────────────
_env = load_dev_env()
PAPERLESS_URL = _env.url
PAPERLESS_TOKEN = _env.token.get_secret_value()
TEST_DOC_ID = _env.test_doc

# ── terminal colours ──────────────────────────────────────────────────────────
RED = "\033[31m"
GREEN = "\033[32m"
YELLOW = "\033[33m"
CYAN = "\033[36m"
BOLD = "\033[1m"
DIM = "\033[2m"
RESET = "\033[0m"

# ── Fields intentionally omitted from the model (API key not in model_fields) ─
# These are keys the live API DOES return but the model deliberately omits.
# Map: model_class_name → {api_key: explanation}
KNOWN_OMISSIONS: dict[str, dict[str, str]] = {
    "Document": {
        "created_date": "legacy alias for `created`; model exposes a `created_date` property instead",
        "notes": "note IDs returned in list view; use document.notes service for full objects",
        "versions": "version IDs returned in list view; use document.versions service for full objects",
    },
    "User": {
        "password": "write-only credential; intentionally excluded from model",
    },
    "Profile": {
        "password": "write-only credential; intentionally excluded from model",
    },
    "MailAccount": {
        "password": "write-only credential; intentionally excluded from model",
    },
}

# ── Model fields NOT returned by the standard live API endpoint ───────────────
# These fields ARE declared in the model but require special query parameters
# (e.g. ?full_perms=true) or are injected by the pypaperless service layer.
# Map: model_class_name → {model_field: explanation}
KNOWN_EXTRAS: dict[str, dict[str, str]] = {
    "Document": {
        "search_hit_": "synthesised from __search_hit__ alias only present in search results",
        "permissions": "only returned with ?full_perms=true; accessed via SecurableMixin",
        "notes_": "injected by service from separate notes endpoint; not in standard Document response",
        "versions_": "injected by service from separate versions endpoint; not in standard Document response",
    },
    "Correspondent": {
        "permissions": "only returned with ?full_perms=true; accessed via SecurableMixin",
        "last_correspondence": "not returned in standard list; confirmed in schema, requires custom query",
    },
    "DocumentType": {
        "permissions": "only returned with ?full_perms=true",
    },
    "StoragePath": {
        "permissions": "only returned with ?full_perms=true",
    },
    "Tag": {
        "permissions": "only returned with ?full_perms=true",
    },
    "DocumentNote": {
        "document": "injected by service from URL path; not part of the Notes JSON body",
    },
    "DocumentHistory": {
        "document": "injected by service from URL path; not part of the LogEntry JSON body",
    },
    "DocumentMeta": {
        "id": "injected by service from path parameter; not in API response",
    },
    "DocumentSuggestions": {
        "id": "injected by service from path parameter; not in API response",
    },
    "DocumentAISuggestions": {
        "id": "injected by service from path parameter; not in API response",
    },
    "SavedView": {
        "permissions": "only returned with ?full_perms=true",
    },
    "MailAccount": {
        "permissions": "only returned with ?full_perms=true",
    },
    "MailRule": {
        "permissions": "only returned with ?full_perms=true",
    },
}

# ── Model fields that the SCHEMA does not document (schema comparison) ────────
# These fields exist in the model but NOT in the OpenAPI component schema.
KNOWN_SCHEMA_EXTRAS: dict[str, dict[str, str]] = {
    "Document": {
        "search_hit_": "synthesised from __search_hit__ alias; not in OpenAPI schema",
        "permissions": "only returned with full_perms=true; not in standard Document schema",
        "versions_": "injected by service from separate versions endpoint; not in OpenAPI Document schema",
    },
    "Correspondent": {
        "permissions": "only returned with full_perms=true; not in standard Correspondent schema",
    },
    "DocumentType": {
        "permissions": "only returned with full_perms=true; not in standard DocumentType schema",
    },
    "StoragePath": {
        "permissions": "only returned with full_perms=true; not in standard StoragePath schema",
    },
    "Tag": {
        "permissions": "only returned with full_perms=true; not in standard Tag schema",
        "children": "pypaperless extension for nested tags; not in OpenAPI schema",
        "parent": "pypaperless extension for nested tags; not in OpenAPI schema",
    },
    "SavedView": {
        "permissions": "only returned with full_perms=true; not in standard SavedView schema",
    },
    "MailAccount": {
        "permissions": "only returned with full_perms=true; not in standard MailAccount schema",
        "account_type": "not listed in MailAccount schema properties",
        "expiration": "not listed in MailAccount schema properties",
    },
    "MailRule": {
        "permissions": "only returned with full_perms=true; not in standard MailRule schema",
    },
    "DocumentNote": {
        "document": "injected by service; Notes schema has no 'document' property",
    },
    "DocumentHistory": {
        "document": "injected by service; LogEntry schema has no 'document' property",
    },
    "DocumentMeta": {
        "id": "injected by service from path parameter; Metadata schema has no 'id'",
    },
    "DocumentSuggestions": {
        "id": "injected by service from path parameter; Suggestions schema has no 'id'",
    },
    "DocumentAISuggestions": {
        "id": "injected by service from path parameter; AISuggestions schema has no 'id'",
    },
}

# ── Schema fields that the model deliberately omits (schema comparison) ───────
# These appear in the OpenAPI component schema but are intentionally absent.
KNOWN_SCHEMA_OMISSIONS: dict[str, dict[str, str]] = {
    "MailAccount": {
        "password": "write-only credential; API never returns it",
    },
    "User": {
        "password": "write-only credential; API never returns it",
    },
    "Document": {
        "created_date": "legacy alias for `created`; model uses `created` with a property alias",
        "notes": "inline note IDs only; full objects accessed via the document.notes service",
        "versions": "inline version IDs only; full objects accessed via the document.versions service",
    },
    "Status": {
        "index": "SystemStatus.index sub-object not yet modelled",
        "classifier": "SystemStatus.classifier sub-object not yet modelled",
        "sanity_check": "SystemStatus.sanity_check sub-object not yet modelled",
    },
}

# ── Sub-model classes mapped to their OpenAPI schema component ───────────────
# (pypaperless_class, openapi_schema_name, description)
SUBMODEL_MAPPINGS: list[tuple[type, str, str]] = [
    (StatusDatabaseMigration, "MigrationStatus", "Status.database.migration_status"),
    (StatusDatabase, "Database", "Status.database"),
    (StatusStorage, "Storage", "Status.storage"),
    (StatusTasks, "Tasks", "Status.tasks"),
    (CustomFieldValue, "CustomFieldInstance", "Document.custom_fields[]"),
    (WorkflowActionEmail, "WorkflowActionEmail", "WorkflowAction.email"),
    (WorkflowActionWebhook, "WorkflowActionWebhook", "WorkflowAction.webhook"),
    (SavedViewFilterRule, "SavedViewFilterRule", "SavedView.filter_rules[]"),
]

# Fields in the pypaperless sub-model class that are not in the schema component
# (intentional extras – e.g. injected by service layer or schema is incomplete)
SUBMODEL_KNOWN_EXTRAS: dict[str, dict[str, str]] = {
    "StatusTasks": {
        # Schema Tasks only documents 4 fields; live API returns all 15 flat fields.
        # PaperlessClient consolidates index/classifier/sanity_check into this object.
        "celery_url": "not in schema; live API returns it in tasks object",
        "celery_error": "not in schema; live API returns it in tasks object",
        "index_status": "schema has separate Index component but API flattens into tasks",
        "index_last_modified": "schema has separate Index component but API flattens into tasks",
        "index_error": "schema has separate Index component but API flattens into tasks",
        "classifier_status": "schema has separate Classifier component but API flattens into tasks",
        "classifier_last_trained": "schema has separate Classifier component but API flattens into tasks",
        "classifier_error": "schema has separate Classifier component but API flattens into tasks",
        "sanity_check_status": "schema has separate SanityCheck component but API flattens into tasks",
        "sanity_check_last_run": "schema has separate SanityCheck component but API flattens into tasks",
        "sanity_check_error": "schema has separate SanityCheck component but API flattens into tasks",
    },
    "CustomFieldValue": {
        "name": "injected from custom field cache by pypaperless service layer",
        "data_type": "injected from custom field cache by pypaperless service layer",
        "extra_data": "injected from custom field cache by pypaperless service layer",
    },
}

# Fields in the schema component that are intentionally absent from the sub-model
SUBMODEL_KNOWN_OMISSIONS: dict[str, dict[str, str]] = {}

# Sub-models that have no direct OpenAPI schema component (cannot verify field-by-field)
# (class_name, source_file, usage / why it has no schema)
UNSCHEMA_SUBMODELS: list[tuple[str, str, str]] = [
    (
        "DocumentMetaEntry",
        "documents.py",
        "DocumentMeta.original_metadata/archive_metadata — schema: untyped 'object'",
    ),
    (
        "DocumentSearchHit",
        "documents.py",
        "Document.__search_hit__ — internal search enrichment, not in schema",
    ),
    (
        "StatisticDocumentFileTypeCount",
        "statistics.py",
        "Statistic.document_file_type_counts[] — schema: Statistic component is untyped",
    ),
    (
        "CustomFieldExtraData",
        "custom_fields.py",
        "CustomField.extra_data — schema: untyped nullable object",
    ),
    (
        "CustomFieldSelectOptions",
        "custom_fields.py",
        "CustomFieldExtraData.select_options[] — schema: untyped nullable object",
    ),
    (
        "_PermissionScope",
        "mixins/securable.py",
        "Permissions.view/.change — schema: inlined, no standalone component",
    ),
    (
        "Permissions",
        "mixins/securable.py",
        "SecurableMixin.permissions — schema: inlined, no standalone component",
    ),
]

# ── Typed enum classes mapped to their OpenAPI schema component ──────────────
# (pypaperless_class, openapi_schema_name, description)
ENUM_MAPPINGS: list[tuple[type[Enum], str, str]] = [
    (MatchingAlgorithm, "MatchingAlgorithm", "Correspondent / DocumentType / StoragePath / Tag"),
    (CustomFieldType, "DataTypeEnum", "CustomField.data_type"),
    (ShareLinkFileVersion, "FileVersionEnum", "ShareLink.file_version"),
    (TaskType, "TaskTypeEnum", "Task.task_type"),
    (TaskStatus, "TaskSerializerV10StatusEnum", "Task.status"),
    (TaskTriggerSource, "TriggerSourceEnum", "Task.trigger_source"),
    (WorkflowActionType, "WorkflowActionTypeEnum", "WorkflowAction.type"),
    (WorkflowTriggerType, "WorkflowTriggerTypeEnum", "WorkflowTrigger.type"),
    (
        WorkflowTriggerScheduleDateField,
        "ScheduleDateFieldEnum",
        "WorkflowTrigger.schedule_date_field",
    ),
    (WorkflowTriggerSource, "SourcesEnum", "WorkflowTrigger.sources"),
    (OutputType, "OutputTypeEnum", "Config.output_type"),
    (OcrMode, "ModeEnum", "Config.mode"),
    (ArchiveFileGeneration, "ArchiveFileGenerationEnum", "Config.archive_file_generation"),
    (UnpaperClean, "UnpaperCleanEnum", "Config.unpaper_clean"),
    (ColorConversionStrategy, "ColorConversionStrategyEnum", "Config.color_conversion_strategy"),
    (SavedViewDisplayMode, "DisplayModeEnum", "SavedView.display_mode"),
    (ImapSecurity, "ImapSecurityEnum", "MailAccount.imap_security"),
    (MailAccountType, "AccountTypeEnum", "MailAccount.account_type"),
    (MailRuleAction, "MailRuleActionEnum", "MailRule.action"),
    (MailRuleTitleSource, "AssignTitleFromEnum", "MailRule.assign_title_from"),
    (
        MailRuleCorrespondentSource,
        "AssignCorrespondentFromEnum",
        "MailRule.assign_correspondent_from",
    ),
    (MailRuleAttachmentType, "AttachmentTypeEnum", "MailRule.attachment_type"),
    (MailRuleConsumptionScope, "ConsumptionScopeEnum", "MailRule.consumption_scope"),
    (MailRulePdfLayout, "PdfLayoutEnum", "MailRule.pdf_layout"),
]

# ── Model fields that use raw int/str where schema defines a typed enum ───────
# (model_name, field_name, openapi_schema_name)
UNTYPED_ENUM_FIELDS: list[tuple[str, str, str]] = []

# ── Schema paths that are action/utility endpoints (no resource model needed) ─
KNOWN_UTILITY_PATHS: set[str] = {
    "/api/bulk_edit_objects/",
    "/api/schema/",
    "/api/token/",
    "/api/oauth/callback/",
    "/api/documents/bulk_download/",
    "/api/documents/bulk_edit/",
    "/api/documents/delete/",
    "/api/documents/edit_pdf/",
    "/api/documents/email/",
    "/api/documents/merge/",
    "/api/documents/remove_password/",
    "/api/documents/reprocess/",
    "/api/documents/rotate/",
    "/api/documents/next_asn/",
    "/api/documents/post_document/",
    "/api/documents/selection_data/",
    "/api/documents/{id}/download/",
    "/api/documents/{id}/email/",
    "/api/documents/{id}/preview/",
    "/api/documents/{id}/thumb/",
    "/api/documents/{id}/update_version/",
    "/api/documents/{id}/versions/{version_id}/",
    "/api/mail_accounts/test/",
    "/api/mail_accounts/{id}/process/",
    "/api/processed_mail/bulk_delete/",
    "/api/profile/disconnect_social_account/",
    "/api/profile/generate_auth_token/",
    "/api/profile/social_account_providers/",
    "/api/profile/totp/",
    "/api/search/autocomplete/",
    "/api/storage_paths/test/",
    "/api/tasks/acknowledge/",
    "/api/tasks/active/",
    "/api/tasks/run/",
    "/api/tasks/status_counts/",
    "/api/tasks/summary/",
    "/api/users/{id}/deactivate_totp/",
}

# ── Schema paths intentionally out of scope for pypaperless ──────────────────
INTENTIONALLY_SKIPPED_PATHS: set[str] = {
    "/api/logs/",  # server log streaming, not a resource
    "/api/logs/{id}/",  # server log streaming, not a resource
    "/api/ui_settings/",  # paperless-ngx front-end settings, not exposed
}


# ─────────────────────────────────────────────────────────────────────────────
# Data structures
# ─────────────────────────────────────────────────────────────────────────────


@dataclass
class EndpointSpec:
    """Describe how to fetch and parse one API endpoint."""

    label: str
    path: str
    model_cls: type
    unwrap: str = "paginated"  # "paginated" | "direct" | "list_index0"
    schema_component: str | None = None  # OpenAPI components/schemas/<name>


@dataclass
class AuditResult:
    label: str
    model_name: str
    api_keys: set[str] = field(default_factory=set)
    schema_keys: set[str] = field(default_factory=set)
    model_keys: set[str] = field(default_factory=set)
    missing_in_model: list[str] = field(default_factory=list)  # in reference, not in model
    extra_in_model: list[str] = field(default_factory=list)  # in model, not in reference
    known_omissions: list[str] = field(default_factory=list)
    known_extras: list[str] = field(default_factory=list)
    fetch_error: str | None = None
    no_data: bool = False  # empty endpoint AND no schema component
    schema_only: bool = False  # no live data, verified via schema only


# ─────────────────────────────────────────────────────────────────────────────
# Endpoint catalogue
# ─────────────────────────────────────────────────────────────────────────────

ENDPOINTS: list[EndpointSpec] = [
    EndpointSpec("Config", "/api/config/1/", Config, "direct", "ApplicationConfiguration"),
    EndpointSpec(
        "Correspondent",
        "/api/correspondents/?page_size=1",
        Correspondent,
        "paginated",
        "Correspondent",
    ),
    EndpointSpec(
        "CustomField", "/api/custom_fields/?page_size=1", CustomField, "paginated", "CustomField"
    ),
    EndpointSpec("Document", f"/api/documents/{TEST_DOC_ID}/", Document, "direct", "Document"),
    EndpointSpec(
        "DocumentMeta",
        f"/api/documents/{TEST_DOC_ID}/metadata/",
        DocumentMeta,
        "direct",
        "Metadata",
    ),
    EndpointSpec(
        "DocumentNote", f"/api/documents/{TEST_DOC_ID}/notes/", DocumentNote, "list_index0", "Notes"
    ),
    EndpointSpec(
        "DocumentHistory",
        f"/api/documents/{TEST_DOC_ID}/history/",
        DocumentHistory,
        "list_index0",
        "LogEntry",
    ),
    EndpointSpec(
        "DocumentSuggestions",
        f"/api/documents/{TEST_DOC_ID}/suggestions/",
        DocumentSuggestions,
        "direct",
        "Suggestions",
    ),
    EndpointSpec(
        "DocumentRoot",
        f"/api/documents/{TEST_DOC_ID}/root/",
        DocumentRoot,
        "direct",
        "DocumentRootResponse",
    ),
    EndpointSpec(
        "DocumentType",
        "/api/document_types/?page_size=1",
        DocumentType,
        "paginated",
        "DocumentType",
    ),
    EndpointSpec("Group", "/api/groups/?page_size=1", Group, "paginated", "Group"),
    EndpointSpec(
        "MailAccount", "/api/mail_accounts/?page_size=1", MailAccount, "paginated", "MailAccount"
    ),
    EndpointSpec("MailRule", "/api/mail_rules/?page_size=1", MailRule, "paginated", "MailRule"),
    EndpointSpec(
        "ProcessedMail",
        "/api/processed_mail/?page_size=1",
        ProcessedMail,
        "paginated",
        "ProcessedMail",
    ),
    EndpointSpec("Profile", "/api/profile/", Profile, "direct", "Profile"),
    EndpointSpec(
        "RemoteVersion", "/api/remote_version/", RemoteVersion, "direct", None
    ),  # untyped in schema
    EndpointSpec("SavedView", "/api/saved_views/?page_size=1", SavedView, "paginated", "SavedView"),
    EndpointSpec(
        "SearchResult", "/api/search/?query=invoice", SearchResult, "direct", "SearchResult"
    ),
    EndpointSpec("ShareLink", "/api/share_links/?page_size=1", ShareLink, "paginated", "ShareLink"),
    EndpointSpec(
        "ShareLinkBundle",
        "/api/share_link_bundles/?page_size=1",
        ShareLinkBundle,
        "paginated",
        "ShareLinkBundle",
    ),
    EndpointSpec("Statistic", "/api/statistics/", Statistic, "direct", None),  # untyped in schema
    EndpointSpec("Status", "/api/status/", Status, "direct", "SystemStatus"),
    EndpointSpec(
        "StoragePath", "/api/storage_paths/?page_size=1", StoragePath, "paginated", "StoragePath"
    ),
    EndpointSpec("Tag", "/api/tags/?page_size=1", Tag, "paginated", "Tag"),
    EndpointSpec("Task", "/api/tasks/?page_size=1", Task, "paginated", "TaskSerializerV10"),
    EndpointSpec("Trash", "/api/trash/?page_size=1", Document, "paginated", "Document"),
    EndpointSpec("User", "/api/users/?page_size=1", User, "paginated", "User"),
    EndpointSpec("Workflow", "/api/workflows/?page_size=1", Workflow, "paginated", "Workflow"),
    EndpointSpec(
        "WorkflowAction",
        "/api/workflow_actions/?page_size=1",
        WorkflowAction,
        "paginated",
        "WorkflowAction",
    ),
    EndpointSpec(
        "WorkflowTrigger",
        "/api/workflow_triggers/?page_size=1",
        WorkflowTrigger,
        "paginated",
        "WorkflowTrigger",
    ),
]

# ── AI endpoints — only audited when --with-ai is passed ──────────────────────
AI_ENDPOINTS: list[EndpointSpec] = [
    EndpointSpec(
        "DocumentAISuggestions",
        f"/api/documents/{TEST_DOC_ID}/ai_suggestions/",
        DocumentAISuggestions,
        "direct",
        "AISuggestions",
    ),
]

# Schema paths that belong to AI features — excluded from the unimplemented-path
# check unless --with-ai is active.
AI_PATHS: set[str] = {
    "/api/documents/chat/",
    "/api/documents/{id}/ai_suggestions/",
}


# ─────────────────────────────────────────────────────────────────────────────
# Schema helpers
# ─────────────────────────────────────────────────────────────────────────────


async def _fetch_openapi_schema(url: str, token: str) -> dict[str, Any]:
    """Fetch and return the full OpenAPI schema JSON."""
    async with httpx.AsyncClient(verify=False) as client:
        resp = await client.get(
            f"{url}/api/schema/?format=json",
            headers={"Authorization": f"Token {token}"},
        )
        resp.raise_for_status()
        return resp.json()


def _schema_keys(schema: dict[str, Any], component: str | None) -> set[str]:
    """Return schema property names for a given component, or empty set."""
    if component is None:
        return set()
    return set(
        schema.get("components", {})
        .get("schemas", {})
        .get(component, {})
        .get("properties", {})
        .keys()
    )


def _unimplemented_paths(schema: dict[str, Any], *, with_ai: bool = False) -> list[tuple[str, str]]:
    """Return (path, http_methods) for schema paths not in const.EndpointPath.

    Normalises ``{id}`` placeholders to ``{pk}`` before comparing.
    Excludes utility/action paths from KNOWN_UTILITY_PATHS.
    When *with_ai* is ``False``, AI-related paths are also excluded.
    """

    def _norm(p: str) -> str:
        return re.sub(r"\{[^}]+\}", "{pk}", p)

    known_normalised = {_norm(v) for v in EndpointPath}
    skip = KNOWN_UTILITY_PATHS | INTENTIONALLY_SKIPPED_PATHS
    if not with_ai:
        skip = skip | AI_PATHS
    result: list[tuple[str, str]] = []
    for path, path_item in sorted(schema["paths"].items()):
        if path in skip:
            continue
        if _norm(path) not in known_normalised:
            methods = ", ".join(
                m.upper() for m in path_item if m in {"get", "post", "put", "patch", "delete"}
            )
            result.append((path, methods))
    return result


# ─────────────────────────────────────────────────────────────────────────────
# Fetch helpers
# ─────────────────────────────────────────────────────────────────────────────


async def _fetch_item(p: PaperlessClient, spec: EndpointSpec) -> dict[str, Any] | None:
    """Return a representative raw JSON object for ``spec``, or None on error/no-data."""
    raw = await p._runtime.transport.get(spec.path)

    if spec.unwrap == "paginated":
        results = raw.get("results", [])
        if not results:
            return None  # no data on this instance
        return results[0]

    if spec.unwrap == "list_index0":
        if not isinstance(raw, list) or not raw:
            return None
        return raw[0]

    # "direct" — raw IS the object
    return raw


# ─────────────────────────────────────────────────────────────────────────────
# Audit logic
# ─────────────────────────────────────────────────────────────────────────────


def _audit_one(spec: EndpointSpec, item: dict[str, Any] | None, s_keys: set[str]) -> AuditResult:
    result = AuditResult(label=spec.label, model_name=spec.model_cls.__name__)

    model_keys = set(spec.model_cls.model_fields.keys())
    result.model_keys = model_keys
    result.schema_keys = s_keys

    # ── No live data – fall back to schema if available ───────────────────────
    if item is None:
        if not s_keys:
            result.no_data = True
            return result

        result.schema_only = True
        ref_keys = s_keys
        omissions = KNOWN_SCHEMA_OMISSIONS.get(spec.model_cls.__name__, {})
        extras = KNOWN_SCHEMA_EXTRAS.get(spec.model_cls.__name__, {})
    else:
        # ── Live data available ───────────────────────────────────────────────
        ref_keys = set(item.keys())
        result.api_keys = ref_keys
        omissions = KNOWN_OMISSIONS.get(spec.model_cls.__name__, {})
        extras = KNOWN_EXTRAS.get(spec.model_cls.__name__, {})

    raw_missing = sorted(ref_keys - model_keys)
    raw_extra = sorted(model_keys - ref_keys)

    for k in raw_missing:
        if k in omissions:
            result.known_omissions.append(k)
        else:
            result.missing_in_model.append(k)

    for k in raw_extra:
        if k in extras:
            result.known_extras.append(k)
        else:
            result.extra_in_model.append(k)

    return result


# ─────────────────────────────────────────────────────────────────────────────
# Reporting
# ─────────────────────────────────────────────────────────────────────────────


def _print_result(r: AuditResult) -> None:
    if r.schema_only:
        omissions = KNOWN_SCHEMA_OMISSIONS.get(r.model_name, {})
        extras = KNOWN_SCHEMA_EXTRAS.get(r.model_name, {})
        ref_label = "schema"
    else:
        omissions = KNOWN_OMISSIONS.get(r.model_name, {})
        extras = KNOWN_EXTRAS.get(r.model_name, {})
        ref_label = "API"

    has_gaps = bool(r.missing_in_model or r.extra_in_model)

    if r.fetch_error:
        status = f"{RED}FETCH ERROR{RESET}"
    elif r.no_data:
        status = f"{YELLOW}NO DATA{RESET}   (no live data and no typed schema component)"
    elif has_gaps:
        status = f"{RED}GAPS FOUND{RESET}"
    elif r.schema_only:
        status = f"{CYAN}SCHEMA ✓{RESET}   (no live data; verified via OpenAPI schema)"
    else:
        status = f"{GREEN}OK{RESET}"

    print(f"\n{BOLD}{r.label}{RESET} ({DIM}{r.model_name}{RESET})  →  {status}")

    if r.fetch_error:
        print(f"  {RED}  {r.fetch_error}{RESET}")
        return

    if r.no_data:
        print(f"  {DIM}model fields: {sorted(r.model_keys)}{RESET}")
        return

    if r.missing_in_model:
        print(f"  {RED}MISSING IN MODEL{RESET} – {ref_label} field not captured by model:")
        for k in r.missing_in_model:
            print(f"    {RED}✗  {k!r}{RESET}")

    if r.extra_in_model:
        print(f"  {YELLOW}EXTRA IN MODEL{RESET} – model field not present in {ref_label}:")
        for k in r.extra_in_model:
            print(f"    {YELLOW}⚠  {k!r}{RESET}")

    if r.known_omissions:
        print(f"  {DIM}known omissions (intentional, not in model):{RESET}")
        for k in r.known_omissions:
            print(f"    {DIM}·  {k!r}  →  {omissions[k]}{RESET}")

    if r.known_extras:
        print(f"  {DIM}known extras (intentional, not in {ref_label}):{RESET}")
        for k in r.known_extras:
            print(f"    {DIM}·  {k!r}  →  {extras[k]}{RESET}")

    if not has_gaps:
        ref_count = len(r.schema_keys) if r.schema_only else len(r.api_keys)
        print(
            f"  {DIM}all {ref_count} {ref_label} fields covered  "
            f"(+ {len(r.known_omissions)} known omissions, "
            f"{len(r.known_extras)} known extras){RESET}"
        )

    # ── Schema cross-validation (when live data was used) ────────────────────
    if not r.schema_only and r.schema_keys and not r.fetch_error:
        schema_vs_model_missing = sorted(
            r.schema_keys - r.model_keys - set(KNOWN_SCHEMA_OMISSIONS.get(r.model_name, {}))
        )
        schema_vs_model_extra = sorted(
            r.model_keys - r.schema_keys - set(KNOWN_SCHEMA_EXTRAS.get(r.model_name, {}))
        )
        if schema_vs_model_missing or schema_vs_model_extra:
            print(f"  {DIM}┄ schema cross-check:{RESET}")
            for k in schema_vs_model_missing:
                print(f"    {DIM}schema has {k!r} but model doesn't (consider adding){RESET}")
            for k in schema_vs_model_extra:
                print(f"    {DIM}model has {k!r} but schema doesn't (extension or stale){RESET}")


def _audit_submodels(
    oa_schema: dict[str, Any],
) -> list[tuple[str, str, list[str], list[str]]]:
    """Compare pypaperless sub-model classes against OpenAPI schema component fields."""
    schemas = oa_schema.get("components", {}).get("schemas", {})

    print(f"\n{'─' * 70}")
    print(f"  {BOLD}Sub-model Field Audit{RESET}")
    print(f"{'─' * 70}")

    all_ok = True
    gaps: list[tuple[str, str, list[str], list[str]]] = []
    for cls, schema_name, usage in SUBMODEL_MAPPINGS:
        schema_component = schemas.get(schema_name, {})
        if not schema_component:
            print(
                f"  {YELLOW}MISSING{RESET}  {cls.__name__}  ↔  {DIM}{schema_name}{RESET}  ({DIM}schema component not found{RESET})"
            )
            all_ok = False
            continue

        schema_keys = set(schema_component.get("properties", {}).keys())
        model_keys = set(cls.model_fields.keys())
        known_omissions = SUBMODEL_KNOWN_OMISSIONS.get(cls.__name__, {})
        known_extras = SUBMODEL_KNOWN_EXTRAS.get(cls.__name__, {})

        raw_missing = sorted(schema_keys - model_keys)
        raw_extra = sorted(model_keys - schema_keys)
        missing = [k for k in raw_missing if k not in known_omissions]
        extra = [k for k in raw_extra if k not in known_extras]
        k_omissions = [k for k in raw_missing if k in known_omissions]
        k_extras = [k for k in raw_extra if k in known_extras]

        if missing or extra:
            all_ok = False
            gaps.append((cls.__name__, schema_name, missing, extra))
            print(
                f"\n  {RED}GAPS{RESET}   {cls.__name__}  ↔  {DIM}{schema_name}{RESET}  ({DIM}{usage}{RESET})"
            )
            for k in missing:
                print(f"    {RED}✗  schema has {k!r} — missing in {cls.__name__}{RESET}")
            for k in extra:
                print(f"    {YELLOW}⚠  {cls.__name__} has {k!r} — not in schema{RESET}")
        else:
            note = f"{len(schema_keys)} fields"
            if k_omissions or k_extras:
                note += f"  ·  +{len(k_omissions)} omissions, +{len(k_extras)} extras"
            print(
                f"  {GREEN}OK{RESET}     {cls.__name__}  ↔  {DIM}{schema_name}{RESET}  ({DIM}{note}  ·  {usage}{RESET})"
            )

        for k in k_omissions:
            print(f"    {DIM}·  omission: {k!r}  →  {known_omissions[k]}{RESET}")
        for k in k_extras:
            print(f"    {DIM}·  extra:    {k!r}  →  {known_extras[k]}{RESET}")

    if UNSCHEMA_SUBMODELS:
        print(
            f"\n  {CYAN}Sub-models without a direct schema component (field-by-field verification not possible):{RESET}"
        )
        for cls_name, src, usage in UNSCHEMA_SUBMODELS:
            print(f"  {DIM}·  {cls_name:<35s}  ({src})")
            print(f"  {DIM}   └─ {usage}{RESET}")

    if all_ok:
        print(f"\n  {GREEN}{BOLD}All mappable sub-models match schema fields.{RESET}")

    return gaps


def _audit_enums(oa_schema: dict[str, Any]) -> list[tuple[str, str, list, list]]:
    """Compare pypaperless typed enum classes against OpenAPI schema enum values."""
    schemas = oa_schema.get("components", {}).get("schemas", {})

    print(f"\n{'─' * 70}")
    print(f"  {BOLD}Enum Value Audit{RESET}")
    print(f"{'─' * 70}")

    all_ok = True
    gaps: list[tuple[str, str, list, list]] = []
    for cls, schema_name, usage in ENUM_MAPPINGS:
        schema_component = schemas.get(schema_name, {})
        if not schema_component:
            print(
                f"  {YELLOW}MISSING{RESET}  {cls.__name__}  ↔  {DIM}{schema_name}{RESET}  ({DIM}schema component not found{RESET})"
            )
            all_ok = False
            gaps.append((cls.__name__, schema_name, ["<schema component not found>"], []))
            continue

        schema_vals: set = set(schema_component.get("enum", []))
        model_vals: set = {m.value for m in cls if m.name != "UNKNOWN"}

        missing = sorted(schema_vals - model_vals, key=str)
        extra = sorted(model_vals - schema_vals, key=str)

        if missing or extra:
            all_ok = False
            gaps.append((cls.__name__, schema_name, missing, extra))
            print(
                f"\n  {RED}GAPS{RESET}   {cls.__name__}  ↔  {DIM}{schema_name}{RESET}  ({DIM}{usage}{RESET})"
            )
            for v in missing:
                print(f"    {RED}✗  schema has {v!r} — missing in {cls.__name__}{RESET}")
            for v in extra:
                print(f"    {YELLOW}⚠  {cls.__name__} has {v!r} — not in schema{RESET}")
        else:
            print(
                f"  {GREEN}OK{RESET}     {cls.__name__}  ↔  {DIM}{schema_name}{RESET}  ({DIM}{len(schema_vals)} values  ·  {usage}{RESET})"
            )

    if UNTYPED_ENUM_FIELDS:
        print(f"\n  {CYAN}Untyped enum fields (raw int/str; schema has a typed enum):{RESET}")
        for model_name, field_name, schema_name in UNTYPED_ENUM_FIELDS:
            vals = schemas.get(schema_name, {}).get("enum", ["schema not found"])
            print(f"  {DIM}·  {model_name}.{field_name:<35s}  →  {schema_name}: {vals}{RESET}")

    if all_ok:
        print(f"\n  {GREEN}{BOLD}All typed enums match schema values.{RESET}")

    return gaps


def _print_summary(
    results: list[AuditResult],
    schema: dict[str, Any],
    submodel_gaps: list[tuple[str, str, list[str], list[str]]],
    enum_gaps: list[tuple[str, str, list, list]],
    *,
    with_ai: bool = False,
) -> None:
    total = len(results)
    live_ok = sum(
        1
        for r in results
        if not r.fetch_error
        and not r.no_data
        and not r.schema_only
        and not r.missing_in_model
        and not r.extra_in_model
    )
    schema_ok = sum(
        1 for r in results if r.schema_only and not r.missing_in_model and not r.extra_in_model
    )
    gap_count = sum(1 for r in results if r.missing_in_model or r.extra_in_model)
    no_data_count = sum(1 for r in results if r.no_data)
    error_count = sum(1 for r in results if r.fetch_error)

    print(f"\n{'═' * 70}")
    print(f"  {BOLD}Summary{RESET}")
    print(f"{'─' * 70}")
    print(f"  Total endpoints audited  : {total}")
    print(f"  {GREEN}Fully covered (live)     : {live_ok}{RESET}")
    if schema_ok:
        print(f"  {CYAN}Verified via schema only : {schema_ok}{RESET}")
    if gap_count:
        print(f"  {RED}Gaps found               : {gap_count}{RESET}")
    if no_data_count:
        print(f"  {YELLOW}Unverifiable (no data)   : {no_data_count}{RESET}")
    if error_count:
        print(f"  {RED}Fetch errors             : {error_count}{RESET}")
    if submodel_gaps:
        print(f"  {RED}Sub-model gaps           : {len(submodel_gaps)}{RESET}")
    if enum_gaps:
        print(f"  {RED}Enum gaps                : {len(enum_gaps)}{RESET}")

    # Collect all gaps
    all_missing: list[tuple[str, str, bool]] = []
    all_extra: list[tuple[str, str, bool]] = []
    for r in results:
        for k in r.missing_in_model:
            all_missing.append((r.label, k, r.schema_only))
        for k in r.extra_in_model:
            all_extra.append((r.label, k, r.schema_only))

    # ── Schema-only endpoints ─────────────────────────────────────────────────
    schema_only_list = [
        r for r in results if r.schema_only and not r.missing_in_model and not r.extra_in_model
    ]
    if schema_only_list:
        print(f"\n  {CYAN}{BOLD}Verified via schema only (no live data available):{RESET}")
        for r in schema_only_list:
            print(
                f"    {CYAN}·  {r.label:<30s}  {DIM}({r.model_name}  ·  {len(r.schema_keys)} schema fields){RESET}"
            )

    # ── No-data endpoints ─────────────────────────────────────────────────────
    no_data_list = [r for r in results if r.no_data]
    if no_data_list:
        print(
            f"\n  {YELLOW}{BOLD}Unverifiable — endpoint empty and no typed schema component:{RESET}"
        )
        for r in no_data_list:
            print(f"    {YELLOW}·  {r.label:<30s}  {DIM}({r.model_name}){RESET}")

    # ── Fetch errors ──────────────────────────────────────────────────────────
    error_list = [r for r in results if r.fetch_error]
    if error_list:
        print(f"\n  {RED}{BOLD}Fetch errors:{RESET}")
        for r in error_list:
            print(f"    {RED}✗  {r.label:<30s}  {DIM}{r.fetch_error}{RESET}")

    if all_missing:
        print(f"\n  {RED}{BOLD}Fields missing in model (not captured by any field):{RESET}")
        for label, k, via_schema in all_missing:
            badge = f" {DIM}(schema){RESET}" if via_schema else ""
            print(f"    {RED}✗  {label}.{k}{badge}{RESET}")

    if all_extra:
        print(
            f"\n  {YELLOW}{BOLD}Fields extra in model (declared but reference never returns):{RESET}"
        )
        for label, k, via_schema in all_extra:
            badge = f" {DIM}(schema){RESET}" if via_schema else ""
            print(f"    {YELLOW}⚠  {label}.{k}{badge}{RESET}")

    # ── Sub-model gaps ────────────────────────────────────────────────────────
    if submodel_gaps:
        print(f"\n  {RED}{BOLD}Sub-model field gaps:{RESET}")
        for cls_name, schema_name, missing, extra in submodel_gaps:
            print(f"    {RED}↔  {cls_name}  ↔  {DIM}{schema_name}{RESET}")
            for k in missing:
                print(f"      {RED}✗  schema has {k!r} — missing in model{RESET}")
            for k in extra:
                print(f"      {YELLOW}⚠  model has {k!r} — not in schema{RESET}")

    # ── Enum gaps ─────────────────────────────────────────────────────────────
    if enum_gaps:
        print(f"\n  {RED}{BOLD}Enum value gaps:{RESET}")
        for cls_name, schema_name, missing, extra in enum_gaps:
            print(f"    {RED}↔  {cls_name}  ↔  {DIM}{schema_name}{RESET}")
            for v in missing:
                print(f"      {RED}✗  schema has {v!r} — missing in enum{RESET}")
            for v in extra:
                print(f"      {YELLOW}⚠  enum has {v!r} — not in schema{RESET}")

    # ── Unimplemented schema paths ────────────────────────────────────────────
    unimplemented = _unimplemented_paths(schema, with_ai=with_ai)
    if not with_ai:
        print(
            f"  {DIM}AI paths excluded from unimplemented check (pass --with-ai to include){RESET}"
        )
    if unimplemented:
        print(f"\n  {CYAN}{BOLD}Schema paths not covered by const.EndpointPath:{RESET}")
        for path, methods in unimplemented:
            print(f"    {CYAN}·  {path:<55s} {DIM}[{methods}]{RESET}")

    print(f"\n{'═' * 70}\n")


# ─────────────────────────────────────────────────────────────────────────────
# Main
# ─────────────────────────────────────────────────────────────────────────────


async def main(*, with_ai: bool = False) -> None:
    print(f"\n{BOLD}pypaperless  ·  API coverage audit  (live + OpenAPI schema){RESET}")
    print(f"Target : {PAPERLESS_URL}")
    if not with_ai:
        print(f"{BOLD}AI endpoints skipped{RESET} (pass --with-ai to include)")
    print(f"{'─' * 70}")

    # ── Fetch OpenAPI schema first ────────────────────────────────────────────
    print("Fetching OpenAPI schema …", end=" ", flush=True)
    try:
        oa_schema = await _fetch_openapi_schema(PAPERLESS_URL, PAPERLESS_TOKEN)
        oa_version = oa_schema.get("info", {}).get("version", "?")
        print(f"{GREEN}OK{RESET}  (server version {oa_version})")
    except Exception as exc:
        print(f"{RED}FAILED{RESET}: {exc}")
        oa_schema = {}

    p = PaperlessClient(
        PAPERLESS_URL,
        PAPERLESS_TOKEN,
        client=httpx.AsyncClient(verify=False),
    )

    results: list[AuditResult] = []
    endpoints = ENDPOINTS + (AI_ENDPOINTS if with_ai else [])

    async with p:
        p.runtime.cache.custom_fields = await p.custom_fields.as_dict()

        for spec in endpoints:
            try:
                item = await _fetch_item(p, spec)
                s_keys = _schema_keys(oa_schema, spec.schema_component)
                result = _audit_one(spec, item, s_keys)
            except Exception as exc:
                result = AuditResult(label=spec.label, model_name=spec.model_cls.__name__)
                result.fetch_error = str(exc)

            _print_result(result)
            results.append(result)

    submodel_gaps = _audit_submodels(oa_schema)
    enum_gaps = _audit_enums(oa_schema)
    _print_summary(results, oa_schema, submodel_gaps, enum_gaps, with_ai=with_ai)


if __name__ == "__main__":
    _parser = argparse.ArgumentParser(description="pypaperless API coverage audit")
    _parser.add_argument(
        "--with-ai",
        action="store_true",
        default=False,
        help="Include AI endpoints in the audit (ai_suggestions, chat). Disabled by default.",
    )
    _args = _parser.parse_args()
    asyncio.run(main(with_ai=_args.with_ai))
