"""Audit the 3D asset library: coverage, embeddings, broken entries, rigs, variants, usage.

usage:
  python3 tools/audit_library.py --blobs <store/blobs> [--blobs <more>] [--thumbs <dir>]
                                 [--extra-cards <cards dir>] [--out audit]

Reads only (no network). Writes <out>/audit.json and <out>/audit.md.
Vector lookup matches lib/search.py: vectors/<VECTOR_SET>/<sha256(id@release)>.npz with 'image' and 'text' members.
The thumbnail check looks for <thumbs dir>/<glb sha>.jpg (site/out is gitignored, so pass the dir that has them).
--extra-cards reads another worktree's cards/ dir (e.g. the creatures worktree). Those cards are not in index.json,
so they are counted in coverage and checked for validity, but not for embeddings or rigs.
"""
import argparse, glob, hashlib, json, os, re, sys, time, zipfile

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(ROOT, 'lib'))
import card as cardlib  # noqa: E402  (stdlib only)

VECTOR_SET = 'gemini-embedding-2-preview'  # lib/embeddings.py VECTOR_SET
DRIVABLE = ('car', 'kart', 'two-wheel', 'service', 'utility', 'military-land', 'sea')
OPENING = ('cabinet', 'wardrobe', 'sideboard', 'dresser', 'chest', 'drawer', 'fridge', 'desk', 'box', 'door')
VARIANT_RE = re.compile(r'painted|repaint|retexture|recolou?r|-g4\b|-astra\b|rerun|resiz', re.I)
PARENT_FIELDS = ('parent', 'lineage', 'variant_of')
USAGE_RE = re.compile(r'(used|use)_?(by|in)|usage|^games?$|^apps?$|consumers?|deployed', re.I)
EXAMPLES = 10
CATEGORIES = [  # (label, test on id)
    ('humans (character.*)', lambda k, c: k == 'character'),
    ('animals', lambda k, c: k == 'animal'),
    ('monsters', lambda k, c: k == 'monster'),
    ('vehicles', lambda k, c: k == 'vehicle'),
    ('furniture (prop.furniture.*)', lambda k, c: c.startswith('prop.furniture')),
    ('props (prop.*, not furniture)', lambda k, c: k == 'prop' and not c.startswith('prop.furniture')),
    ('buildings', lambda k, c: k == 'building'),
    ('weapons', lambda k, c: k == 'weapon'),
]


def kind_of(cid):
    return cid.split('/')[0].split('.')[0]


def class_of(cid):
    return cid.split('/')[0]


def load_cards_dir(cards_dir):
    """{id: [versions sorted]} from a bare cards/ directory (no index)."""
    out = {}
    for p in glob.glob(os.path.join(cards_dir, '**', 'v*.json'), recursive=True):
        c = json.load(open(p))
        out.setdefault(c['id'], []).append(c)
    for v in out.values():
        v.sort(key=lambda c: c['version'])
    return out


def walk_keys(o, depth=0):
    if depth > 3 or not isinstance(o, dict):
        return
    for k, v in o.items():
        yield k
        yield from walk_keys(v, depth + 1)


def audit(root, blob_dirs, thumb_dirs, extra_dirs):
    index = json.load(open(os.path.join(root, 'index.json')))
    rows = index['assets']
    own_cards = cardlib.load_all(root)
    vec_dir = os.path.join(root, 'vectors', VECTOR_SET)
    blob_names = set()
    for d in blob_dirs:
        if os.path.isdir(d):
            blob_names.update(os.listdir(d))
    thumb_names = set()
    for d in thumb_dirs:
        if os.path.isdir(d):
            thumb_names.update(os.listdir(d))

    # ---- per asset pass over the index (active assets) ----
    cls_stats, kind_stats = {}, {}
    coverage = {label: {'active': [], 'extra': [], 'versions': 0, 'statuses': {}} for label, _ in CATEGORIES}
    broken = {'missing_blob': [], 'invalid_card': [], 'no_card_file': []}
    emb = {'missing_visual_description': [], 'missing_text_vector': [], 'missing_image_vector': [],
           'missing_description_vector': [], 'missing_thumbnail': []}
    rigs = {'vehicles_no_wheels_steer': [], 'vehicles_no_doors': [], 'furniture_opening_no_nodes': [],
            'characters_no_rig': [], 'characters_no_clips': []}
    variants = {'candidates': [], 'with_parent_link': [], 'without_parent_link': []}
    usage = {'fields_found': set(), 'assets_with_usage_field': []}
    statuses = {}

    for r in rows:
        cid, kind, cls = r['id'], kind_of(r['id']), class_of(r['id'])
        statuses[r['status']] = statuses.get(r['status'], 0) + 1
        ks = kind_stats.setdefault(kind, {'active': 0, 'card_versions': 0, 'statuses': {}})
        ks['active'] += 1
        ks['statuses'][r['status']] = ks['statuses'].get(r['status'], 0) + 1
        cs = cls_stats.setdefault(cls, {'active': 0, 'card_versions': 0, 'statuses': {}})
        cs['active'] += 1
        cs['statuses'][r['status']] = cs['statuses'].get(r['status'], 0) + 1
        versions = own_cards.get(cid, [])
        ks['card_versions'] += len(versions)
        cs['card_versions'] += len(versions)
        for label, test in CATEGORIES:
            if test(kind, cls):
                cov = coverage[label]
                cov['active'].append(cid)
                cov['versions'] += len(versions)
                cov['statuses'][r['status']] = cov['statuses'].get(r['status'], 0) + 1

        # broken: blob, card file, card validity
        if r['glb'] + '.glb' not in blob_names:
            broken['missing_blob'].append(cid)
        card_path = os.path.join(root, r.get('card', ''))
        card = None
        if not r.get('card') or not os.path.exists(card_path):
            broken['no_card_file'].append(cid)
        else:
            card = json.load(open(card_path))
            errs = cardlib.validate(card)
            if errs:
                broken['invalid_card'].append({'id': cid, 'errors': errs})

        # embeddings (same key as lib/search.py)
        key = hashlib.sha256((cid + '@' + r['release']).encode()).hexdigest()
        if not r.get('visual_description'):
            emb['missing_visual_description'].append(cid)
        npz = os.path.join(vec_dir, key + '.npz')
        members = set(zipfile.ZipFile(npz).namelist()) if os.path.exists(npz) else set()
        if 'text.npy' not in members:
            emb['missing_text_vector'].append(cid)
        if 'image.npy' not in members:
            emb['missing_image_vector'].append(cid)
        if 'description.npy' not in members:
            emb['missing_description_vector'].append(cid)
        thumb_ok = os.path.exists(os.path.join(root, r.get('thumbnail', ''))) or (r['glb'] + '.jpg') in thumb_names
        if not thumb_ok:
            emb['missing_thumbnail'].append(cid)

        if card is None:
            continue
        nodes = card.get('nodes') or {}
        names = ' '.join([r['name'], cid, ' '.join(r.get('tags', []))])
        if kind == 'vehicle' and cls.split('.')[1] in DRIVABLE:
            boat = cls.split('.')[1] == 'sea'  # boats steer with a rudder (D2 convention: hull + rudder + propeller/oars)
            if boat:
                if not (any(k.startswith('hull') for k in nodes) and any(k.startswith(('rudder', 'oar')) for k in nodes)):
                    rigs.setdefault('boats_no_hull_rudder', []).append(cid)
            elif not (any(k.startswith('wheel') for k in nodes) and any(k.startswith('steer') for k in nodes)):
                rigs['vehicles_no_wheels_steer'].append(cid)
            if not boat and not any(k.startswith('door') for k in nodes):
                rigs['vehicles_no_doors'].append(cid)
        if cls.startswith('prop.furniture'):
            leaf = cls.split('.')[-1]
            opens = leaf in OPENING
            if leaf == 'bookcase' and re.search(r'door|glass', names, re.I):
                opens = True
            if opens and not any(k.startswith(('door', 'drawer', 'lid')) for k in nodes):
                rigs['furniture_opening_no_nodes'].append(cid)
        if kind == 'character':
            if not r.get('rig'):
                rigs['characters_no_rig'].append(cid)
            if not r.get('clips'):
                rigs['characters_no_clips'].append(cid)

        m = VARIANT_RE.search(cid.split('/')[1] + ' ' + names + ' ' + json.dumps(card.get('provenance', {})))
        if m:
            variants['candidates'].append({'id': cid, 'hint': m.group(0).lower()})
            linked = any(card.get(f) or (card.get('provenance') or {}).get(f) for f in PARENT_FIELDS)
            (variants['with_parent_link'] if linked else variants['without_parent_link']).append(cid)

        hits = {k for k in walk_keys(card) if USAGE_RE.search(k)}
        if hits:
            usage['fields_found'].update(hits)
            usage['assets_with_usage_field'].append(cid)

    # ---- extra cards (other worktree, read-only): coverage + validity only ----
    extra_ids = {}
    for d in extra_dirs:
        for cid, versions in load_cards_dir(d).items():
            if cid in extra_ids or cid in {r['id'] for r in rows}:
                continue
            extra_ids[cid] = (versions, cardlib.active(versions))
    extra_errors = []
    for cid, (versions, act) in sorted(extra_ids.items()):
        kind, cls = kind_of(cid), class_of(cid)
        for label, test in CATEGORIES:
            if test(kind, cls):
                cov = coverage[label]
                cov['extra'].append(cid)
                cov['versions'] += len(versions)
        errs = cardlib.validate(act)
        if errs:
            extra_errors.append({'id': cid, 'errors': errs})
        if act['files']['glb'].split(':')[1] + '.glb' not in blob_names:
            broken['missing_blob'].append(cid + ' (extra)')
        m = VARIANT_RE.search(cid.split('/')[1] + ' ' + act.get('name', '') + ' ' + json.dumps(act.get('provenance', {})))
        if m:
            variants['candidates'].append({'id': cid, 'hint': m.group(0).lower(), 'extra': True})
    broken['invalid_extra_card'] = extra_errors

    cov_rows = []
    for label, _ in CATEGORIES:
        cov = coverage[label]
        n = len(cov['active'])
        cov_rows.append({'category': label, 'active': n, 'extra_not_indexed': len(cov['extra']),
                         'card_versions': cov['versions'], 'statuses': cov['statuses'],
                         'gap': n == 0 or n < 5})
    return {
        'generated': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
        'root': root, 'blob_dirs': blob_dirs, 'thumb_dirs': thumb_dirs, 'extra_card_dirs': extra_dirs,
        'index_built': index.get('built'), 'active_assets': len(rows), 'statuses': statuses,
        'extra_ids_not_indexed': len(extra_ids),
        'coverage': cov_rows,
        'per_kind': dict(sorted(kind_stats.items())),
        'per_class': dict(sorted(cls_stats.items())),
        'embeddings': {k: v for k, v in emb.items()},
        'broken': broken,
        'rigs': rigs,
        'variants': {'candidates': variants['candidates'],
                     'with_parent_link': variants['with_parent_link'],
                     'without_parent_link': variants['without_parent_link']},
        'usage': {'fields_found': sorted(usage['fields_found']),
                  'assets_with_usage_field': usage['assets_with_usage_field']},
    }


def ex(ids):
    return ', '.join(ids[:EXAMPLES]) + (f' ... (+{len(ids) - EXAMPLES} more)' if len(ids) > EXAMPLES else '')


def write_md(a, path):
    L = []
    L.append('# Library audit\n')
    L.append(f"Generated {a['generated']} (UTC). Active assets in index: **{a['active_assets']}**. "
             f"Assets from the extra cards dir not in the index yet: **{a['extra_ids_not_indexed']}**.\n")
    L.append('## 1. Coverage\n')
    L.append('| Category | Active (in index) | Extra (not indexed) | Card versions | Gap? |')
    L.append('| --- | ---: | ---: | ---: | --- |')
    for c in a['coverage']:
        L.append(f"| {c['category']} | {c['active']} | {c['extra_not_indexed']} | {c['card_versions']} | "
                 f"{'**GAP** (under 5)' if c['gap'] else 'ok'} |")
    L.append('')
    L.append('Status of active assets: ' + ', '.join(f'{k} {v}' for k, v in sorted(a['statuses'].items())) + '.\n')
    L.append('Per top kind: ' + ', '.join(f"{k} {v['active']}" for k, v in a['per_kind'].items()) + '.\n')
    e = a['embeddings']
    L.append('## 2. Embeddings and thumbnails (active assets)\n')
    for key, label in [('missing_visual_description', 'no visual description'),
                       ('missing_text_vector', 'no text vector'),
                       ('missing_image_vector', 'no image vector'),
                       ('missing_description_vector', 'no description vector (search uses it)'),
                       ('missing_thumbnail', 'no thumbnail')]:
        L.append(f"- **{len(e[key])}** {label}. e.g. {ex(e[key]) or 'none'}")
    L.append('')
    b = a['broken']
    L.append('## 3. Broken\n')
    L.append(f"- **{len(b['missing_blob'])}** GLB blob missing from every blobs dir. e.g. {ex(b['missing_blob']) or 'none'}")
    L.append(f"- **{len(b['invalid_card'])}** card fails validation. e.g. "
             f"{ex([x['id'] for x in b['invalid_card']]) or 'none'}")
    L.append(f"- **{len(b['no_card_file'])}** index entry with no card file. e.g. {ex(b['no_card_file']) or 'none'}")
    L.append(f"- **{len(b.get('invalid_extra_card', []))}** extra card fails validation. e.g. "
             f"{ex([x['id'] for x in b.get('invalid_extra_card', [])]) or 'none'}\n")
    r = a['rigs']
    L.append('## 4. Rigs\n')
    L.append(f"- **{len(r['vehicles_no_wheels_steer'])}** drivable vehicles without wheels + steer nodes. "
             f"e.g. {ex(r['vehicles_no_wheels_steer']) or 'none'}")
    L.append(f"- **{len(r['vehicles_no_doors'])}** drivable vehicles without doors. e.g. {ex(r['vehicles_no_doors']) or 'none'}")
    L.append(f"- **{len(r['furniture_opening_no_nodes'])}** furniture that opens, without door/drawer/lid nodes. "
             f"e.g. {ex(r['furniture_opening_no_nodes']) or 'none'}")
    L.append(f"- **{len(r['characters_no_rig'])}** characters without a rig. e.g. {ex(r['characters_no_rig']) or 'none'}")
    L.append(f"- **{len(r['characters_no_clips'])}** characters without clips. e.g. {ex(r['characters_no_clips']) or 'none'}\n")
    v = a['variants']
    L.append('## 5. Variants\n')
    L.append(f"- **{len(v['candidates'])}** cards look like a variant (painted, repaint, recolour, -g4, -astra, rerun, resized).")
    L.append(f"- **{len(v['with_parent_link'])}** of them link to a parent. "
             f"**{len(v['without_parent_link'])}** do not. e.g. {ex(v['without_parent_link']) or 'none'}\n")
    u = a['usage']
    L.append('## 6. Usage\n')
    L.append(f"- Fields that record which game or app uses an asset: **{', '.join(u['fields_found']) or 'none'}**.")
    L.append(f"- Assets with such a field: **{len(u['assets_with_usage_field'])}**.\n")
    with open(path, 'w') as f:
        f.write('\n'.join(L) + '\n')


def main():
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument('--root', default=ROOT, help='asset repo (default: this checkout)')
    p.add_argument('--blobs', action='append', default=[], help='store/blobs dir (repeatable)')
    p.add_argument('--thumbs', action='append', default=[], help='dir with <glb sha>.jpg thumbnails (repeatable)')
    p.add_argument('--extra-cards', action='append', default=[], help='cards/ dir of another worktree, read-only')
    p.add_argument('--out', default=None, help='output dir (default <root>/audit)')
    a = p.parse_args()
    root = os.path.abspath(a.root)
    thumbs = a.thumbs or [os.path.join(root, 'site', 'out', 'measure', 'thumbs')]
    result = audit(root, [os.path.abspath(x) for x in a.blobs], [os.path.abspath(x) for x in thumbs],
                   [os.path.abspath(x) for x in a.extra_cards])
    out = os.path.abspath(a.out or os.path.join(root, 'audit'))
    os.makedirs(out, exist_ok=True)
    with open(os.path.join(out, 'audit.json'), 'w') as f:
        json.dump(result, f, indent=1, ensure_ascii=False)
        f.write('\n')
    write_md(result, os.path.join(out, 'audit.md'))
    print(f"audit: {result['active_assets']} active, {result['extra_ids_not_indexed']} extra -> {out}")


if __name__ == '__main__':
    main()