#!/usr/bin/env python3
"""Fetch a list of URLs politely. Prints status per URL. Skips robots-disallowed."""
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent))
from fetch import get

def main(list_path):
    urls = [u.strip() for u in Path(list_path).read_text().splitlines() if u.strip() and not u.startswith("#")]
    ok = fail = skip = 0
    for i, url in enumerate(urls, 1):
        try:
            status, final, content, cached = get(url, timeout=50)
            print(f"{i}/{len(urls)} {status} cached={cached} bytes={len(content)} {url}", flush=True)
            if status >= 400:
                fail += 1
            else:
                ok += 1
            # login wall signal
            head = content[:2500].decode("utf-8", "replace").lower()
            if status in (401, 403) or "you need to be logged in" in head or "please log in" in head or "sign in to continue" in head:
                print(f"  LOGIN_OR_FORBIDDEN {status} {final}", flush=True)
        except PermissionError as e:
            print(f"{i}/{len(urls)} ROBOTS_SKIP {url} {e}", flush=True)
            skip += 1
        except Exception as e:
            print(f"{i}/{len(urls)} ERR {type(e).__name__} {e} {url}", flush=True)
            fail += 1
    print(f"DONE ok={ok} fail={fail} skip={skip}", flush=True)

if __name__ == "__main__":
    main(sys.argv[1])