"""fetch.py - page through All employers's active jobs into jobs.json for fit.py. Python 3.9+, stdlib only.

usage: python3 fetch.py [--ids 1,2,3] [--category C ...] [--location L ...] [--title T] [--time-frame 6m] [--text] [--out jobs.json]

Keyless. Cursor paging over https://www.openrolesindex.com/orgs/all/v1/active-ats with include_beta=true (requirements included), 100 per page,
until a page comes back short.
--ids fetches exactly the listed job ids (the ids in an alert digest) and ignores the other filters; closed ids are skipped. Honours 429 retry-after. Several --category / --location values are ORed.
"""
import argparse, json, sys, time, urllib.error, urllib.parse, urllib.request

API = "https://www.openrolesindex.com/orgs/all/v1/active-ats"
PAGE = 100


def get(url):
    for attempt in range(6):
        req = urllib.request.Request(url, headers={"Accept": "application/json", "User-Agent": "agent-pack-fetch/1"})
        try:
            with urllib.request.urlopen(req, timeout=60) as res:
                return json.load(res)
        except urllib.error.HTTPError as e:
            if e.code == 429 and attempt < 5:
                time.sleep(int(e.headers.get("retry-after") or 5))
                continue
            detail = e.read().decode("utf-8", "replace")
            sys.exit(f"HTTP {e.code} for {url}\n{detail}")
    sys.exit("gave up after repeated 429s")


def main():
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--ids", help="comma-separated job ids (from an alert digest); replaces the other filters")
    ap.add_argument("--category", action="append", default=[], help="source_category value (exact); repeatable")
    ap.add_argument("--location", action="append", default=[], help="city, region or country, full names; repeatable")
    ap.add_argument("--title", help="title search, Google-style")
    ap.add_argument("--time-frame", default="6m", choices=["1h", "24h", "7d", "6m"],
                    help="6m = every active job (default); 24h = new in the last 24 complete hours")
    ap.add_argument("--text", action="store_true", help="include description_text (about 10 KB a job)")
    ap.add_argument("--max", type=int, default=2000, help="stop after this many jobs (default 2000)")
    ap.add_argument("--out", default="jobs.json")
    a = ap.parse_args()
    q = {"time_frame": a.time_frame, "include_beta": "true", "limit": PAGE}
    if a.category:
        q["source_category"] = ",".join('"%s"' % c if "," in c else c for c in a.category)
    if a.location:
        q["location"] = " OR ".join('"%s"' % loc for loc in a.location)
    if a.title:
        q["title"] = a.title
    if a.text:
        q["description_format"] = "text"
    if a.ids:
        try:
            ids = [int(x) for x in a.ids.split(",") if x.strip()]
        except ValueError:
            sys.exit("--ids takes comma-separated integers, e.g. --ids 101,102,103")
        if not ids:
            sys.exit("--ids is empty")
        q = {k: v for k, v in q.items() if k in ("include_beta", "description_format")}
        q.update(time_frame="6m", limit=PAGE)
        jobs = []
        for i in range(0, len(ids), PAGE):
            jobs += get(API + "?" + urllib.parse.urlencode(dict(q, id=",".join(map(str, ids[i:i + PAGE]))), quote_via=urllib.parse.quote))
        missing = sorted(set(ids) - {j["id"] for j in jobs})
        if missing:
            print("not active (skipped): " + ", ".join(map(str, missing)), file=sys.stderr)
        with open(a.out, "w", encoding="utf-8") as f:
            json.dump(jobs, f, ensure_ascii=False)
        print(f"wrote {len(jobs)} jobs to {a.out}", file=sys.stderr)
        return
    jobs, cursor = [], 0
    while len(jobs) < a.max:
        page = get(API + "?" + urllib.parse.urlencode(dict(q, cursor=cursor), quote_via=urllib.parse.quote))
        jobs += page
        print(f"{len(jobs)} jobs", file=sys.stderr)
        if len(page) < PAGE:
            break
        cursor = page[-1]["id"]
    with open(a.out, "w", encoding="utf-8") as f:
        json.dump(jobs[:a.max], f, ensure_ascii=False)
    print(f"wrote {min(len(jobs), a.max)} jobs to {a.out}", file=sys.stderr)


if __name__ == "__main__":
    main()