# Same as export_to_markdown.py, but streams the file with ijson so a huge
# export (hundreds of MB) never has to fit in memory at once.
# Usage: pip install ijson
#        python export_to_markdown_streaming.py conversations.json out_folder
import datetime
import pathlib
import re
import sys

import ijson

src, out = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2])
out.mkdir(parents=True, exist_ok=True)


def kept_branch(conv):
    mapping, node, msgs = conv.get("mapping", {}), conv.get("current_node"), []
    while node in mapping:
        msg = mapping[node].get("message")
        if msg:
            msgs.append(msg)
        node = mapping[node].get("parent")
    return list(reversed(msgs))


used = set()
with open(src, "rb") as f:
    # "item" = each element of the top-level array, one at a time.
    for conv in ijson.items(f, "item", use_float=True):
        title = (conv.get("title") or "").strip() or "Untitled chat"
        lines = [f"# {title}", ""]
        for msg in kept_branch(conv):
            role = msg.get("author", {}).get("role")
            if (msg.get("metadata") or {}).get("is_visually_hidden_from_conversation"):
                continue
            if role == "assistant" and msg.get("recipient", "all") != "all":
                continue
            parts = (msg.get("content") or {}).get("parts") or []
            text = "\n\n".join(p for p in parts if isinstance(p, str)).strip()
            if role in ("user", "assistant") and text:
                lines += [f"## {'You' if role == 'user' else 'ChatGPT'}", "", text, ""]
        day = datetime.datetime.fromtimestamp(conv.get("create_time") or 0).strftime("%Y-%m-%d")
        name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', " ", title)[:80].strip() or "Untitled chat"
        path = out / f"{day} {name}.md"
        n = 2
        while path.name.lower() in used:
            path = out / f"{day} {name} ({n}).md"
            n += 1
        used.add(path.name.lower())
        path.write_text("\n".join(lines), encoding="utf-8")

print(f"wrote {len(used)} files to {out}")