दस्तावेज़ों की अच्छी चंकिंग (Chunking)

def split_sections(text):
    """Split text into (heading, body) pairs at lines starting with '#'."""
    sections, heading, lines = [], "Untitled", []
    for line in text.splitlines():
        if line.startswith("#"):
            if lines:
                sections.append((heading, " ".join(lines)))
            heading, lines = line.lstrip("#").strip(), []
        elif line.strip():
            lines.append(line.strip())
    if lines:
        sections.append((heading, " ".join(lines)))
    return sections

def chunk(text, doc_id, title, max_words=80, overlap=15):
    chunks = []
    for heading, body in split_sections(text):
        words = body.split()
        start = 0
        while True:
            piece = " ".join(words[start:start + max_words])
            chunks.append({
                "id": f"{doc_id}#{len(chunks)}",
                "doc_id": doc_id,
                "section": heading,
                # Title and heading go into the text that gets embedded.
                "text": f"{title} > {heading}: {piece}",
            })
            if start + max_words >= len(words):
                break
            start += max_words - overlap
    return chunks

policy = """# Leave policy
Staff get 12 days of casual leave each year.
# Carry forward
Unused casual leave cannot be carried to the next year."""
for c in chunk(policy, "hr-leave-v3", "HR Leave Policy 2025"):
    print(c["id"], "|", c["text"])