IETM_FIXE\ietm6
4d049274f9
Some checks failed
E2E Tests / e2e (push) Waiting to run
Tests unitaires / Web (Angular · vitest + couverture) (push) Successful in 26s
Build & Push Images / tests (push) Failing after 13s
Build & Push Images / build (brain) (push) Has been skipped
Build & Push Images / build (core) (push) Has been skipped
Build & Push Images / build (web) (push) Has been skipped
Build & Push Images / build-switcher (push) Has been skipped
Tests unitaires / Brain (Python · pytest + couverture) (push) Successful in 1m12s
Tests unitaires / Core (Java · mvn test + JaCoCo) (push) Failing after 1m28s
Mise en place de tests unitaires coté Python et Angular Mise en place de la couverture de test directement dans le workflow : le programme ne build pas si jamais un test échoue Passage en v0.16.2 en conséquence
73 lines
2.4 KiB
Python
73 lines
2.4 KiB
Python
"""Tests du découpage de texte (app.application.chunking).
|
|
|
|
Vérifie le découpage par paragraphes vers une cible de tokens, le découpage des
|
|
paragraphes géants, le recouvrement (overlap), et le split_in_half du repli
|
|
anti-troncature. tiktoken (cl100k_base) est déterministe → assertions stables.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from app.application.chunking import chunk_text, split_in_half
|
|
|
|
|
|
def test_empty_text_returns_no_chunk():
|
|
assert chunk_text("") == []
|
|
assert chunk_text(" \n\n ") == []
|
|
|
|
|
|
def test_short_text_stays_single_chunk():
|
|
chunks = chunk_text("Paragraphe un.\n\nParagraphe deux.", target_tokens=1000)
|
|
assert len(chunks) == 1
|
|
assert "Paragraphe un." in chunks[0]
|
|
assert "Paragraphe deux." in chunks[0]
|
|
|
|
|
|
def test_splits_into_several_chunks_when_exceeding_target():
|
|
paras = [f"Paragraphe numero {i} avec un peu de contenu." for i in range(20)]
|
|
full = "\n\n".join(paras)
|
|
chunks = chunk_text(full, target_tokens=20)
|
|
assert len(chunks) > 1
|
|
# Aucun paragraphe perdu : tous présents quelque part.
|
|
joined = "\n\n".join(chunks)
|
|
for p in paras:
|
|
assert p in joined
|
|
|
|
|
|
def test_oversized_single_paragraph_is_split():
|
|
# Un seul paragraphe (aucun "\n\n") plus gros que la cible → plusieurs sous-blocs.
|
|
huge = "mot " * 500
|
|
chunks = chunk_text(huge, target_tokens=50)
|
|
assert len(chunks) > 1
|
|
|
|
|
|
def test_overlap_repeats_content_without_losing_paragraphs():
|
|
paras = [f"Bloc {i} de texte distinct." for i in range(12)]
|
|
full = "\n\n".join(paras)
|
|
chunks = chunk_text(full, target_tokens=20, overlap_tokens=10)
|
|
assert len(chunks) > 1
|
|
joined = "\n\n".join(chunks)
|
|
for p in paras:
|
|
assert p in joined
|
|
|
|
|
|
# --- split_in_half -------------------------------------------------------------
|
|
|
|
def test_split_in_half_too_short_returns_empty():
|
|
assert split_in_half("court") == ("", "")
|
|
|
|
|
|
def test_split_in_half_splits_on_newline_near_middle():
|
|
text = "A" * 300 + "\n" + "B" * 300
|
|
left, right = split_in_half(text)
|
|
assert left and right
|
|
assert left.startswith("A")
|
|
assert right.startswith("B")
|
|
|
|
|
|
def test_split_in_half_halves_cover_all_content():
|
|
text = "\n".join(f"ligne {i} " + "x" * 20 for i in range(40))
|
|
left, right = split_in_half(text)
|
|
assert left and right
|
|
# Le découpage ne perd rien : la concaténation contient début et fin.
|
|
assert "ligne 0" in left
|
|
assert "ligne 39" in right
|