Refacto du code coté Python, java et coté angular afin de mieux séparer les responsabilité et d'avoir moins de répétitivité dans le code.
Some checks failed
E2E Tests / e2e (push) Waiting to run
Tests unitaires / Web (Angular · vitest + couverture) (push) Successful in 26s
Build & Push Images / tests (push) Failing after 13s
Build & Push Images / build (brain) (push) Has been skipped
Build & Push Images / build (core) (push) Has been skipped
Build & Push Images / build (web) (push) Has been skipped
Build & Push Images / build-switcher (push) Has been skipped
Tests unitaires / Brain (Python · pytest + couverture) (push) Successful in 1m12s
Tests unitaires / Core (Java · mvn test + JaCoCo) (push) Failing after 1m28s
Some checks failed
E2E Tests / e2e (push) Waiting to run
Tests unitaires / Web (Angular · vitest + couverture) (push) Successful in 26s
Build & Push Images / tests (push) Failing after 13s
Build & Push Images / build (brain) (push) Has been skipped
Build & Push Images / build (core) (push) Has been skipped
Build & Push Images / build (web) (push) Has been skipped
Build & Push Images / build-switcher (push) Has been skipped
Tests unitaires / Brain (Python · pytest + couverture) (push) Successful in 1m12s
Tests unitaires / Core (Java · mvn test + JaCoCo) (push) Failing after 1m28s
Mise en place de tests unitaires coté Python et Angular Mise en place de la couverture de test directement dans le workflow : le programme ne build pas si jamais un test échoue Passage en v0.16.2 en conséquence
This commit is contained in:
72
brain/tests/test_chunking.py
Normal file
72
brain/tests/test_chunking.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""Tests du découpage de texte (app.application.chunking).
|
||||
|
||||
Vérifie le découpage par paragraphes vers une cible de tokens, le découpage des
|
||||
paragraphes géants, le recouvrement (overlap), et le split_in_half du repli
|
||||
anti-troncature. tiktoken (cl100k_base) est déterministe → assertions stables.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.application.chunking import chunk_text, split_in_half
|
||||
|
||||
|
||||
def test_empty_text_returns_no_chunk():
|
||||
assert chunk_text("") == []
|
||||
assert chunk_text(" \n\n ") == []
|
||||
|
||||
|
||||
def test_short_text_stays_single_chunk():
|
||||
chunks = chunk_text("Paragraphe un.\n\nParagraphe deux.", target_tokens=1000)
|
||||
assert len(chunks) == 1
|
||||
assert "Paragraphe un." in chunks[0]
|
||||
assert "Paragraphe deux." in chunks[0]
|
||||
|
||||
|
||||
def test_splits_into_several_chunks_when_exceeding_target():
|
||||
paras = [f"Paragraphe numero {i} avec un peu de contenu." for i in range(20)]
|
||||
full = "\n\n".join(paras)
|
||||
chunks = chunk_text(full, target_tokens=20)
|
||||
assert len(chunks) > 1
|
||||
# Aucun paragraphe perdu : tous présents quelque part.
|
||||
joined = "\n\n".join(chunks)
|
||||
for p in paras:
|
||||
assert p in joined
|
||||
|
||||
|
||||
def test_oversized_single_paragraph_is_split():
|
||||
# Un seul paragraphe (aucun "\n\n") plus gros que la cible → plusieurs sous-blocs.
|
||||
huge = "mot " * 500
|
||||
chunks = chunk_text(huge, target_tokens=50)
|
||||
assert len(chunks) > 1
|
||||
|
||||
|
||||
def test_overlap_repeats_content_without_losing_paragraphs():
|
||||
paras = [f"Bloc {i} de texte distinct." for i in range(12)]
|
||||
full = "\n\n".join(paras)
|
||||
chunks = chunk_text(full, target_tokens=20, overlap_tokens=10)
|
||||
assert len(chunks) > 1
|
||||
joined = "\n\n".join(chunks)
|
||||
for p in paras:
|
||||
assert p in joined
|
||||
|
||||
|
||||
# --- split_in_half -------------------------------------------------------------
|
||||
|
||||
def test_split_in_half_too_short_returns_empty():
|
||||
assert split_in_half("court") == ("", "")
|
||||
|
||||
|
||||
def test_split_in_half_splits_on_newline_near_middle():
|
||||
text = "A" * 300 + "\n" + "B" * 300
|
||||
left, right = split_in_half(text)
|
||||
assert left and right
|
||||
assert left.startswith("A")
|
||||
assert right.startswith("B")
|
||||
|
||||
|
||||
def test_split_in_half_halves_cover_all_content():
|
||||
text = "\n".join(f"ligne {i} " + "x" * 20 for i in range(40))
|
||||
left, right = split_in_half(text)
|
||||
assert left and right
|
||||
# Le découpage ne perd rien : la concaténation contient début et fin.
|
||||
assert "ligne 0" in left
|
||||
assert "ligne 39" in right
|
||||
Reference in New Issue
Block a user