"""Unit tests for the slide parser: order, shapes, tables, and one chunk a slide."""
from __future__ import annotations
import asyncio
import io
import zipfile
import pytest
from raven.knowledge._naive_chunker import NaiveChunker
from raven.knowledge._structure import HeadingAwareChunker
from raven.knowledge._types import Section
from raven.knowledge.parser import ATOMIC, ELEMENTS, PAGE_NUMBER, READING_ORDER, LayoutType
from raven.knowledge.parser.ppt_parser import PptParser
_NS = (
'xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" '
'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" '
'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"'
)
def _text_shape(text: str, *, x: int = 0, y: int = 0, placeholder: str = "", bullets: bool = False) -> str:
"""One text box on a slide, at a stated point."""
ph = f'' if placeholder else ""
frame = f''
paragraphs = "".join(
f"{'' if bullets else ''}{line}"
for line in text.split("\n")
)
return f"{ph}{frame}{paragraphs}"
def _table_shape(rows: "list[list[str | tuple[str, str]]]", *, x: int = 0, y: int = 0) -> str:
"""A table shape. A cell given as a pair carries that raw attribute string,
which is how a merge is written: `gridSpan="2"` on the cell that begins it
and `hMerge="1"` on each cell it covers."""
def one(cell: "str | tuple[str, str]") -> str:
text, marks = cell if isinstance(cell, tuple) else (cell, "")
return f"{text}"
grid = "".join("" + "".join(one(cell) for cell in row) + "" for row in rows)
return (
f''
f"{grid}"
)
def _deck(slides: list[str], *, order: list[int] | None = None, layouts: dict[str, str] | None = None) -> bytes:
"""A package holding these slides, presented in ``order``.
``order`` indexes ``slides`` and defaults to the order given. It exists
because the running order is the one thing about a deck that cannot be read
off the part names.
"""
order = list(range(len(slides))) if order is None else order
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w") as package:
entries = "".join(f'' for at in order)
package.writestr(
"ppt/presentation.xml",
f"{entries}",
)
rels = "".join(
f''
for at in range(len(slides))
)
package.writestr(
"ppt/_rels/presentation.xml.rels",
''
f''
f"{rels}",
)
for at, body in enumerate(slides):
package.writestr(
f"ppt/slides/slide{at + 1}.xml",
f"{body}",
)
if layouts:
package.writestr(
f"ppt/slides/_rels/slide{at + 1}.xml.rels",
''
''
"",
)
for name, body in (layouts or {}).items():
package.writestr(name, f"{body}")
return buffer.getvalue()
def _parse(payload: bytes, name: str = "deck.pptx") -> list[Section]:
return asyncio.run(PptParser().parse(payload, name))
def _texts(sections: list[Section]) -> list[str]:
return [section.content.text for section in sections]
# -- what it claims -------------------------------------------------
def test_it_claims_both_deck_formats() -> None:
assert PptParser.supported_extensions() == [".ppt", ".pptx"]
assert "application/vnd.ms-powerpoint" in PptParser.supported_media_types
# -- one slide, one section -----------------------------------------
def test_each_slide_is_a_section_numbered_from_one() -> None:
sections = _parse(_deck([_text_shape("First slide"), _text_shape("Second slide")]))
assert _texts(sections) == ["First slide", "Second slide"]
assert [section.metadata[PAGE_NUMBER] for section in sections] == [1, 2]
assert [section.metadata[READING_ORDER] for section in sections] == [0, 1]
def test_the_running_order_comes_from_the_deck_not_the_filenames() -> None:
"""`slide10.xml` sorts before `slide2.xml`, and a deck whose slides were
reordered keeps the numbering it was first saved with -- so the part names
say nothing about what a reader sees first."""
slides = [_text_shape(f"Slide {n}") for n in range(3)]
sections = _parse(_deck(slides, order=[2, 0, 1]))
assert _texts(sections) == ["Slide 2", "Slide 0", "Slide 1"]
assert [section.metadata[PAGE_NUMBER] for section in sections] == [1, 2, 3]
def test_a_slide_with_nothing_on_it_is_skipped() -> None:
"""A slide that is one picture has nothing to embed, and a chunk of nothing
is a row no query can match. The pages that remain still name themselves
correctly, which is what the preview needs."""
sections = _parse(_deck([_text_shape("First"), "", _text_shape("Third")]))
assert _texts(sections) == ["First", "Third"]
assert [section.metadata[PAGE_NUMBER] for section in sections] == [1, 3]
def test_a_deck_with_no_slides_parses_to_nothing() -> None:
assert _parse(_deck([])) == []
# -- the order shapes are read in -----------------------------------
def test_shapes_are_read_down_the_slide_then_across() -> None:
"""The order they sit in, not the order the file lists them: a deck writes
shapes in creation order, and nobody builds a slide top to bottom."""
body = (
_text_shape("bottom", x=0, y=4_000_000)
+ _text_shape("top right", x=5_000_000, y=0)
+ _text_shape("top left", x=0, y=0)
)
assert _texts(_parse(_deck([body]))) == ["top left\ntop right\nbottom"]
def test_shapes_within_a_line_of_each_other_count_as_one_row() -> None:
"""Two shapes meant to be read side by side are never at exactly the same
height, and sorting on the exact top puts the right-hand one first whenever
it sits a rounding error higher."""
body = _text_shape("right", x=5_000_000, y=0) + _text_shape("left", x=0, y=20_000)
assert _texts(_parse(_deck([body]))) == ["left\nright"]
def test_a_slide_that_states_no_positions_keeps_its_own_order() -> None:
body = "".join(_text_shape(word) for word in ("one", "two", "three"))
assert _texts(_parse(_deck([body]))) == ["one\ntwo\nthree"]
def test_a_placeholder_takes_its_position_from_the_layout() -> None:
"""Most title and body placeholders carry no geometry of their own -- it
lives in the layout. Read as unplaced, a title sorts after every shape that
does state a position, which on a real deck is most of them."""
slide = (
''
"The title"
) + _text_shape("body text", x=0, y=3_000_000)
layout = (
''
''
""
)
sections = _parse(_deck([slide], layouts={"ppt/slideLayouts/slideLayout1.xml": layout}))
assert _texts(sections) == ["The title\nbody text"]
# -- what a shape says ----------------------------------------------
def test_a_bulleted_paragraph_is_marked_and_indented() -> None:
"""RAGFlow's rendering: once the styling is gone a list has to still read
as a list."""
sections = _parse(_deck([_text_shape("first\nsecond", bullets=True)]))
assert sections[0].content.text == " .first\n .second"
def test_a_table_is_kept_as_a_grid() -> None:
"""The shape the Word and PDF parsers keep too: a slide states where its
cell boundaries are, and that is what a flattened table throws away."""
body = _table_shape([["Region", "Revenue"], ["EU", "1.2M"], ["US", "3.4M"]])
assert _texts(_parse(_deck([body]))) == [
"
\n"
"| Region | Revenue |
\n"
"| EU | 1.2M |
\n"
"| US | 3.4M |
\n"
"
"
]
def test_a_merged_cell_spans_instead_of_leaving_a_blank() -> None:
"""DrawingML puts the span on the cell that begins the merge and marks the
cells it covers as carrying nothing. Read straight through, those covered
cells come out as blanks -- a grid saying the slide left a cell empty,
which is a different claim from a cell being part of its neighbour."""
body = _table_shape(
[
[("Revenue", 'gridSpan="2"'), ("", 'hMerge="1"')],
[("EU", 'rowSpan="2"'), "1.2M"],
[("", 'vMerge="1"'), "1.4M"],
]
)
text = _texts(_parse(_deck([body])))[0]
assert 'Revenue | ' in text
assert 'EU | ' in text
assert "| 1.4M |
" in text, "the covered cell is gone, not blank"
def test_a_group_is_read_through() -> None:
inner = _text_shape("inside one", x=0, y=0) + _text_shape("inside two", x=0, y=1_000_000)
body = (
f'{inner}'
)
assert _texts(_parse(_deck([body]))) == ["inside one\ninside two"]
def test_the_title_placeholder_becomes_the_heading_path() -> None:
body = _text_shape("Quarterly review", placeholder="ctrTitle") + _text_shape("body", y=2_000_000)
sections = _parse(_deck([body]))
assert sections[0].metadata["heading_path"] == ["Quarterly review"]
def test_a_slide_with_no_title_carries_no_heading() -> None:
"""Rather than borrowing its first line, which is a caption as often as a
title and would be cited as one."""
assert "heading_path" not in _parse(_deck([_text_shape("just some text")]))[0].metadata
# -- where each shape sits ------------------------------------------
def test_every_shape_records_its_own_box_and_page() -> None:
"""Complete boxes, unlike a Word paragraph's: a slide states absolute
coordinates for everything on it, so a hit can be pointed at."""
body = _text_shape("top", x=127_000, y=254_000) + _text_shape("under", x=0, y=2_540_000)
spans = _parse(_deck([body]))[0].metadata[ELEMENTS]
assert [span["page_number"] for span in spans] == [1, 1]
# 127000 EMU is 10pt, and the shape is 100000 EMU wide. Rounded because the
# conversion is a division and the exact tail is not the point.
box = spans[0]["bbox"]
assert (round(box["x0"], 3), round(box["top"], 3)) == (10.0, 20.0)
assert (round(box["x1"], 3), round(box["bottom"], 3)) == (17.874, 23.937)
def test_a_table_span_says_it_is_a_table() -> None:
body = _text_shape("intro") + _table_shape([["a", "b"], ["1", "2"]], y=1_000_000)
spans = _parse(_deck([body]))[0].metadata[ELEMENTS]
assert [span["layout_type"] for span in spans] == [LayoutType.TEXT, LayoutType.TABLE]
def test_the_spans_address_the_section_text() -> None:
body = _text_shape("first", y=0) + _text_shape("second", y=1_000_000)
section = _parse(_deck([body]))[0]
quoted = [section.content.text[span["char_start"] : span["char_end"]] for span in section.metadata[ELEMENTS]]
assert quoted == ["first", "second"]
# -- one slide, one chunk -------------------------------------------
def test_a_slide_is_marked_as_one_chunk() -> None:
assert _parse(_deck([_text_shape("anything")]))[0].metadata[ATOMIC] is True
def test_a_long_slide_is_still_one_chunk() -> None:
"""The guarantee the parser exists under. A reader says "slide 12"; half a
slide is not something they can point at, and the preview beside the list
can only scroll to a page."""
body = _text_shape(". ".join(f"sentence {n}" for n in range(200)))
sections = _parse(_deck([body]))
chunks = asyncio.run(NaiveChunker(chunk_size=16).chunk(sections))
assert len(chunks) == 1
def test_two_slides_are_never_merged_into_one_chunk() -> None:
"""The other half: a chunk holding two slides puts one piece in front of a
viewer that can only show one page."""
sections = _parse(_deck([_text_shape("first"), _text_shape("second")]))
chunks = asyncio.run(NaiveChunker(chunk_size=4096).chunk(sections))
assert [chunk.content.text for chunk in chunks] == ["first", "second"]
def test_a_slide_takes_no_overlap_from_its_neighbours() -> None:
"""Overlap repeats the neighbours at each end, which on a deck would put
the previous slide's words into this slide's chunk and make the page it
names a lie about half its text."""
sections = _parse(_deck([_text_shape(f"slide {n} text") for n in range(3)]))
chunks = asyncio.run(NaiveChunker(chunk_size=4096, overlap_size=20).chunk(sections))
assert [chunk.content.text for chunk in chunks] == ["slide 0 text", "slide 1 text", "slide 2 text"]
def test_the_page_travels_to_the_chunk() -> None:
"""Which is what the preview reads to scroll: the chunk has to carry the
slide number, not merely have been cut from it."""
sections = _parse(_deck([_text_shape("one"), _text_shape("two"), _text_shape("three")]))
chunks = asyncio.run(NaiveChunker(chunk_size=4096).chunk(sections))
assert [chunk.metadata[PAGE_NUMBER] for chunk in chunks] == [1, 2, 3]
def test_the_structural_chunker_keeps_the_slide_whole_too() -> None:
"""Which chunker a base runs is a setting, and a guarantee a parser relies
on cannot depend on a setting."""
body = _text_shape(". ".join(f"sentence {n}" for n in range(200)))
chunks = asyncio.run(HeadingAwareChunker(chunk_size=16, overlap=4).chunk(_parse(_deck([body]))))
assert len(chunks) == 1
# -- the awkward payloads -------------------------------------------
def test_a_file_that_is_not_a_deck_says_so() -> None:
with pytest.raises(ValueError, match="slide deck"):
_parse(b"PK\x03\x04 and then nonsense")
def test_a_zip_with_no_presentation_part_says_so() -> None:
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w") as package:
package.writestr("docProps/app.xml", "")
with pytest.raises(ValueError, match="presentation"):
_parse(buffer.getvalue())
def test_a_path_is_read_from_disk(tmp_path) -> None:
deck = tmp_path / "deck.pptx"
deck.write_bytes(_deck([_text_shape("from disk")]))
assert _texts(asyncio.run(PptParser().parse(str(deck), "deck.pptx"))) == ["from disk"]
def test_a_missing_path_is_reported_as_missing(tmp_path) -> None:
with pytest.raises(FileNotFoundError):
asyncio.run(PptParser().parse(str(tmp_path / "gone.pptx"), "gone.pptx"))