Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions core/llm/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class DescriptiveCodeBlockParser:
"""

def __init__(self):
self.pattern = re.compile(r"^(.*?)\n```([a-z0-9]+\n)?(.*?)^```\s*", re.DOTALL | re.MULTILINE)
self.pattern = re.compile(r"^(.*?)\n```([a-z0-9]+\r?\n)?(.*?)^```\s*", re.DOTALL | re.MULTILINE)

def __call__(self, text: str) -> ParsedBlocks:
# Store original response
Expand Down Expand Up @@ -86,7 +86,7 @@ class MultiCodeBlockParser:
"""

def __init__(self):
self.pattern = re.compile(r"^```([a-z0-9]+\n)?(.*?)^```\s*", re.DOTALL | re.MULTILINE)
self.pattern = re.compile(r"^```([a-z0-9]+\r?\n)?(.*?)^```\s*", re.DOTALL | re.MULTILINE)

def __call__(self, text: str) -> list[str]:
blocks = []
Expand Down
37 changes: 36 additions & 1 deletion tests/llm/test_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,14 @@
import pytest
from pydantic import BaseModel, field_validator

from core.llm.parser import CodeBlockParser, EnumParser, JSONParser, MultiCodeBlockParser, OptionalCodeBlockParser
from core.llm.parser import (
CodeBlockParser,
DescriptiveCodeBlockParser,
EnumParser,
JSONParser,
MultiCodeBlockParser,
OptionalCodeBlockParser,
)


@pytest.mark.parametrize(
Expand Down Expand Up @@ -45,6 +52,34 @@ def test_code_block_parser(input, expected):
assert parser(input) == expected


@pytest.mark.parametrize("newline", ["\n", "\r\n"])
@pytest.mark.parametrize("parser_type", [CodeBlockParser, MultiCodeBlockParser])
def test_code_block_parser_line_endings(newline, parser_type):
content = f"first{newline}second"
text = f"```python{newline}{content}{newline}```"

expected = [content] if parser_type is MultiCodeBlockParser else content
assert parser_type()(text) == expected


@pytest.mark.parametrize("newline", ["\n", "\r\n"])
def test_descriptive_code_block_parser_line_endings(newline):
text = f"file: main.py{newline}```python{newline}print('hello'){newline}```"

blocks = DescriptiveCodeBlockParser()(text).blocks

assert len(blocks) == 1
assert blocks[0].description == "file: main.py"
assert blocks[0].content == "print('hello')"


@pytest.mark.parametrize("newline", ["\n", "\r\n"])
def test_json_parser_fenced_line_endings(newline):
text = f'```json{newline}{{"value": 42}}{newline}```'

assert JSONParser()(text) == {"value": 42}


@pytest.mark.parametrize(
("input", "strict", "expected"),
[
Expand Down