diff --git a/core/llm/parser.py b/core/llm/parser.py index e233069016..450f4c8510 100644 --- a/core/llm/parser.py +++ b/core/llm/parser.py @@ -38,7 +38,7 @@ class DescriptiveCodeBlockParser: """ def __init__(self): - self.pattern = re.compile(r"^(.*?)\n```([a-z0-9]+\n)?(.*?)^```\s*", re.DOTALL | re.MULTILINE) + self.pattern = re.compile(r"^(.*?)\n```([a-z0-9]+\r?\n)?(.*?)^```\s*", re.DOTALL | re.MULTILINE) def __call__(self, text: str) -> ParsedBlocks: # Store original response @@ -86,7 +86,7 @@ class MultiCodeBlockParser: """ def __init__(self): - self.pattern = re.compile(r"^```([a-z0-9]+\n)?(.*?)^```\s*", re.DOTALL | re.MULTILINE) + self.pattern = re.compile(r"^```([a-z0-9]+\r?\n)?(.*?)^```\s*", re.DOTALL | re.MULTILINE) def __call__(self, text: str) -> list[str]: blocks = [] diff --git a/tests/llm/test_parser.py b/tests/llm/test_parser.py index 16f8aadc7f..38233d4a61 100644 --- a/tests/llm/test_parser.py +++ b/tests/llm/test_parser.py @@ -4,7 +4,14 @@ import pytest from pydantic import BaseModel, field_validator -from core.llm.parser import CodeBlockParser, EnumParser, JSONParser, MultiCodeBlockParser, OptionalCodeBlockParser +from core.llm.parser import ( + CodeBlockParser, + DescriptiveCodeBlockParser, + EnumParser, + JSONParser, + MultiCodeBlockParser, + OptionalCodeBlockParser, +) @pytest.mark.parametrize( @@ -45,6 +52,34 @@ def test_code_block_parser(input, expected): assert parser(input) == expected +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +@pytest.mark.parametrize("parser_type", [CodeBlockParser, MultiCodeBlockParser]) +def test_code_block_parser_line_endings(newline, parser_type): + content = f"first{newline}second" + text = f"```python{newline}{content}{newline}```" + + expected = [content] if parser_type is MultiCodeBlockParser else content + assert parser_type()(text) == expected + + +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_descriptive_code_block_parser_line_endings(newline): + text = f"file: main.py{newline}```python{newline}print('hello'){newline}```" + + blocks = DescriptiveCodeBlockParser()(text).blocks + + assert len(blocks) == 1 + assert blocks[0].description == "file: main.py" + assert blocks[0].content == "print('hello')" + + +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_json_parser_fenced_line_endings(newline): + text = f'```json{newline}{{"value": 42}}{newline}```' + + assert JSONParser()(text) == {"value": 42} + + @pytest.mark.parametrize( ("input", "strict", "expected"), [