From 896763550e07b883afc09265043549d77a34181d Mon Sep 17 00:00:00 2001 From: James Vorderbruggen Date: Sat, 13 Jul 2024 08:40:24 -0500 Subject: [PATCH 1/9] Pytest plugins in their own file --- generated/test_class.py | 1 - main.py | 48 +--------------------------------------- pytest_plugins.py | 49 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 48 deletions(-) delete mode 100644 generated/test_class.py create mode 100644 pytest_plugins.py diff --git a/generated/test_class.py b/generated/test_class.py deleted file mode 100644 index 1909353..0000000 --- a/generated/test_class.py +++ /dev/null @@ -1 +0,0 @@ -# THIS FILE INTENTIONALLY LEFT BLANK \ No newline at end of file diff --git a/main.py b/main.py index c7a82bf..f0c7ca6 100644 --- a/main.py +++ b/main.py @@ -2,7 +2,6 @@ import typer # for the test runner -import time import pytest # ------------------ @@ -10,9 +9,7 @@ from langroid.utils.configuration import set_global, Settings from langroid.utils.logging import setup_colored_logging -# import the empty generated code -import generated.test_class -# ------------------ +from pytest_plugins import ResultsCollector, SessionStartPlugin app = typer.Typer() setup_colored_logging() @@ -43,49 +40,6 @@ def generate_first_attempt() -> None: _out.write(response.content) -class ResultsCollector: - def __init__(self): - self.reports = [] - self.collected = 0 - self.exitcode = 0 - self.passed = 0 - self.failed = 0 - self.xfailed = 0 - self.skipped = 0 - self.total_duration = 0 - - @pytest.hookimpl(hookwrapper=True) - def pytest_runtest_makereport(self, item, call): - outcome = yield - report = outcome.get_result() - if report.when == 'call': - self.reports.append(report) - - def pytest_collection_modifyitems(self, items): - self.collected = len(items) - - def pytest_terminal_summary(self, terminalreporter, exitstatus): - self.exitcode = exitstatus - self.passed = len(terminalreporter.stats.get('passed', [])) - self.failed = len(terminalreporter.stats.get('failed', [])) - self.xfailed = len(terminalreporter.stats.get('xfailed', [])) - self.skipped = len(terminalreporter.stats.get('skipped', [])) - - self.total_duration = time.time() - terminalreporter._sessionstarttime - - -class SessionStartPlugin: - """ - The goal of this plugin is to allow us to run pytest multiple times - and have it pick up the changes we generate in `generated/test_class.py` - """ - def pytest_sessionstart(self): - if globals().get('generated', None) is not None: - import importlib - print("Reloading generated.test_class module...") - importlib.reload(generated.test_class) - - def get_test_results() -> str: collector = ResultsCollector() setup = SessionStartPlugin() diff --git a/pytest_plugins.py b/pytest_plugins.py new file mode 100644 index 0000000..414dd5e --- /dev/null +++ b/pytest_plugins.py @@ -0,0 +1,49 @@ +import pytest +import time +try: + import generated.test_class +except ImportError: + pass # Since this is a generated file, it sometimes doesn't exist. + + +class ResultsCollector: + def __init__(self): + self.reports = [] + self.collected = 0 + self.exitcode = 0 + self.passed = 0 + self.failed = 0 + self.xfailed = 0 + self.skipped = 0 + self.total_duration = 0 + + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_makereport(self, item, call): + outcome = yield + report = outcome.get_result() + if report.when == 'call': + self.reports.append(report) + + def pytest_collection_modifyitems(self, items): + self.collected = len(items) + + def pytest_terminal_summary(self, terminalreporter, exitstatus): + self.exitcode = exitstatus + self.passed = len(terminalreporter.stats.get('passed', [])) + self.failed = len(terminalreporter.stats.get('failed', [])) + self.xfailed = len(terminalreporter.stats.get('xfailed', [])) + self.skipped = len(terminalreporter.stats.get('skipped', [])) + + self.total_duration = time.time() - terminalreporter._sessionstarttime + + +class SessionStartPlugin: + """ + The goal of this plugin is to allow us to run pytest multiple times + and have it pick up the changes we generate in `generated/test_class.py` + """ + def pytest_sessionstart(self): + if globals().get('generated', None) is not None: + import importlib + print("Reloading generated.test_class module...") + importlib.reload(generated.test_class) \ No newline at end of file From b5fee83f30d06cdf1642d2768af18fbc3a9bed60 Mon Sep 17 00:00:00 2001 From: James Vorderbruggen Date: Sat, 13 Jul 2024 08:52:05 -0500 Subject: [PATCH 2/9] Empty file required on first run --- generated/test_class.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 generated/test_class.py diff --git a/generated/test_class.py b/generated/test_class.py new file mode 100644 index 0000000..e69de29 From b0fd195434140be537b9e250118a5f48a4cfde83 Mon Sep 17 00:00:00 2001 From: James Vorderbruggen Date: Sat, 13 Jul 2024 09:03:56 -0500 Subject: [PATCH 3/9] Clean context on each invocation --- .gitignore | 3 +++ main.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/.gitignore b/.gitignore index 7b6caf3..64d0815 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Project-specific rules +build/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/main.py b/main.py index f0c7ca6..5988e29 100644 --- a/main.py +++ b/main.py @@ -87,6 +87,17 @@ def generate_next_attempt(test_results: str) -> None: _out.write(response.content) +def teardown() -> None: + codegen_path = os.path.join(".", "generated") + build_path = os.path.join(".", "build") + if not os.path.exists(build_path): + os.makedirs(build_path) + with open(os.path.join(codegen_path, "test_class.py"), "r+") as generated_file: + with open(os.path.join(build_path, "test_class.py"), "w+") as _out: + _out.write(generated_file.read()) + generated_file.truncate(0) + + def chat() -> None: generate_first_attempt() for _ in range(5): @@ -95,6 +106,7 @@ def chat() -> None: generate_next_attempt(test_results) else: break + teardown() print("Done! All tests are passing, or there is some problem with the test suite itself.") @app.command() From 386062d7ff907071f7125d05ce826c4c00e48b35 Mon Sep 17 00:00:00 2001 From: James Vorderbruggen Date: Sun, 14 Jul 2024 22:53:37 -0500 Subject: [PATCH 4/9] Pull test runners into classes --- TestRunner/GenericTestRunner.py | 47 ++++++++++++++++++++++++++++++ TestRunner/SubProcessTestRunner.py | 1 + TestRunner/__init__.py | 0 main.py | 26 ++++------------- 4 files changed, 53 insertions(+), 21 deletions(-) create mode 100644 TestRunner/GenericTestRunner.py create mode 100644 TestRunner/SubProcessTestRunner.py create mode 100644 TestRunner/__init__.py diff --git a/TestRunner/GenericTestRunner.py b/TestRunner/GenericTestRunner.py new file mode 100644 index 0000000..2adef40 --- /dev/null +++ b/TestRunner/GenericTestRunner.py @@ -0,0 +1,47 @@ +from abc import ABCMeta, abstractmethod + +import pytest + +from pytest_plugins import ResultsCollector, SessionStartPlugin + + +class GenericTestRunner(metaclass=ABCMeta): + @abstractmethod + def run(self, *args, **kwargs) -> (int, str): pass + +import subprocess + + +class SubProcessTestRunner(GenericTestRunner): + code_dir: str + test_dir: str + + def __init__(self, _code, _test) -> None: + self.code_dir = _code + self.test_dir = _test + + def run(self, *args, **kwargs) -> (int, str): + # TODO: check that code_dir and test_dir exist + proc = subprocess.run(["pytest", self.test_dir], capture_output=True) + return proc.returncode, proc.stdout + + +class InlineTestRunner(GenericTestRunner): + + def __init__(self, _code, _test) -> None: + self.code_dir = _code + self.test_dir = _test + + def run(self, *args, **kwargs) -> (int, str): + collector = ResultsCollector() + setup = SessionStartPlugin() + pytest.main(args=["-k", "ExampleClass"], plugins=[collector, setup]) + _out = "" + + if collector.exitcode > 0: + for report in collector.reports: + _out += f"{report.outcome.upper()} {report.nodeid} ... Outcome: - {report.longrepr.reprcrash.message}" + _out += "\n" + _out += report.longreprtext + _out += "\n" + return collector.exitcode, _out diff --git a/TestRunner/SubProcessTestRunner.py b/TestRunner/SubProcessTestRunner.py new file mode 100644 index 0000000..0fe84b4 --- /dev/null +++ b/TestRunner/SubProcessTestRunner.py @@ -0,0 +1 @@ +from . import GenericTestRunner diff --git a/TestRunner/__init__.py b/TestRunner/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/main.py b/main.py index 5988e29..6f43abb 100644 --- a/main.py +++ b/main.py @@ -1,15 +1,11 @@ import os import typer -# for the test runner -import pytest -# ------------------ - import langroid as lr from langroid.utils.configuration import set_global, Settings from langroid.utils.logging import setup_colored_logging -from pytest_plugins import ResultsCollector, SessionStartPlugin +from TestRunner.GenericTestRunner import InlineTestRunner app = typer.Typer() setup_colored_logging() @@ -40,21 +36,6 @@ def generate_first_attempt() -> None: _out.write(response.content) -def get_test_results() -> str: - collector = ResultsCollector() - setup = SessionStartPlugin() - pytest.main(args=["-k", "ExampleClass"], plugins=[collector, setup]) - _out = "" - - if collector.exitcode > 0: - for report in collector.reports: - _out += f"{report.outcome.upper()} {report.nodeid} ... Outcome: - {report.longrepr.reprcrash.message}" - _out += "\n" - _out += report.longreprtext - _out += "\n" - return collector.exitcode, _out - - def generate_next_attempt(test_results: str) -> None: cfg = lr.ChatAgentConfig( llm=lr.language_models.OpenAIGPTConfig( @@ -100,8 +81,11 @@ def teardown() -> None: def chat() -> None: generate_first_attempt() + test_runner = InlineTestRunner("", os.path.join(".", "test")) for _ in range(5): - test_exit_code, test_results = get_test_results() + # test_exit_code, test_results = get_test_results() + test_exit_code, test_results = test_runner.run() + print(test_results) if test_exit_code == 1: generate_next_attempt(test_results) else: From 9dfc62ad7ce4b94e54fe2f5e4c1b519a55170d59 Mon Sep 17 00:00:00 2001 From: James Vorderbruggen Date: Mon, 15 Jul 2024 22:22:15 -0500 Subject: [PATCH 5/9] Make test dir and class skel location configurable --- main.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/main.py b/main.py index 6f43abb..43cae67 100644 --- a/main.py +++ b/main.py @@ -5,16 +5,14 @@ from langroid.utils.configuration import set_global, Settings from langroid.utils.logging import setup_colored_logging -from TestRunner.GenericTestRunner import InlineTestRunner +from TestRunner.GenericTestRunner import GenericTestRunner, InlineTestRunner, SubProcessTestRunner app = typer.Typer() setup_colored_logging() -def generate_first_attempt() -> None: - # code_prompt = typer.Prompt("Describe what kind of code you want.") - # class_skeleton: str = "" - with open(os.path.join(".", "assets", "test_class.py"), "r") as f: +def generate_first_attempt(class_skeleton: str) -> None: + with open(class_skeleton, "r") as f: class_skeleton = f.read() cfg = lr.ChatAgentConfig( @@ -79,25 +77,34 @@ def teardown() -> None: generated_file.truncate(0) -def chat() -> None: - generate_first_attempt() - test_runner = InlineTestRunner("", os.path.join(".", "test")) - for _ in range(5): +def chat(class_skeleton: str, test_dir: str, test_runner: GenericTestRunner, max_epochs: int=5) -> None: + generate_first_attempt(class_skeleton) + solved = False + for _ in range(max_epochs): # test_exit_code, test_results = get_test_results() test_exit_code, test_results = test_runner.run() print(test_results) - if test_exit_code == 1: + if test_exit_code == 0: + solved = True + print("Done!") + break + elif test_exit_code == 1: generate_next_attempt(test_results) else: + solved = True + print("There is some problem with the test suite itself.") break teardown() - print("Done! All tests are passing, or there is some problem with the test suite itself.") + if not solved: + print(f"Reached the end of epoch {max_epochs} without finding a solution :(") @app.command() def main( debug: bool = typer.Option(False, "--debug", "-d", help="debug mode"), no_stream: bool = typer.Option(False, "--nostream", "-ns", help="no streaming"), nocache: bool = typer.Option(False, "--nocache", "-nc", help="don't use cache"), + class_skeleton: str = typer.Option(None, "--class-skeleton", "-c", help="You must provide a class skeleton."), + test_dir: str = typer.Option(os.path.join(".", "test"), "--test-dir", "-t", help=""), ) -> None: set_global( Settings( @@ -106,7 +113,15 @@ def main( stream=not no_stream, ) ) - chat() + assert os.path.isfile(class_skeleton), f"The class skeleton file provided does not exist! Got {class_skeleton}" + assert os.path.exists(test_dir), f"The test-dir provided does not exist! Got {test_dir}" + + tr: GenericTestRunner = InlineTestRunner("", test_dir) + chat( + class_skeleton=class_skeleton, + test_dir=test_dir, + test_runner=tr + ) if __name__ == "__main__": From 11e004a516e3396bc614e73d14e18d32326b606a Mon Sep 17 00:00:00 2001 From: James Vorderbruggen Date: Mon, 15 Jul 2024 22:35:24 -0500 Subject: [PATCH 6/9] Fix subprocess runner so it runs --- TestRunner/GenericTestRunner.py | 2 +- test/test_ExampleClass.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TestRunner/GenericTestRunner.py b/TestRunner/GenericTestRunner.py index 2adef40..3fe9abe 100644 --- a/TestRunner/GenericTestRunner.py +++ b/TestRunner/GenericTestRunner.py @@ -22,7 +22,7 @@ def __init__(self, _code, _test) -> None: def run(self, *args, **kwargs) -> (int, str): # TODO: check that code_dir and test_dir exist - proc = subprocess.run(["pytest", self.test_dir], capture_output=True) + proc = subprocess.run(["pytest", self.test_dir], capture_output=True, universal_newlines=True) return proc.returncode, proc.stdout diff --git a/test/test_ExampleClass.py b/test/test_ExampleClass.py index e96e44d..0d59a4d 100644 --- a/test/test_ExampleClass.py +++ b/test/test_ExampleClass.py @@ -1,5 +1,5 @@ import pytest -import generated +import generated.test_class @pytest.mark.parametrize("num1, num2, expected1, expected2", [ From 4c617776a8076ca93cf1e95fa9464a7b108c5c2b Mon Sep 17 00:00:00 2001 From: James Vorderbruggen Date: Tue, 16 Jul 2024 07:36:39 -0500 Subject: [PATCH 7/9] Provide insights about test results to help code gen --- TestRunner/GenericTestRunner.py | 7 ++----- TestRunner/SubProcessTestRunner.py | 1 - main.py | 30 +++++++++++++++++++++++++++--- 3 files changed, 29 insertions(+), 9 deletions(-) delete mode 100644 TestRunner/SubProcessTestRunner.py diff --git a/TestRunner/GenericTestRunner.py b/TestRunner/GenericTestRunner.py index 3fe9abe..b93e13a 100644 --- a/TestRunner/GenericTestRunner.py +++ b/TestRunner/GenericTestRunner.py @@ -1,7 +1,6 @@ -from abc import ABCMeta, abstractmethod - import pytest - +import subprocess +from abc import ABCMeta, abstractmethod from pytest_plugins import ResultsCollector, SessionStartPlugin @@ -9,8 +8,6 @@ class GenericTestRunner(metaclass=ABCMeta): @abstractmethod def run(self, *args, **kwargs) -> (int, str): pass -import subprocess - class SubProcessTestRunner(GenericTestRunner): code_dir: str diff --git a/TestRunner/SubProcessTestRunner.py b/TestRunner/SubProcessTestRunner.py deleted file mode 100644 index 0fe84b4..0000000 --- a/TestRunner/SubProcessTestRunner.py +++ /dev/null @@ -1 +0,0 @@ -from . import GenericTestRunner diff --git a/main.py b/main.py index 43cae67..92defee 100644 --- a/main.py +++ b/main.py @@ -34,7 +34,7 @@ def generate_first_attempt(class_skeleton: str) -> None: _out.write(response.content) -def generate_next_attempt(test_results: str) -> None: +def generate_next_attempt(test_results: str, test_results_insights: str) -> None: cfg = lr.ChatAgentConfig( llm=lr.language_models.OpenAIGPTConfig( chat_model="ollama/llama3:latest", @@ -53,6 +53,8 @@ def generate_next_attempt(test_results: str) -> None: {code_snippet} Here are the test results: {test_results} + In addition, you may consider these insights about the test results when coming up with your solution: + {test_results_insights} Update the code so that the tests will pass. Your output MUST contain all the same classes and methods as the input code. Do NOT add any other methods or commentary. @@ -66,6 +68,27 @@ def generate_next_attempt(test_results: str) -> None: _out.write(response.content) +def interpret_test_results(results: str) -> str: + cfg = lr.ChatAgentConfig( + llm=lr.language_models.OpenAIGPTConfig( + chat_model="ollama/llama3:latest", + chat_context_length=8000, + ), + vecdb=None + ) + agent = lr.ChatAgent(cfg) + prompt = f""" + You are an expert at interpreting the results of unit tests, and providing insight into what they mean. + You should be descriptive about what variables are incorrect, and in what way. + You should include information about which methods should be modified, and in what way. + You should generally not provide code. + Please provide insights about the following test results: + {results} + """ + response = agent.llm_response(prompt) + return response.content + + def teardown() -> None: codegen_path = os.path.join(".", "generated") build_path = os.path.join(".", "build") @@ -89,7 +112,8 @@ def chat(class_skeleton: str, test_dir: str, test_runner: GenericTestRunner, max print("Done!") break elif test_exit_code == 1: - generate_next_attempt(test_results) + results_insights = interpret_test_results(test_results) + generate_next_attempt(test_results, results_insights) else: solved = True print("There is some problem with the test suite itself.") @@ -116,7 +140,7 @@ def main( assert os.path.isfile(class_skeleton), f"The class skeleton file provided does not exist! Got {class_skeleton}" assert os.path.exists(test_dir), f"The test-dir provided does not exist! Got {test_dir}" - tr: GenericTestRunner = InlineTestRunner("", test_dir) + tr: GenericTestRunner = SubProcessTestRunner("", test_dir) chat( class_skeleton=class_skeleton, test_dir=test_dir, From df727dc1666302274f5053e23b14c666a5f464aa Mon Sep 17 00:00:00 2001 From: James Vorderbruggen Date: Tue, 16 Jul 2024 07:39:40 -0500 Subject: [PATCH 8/9] Forbid annoying backtick behavior --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 92defee..60ba5fa 100644 --- a/main.py +++ b/main.py @@ -61,6 +61,7 @@ def generate_next_attempt(test_results: str, test_results_insights: str) -> None Your response should be ONLY the python code. Do not say 'here is the python code' Do not surround your response with quotes or backticks. + Your response should NEVER start or end with ``` Your output MUST be valid, runnable python code and NOTHING else. """ response = agent.llm_response(prompt) From ff318e94a148a2a144c1669258a9b25923f0a84e Mon Sep 17 00:00:00 2001 From: James Vorderbruggen Date: Tue, 16 Jul 2024 21:44:33 -0500 Subject: [PATCH 9/9] Regular imports --- test/test_ExampleClass.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_ExampleClass.py b/test/test_ExampleClass.py index 0d59a4d..49953a3 100644 --- a/test/test_ExampleClass.py +++ b/test/test_ExampleClass.py @@ -1,5 +1,5 @@ import pytest -import generated.test_class +from generated import test_class @pytest.mark.parametrize("num1, num2, expected1, expected2", [ @@ -11,6 +11,6 @@ def test_init(num1, num2, expected1, expected2): # Let's make a test with a tricky expectation. # When we initialize TestClass, we should subtract 1 from x and y. - instance = generated.test_class.ExampleClass(num1, num2) + instance = test_class.ExampleClass(num1, num2) assert instance.x == expected1 assert instance.y == expected2