with cmd

Author: Yasu-umi

.github/actions/extract_error/action.yml

@@ -1,21 +1,27 @@
 name: 'Extract Error'
 description: 'extract test error messaget to pull request comment'
 inputs:
-  error_filepath:
-    description: 'error mesasge file path'
-    required: true
   openai_api_key:
     description: 'OpenAI API Key'
     required: true
+  cmd:
+    description: 'test cmd'
+    required: true
 
 runs:
   using: "composite"
   steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        `${{ inputs.cmd }}` | tee /tmp/test_output.txt
+
     - name: Generate Test Output Summary
       shell: bash
-      if: github.event_name == 'pull_request'
+      env:
+        OPENAI_API_KEY: ${{ inputs.openai_api_key }}
       run: |
-        OPENAI_API_KEY=${{ inputs.openai_api_key }} python .github/actions/extract_error/extract_error.py ${{ inputs.error_filepath }} /tmp/output.txt
+        python .github/actions/extract_error/extract_error.py /tmp/test_output.txt /tmp/output.txt
 
     - name: Get Test Output Summary
       shell: bash

.github/actions/extract_error/extract_error.py

@@ -1,27 +1,78 @@
 import argparse
+import csv
 import os
-from prompttrail.agent.templates import LinearTemplate
+from io import StringIO
+
 from prompttrail.agent.runners import CommandLineRunner
-from prompttrail.models.openai import OpenAIChatCompletionModel, OpenAIModelConfiguration, OpenAIModelParameters
+from prompttrail.agent.templates import LinearTemplate
+from prompttrail.agent.templates.openai import (
+    OpenAIGenerateTemplate,
+    OpenAIMessageTemplate,
+)
 from prompttrail.agent.user_interaction import UserInteractionTextCLIProvider
+from prompttrail.models.openai import (
+    OpenAIChatCompletionModel,
+    OpenAIModelConfiguration,
+    OpenAIModelParameters,
+)
 
 
 def extract(error_message: str):
-    return f"""# Error Summary
+    template = LinearTemplate(
+        [
+            OpenAIMessageTemplate(
+                role="system",
+                content="""
+You're an AI assistant that helps software engineer to understand test error message.
+Your input is error message that emitted by test codes.
+Your output is the summary csv of the error message.
+Each line in the summary csv descributes each error in the error message.
+the summary csv first column indicates where the error occurred, second column is summary of error.
+You emit ONLY the summary csv. No explanation is needed.
+""",
+            ),
+            OpenAIMessageTemplate(
+                role="user",
+                content=error_message,
+            ),
+            OpenAIGenerateTemplate(role="assistant"),
+        ]
+    )
+    runner = CommandLineRunner(
+        model=OpenAIChatCompletionModel(
+            configuration=OpenAIModelConfiguration(
+                api_key=os.environ.get("OPENAI_API_KEY", "")
+            )
+        ),
+        parameters=OpenAIModelParameters(model_name="gpt-3.5-turbo-0301"),
+        template=template,
+        user_interaction_provider=UserInteractionTextCLIProvider(),
+    )
+    state = runner.run()
+    last_message_content = state.get_last_message().content
+    # remove response message header
+    csv_str = "\n".join(last_message_content.splitlines()[2:])
+    with StringIO() as f:
+        f.write(csv_str)
+        f.seek(0)
+        csv_content = list(csv.reader(f))
+        content = "\n".join([f"| {row[0]} | {row[1]} |" for row in csv_content])
+    return f"""# Error Summaries
 
-TODO: impl summarization of error_message by PromptTail.
-error_mesasge len: {len(error_message)}
+| where | summary |
+| ----- | ------- |
+{content}
 """
 
 
 def main() -> None:
     parser = argparse.ArgumentParser()
-    parser.add_argument('error_filepath') 
-    parser.add_argument('output_filepath') 
+    parser.add_argument("error_filepath")
+    parser.add_argument("output_filepath")
     args = parser.parse_args()
     with open(args.error_filepath) as f:
         error_message = f.read()
-    with open(args.output_filepath, 'w') as f:
+    with open(args.output_filepath, "w") as f:
         f.write(extract(error_message))
 
 

.github/workflows/test.yml

@@ -29,19 +29,12 @@ jobs:
         run: flake8 --ignore=E121,E123,E126,E226,E24,E704,E203,W503,W504,E501,F401,F403 src tests examples
       - name: Run black
         run: black --check src examples tests
-      - name: Run pytest
-        run: pytest --cov=src --cov-report=xml tests > /tmp/test_output.txt
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          GOOGLE_CLOUD_API_KEY: ${{ secrets.GOOGLE_CLOUD_API_KEY }}
 
       - name: Extract Error
         uses: ./.github/actions/extract_error
-        if: failure()
         with:
-          error_filepath: /tmp/test_output.txt
           openai_api_key: ${{ secrets.OPENAI_API_KEY }}
-
+          cmd: OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} GOOGLE_CLOUD_API_KEY=${{ secrets.GOOGLE_CLOUD_API_KEY }} pytest --cov=src --cov-report=xml tests
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
         env: