Update PRReview patchflow (#1270)

CTY-git · web-flow · commit 87706a119167 · 2025-02-05T14:30:31.000+08:00
* Update prreview patchflow

* update

* fix output model

* fix typings

* update inputs validation

* set model via default config instead

* changed to use actual anthropic key

* lint and update

* bump version
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,10 +1,6 @@
 name: Tests
 
 on:
-  pull_request:
-    types:
-      - ready_for_review
-      - review_requested
   push:
     branches-ignore:
       - main
@@ -15,15 +11,12 @@ on:
       - generateunittests-*
       - generatecodeusageexample-*
       - resolveissue-*
-
       - demo*
 
 # Credits to https://blog.maximeheckel.com/posts/building-perfect-github-action-frontend-teams/#you-are-terminated
 concurrency:
   # Here the group is defined by the head_ref of the PR
   group: ${{ github.head_ref || github.ref_name }}
-  # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push
-  # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts
   cancel-in-progress: true
 
 jobs:
@@ -211,7 +204,7 @@ jobs:
         run: |
           source .venv/bin/activate
           patchwork PRReview --log debug \
-          --patched_api_key=${{ secrets.PATCHED_API_KEY }} \
+          --anthropic_api_key=${{ secrets.ANTHROPIC_API_KEY }} \
           --github_api_key=${{ secrets.SCM_GITHUB_KEY }} \
           --pr_url=https://github.com/patched-codes/patchwork/pull/${{ steps.findPr.outputs.number }} \
           --disable_telemetry
diff --git a/patchwork/patchflows/PRReview/PRReview.py b/patchwork/patchflows/PRReview/PRReview.py
@@ -1,124 +1,144 @@
-import json
 from pathlib import Path
 
 import yaml
 
 from patchwork.common.utils.progress_bar import PatchflowProgressBar
 from patchwork.common.utils.step_typing import validate_steps_with_inputs
 from patchwork.step import Step
-from patchwork.steps import (
-    LLM,
-    CallLLM,
-    CreatePRComment,
-    ExtractModelResponse,
-    PreparePR,
-    PreparePrompt,
-    ReadPRDiffs,
-)
+from patchwork.steps import CreatePRComment, ReadPRDiffs, SimplifiedLLMOnce
 
 _DEFAULT_PROMPT_JSON = Path(__file__).parent / "pr_review_prompt.json"
 _DEFAULT_INPUT_FILE = Path(__file__).parent / "defaults.yml"
 
 
-_NONE = "none"
-_SHORT = "short"
-_LONG = "long"
-_SUMMARY_LEVEL = {
-    _NONE: 0,
-    _SHORT: 1,
-    _LONG: 2,
-}
-
-
 class PRReview(Step):
     def __init__(self, inputs: dict):
         PatchflowProgressBar(self).register_steps(
-            CallLLM,
-            CreatePRComment,
-            ExtractModelResponse,
-            PreparePR,
-            PreparePrompt,
             ReadPRDiffs,
+            SimplifiedLLMOnce,
+            CreatePRComment,
         )
         final_inputs = yaml.safe_load(_DEFAULT_INPUT_FILE.read_text())
         final_inputs.update(inputs)
 
-        if "prompt_template_file" not in final_inputs.keys():
-            final_inputs["prompt_template_file"] = _DEFAULT_PROMPT_JSON
-
-        diff_summary = final_inputs.get("diff_summary", _LONG)
-        if diff_summary.lower() not in _SUMMARY_LEVEL.keys():
-            raise ValueError(f"Invalid diff_summary, accepted diff_summary values: {_SUMMARY_LEVEL.keys()}")
-        self.verbosity = _SUMMARY_LEVEL[diff_summary.lower()]
-
-        self.is_suggestion_required = bool(final_inputs.get("diff_suggestion"))
-
         validate_steps_with_inputs(
             set(final_inputs.keys()).union(
                 {
-                    "prompt_id",
-                    "prompt_values",
-                    "modified_code_files",
+                    "user_prompt",
+                    "prompt_value",
+                    "json_schema",
                     "pr_comment",
                 }
             ),
             ReadPRDiffs,
-            LLM,
-            PreparePR,
+            SimplifiedLLMOnce,
             CreatePRComment,
         )
 
         self.inputs = final_inputs
 
     def run(self) -> dict:
-        if self.verbosity == _SUMMARY_LEVEL[_NONE]:
-            return dict()
-
-        outputs = ReadPRDiffs(self.inputs).run()
-        self.inputs["prompt_values"] = outputs["diffs"]
-
-        outputs = LLM(
-            dict(
-                prompt_id="diffreview-suggestion" if self.is_suggestion_required else "diffreview",
-                model_response_format=dict(type="json_object"),
-                **self.inputs,
-            )
-        ).run()
-        self.inputs.update(outputs)
-
-        summaries = []
-        for raw_response, prompt_values in zip(self.inputs["openai_responses"], self.inputs["prompt_values"]):
-            response = json.loads(raw_response)
-            summary = {}
-            if "path" in prompt_values.keys():
-                summary["path"] = prompt_values["path"]
-            if "review" in response.keys():
-                summary["commit_message"] = response["review"]
-            if "suggestion" in response.keys():
-                summary["patch_message"] = response["suggestion"]
-            summaries.append(summary)
-
-        header = ""
-        if self.verbosity > _SUMMARY_LEVEL[_SHORT]:
-            filtered_summaries = [
-                str(summary["commit_message"]) for summary in summaries if summary.get("commit_message")
-            ]
-            self.inputs["prompt_id"] = "diffreview_summary"
-            self.inputs["prompt_values"] = [{"diffreviews": "\n".join(filtered_summaries)}]
-
-            outputs = PreparePrompt(self.inputs).run()
-            self.inputs.update(outputs)
-            outputs = CallLLM(self.inputs).run()
-            self.inputs.update(outputs)
-            header = self.inputs["openai_responses"][0]
-
-        self.inputs["pr_header"] = header
-        self.inputs["modified_code_files"] = summaries
-        outputs = PreparePR(self.inputs).run()
-        self.inputs.update(outputs)
-
-        self.inputs["pr_comment"] = self.inputs["pr_body"]
-        outputs = CreatePRComment(self.inputs).run()
-        self.inputs.update(outputs)
-
-        return self.inputs
+        pr_diffs_outputs = ReadPRDiffs(self.inputs).run()
+
+        reviews = []
+        for diffs in iter(pr_diffs_outputs["diffs"]):
+            llm1_outputs = SimplifiedLLMOnce(
+                dict(
+                    prompt_value=diffs,
+                    user_prompt="""\
+Analyze the following code diff against the provided rules:
+
+<CODE_DIFF>
+{{diff}}
+</CODE_DIFF>
+
+<RULES>
+- Do not ignore potential bugs in the code.
+- Do not overlook possible security vulnerabilities introduced by code modifications.
+- Do not deviate from the original coding standards established in the pull request.
+</RULES>
+
+For each rule, determine if there\'s a violation. Use the following chain of thought process:
+
+1. Understand the rule
+2. Examine the diff line by line
+3. Identify any potential violations
+4. Determine the specific line numbers of violations
+5. Summarize your findings
+
+Rule 1:
+1. Rule understanding: [Briefly explain the rule]
+2. Diff examination: [Describe how you\'re examining the diff]
+3. Potential violations: [List any potential violations you\'ve identified]
+4. Line numbers: [If violations exist, list the specific line numbers]
+5. Summary: [Summarize your findings for this rule]
+
+Rule 2:
+[Repeat the above structure for each rule]
+
+Now, carefully review your reasoning in the section above. Ensure that your conclusions are consistent with the analysis you\'ve done for each rule.
+
+Your review should have the following markdown format:
+
+<REVIEW_FORMAT>
+## File Changed: `{{path}}`
+
+Details: [If rule violation include brief prescriptive explanation]
+
+Affected Code Snippet: 
+[Original code enclosed in a code block from the file that is affected by this violation. If no violation, write "N/A"]
+
+Start Line: [Starting Line number of the affected code. If no violation, write "N/A"]
+
+End Line: [Ending Line number of the affected code. If no violation, write "N/A"]
+
+-------------
+
+Details: [If rule violation include brief prescriptive explanation]
+
+Affected Code Snippet: 
+[Original code enclosed in a code block from the file that is affected by this violation. If no violation, write "N/A"]
+
+Start Line: [Starting Line number of the affected code. If no violation, write "N/A"]
+
+End Line: [Ending Line number of the affected code. If no violation, write "N/A"]
+
+-------------
+
+... (continue for all rules)
+</REVIEW_FORMAT>
+
+Ensure that you include all rules in your response, even if there\'s no violation. The output should directly reflect the reasoning in your thinking section.
+""",
+                    json_schema={"review": "The markdown text of the reviews"},
+                    **self.inputs,
+                )
+            ).run()
+
+            llm2_outputs = SimplifiedLLMOnce(
+                dict(
+                    prompt_value=llm1_outputs,
+                    user_prompt="""\
+You are a software manager compiling code reviews from all teams. You are given a list of code reviews. You have to remove code reviews that is either not actionable or useful. Do not change the accepted reviews, return the original review for the response. Do not remove the path from the review.
+
+<code_reviews>
+{{review}}
+<code_reviews>
+
+You should return an empty response if there are no code reviews that is actionable or useful.
+""",
+                    json_schema={"review": "The reviews curated"},
+                    **self.inputs,
+                )
+            ).run()
+
+            review = llm2_outputs.get("review")
+            if review is not None and len(str(review)) > 0:
+                reviews.append(review)
+
+        if len(reviews) > 0:
+            reviews_str = "\n".join(reviews)
+        else:
+            reviews_str = "No issues found."
+
+        return CreatePRComment(dict(pr_comment=reviews_str, **self.inputs)).run()
diff --git a/patchwork/patchflows/PRReview/defaults.yml b/patchwork/patchflows/PRReview/defaults.yml
@@ -1,7 +1,4 @@
 # PRReview Inputs
-diff_summary: long
-diff_suggestion: false
-
 
 # ReadPRDiffs Inputs
 # github_api_key: required-for-github-scm
@@ -14,7 +11,7 @@ diff_suggestion: false
 # CallLLM Inputs
 # openai_api_key: required-for-chatgpt
 # google_api_key: required-for-gemini
-# model: gpt-4o
+ model: claude-3-5-sonnet-latest
 # client_base_url: https://api.openai.com/v1
 # Example HF model
 # client_base_url: https://api-inference.huggingface.co/models/codellama/CodeLlama-70b-Instruct-hf/v1
diff --git a/patchwork/step.py b/patchwork/step.py
@@ -73,8 +73,6 @@ def __init__(self, inputs: DataPoint):
         self.run = self.__managed_run
 
     def __init_subclass__(cls, input_class: Optional[Type] = None, output_class: Optional[Type] = None, **kwargs):
-        if cls.__name__ == "PreparePR":
-            print(1)
         input_class = input_class or getattr(cls, "input_class", None)
         if input_class is not None and not is_typeddict(input_class):
             input_class = None
diff --git a/patchwork/steps/CallShell/CallShell.py b/patchwork/steps/CallShell/CallShell.py
@@ -8,7 +8,6 @@
 from patchwork.common.utils.utils import mustache_render
 from patchwork.logger import logger
 from patchwork.step import Step, StepStatus
-from patchwork.steps import CallSQL
 from patchwork.steps.CallShell.typed import CallShellInputs, CallShellOutputs
 
 
diff --git a/patchwork/steps/SimplifiedLLMOnce/SimplifiedLLMOnce.py b/patchwork/steps/SimplifiedLLMOnce/SimplifiedLLMOnce.py
@@ -1,9 +1,12 @@
 from patchwork.step import Step
 from patchwork.steps.SimplifiedLLM.SimplifiedLLM import SimplifiedLLM
-from patchwork.steps.SimplifiedLLMOnce.typed import SimplifiedLLMOnceInputs
+from patchwork.steps.SimplifiedLLMOnce.typed import (
+    SimplifiedLLMOnceInputs,
+    SimplifiedLLMOnceOutputs,
+)
 
 
-class SimplifiedLLMOnce(Step, input_class=SimplifiedLLMOnceInputs):
+class SimplifiedLLMOnce(Step, input_class=SimplifiedLLMOnceInputs, output_class=SimplifiedLLMOnceOutputs):
     def __init__(self, inputs):
         super().__init__(inputs)
 
diff --git a/patchwork/steps/SimplifiedLLMOnce/typed.py b/patchwork/steps/SimplifiedLLMOnce/typed.py
@@ -38,3 +38,8 @@ class SimplifiedLLMOnceInputs(__SimplifiedLLMOncePBInputsRequired, total=False):
     google_api_key: Annotated[
         str, StepTypeConfig(is_config=True, or_op=["patched_api_key", "openai_api_key", "anthropic_api_key"])
     ]
+
+
+class SimplifiedLLMOnceOutputs(TypedDict):
+    request_tokens: int
+    response_tokens: int
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml

Original file line number	Diff line number	Diff line change
`@@ -38,3 +38,8 @@ class SimplifiedLLMOnceInputs(__SimplifiedLLMOncePBInputsRequired, total=False):`
`38`	`38`	`google_api_key: Annotated[`
`39`	`39`	`str, StepTypeConfig(is_config=True, or_op=["patched_api_key", "openai_api_key", "anthropic_api_key"])`
`40`	`40`	`]`
	`41`	`+`
	`42`	`+`
	`43`	`+class SimplifiedLLMOnceOutputs(TypedDict):`
	`44`	`+ request_tokens: int`
	`45`	`+ response_tokens: int`