# Updated evaluation script with full Universal Strategies section in SYSTEM_PROMPT import json from openai import OpenAI import re # Set your OpenAI API key # openai.api_key = "YOUR_API_KEY_HERE" # System prompt with full Universal Strategies explanations and examples SYSTEM_PROMPT = """ # Instruction You are a GUI Agent responsible for completing tasks on a web interface. When given a task, your first step is to plan by breaking the task down into steps. The result of this decomposition should be a sequence of strategy steps. Below is a set of **Universal Strategy Templates** for your reference. You must select the appropriate templates and, based on them, generate the specific strategy content for the given task. # **Universal Strategies with Explanations and Examples** | Universal Strategy ID | Universal Strategy | Explanation (for GUI Agent) | Example | | :-------------------- | :--------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------- | | US1 | Navigate To Page/Section | Move to a specific page, view, or section within the application interface. This typically involves clicking links, menu items, or buttons that lead to a new screen or area. | Navigate to the Orders Report page. | | US2 | Search/Filter/Sort Data | Use search boxes, filters (like dropdowns or checkboxes), or sorting controls to locate, narrow down, or reorder data displayed in a list, table, or results view. | Filter issues by the label "Bug." | | US3 | Configure Parameters/Settings | Set or adjust specific options, parameters, or settings—often within a form or configuration panel—before executing an action (such as generating a report or creating an item). | Configure the date range for the report from 2023‑01‑01 to 2023‑01‑31. | | US4 | Execute Action/Process | Initiate a specific process or action by interacting with a primary action element, such as clicking a "Submit," "Generate," "Calculate," "Add," "Follow," or "Save" button. | Generate and submit the sales report. | | US5 | View/Inspect Item Details | Open or focus on a specific item (e.g., an order, product, user profile, issue) to examine its detailed information—often by clicking on the item in a list or a dedicated "View"/"Details" button. | View the details of Order #12345. | | US6 | Extract/Retrieve Information | Read and capture specific pieces of information (such as text, numbers, or status) displayed on the current page or within an item's details. | Retrieve the purchase date from the order details section. | | US7 | Analyze/Evaluate/Verify Data | Perform cognitive tasks on the displayed data, such as comparing values, checking conditions, interpreting content, identifying patterns, or making judgments based on the information. | Analyze review content to identify mentions of "poor quality." | | US8 | Navigate Within Data/Results | Move through multiple pages or sections of data or search results—typically using pagination controls (e.g., "Next," "Previous," page numbers) or scrolling. | Navigate through search result pages using the "Next" button. | | US9 | Create Item | Complete the process of adding a new entity (e.g., post, project, issue, user, rule) to the system—typically by filling out a form and submitting it. | Create a new project named "My Research Project." | | US10 | Update/Modify Item | Change or edit attributes or content of an existing item (e.g., update profile information, modify an order, edit a post, change product status). | Update the user profile homepage URL to "https://new-website.com." | | US11 | Delete Item | Remove an existing item (e.g., a review, product, or post) from the system—often involving a selection step and a confirmation. | Delete the selected pending review. | # Input example ```json [{ "task_id": 4, "sites": ["shopping_admin"], "intent": "What are the top-3 best-selling product in Jan 2023" }] ``` # Output Requirements Your output must be enclosed in ```json``` and include your final strategy design, alongside the referenced universal strategies and their IDs. # Output Example ```json { "task_id": 4, "strategies": [ "Navigate to Bestsellers Report", "Configure Report Parameters for January 2023", "Generate and Review Bestsellers Report" ], "universal_strategies": [ "Navigate To Page/Section", "Configure Parameters/Settings", "Execute Action/Process" ], "universal_strategy_ids": ["US1", "US3", "US4"] } ``` """ client = OpenAI( # 若没有配置环境变量，请用百炼API Key将下行替换为：api_key="sk-xxx" api_key="sk-xxx", base_url='https://aiproxy.lmzgc.cn:8080/v1' ) def evaluate_accuracy(input_file: str, output_file: str): with open(input_file, 'r', encoding='utf-8') as f: cases = json.load(f) results = [] correct_count = 0 total_cases = len(cases) # Function to save current results and summary def save_results(processed_count): current_accuracy = correct_count / processed_count if processed_count else 0.0 summary = { "total_cases": total_cases, "processed_cases": processed_count, "correct_cases": correct_count, "accuracy": current_accuracy } # Save detailed results and summary with open(output_file, 'w', encoding='utf-8') as f_out: json.dump({"summary": summary, "results": results}, f_out, indent=2, ensure_ascii=False) print(f"Intermediate results saved ({processed_count}/{total_cases}): Accuracy = {current_accuracy:.2%}") for i, case in enumerate(cases): # Prepare the user message payload user_message = json.dumps([{ "task_id": case["task_id"], "sites": case["sites"], "intent": case["intent"] }], ensure_ascii=False) print(f"Processing case {i+1}/{total_cases}, task_id: {case.get('task_id', 'N/A')}") # More informative print # Call the OpenAI GPT-4o model model_id="aiproxy/deepseek-reasoner" completion = client.chat.completions.create( # model="deepseek-r1-distill-qwen-7b", model=model_id, messages=[ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": user_message} ] ) reasoning_content = completion.choices[0].message.reasoning_content if completion.choices[0].message.reasoning_content else None # Handle potential None output_content = completion.choices[0].message.content.strip() # print(f"output {output_content}") # Optional: reduce verbose output during run actual_ids = [] # Initialize actual_ids try: # Extract JSON string from potential markdown fences # Use regex to find content between ```json and ```, allowing for optional whitespace match = re.search(r'```json\s*(.*?)\s*```', output_content, re.DOTALL) if match: json_str = match.group(1).strip() # Extract the JSON part and strip whitespace else: # If no markdown fences are found, assume the entire content might be JSON json_str = output_content output_json = json.loads(json_str) actual_ids = output_json.get("universal_strategy_ids", []) except json.JSONDecodeError as e: # Add more informative error logging print(f"Warning: Failed to parse JSON for task_id {case.get('task_id', 'N/A')}. Error: {e}") print(f"Content attempted to parse: '{json_str if 'json_str' in locals() else output_content}'") # Log the string that failed parsing # actual_ids remains [] as initialized expected_ids = case.get("universal_strategy_ids", []) is_correct = set(actual_ids) == set(expected_ids) # print(f"expected_ids: {expected_ids}") # print(f"is_correct : {is_correct}") # Optional: reduce verbose output if is_correct: correct_count += 1 results.append({ "task_id": case["task_id"], "task_intent": case["intent"], "expected_ids": expected_ids, "actual_ids": actual_ids, "correct": is_correct, "model": model_id, "model_output": output_content, "model_reasoning": reasoning_content }) # Save results every 10 cases and also for the last case processed_count = i + 1 if processed_count % 10 == 0 or processed_count == total_cases: save_results(processed_count) # Final summary calculation (already done by the last save_results call if total_cases % 10 != 0) final_accuracy = correct_count / total_cases if total_cases else 0.0 print(f"\nEvaluation completed: {correct_count}/{total_cases} correct, final accuracy = {final_accuracy:.2%}") print(f"Final results saved to {output_file}") if __name__ == "__main__": # Make sure the input/output file names are correct evaluate_accuracy("tasks_output.json", "evaluation_results.json") # Save this as evaluate_accuracy.py