# Updated evaluation script with full Universal Strategies section in SYSTEM_PROMPT

import json
from openai import OpenAI
import re


# Set your OpenAI API key
# openai.api_key = "YOUR_API_KEY_HERE"

# System prompt with full Universal Strategies explanations and examples
SYSTEM_PROMPT = """
# Instruction  
You are a GUI Agent responsible for completing tasks on a web interface. When given a task, your first step is to plan by breaking the task down into steps. The result of this decomposition should be a sequence of strategy steps. Below is a set of **Universal Strategy Templates** for your reference. You must select the appropriate templates and, based on them, generate the specific strategy content for the given task.

# **Universal Strategies with Explanations and Examples**

| Universal Strategy ID | Universal Strategy           | Explanation (for GUI Agent)                                                                                                                                 | Example                                                        |
| :-------------------- | :--------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------- |
| US1                   | Navigate To Page/Section     | Move to a specific page, view, or section within the application interface. This typically involves clicking links, menu items, or buttons that lead to a new screen or area. | Navigate to the Orders Report page.                            |
| US2                   | Search/Filter/Sort Data      | Use search boxes, filters (like dropdowns or checkboxes), or sorting controls to locate, narrow down, or reorder data displayed in a list, table, or results view. | Filter issues by the label "Bug."                              |
| US3                   | Configure Parameters/Settings | Set or adjust specific options, parameters, or settings—often within a form or configuration panel—before executing an action (such as generating a report or creating an item). | Configure the date range for the report from 2023‑01‑01 to 2023‑01‑31. |
| US4                   | Execute Action/Process       | Initiate a specific process or action by interacting with a primary action element, such as clicking a "Submit," "Generate," "Calculate," "Add," "Follow," or "Save" button. | Generate and submit the sales report.                          |
| US5                   | View/Inspect Item Details    | Open or focus on a specific item (e.g., an order, product, user profile, issue) to examine its detailed information—often by clicking on the item in a list or a dedicated "View"/"Details" button. | View the details of Order #12345.                              |
| US6                   | Extract/Retrieve Information | Read and capture specific pieces of information (such as text, numbers, or status) displayed on the current page or within an item's details.             | Retrieve the purchase date from the order details section.     |
| US7                   | Analyze/Evaluate/Verify Data | Perform cognitive tasks on the displayed data, such as comparing values, checking conditions, interpreting content, identifying patterns, or making judgments based on the information. | Analyze review content to identify mentions of "poor quality." |
| US8                   | Navigate Within Data/Results | Move through multiple pages or sections of data or search results—typically using pagination controls (e.g., "Next," "Previous," page numbers) or scrolling. | Navigate through search result pages using the "Next" button. |
| US9                   | Create Item                  | Complete the process of adding a new entity (e.g., post, project, issue, user, rule) to the system—typically by filling out a form and submitting it.      | Create a new project named "My Research Project."              |
| US10                  | Update/Modify Item           | Change or edit attributes or content of an existing item (e.g., update profile information, modify an order, edit a post, change product status).         | Update the user profile homepage URL to "https://new-website.com." |
| US11                  | Delete Item                  | Remove an existing item (e.g., a review, product, or post) from the system—often involving a selection step and a confirmation.                            | Delete the selected pending review.                            |

# Input example
```json
[{
    "task_id": 4,
    "sites": ["shopping_admin"],
    "intent": "What are the top-3 best-selling product in Jan 2023"
}]
```

# Output Requirements  
Your output must be enclosed in ```json``` and include your final strategy design, alongside the referenced universal strategies and their IDs.

# Output Example  
```json
{
    "task_id": 4,
    "strategies": [
        "Navigate to Bestsellers Report",
        "Configure Report Parameters for January 2023",
        "Generate and Review Bestsellers Report"
    ],
    "universal_strategies": [
        "Navigate To Page/Section",
        "Configure Parameters/Settings",
        "Execute Action/Process"
    ],
    "universal_strategy_ids": ["US1", "US3", "US4"]
}
```
"""

client = OpenAI(
        # 若没有配置环境变量，请用百炼API Key将下行替换为：api_key="sk-xxx"
        api_key="sk-xxx",
        base_url='https://aiproxy.lmzgc.cn:8080/v1'
    )

def evaluate_accuracy(input_file: str, output_file: str):
    with open(input_file, 'r', encoding='utf-8') as f:
        cases = json.load(f)

    results = []
    correct_count = 0
    total_cases = len(cases)

    # Function to save current results and summary
    def save_results(processed_count):
        current_accuracy = correct_count / processed_count if processed_count else 0.0
        summary = {
            "total_cases": total_cases,
            "processed_cases": processed_count,
            "correct_cases": correct_count,
            "accuracy": current_accuracy
        }
        # Save detailed results and summary
        with open(output_file, 'w', encoding='utf-8') as f_out:
            json.dump({"summary": summary, "results": results}, f_out, indent=2, ensure_ascii=False)
        print(f"Intermediate results saved ({processed_count}/{total_cases}): Accuracy = {current_accuracy:.2%}")


    for i, case in enumerate(cases):
        # Prepare the user message payload
        user_message = json.dumps([{
            "task_id": case["task_id"],
            "sites": case["sites"],
            "intent": case["intent"]
        }], ensure_ascii=False)

        print(f"Processing case {i+1}/{total_cases}, task_id: {case.get('task_id', 'N/A')}") # More informative print

        # Call the OpenAI GPT-4o model
        model_id="aiproxy/deepseek-reasoner"

        completion = client.chat.completions.create(
            # model="deepseek-r1-distill-qwen-7b",
            model=model_id,
            messages=[
                {"role": "system", "content": SYSTEM_PROMPT},
                {"role": "user", "content": user_message}
            ]
        )

        reasoning_content = completion.choices[0].message.reasoning_content if completion.choices[0].message.reasoning_content else None # Handle potential None
        output_content = completion.choices[0].message.content.strip()

        # print(f"output {output_content}") # Optional: reduce verbose output during run

        actual_ids = [] # Initialize actual_ids
        try:
            # Extract JSON string from potential markdown fences
            # Use regex to find content between ```json and ```, allowing for optional whitespace
            match = re.search(r'```json\s*(.*?)\s*```', output_content, re.DOTALL)
            if match:
                json_str = match.group(1).strip() # Extract the JSON part and strip whitespace
            else:
                # If no markdown fences are found, assume the entire content might be JSON
                json_str = output_content

            output_json = json.loads(json_str)
            actual_ids = output_json.get("universal_strategy_ids", [])
        except json.JSONDecodeError as e:
            # Add more informative error logging
            print(f"Warning: Failed to parse JSON for task_id {case.get('task_id', 'N/A')}. Error: {e}")
            print(f"Content attempted to parse: '{json_str if 'json_str' in locals() else output_content}'") # Log the string that failed parsing
            # actual_ids remains [] as initialized

        expected_ids = case.get("universal_strategy_ids", [])
        is_correct = set(actual_ids) == set(expected_ids)

        # print(f"expected_ids: {expected_ids}")
        # print(f"is_correct : {is_correct}") # Optional: reduce verbose output

        if is_correct:
            correct_count += 1

        results.append({
            "task_id": case["task_id"],
            "task_intent": case["intent"],
            "expected_ids": expected_ids,
            "actual_ids": actual_ids,
            "correct": is_correct,
            "model": model_id,
            "model_output": output_content,
            "model_reasoning": reasoning_content
        })

        # Save results every 10 cases and also for the last case
        processed_count = i + 1
        if processed_count % 10 == 0 or processed_count == total_cases:
             save_results(processed_count)

    # Final summary calculation (already done by the last save_results call if total_cases % 10 != 0)
    final_accuracy = correct_count / total_cases if total_cases else 0.0
    print(f"\nEvaluation completed: {correct_count}/{total_cases} correct, final accuracy = {final_accuracy:.2%}")
    print(f"Final results saved to {output_file}")

if __name__ == "__main__":
    # Make sure the input/output file names are correct
    evaluate_accuracy("tasks_output.json", "evaluation_results.json")

# Save this as evaluate_accuracy.py
