初始化

misc/temp_analysis/exam_gen.py

@@ -0,0 +1,55 @@
+import json
+import os
+import uuid
+
+def create_exam_cases():
+    # 读取process_3.json文件
+    with open('temp_analysis/process_3.json', 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    
+    exam_cases = []
+    num = 0
+    
+    # 遍历每个URL条目
+    for url, url_data in data.items():
+        # 获取title作为question
+        question = url_data.get('title', '').strip()
+        
+        # 遍历所有最短路径的元数据
+        for path_meta in url_data.get('shortestPathsMeta', []):
+            chain_ids = path_meta.get('chainIDs', [])
+            chain_urls = path_meta.get('chainUrls', [])
+            chain_ax_tree_ids = path_meta.get('chainAxTreeID', [])
+            chain_texts = path_meta.get('chainTexts', [])
+
+            trajectory_id = str(uuid.uuid4())
+            trajectory_step_num = 0
+            
+            # 确保所有列表长度一致
+            min_length = min(len(chain_ids), len(chain_urls), len(chain_ax_tree_ids), len(chain_texts))
+            
+            # 为每个步骤创建一个测试用例
+            for i in range(min_length):
+                num +=1
+                trajectory_step_num += 1
+                case = {
+                    "num": num,
+                    "id": str(uuid.uuid4()),
+                    "trajectory_id": trajectory_id,
+                    "trajectory_step_num": trajectory_step_num,
+                    "page_id": chain_ids[i] if i < len(chain_ids) else "",
+                    "url": chain_urls[i] if i < len(chain_urls) else "",
+                    "question": question,
+                    "answer": str(chain_ax_tree_ids[i]) if i < len(chain_ax_tree_ids) else "",
+                    "answer_text": chain_texts[i] if i < len(chain_texts) else ""
+                }
+                exam_cases.append(case)
+    
+    # 将结果保存到exam.json
+    with open('temp_analysis/exam.json', 'w', encoding='utf-8') as f:
+        json.dump(exam_cases, f, ensure_ascii=False, indent=2)
+    
+    print(f"已生成 {len(exam_cases)} 个测试用例并保存到 temp_analysis/exam.json")
+
+if __name__ == "__main__":
+    create_exam_cases()

misc/temp_analysis/exam_run_axtree.py

@@ -0,0 +1,331 @@
+import json
+import os
+import re
+import logging
+import concurrent.futures
+import argparse
+from openai import OpenAI
+from dotenv import load_dotenv
+import datetime
+
+# 设置日志
+# 创建logger
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+# 创建文件处理器
+file_handler = logging.FileHandler('temp_analysis/test_run.log')
+file_handler.setLevel(logging.INFO)
+
+# 创建控制台处理器
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.INFO)
+
+# 创建格式器并添加到处理器
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+
+# 将处理器添加到logger
+log.addHandler(file_handler)
+log.addHandler(console_handler)
+
+# 加载环境变量
+load_dotenv()
+
+PARALLEL_WORKERS=4
+
+MODEL_NAME=os.getenv("MODEL_NAME")
+print(f"MODEL_NAME: {MODEL_NAME}")
+
+# 配置OpenAI客户端
+# client = OpenAI()
+client = OpenAI(
+    api_key=os.getenv("OPENAI_API_KEY"),
+    base_url=os.getenv("OPENAI_API_BASE_URL")  # 如果使用其他兼容服务，可以设置基础URL
+)
+
+# modified version of the task proposer agent prompt from https://arxiv.org/pdf/2502.11357
+
+# System prompt
+cot_system_prompt = """
+What does this webpage show? Imagine you are a real user on this webpage. Given the webpage
+screenshot or ocr result and parsed HTML/accessibility tree and the task description, please provide 
+the first action towards completing that task.
+
+Do the following step by step:
+1. Given the webpage screenshot or ocr result and parsed HTML/accessibility tree, generate the first action
+towards completing that task (in natural language form).
+2. Given the webpage screenshot or ocr result, parsed HTML/accessibility tree, and the natural language
+action, generate the grounded version of that action.
+
+ACTION SPACE: Your action space is: ['click [element ID]', 'type [element ID] [content]',
+'select [element ID] [content of option to select]', 'scroll [up]', 'scroll [down]', and 'stop'].
+Action output should follow the syntax as given below:
+click [element ID]: This action clicks on an element with a specific ID on the webpage.
+type [element ID] [content]: Use this to type the content into the field with id. By default, the
+"Enter" key is pressed after typing. Both the content and the ID should be within square braces
+as per the syntax.
+select [element ID] [content of option to select]: Select an option from a dropdown menu. The
+content of the option to select should be within square braces. When you get (select an option)
+tags from the accessibility tree, you need to select the serial number (element_id) corresponding
+to the select tag, not the option, and select the most likely content corresponding to the option as
+input.
+scroll [down]: Scroll the page down.
+scroll [up]: Scroll the page up.
+
+IMPORTANT: 
+To be successful, it is important to STRICTLY follow the below rules:
+Action generation rules:
+1. You should generate a single atomic action at each step.
+2. The action should be an atomic action from the given vocabulary - click, type, select, scroll
+(up or down), or stop.
+3. The arguments to each action should be within square braces. For example, "click [127]",
+"type [43] [content to type]", "scroll [up]", "scroll [down]".
+4. The natural language form of action (corresponding to the field "action_in_natural_language")
+should be consistent with the grounded version of the action (corresponding to the field "grounded
+_action"). Do NOT add any additional information in the grounded action. For example, if a
+particular element ID is specified in the grounded action, a description of that element must be
+present in the natural language action.
+5. If the type action is selected, the natural language form of action ("action_in_natural_language") should always specify the actual text to be typed.
+6. You should issue a "stop" action if the current webpage asks to log in or for credit card
+information.
+7. To input text, there is NO need to click the textbox first, directly type content. After typing,
+the system automatically hits the 'ENTER' key.
+8. STRICTLY Avoid repeating the same action (click/type) if the webpage remains unchanged.
+You may have selected the wrong web element.
+9. Do NOT use quotation marks in the action generation.
+
+OUTPUT FORMAT: 
+Please give a short analysis of the screenshot, parsed
+HTML/accessibility tree, then put your answer within ``` ```, for example,
+"In summary, the proposed task and the corresponding action is: ```{
+"action_in_natural_language": "<ACTION_IN_NATURAL_LANGUAGE>:str",
+"grounded_action": "<ACTION>:str"}```
+"""
+
+# User prompt
+cot_user_prompt = """
+Website URL: {INIT_URL}
+Parsed HTML/Accessibility Tree: {A11Y_TREE}
+Screenshot ocr result: {SCREENSHOT}
+Task description: {TASK_DESCRIPTION}
+"""
+
+def call_api(messages):
+    """使用openai库调用API接口"""
+    try:
+        response = client.chat.completions.create(
+            messages=messages,
+            # model="gpt-4o-mini"
+            model=MODEL_NAME
+        )
+
+        return response
+    except Exception as e:
+        log.error(f"API调用出错: {e}")
+        return None
+
+def extract_action(response_text):
+    """从API响应中提取action_in_natural_language和grounded_action"""
+    # 使用正则表达式提取JSON部分
+    match = re.search(r'```\s*{\s*(.+?)\s*}\s*```', response_text, re.DOTALL)
+    if match:
+        try:
+            # 构建完整的JSON字符串并解析
+            json_str = "{" + match.group(1) + "}"
+            action_data = json.loads(json_str)
+            return action_data.get("action_in_natural_language"), action_data.get("grounded_action")
+        except json.JSONDecodeError:
+            log.error(f"无法解析JSON: {match.group(1)}")
+    return None, None
+
+def check_answer(grounded_action, answer):
+    try:
+        log.info(f"grounded_action: {grounded_action}, answer: {answer}")
+        """检查grounded_action是否匹配answer"""
+        if not grounded_action or not grounded_action.startswith("click"):
+            return False
+        
+        # 提取element ID
+        match = re.search(r'click \[(\d+)\]', grounded_action)
+        if not match:
+            return False
+        
+        element_id = match.group(1)
+        # 将answer拆分为列表（逗号分隔）
+        answer_ids = [id.strip() for id in answer.split(",")]
+        
+        # 检查element ID是否在answer列表中
+        return element_id in answer_ids
+    except Exception as e:
+        log.error(f"检查答案出错: {e}")
+        return False
+
+def process_item(item):
+    """处理单个测试项目"""
+    item_num = item["num"]
+    item_id = item["id"]
+    page_id = item["page_id"]
+    url = item["url"]
+    task_description = item["question"]
+    answer = item["answer"]
+    answer_text = item["answer_text"]
+    trajectory_id = item["trajectory_id"]
+    trajectory_step_num = item["trajectory_step_num"]
+    
+    log.info(f"处理ID: {item_id}, URL: {url}")
+    
+    # 读取a11y tree
+    try:
+        with open(f"axtrees/{page_id}.txt", "r") as f:
+            a11y_tree = f.read()
+    except FileNotFoundError:
+        log.error(f"找不到文件: axtrees/{page_id}.txt")
+        return None
+    
+    # 构建API请求
+    messages = [
+        {"role": "system", "content": cot_system_prompt},
+        {"role": "user", "content": cot_user_prompt.format(
+            INIT_URL=url,
+            A11Y_TREE=a11y_tree,
+            SCREENSHOT="",  # 这里没有提供截图OCR结果
+            TASK_DESCRIPTION=task_description
+        )}
+    ]
+
+    log.info(f"task_description: {task_description}")
+    log.info(f"answer: {answer}, answer_text: {answer_text}")
+    
+    # 调用API
+    response = call_api(messages)
+
+    if response and hasattr(response, 'choices') and len(response.choices) > 0:
+        content = response.choices[0].message.content
+        if MODEL_NAME == "qwen/qwq-32b:free":
+            reasoning_content = response.choices[0].message.reasoning
+        else:
+            reasoning_content = response.choices[0].message.reasoning_content
+
+        log.info(f"reasoning_content: {reasoning_content}")
+        log.info(f"content: {content}")
+        
+        # 提取action
+        action_nl, grounded_action = extract_action(content)
+
+        log.info(f"action_nl: {action_nl}, grounded_action: {grounded_action}")
+        
+        # 检查答案
+        is_correct = check_answer(grounded_action, answer)
+        log.info(f"is_correct: {is_correct}")
+
+        # 记录结果
+        result = {
+            "num": item_num,
+            "id": item_id,
+            "trajectory_id": trajectory_id,
+            "trajectory_step_num": trajectory_step_num,
+            "page_id": page_id,
+            "url": url,
+            "task": task_description,
+            "expected_answer": answer,
+            "thinking": reasoning_content,
+            "raw_content": content,
+            "action_nl": action_nl,
+            "grounded_action": grounded_action,
+            "is_correct": is_correct,
+            "model": MODEL_NAME,
+            "timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        }
+        
+        log.info(f"ID: {item_id}")
+        log.info(f"任务: {task_description}")
+        log.info(f"动作: {grounded_action}")
+        log.info(f"是否正确: {is_correct}")
+        log.info("-" * 50)
+        
+        return result
+    else:
+        log.error(f"API调用失败: {response}")
+        return None
+
+def main():
+    # 运行完整测试
+    # 读取exam.json
+    with open("temp_analysis/exam.json", "r") as f:
+        exam_data = json.load(f)
+
+    if os.path.exists("temp_analysis/results.json"):
+        # 加载results.json
+        with open("temp_analysis/results.json", "r") as f:
+            results_old = json.load(f)
+        # 提取is_correct为True的测试项
+        results = [r for r in results_old if r.get("is_correct", False)]
+        log.info(f"已经成功完成的测试项目数: {len(results)}")
+    else:
+        results = []
+        log.info(f"results.json文件不存在，将重新执行所有测试")
+    
+    success_ids = [r["id"] for r in results]
+    
+    # 从exam_data中删除已经成功完成的测试项
+    exam_data = [item for item in exam_data if item["id"] not in success_ids]
+
+    total_items = len(exam_data) + len(results)
+    completed = len(results)
+    success_count = len(results)
+    fail_count = 0
+    
+    log.info(f"开始测试，需要执行 {total_items} 个任务, 已经成功 {success_count} 个任务")
+    
+    # 使用线程池并发处理
+    with concurrent.futures.ThreadPoolExecutor(max_workers=PARALLEL_WORKERS) as executor:
+        # 提交所有任务
+        future_to_item = {executor.submit(process_item, item): item for item in exam_data}
+        
+        # 收集结果
+        for future in concurrent.futures.as_completed(future_to_item):
+            result = future.result()
+            
+            # 使用文件锁防止并发写入冲突
+            completed += 1
+            # 重新读取最新结果,避免覆盖其他线程的写入
+            try:
+                with open("temp_analysis/results.json", "r") as f:
+                    results = json.load(f)
+            except FileNotFoundError:
+                results = []
+            
+            if result:
+                results.append(result)
+                if result["is_correct"]:
+                    success_count += 1 
+                else:
+                    fail_count += 1
+            else:
+                fail_count += 1
+
+            # 打印当前进度
+            progress = (completed / total_items) * 100
+            log.info(f"进度: {progress:.2f}% ({completed}/{total_items}) - 成功: {success_count}, 失败: {fail_count}")
+
+            log.info(f"save results to temp_analysis/results.json")
+            # 保存结果
+            with open("temp_analysis/results.json", "w") as f:
+                json.dump(results, f, indent=2)
+    
+    
+    # 计算正确率
+    accuracy = success_count / total_items if total_items > 0 else 0
+    
+    log.info(f"测试完成! 总计: {total_items}题，正确: {success_count}题，错误: {fail_count}题，正确率: {accuracy:.2%}")
+
+if __name__ == "__main__":
+    main()
+
+
+
+
+
+

misc/temp_analysis/exam_run_vision.py

@@ -0,0 +1,418 @@
+import json
+import os
+import re
+import logging
+import concurrent.futures
+import argparse
+from openai import OpenAI
+from dotenv import load_dotenv
+import datetime
+import base64
+from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor
+# from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
+from qwen_vl_utils import process_vision_info
+
+
+os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3"
+
+# 设置日志
+# 创建logger
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+# 创建文件处理器
+file_handler = logging.FileHandler('temp_analysis/exam_run_vision.log')
+file_handler.setLevel(logging.INFO)
+
+# 创建控制台处理器
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.INFO)
+
+# 创建格式器并添加到处理器
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+
+# 将处理器添加到logger
+log.addHandler(file_handler)
+log.addHandler(console_handler)
+
+# 加载环境变量
+load_dotenv()
+
+PARALLEL_WORKERS=1
+
+MODEL_NAME=os.getenv("MODEL_NAME")
+print(f"MODEL_NAME: {MODEL_NAME}")
+
+# 配置OpenAI客户端
+# client = OpenAI()
+client = OpenAI(
+    api_key=os.getenv("OPENAI_API_KEY"),
+    base_url=os.getenv("OPENAI_API_BASE_URL")  # 如果使用其他兼容服务，可以设置基础URL
+)
+
+# modified version of the task proposer agent prompt from https://arxiv.org/pdf/2502.11357
+
+# System prompt
+cot_system_prompt = """
+What does this webpage show? Imagine you are a real user on this webpage. Given the webpage
+screenshot or ocr result and parsed HTML/accessibility tree and the task description, please provide 
+the first action towards completing that task.
+
+Do the following step by step:
+1. Given the webpage screenshot or ocr result and parsed HTML/accessibility tree, generate the first action
+towards completing that task (in natural language form).
+2. Given the webpage screenshot or ocr result, parsed HTML/accessibility tree, and the natural language
+action, generate the grounded version of that action.
+
+ACTION SPACE: Your action space is: ['click [element ID]', 'type [element ID] [content]',
+'select [element ID] [content of option to select]', 'scroll [up]', 'scroll [down]', and 'stop'].
+Action output should follow the syntax as given below:
+click [element text]: This action clicks on an element with a specific text on the webpage.
+type [element ID] [content]: Use this to type the content into the field with id. By default, the
+"Enter" key is pressed after typing. Both the content and the ID should be within square braces
+as per the syntax.
+select [element ID] [content of option to select]: Select an option from a dropdown menu. The
+content of the option to select should be within square braces. When you get (select an option)
+tags from the accessibility tree, you need to select the serial number (element_id) corresponding
+to the select tag, not the option, and select the most likely content corresponding to the option as
+input.
+scroll [down]: Scroll the page down.
+scroll [up]: Scroll the page up.
+
+IMPORTANT: 
+To be successful, it is important to STRICTLY follow the below rules:
+Action generation rules:
+1. You should generate a single atomic action at each step.
+2. The action should be an atomic action from the given vocabulary - click, type, select, scroll
+(up or down), or stop.
+3. The arguments to each action should be within square braces. For example, "click [127]",
+"type [43] [content to type]", "scroll [up]", "scroll [down]".
+4. The natural language form of action (corresponding to the field "action_in_natural_language")
+should be consistent with the grounded version of the action (corresponding to the field "grounded
+_action"). Do NOT add any additional information in the grounded action. For example, if a
+particular element ID is specified in the grounded action, a description of that element must be
+present in the natural language action.
+5. If the type action is selected, the natural language form of action ("action_in_natural_language") should always specify the actual text to be typed.
+6. You should issue a "stop" action if the current webpage asks to log in or for credit card
+information.
+7. To input text, there is NO need to click the textbox first, directly type content. After typing,
+the system automatically hits the 'ENTER' key.
+8. STRICTLY Avoid repeating the same action (click/type) if the webpage remains unchanged.
+You may have selected the wrong web element.
+9. Do NOT use quotation marks in the action generation.
+
+OUTPUT FORMAT: 
+Please give a short analysis of the screenshot, parsed
+HTML/accessibility tree, then put your answer within ``` ```, for example,
+"In summary, the proposed task and the corresponding action is: ```{
+"action_in_natural_language": "<ACTION_IN_NATURAL_LANGUAGE>:str",
+"grounded_action": "<ACTION>:str"}```
+"""
+
+# User prompt
+cot_user_prompt = """
+[
+    {{
+        "type": "text",
+        "text": "Website URL: {INIT_URL}\nParsed HTML/Accessibility Tree: {A11Y_TREE}\nTask description: {TASK_DESCRIPTION}"
+    }},
+    {{
+        "type": "image_url",
+        "image_url": {{
+            "url": f"file://{SCREENSHOT}"
+        }}
+    }}
+]
+"""
+
+def load_qwen_model():
+    """
+    加载Qwen2VL模型和处理器
+    """
+    global qwen_model, processor
+    
+    # 设置本地模型路径
+    model_path = "/data1/yuyr/Qwen2.5-VL-7B-Instruct"
+    # model_path = "/home/yuyr/.cache/huggingface/hub/models--Qwen--Qwen2.5-VL-3B-Instruct/snapshots/1b989f2c63999d7344135894d3cfa8f494116743/"
+    # model_path = "/data1/yuyr/models--bytedance-research--UI-TARS-7B-DPO/snapshots/727b0df39207dafc6cf211a61f29d84b7659c39c/"
+    
+    try:
+        log.info("正在加载Qwen2VL模型和处理器...")
+        
+        # 加载模型
+        qwen_model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+        # qwen_model = Qwen2VLForConditionalGeneration.from_pretrained(
+            model_path, torch_dtype="auto", device_map="cuda:1"
+            # model_path, torch_dtype="auto", device_map="auto"
+        )
+        log.info("Qwen2VL模型加载完成")
+        
+        # 加载processor
+        processor = AutoProcessor.from_pretrained(model_path)
+        log.info("处理器加载完成")
+        
+        log.info(f"Qwen2VL模型和处理器加载完成")
+        
+        return True
+    except Exception as e:
+        log.error(f"Qwen2VL模型加载失败: {e}")
+        return False
+
+
+def call_api(messages):
+    """使用openai库调用API接口"""
+    try:
+        log.info(f"call llm messages ")
+        response = client.chat.completions.create(
+            messages=messages,
+            model=MODEL_NAME
+        )
+        log.info(f"call llm response: {response}")
+        return response
+    except Exception as e:
+        log.error(f"API调用出错: {e}")
+        return None
+
+def call_qwen_model(messages):
+    """
+    调用Qwen2VL模型
+    """
+    # 准备推理输入
+    try:
+        text = processor.apply_chat_template(
+            messages, tokenize=False, add_generation_prompt=True
+        )
+        image_inputs, video_inputs = process_vision_info(messages)
+        inputs = processor(
+            text=[text],
+            images=image_inputs,
+            videos=video_inputs,
+            padding=True,
+            return_tensors="pt",
+        )
+        inputs = inputs.to(qwen_model.device)
+        
+        # 推理：生成输出
+        log.info("正在生成模型回复...")
+        generated_ids = qwen_model.generate(**inputs, max_new_tokens=1024)
+        generated_ids_trimmed = [
+            out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+        ]
+        output_text = processor.batch_decode(
+            generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )[0]
+        
+        log.info(f"模型输出: {output_text}")
+        return output_text
+    except Exception as e:
+        log.error(f"模型推理失败: {e}")
+        return None
+
+
+def extract_action(response_text):
+    """从API响应中提取action_in_natural_language和grounded_action"""
+    # 使用正则表达式提取JSON部分
+    match = re.search(r'```\s*{\s*(.+?)\s*}\s*```', response_text, re.DOTALL)
+    if match:
+        try:
+            # 构建完整的JSON字符串并解析
+            json_str = "{" + match.group(1) + "}"
+            action_data = json.loads(json_str)
+            return action_data.get("action_in_natural_language"), action_data.get("grounded_action")
+        except json.JSONDecodeError:
+            log.error(f"无法解析JSON: {match.group(1)}")
+    return None, None
+
+def check_answer(grounded_action, answer):
+    try:
+        log.info(f"grounded_action: {grounded_action}, answer: {answer}")
+        """检查grounded_action是否匹配answer"""
+        if not grounded_action or not grounded_action.startswith("click"):
+            return False
+        
+        # 提取element ID
+        match = re.search(r'click \[(\d+)\]', grounded_action)
+        if not match:
+            return False
+        
+        element_id = match.group(1)
+        # 将answer拆分为列表（逗号分隔）
+        answer_ids = [id.strip() for id in answer.split(",")]
+        
+        # 检查element ID是否在answer列表中
+        return element_id in answer_ids
+    except Exception as e:
+        log.error(f"检查答案出错: {e}")
+        return False
+
+def process_item(item):
+    """处理单个测试项目"""
+    item_num = item["num"]
+    item_id = item["id"]
+    page_id = item["page_id"]
+    url = item["url"]
+    task_description = item["task"]
+    answer = item["expected_answer"]
+    answer_text = item["answer_text"]
+    trajectory_id = item["trajectory_id"]
+    trajectory_step_num = item["trajectory_step_num"]
+    page_child_num = item["page_child_num"]
+    
+    log.info(f"处理ID: {item_id}, URL: {url}")
+    
+    image_path = f"screenshots/{page_id}_{page_child_num}.png"
+    if not os.path.exists(image_path):
+        log.error(f"找不到文件: {image_path}")
+        return None
+
+    def encode_image_to_base64(image_path):
+        """
+        将图片编码为base64字符串
+        """
+        try:
+            with open(image_path, "rb") as image_file:
+                return base64.b64encode(image_file.read()).decode('utf-8')
+        except Exception as e:
+            log.error(f"图片编码失败: {e}")
+            return None
+
+    SCREENSHOT = image_path
+
+    # 构建API请求
+    messages = [
+        {"role": "system", "content": cot_system_prompt},
+        {"role": "user", "content": cot_user_prompt.format(
+            INIT_URL=url,
+            A11Y_TREE="",
+            SCREENSHOT=SCREENSHOT,  # 这里没有提供截图OCR结果
+            TASK_DESCRIPTION=task_description
+        )}
+    ]
+
+    log.info(f"task_description: {task_description}")
+    log.info(f"answer: {answer}, answer_text: {answer_text}")
+    
+    # 调用API
+    # response = call_api(messages)
+    response = call_qwen_model(messages)
+
+    print(f"response: {response}")
+
+    if response:
+        content = response
+        # if MODEL_NAME == "qwen/qwq-32b:free":
+        #     reasoning_content = response.choices[0].message.reasoning
+        # else:
+        #     reasoning_content = response.choices[0].message.reasoning_content
+
+        # log.info(f"reasoning_content: {reasoning_content}")
+        log.info(f"content: {content}")
+        
+        # 提取action
+        action_nl, grounded_action = extract_action(content)
+
+        log.info(f"action_nl: {action_nl}, grounded_action: {grounded_action}")
+        
+        # 检查答案，使用文字检查
+        is_correct = check_answer(grounded_action, answer_text)
+        log.info(f"is_correct: {is_correct}")
+
+        # 记录结果
+        result = {
+            "num": item_num,
+            "id": item_id,
+            "trajectory_id": trajectory_id,
+            "trajectory_step_num": trajectory_step_num,
+            "page_id": page_id,
+            "url": url,
+            "task": task_description,
+            "expected_answer": answer,
+            "thinking": reasoning_content,
+            "raw_content": content,
+            "action_nl": action_nl,
+            "grounded_action": grounded_action,
+            "is_correct": is_correct,
+            "model": MODEL_NAME,
+            "timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        }
+        
+        log.info(f"ID: {item_id}")
+        log.info(f"任务: {task_description}")
+        log.info(f"动作: {grounded_action}")
+        log.info(f"是否正确: {is_correct}")
+        log.info("-" * 50)
+        
+        return result
+    else:
+        log.error(f"API调用失败: {response}")
+        return None
+
+def main():
+    load_qwen_model()
+
+    # 运行完整测试
+    # 读取exam.json
+    with open("temp_analysis/test.json", "r") as f:
+        exam_data = json.load(f)
+
+    results = []
+
+    total_items = len(exam_data) + len(results)
+    completed = len(results)
+    success_count = len(results)
+    fail_count = 0
+    
+    log.info(f"开始测试，需要执行 {total_items} 个任务, 已经成功 {success_count} 个任务")
+    
+    # 使用线程池并发处理
+    with concurrent.futures.ThreadPoolExecutor(max_workers=PARALLEL_WORKERS) as executor:
+        # 提交所有任务
+        future_to_item = {executor.submit(process_item, item): item for item in exam_data}
+        
+        # 收集结果
+        for future in concurrent.futures.as_completed(future_to_item):
+            result = future.result()
+            
+            # 使用文件锁防止并发写入冲突
+            completed += 1
+            # 重新读取最新结果,避免覆盖其他线程的写入
+            try:
+                with open("temp_analysis/results_vl.json", "r") as f:
+                    results = json.load(f)
+            except FileNotFoundError:
+                results = []
+            
+            if result:
+                results.append(result)
+                if result["is_correct"]:
+                    success_count += 1 
+                else:
+                    fail_count += 1
+            else:
+                fail_count += 1
+
+            # 打印当前进度
+            progress = (completed / total_items) * 100
+            log.info(f"进度: {progress:.2f}% ({completed}/{total_items}) - 成功: {success_count}, 失败: {fail_count}")
+
+            log.info(f"save results to temp_analysis/results_vl.json")
+            # 保存结果
+            with open("temp_analysis/results_vl.json", "w") as f:
+                json.dump(results, f, indent=2)
+    
+    
+    # 计算正确率
+    accuracy = success_count / total_items if total_items > 0 else 0
+    
+    log.info(f"测试完成! 总计: {total_items}题，正确: {success_count}题，错误: {fail_count}题，正确率: {accuracy:.2%}")
+
+if __name__ == "__main__":
+    main()
+
+
+
+
+
+

misc/temp_analysis/merge_process_3.py

@@ -0,0 +1,55 @@
+import json
+import os
+
+def merge_json_files():
+    # 定义输入和输出文件路径
+    process_3_path = "path/processed_3.json"
+    processed_with_analysis_path = "backup/v17_2/path/processed_3_with_analysis_20250324163759.json"
+    output_path = "temp_analysis/process_3.json"
+    
+    # 读取第一个JSON文件
+    with open(process_3_path, 'r', encoding='utf-8') as f:
+        process_3_data = json.load(f)
+    
+    # 读取第二个JSON文件
+    with open(processed_with_analysis_path, 'r', encoding='utf-8') as f:
+        processed_with_analysis_data = json.load(f)
+    
+    # 获取两个文件的键交集
+    common_keys = set(process_3_data.keys()) & set(processed_with_analysis_data.keys())
+    
+    # 创建新的JSON数据
+    merged_data = {}
+    for key in common_keys:
+
+        skip = False
+
+        # 如果shortestPathsMeta为空，则从processed_3_with_analysis文件中获取
+        axTreeIDList = process_3_data[key]['shortestPathsMeta'][0]['chainAxTreeID']
+        for axTreeID in axTreeIDList:
+            if axTreeID is None:
+                skip = True
+                break
+
+        if skip:
+            continue
+        
+        # 以process_3.json为基础
+        merged_data[key] = process_3_data[key]
+
+        # 从processed_3_with_analysis文件中获取raw_result和title并添加
+        if 'raw_result' in processed_with_analysis_data[key]['shortestPathsMeta'][0]:
+            merged_data[key]['raw_result'] = processed_with_analysis_data[key]['shortestPathsMeta'][0]['raw_result']
+        
+        if 'title' in processed_with_analysis_data[key]['shortestPathsMeta'][0]:
+            merged_data[key]['title'] = processed_with_analysis_data[key]['shortestPathsMeta'][0]['title'].strip()
+    
+    # 将合并后的数据写入输出文件
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(merged_data, f, ensure_ascii=False, indent=2)
+    
+    print(f"合并完成！共处理了 {len(common_keys)} 个键。")
+    print(f"结果已保存到 {output_path}")
+
+if __name__ == "__main__":
+    merge_json_files()

misc/temp_analysis/mix.txt

@@ -0,0 +1,47 @@
+"Check for alerts indicating service performance exceeds 100 on Grafana"
+"Check the active notifications for any alerts related to the performance of your Kubernetes deployment and view the corresponding alert rules to ensure you can address any issues promptly on Grafana."
+"Check the active notifications for any alerts related to the performance of your Kubernetes deployment and view the corresponding alert rules to ensure you can address any issues promptly on Grafana."
+"Check the alert history and analyze the performance metrics of the Random Multiple Series dashboard on the Grafana website."
+"Check the alert history for discrepancies in website metrics and view alerts indicating if there are three times more page views than users on Grafana's monitoring dashboard on task website."
+"Check the alert history for the Random Single Serie alert to analyze its performance over time on Grafana"
+"Check the available flowcharting visualization examples, specifically focusing on the gradient color mode, to determine which one best fits your project needs on Grafana Play"
+"Check the current alerts for the do-nyc1-demo-infra Kubernetes cluster on the Grafana website"
+"Check the current alerts for the loadgen workload in the do-nyc1-demo-infra namespace to ensure there are no critical issues on the Grafana monitoring dashboard."
+"Check the historical alert notifications for a specific service and analyze the performance over time on the Grafana dashboard"
+"Check the historical performance metrics of a service using multiple data series on Grafana Play"
+"Check the history of alerts and review the details of the WW Alert on Grafana"
+"Check the history of alerts and review the details of the WW Alert on Grafana"
+"Check the history of alerts and test the alert rule for sun conditions on Grafana"
+"Check the history of alerts and test the alert rule for sun conditions on Grafana"
+"Check the performance and status of synthetic monitoring checks for uptime and latency on Grafana"
+"Check the performance and status of synthetic monitoring checks for uptime and latency on Grafana"
+"Check the performance metrics of application services on Grafana"
+"Check the performance metrics of services to ensure they have 100% uptime and latency under 500ms on Grafana"
+"Check the performance metrics of synthetic monitoring checks for a specific website and view the associated dashboard on Grafana Play"
+"Check the performance metrics of synthetic monitoring checks for a specific website and view the associated dashboard on Grafana Play"
+"Check the performance metrics of the AMQP service in the application monitoring dashboard on Grafana"
+"Check the performance metrics of the AMQP service in the application monitoring dashboard on Grafana"
+"Check the performance metrics of the faro-shop-worker service to ensure it is running optimally on the Grafana observability platform"
+"Check the performance metrics of the Grafana Home Page and ensure it meets the required uptime and response time standards for your website monitoring needs on Grafana Synthetic Monitoring."
+"Check the performance metrics of the Grafana Home Page and ensure its uptime and response time are within acceptable limits on Grafana's synthetic monitoring dashboard"
+"Check the performance metrics of the Grafana Home Page and Grafana Ping Check to ensure uptime is 100% and latency is under 500ms on Grafana's synthetic monitoring platform"
+"Check the performance metrics of the Grafana Home Page service on Grafana Play"
+"Check the performance metrics of the Grafana Home Page service to ensure its uptime and latency are within acceptable limits on Grafana Play"
+"Check the performance metrics of the Grafana website, including uptime and latency, to ensure it meets your requirements for a reliable monitoring service on the Grafana Synthetic Monitoring application."
+"Check the performance metrics of the website monitoring checks to ensure uptime and low latency on Grafana Synthetic Monitoring"
+"Check the performance of services in the application to identify any issues on the Grafana observability app"
+"Check the performance of the faro-shop-frontend service on the Grafana application"
+"Explore and view the flowcharting options demo to understand how to create flowcharts for data visualization on Grafana Play"
+"Explore Grafana's data visualization tools and features for monitoring cloud services on grafana.com"
+"Find and explore flowchart animation examples for dashboard creation on Grafana Play"
+"In summary, the answer is: Check the performance metrics of the faro-shop-backend service to ensure it is running smoothly on the Grafana monitoring dashboard"
+"Reset my password to access my Grafana account and manage my data visualizations on the Grafana website"
+"Reset my password to access my Grafana account and manage my data visualizations on the Grafana website"
+"Set up alert rules based on example dashboards on Grafana Play"
+"View a detailed example of a flowcharting rack diagram to understand its features and functionality on Grafana Play"
+"View a flowcharting example of technical architecture to understand its visualization in Grafana on Grafana Play"
+"View a flowcharting floorplan example for business metrics on Grafana Play"
+"View and compare different flowcharting network diagram examples to understand their features and functionalities on Grafana Play"
+"View and compare the performance metrics of different synthetic monitoring checks for the Grafana service, focusing on uptime and latency, to ensure optimal service availability on play.grafana.org"
+"View examples of hierarchical state level flowcharting capabilities on Grafana"
+"View performance metrics with sparklines to identify trends and make informed decisions on Grafana"

misc/temp_analysis/r1.txt

@@ -0,0 +1,47 @@
+"Check for alerts indicating service performance exceeds 100 on Grafana"
+"Check the active notifications for any alerts related to the performance of your Kubernetes deployment and view the corresponding alert rules to ensure you can address any issues promptly on Grafana."
+"Check the active notifications for any alerts related to the performance of your Kubernetes deployment and view the corresponding alert rules to ensure you can address any issues promptly on Grafana."
+"Check the alert history and analyze the performance metrics of the Random Multiple Series dashboard on the Grafana website."
+"Check the alert history and create a new alert rule to monitor specific events on Grafana"
+"Check the alert history for discrepancies in website metrics and view alerts indicating if there are three times more page views than users on Grafana's monitoring dashboard on task website."
+"Check the alert history for the Random Single Serie alert to analyze its performance over time on Grafana"
+"Check the available flowcharting visualization examples, specifically focusing on the gradient color mode, to determine which one best fits your project needs on Grafana Play"
+"Check the current alerts for the do-nyc1-demo-infra Kubernetes cluster on the Grafana website"
+"Check the current alerts for the loadgen workload in the do-nyc1-demo-infra namespace to ensure there are no critical issues on the Grafana monitoring dashboard."
+"Check the historical alert notifications for a specific service and analyze the performance over time on the Grafana dashboard"
+"Check the historical performance metrics of a service using multiple data series on Grafana Play"
+"Check the history of alerts and review the details of the WW Alert on Grafana"
+"Check the history of alerts and review the details of the WW Alert on Grafana"
+"Check the history of alerts and test the alert rule for sun conditions on Grafana"
+"Check the history of alerts and test the alert rule for sun conditions on Grafana"
+"Check the performance and status of synthetic monitoring checks for uptime and latency on Grafana"
+"Check the performance and status of synthetic monitoring checks for uptime and latency on Grafana"
+"Check the performance metrics of application services on Grafana"
+"Check the performance metrics of synthetic monitoring checks for a specific website and view the associated dashboard on Grafana Play"
+"Check the performance metrics of synthetic monitoring checks for a specific website and view the associated dashboard on Grafana Play"
+"Check the performance metrics of the AMQP service in the application monitoring dashboard on Grafana"
+"Check the performance metrics of the AMQP service in the application monitoring dashboard on Grafana"
+"Check the performance metrics of the faro-shop-worker service to ensure it is running optimally on the Grafana observability platform"
+"Check the performance metrics of the Grafana Home Page and ensure it meets the required uptime and response time standards for your website monitoring needs on Grafana Synthetic Monitoring."
+"Check the performance metrics of the Grafana Home Page and ensure its uptime and response time are within acceptable limits on Grafana's synthetic monitoring dashboard"
+"Check the performance metrics of the Grafana Home Page service on Grafana Play"
+"Check the performance metrics of the Grafana Home Page service to ensure its uptime and latency are within acceptable limits on Grafana Play"
+"Check the performance metrics of the Grafana website, including uptime and latency, to ensure it meets your requirements for a reliable monitoring service on the Grafana Synthetic Monitoring application."
+"Check the performance metrics of the website monitoring checks to ensure uptime and low latency on Grafana Synthetic Monitoring"
+"Check the performance of services in the application to identify any issues on the Grafana observability app"
+"Check the performance of the faro-shop-frontend service on the Grafana application"
+"Explore and view the flowcharting options demo to understand how to create flowcharts for data visualization on Grafana Play"
+"Explore Grafana's data visualization tools and features for monitoring cloud services on grafana.com"
+"Find and explore flowchart animation examples for dashboard creation on Grafana Play"
+"In summary, the answer is: Check the performance metrics of the faro-shop-backend service to ensure it is running smoothly on the Grafana monitoring dashboard"
+"Reset my password to access my Grafana account and manage my data visualizations on the Grafana website"
+"Reset my password to access my Grafana account and manage my data visualizations on the Grafana website"
+"Set up alert rules based on example dashboards on Grafana Play"
+"View a detailed example of a flowcharting rack diagram to understand its features and functionality on Grafana Play"
+"View a flowcharting example of technical architecture to understand its visualization in Grafana on Grafana Play"
+"View a flowcharting floorplan example for business metrics on Grafana Play"
+"View and compare different flowcharting network diagram examples to understand their features and functionalities on Grafana Play"
+"View and compare the performance metrics of different synthetic monitoring checks for the Grafana service, focusing on uptime and latency, to ensure optimal service availability on play.grafana.org"
+"View examples of hierarchical state level flowcharting capabilities on Grafana"
+"View performance metrics with sparklines to identify trends and make informed decisions on Grafana"
+"View performance metrics with sparklines to identify trends and make informed decisions on Grafana"

misc/temp_analysis/result_clean.py

@@ -0,0 +1,60 @@
+import json
+import random
+from pathlib import Path
+
+# 读取 results.json 文件
+with open('temp_analysis/results.json', 'r', encoding='utf-8') as f:
+    results = json.load(f)
+
+# 筛选 is_correct 为 true 的条目
+correct_results = [item for item in results if item.get('is_correct') is True]
+print(f"找到 {len(correct_results)} 个正确的条目")
+
+# 读取 exam.json 文件
+with open('temp_analysis/exam.json', 'r', encoding='utf-8') as f:
+    exam_data = json.load(f)
+
+# 创建 id 到 exam 条目的映射
+exam_map = {item.get('id'): item for item in exam_data}
+
+# 为每个正确的结果添加 answer_text 字段
+for result in correct_results:
+    result_id = result.get('id')
+    if result_id in exam_map:
+        result['answer_text'] = exam_map[result_id].get('answer_text', '')
+    else:
+        print(f"警告: ID {result_id} 在 exam.json 中未找到")
+
+
+with open('temp_analysis/process_3.json', 'r', encoding='utf-8') as f:
+    trajectory_data = json.load(f)
+
+trajectory_map = {item.get('title'): item for key, item in trajectory_data.items()}
+for result in correct_results:
+    task = result.get('task')
+    if task in trajectory_map:
+        step = result.get('trajectory_step_num') - 1
+        result['page_child_num'] = trajectory_map[task]['shortestPathsMeta'][0]['chainChildNum'][step]
+    else:
+        print(f"警告: {task} 在 trajectory_map 中未找到")
+
+# 随机打乱数据
+random.seed(42)  # 设置随机种子以确保结果可重现
+random.shuffle(correct_results)
+
+# 按照 7:3 比例划分训练集和测试集
+split_index = int(len(correct_results) * 0.7)
+train_data = correct_results[:split_index]
+test_data = correct_results[split_index:]
+
+print(f"训练集大小: {len(train_data)}")
+print(f"测试集大小: {len(test_data)}")
+
+# 保存训练集和测试集
+with open('temp_analysis/train.json', 'w', encoding='utf-8') as f:
+    json.dump(train_data, f, ensure_ascii=False, indent=2)
+
+with open('temp_analysis/test.json', 'w', encoding='utf-8') as f:
+    json.dump(test_data, f, ensure_ascii=False, indent=2)
+
+print("处理完成! 数据已保存到 temp_analysis/train.json 和 temp_analysis/test.json")

misc/temp_analysis/results_vl.json

@@ -0,0 +1 @@
+[]

misc/temp_analysis/test_run copy 2.py

@@ -0,0 +1,243 @@
+import json
+import os
+import re
+import logging
+from openai import OpenAI
+from dotenv import load_dotenv
+
+# 设置日志
+# 创建logger
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+# 创建文件处理器
+file_handler = logging.FileHandler('temp_analysis/test_run.log')
+file_handler.setLevel(logging.INFO)
+
+# 创建控制台处理器
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.INFO)
+
+# 创建格式器并添加到处理器
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+
+# 将处理器添加到logger
+log.addHandler(file_handler)
+log.addHandler(console_handler)
+
+# 加载环境变量
+load_dotenv()
+
+# 配置OpenAI客户端
+client = OpenAI(
+    api_key=os.getenv("OPENAI_API_KEY"),
+    base_url=os.getenv("OPENAI_API_BASE_URL")  # 如果使用其他兼容服务，可以设置基础URL
+)
+
+# modified version of the task proposer agent prompt from https://arxiv.org/pdf/2502.11357
+
+# System prompt
+cot_system_prompt = """
+What does this webpage show? Imagine you are a real user on this webpage. Given the webpage
+screenshot or ocr result and parsed HTML/accessibility tree and the task description, please provide 
+the first action towards completing that task.
+
+Do the following step by step:
+1. Given the webpage screenshot or ocr result and parsed HTML/accessibility tree, generate the first action
+towards completing that task (in natural language form).
+2. Given the webpage screenshot or ocr result, parsed HTML/accessibility tree, and the natural language
+action, generate the grounded version of that action.
+
+ACTION SPACE: Your action space is: ['click [element ID]', 'type [element ID] [content]',
+'select [element ID] [content of option to select]', 'scroll [up]', 'scroll [down]', and 'stop'].
+Action output should follow the syntax as given below:
+click [element ID]: This action clicks on an element with a specific ID on the webpage.
+type [element ID] [content]: Use this to type the content into the field with id. By default, the
+"Enter" key is pressed after typing. Both the content and the ID should be within square braces
+as per the syntax.
+select [element ID] [content of option to select]: Select an option from a dropdown menu. The
+content of the option to select should be within square braces. When you get (select an option)
+tags from the accessibility tree, you need to select the serial number (element_id) corresponding
+to the select tag, not the option, and select the most likely content corresponding to the option as
+input.
+scroll [down]: Scroll the page down.
+scroll [up]: Scroll the page up.
+
+IMPORTANT: 
+To be successful, it is important to STRICTLY follow the below rules:
+Action generation rules:
+1. You should generate a single atomic action at each step.
+2. The action should be an atomic action from the given vocabulary - click, type, select, scroll
+(up or down), or stop.
+3. The arguments to each action should be within square braces. For example, "click [127]",
+"type [43] [content to type]", "scroll [up]", "scroll [down]".
+4. The natural language form of action (corresponding to the field "action_in_natural_language")
+should be consistent with the grounded version of the action (corresponding to the field "grounded
+_action"). Do NOT add any additional information in the grounded action. For example, if a
+particular element ID is specified in the grounded action, a description of that element must be
+present in the natural language action.
+5. If the type action is selected, the natural language form of action ("action_in_natural_language") should always specify the actual text to be typed.
+6. You should issue a "stop" action if the current webpage asks to log in or for credit card
+information.
+7. To input text, there is NO need to click the textbox first, directly type content. After typing,
+the system automatically hits the 'ENTER' key.
+8. STRICTLY Avoid repeating the same action (click/type) if the webpage remains unchanged.
+You may have selected the wrong web element.
+9. Do NOT use quotation marks in the action generation.
+
+OUTPUT FORMAT: 
+Please give a short analysis of the screenshot, parsed
+HTML/accessibility tree, then put your answer within ``` ```, for example,
+"In summary, the proposed task and the corresponding action is: ```{
+"action_in_natural_language":<ACTION_IN_NATURAL_LANGUAGE>:str,
+"grounded_action": <ACTION>:str}"```
+"""
+
+# User prompt
+cot_user_prompt = """
+Website URL: {INIT_URL}
+Parsed HTML/Accessibility Tree: {A11Y_TREE}
+Screenshot ocr result: {SCREENSHOT}
+Task description: {TASK_DESCRIPTION}
+"""
+
+def call_api(messages):
+    """使用openai库调用API接口"""
+    try:
+        response = client.chat.completions.create(
+            messages=messages,
+            model="gpt-4o-mini"
+        )
+
+        return response
+    except Exception as e:
+        log.error(f"API调用出错: {e}")
+        return None
+
+def extract_action(response_text):
+    """从API响应中提取action_in_natural_language和grounded_action"""
+    # 使用正则表达式提取JSON部分
+    match = re.search(r'```\s*{(.+?)}```', response_text, re.DOTALL)
+    if match:
+        try:
+            # 构建完整的JSON字符串并解析
+            json_str = "{" + match.group(1) + "}"
+            action_data = json.loads(json_str)
+            return action_data.get("action_in_natural_language"), action_data.get("grounded_action")
+        except json.JSONDecodeError:
+            log.error(f"无法解析JSON: {match.group(1)}")
+    return None, None
+
+def check_answer(grounded_action, answer):
+    """检查grounded_action是否匹配answer"""
+    if not grounded_action or not grounded_action.startswith("click"):
+        return False
+    
+    # 提取element ID
+    match = re.search(r'click \[(\d+)\]', grounded_action)
+    if not match:
+        return False
+    
+    element_id = match.group(1)
+    # 将answer拆分为列表（逗号分隔）
+    answer_ids = [id.strip() for id in answer.split(",")]
+    
+    # 检查element ID是否在answer列表中
+    return element_id in answer_ids
+
+def main():
+    # 读取exam.json
+    with open("temp_analysis/exam.json", "r") as f:
+        exam_data = json.load(f)
+    
+    results = []
+    
+    # 遍历exam.json中的每个项目
+    for item in exam_data:
+        item_id = item["id"]
+        url = item["url"]
+        task_description = item["question"]
+        answer = item["answer"]
+        answer_text = item["answer_text"]
+        
+        log.info(f"处理ID: {item_id}, URL: {url}")
+        
+        # 读取a11y tree
+        try:
+            with open(f"axtrees/{item_id}.txt", "r") as f:  # 修正文件路径
+                a11y_tree = f.read()
+        except FileNotFoundError:
+            log.error(f"找不到文件: axtrees/{item_id}.txt")
+            continue
+        
+        # 构建API请求
+        messages = [
+            {"role": "system", "content": cot_system_prompt},
+            {"role": "user", "content": cot_user_prompt.format(
+                INIT_URL=url,
+                A11Y_TREE=a11y_tree,
+                SCREENSHOT="",  # 这里没有提供截图OCR结果
+                TASK_DESCRIPTION=task_description
+            )}
+        ]
+
+        log.info(f"task_description: {task_description}")
+        log.info(f"answer: {answer}, answer_text: {answer_text}")
+
+        
+        # 调用API
+        response = call_api(messages)
+        
+        if response and hasattr(response, 'choices') and len(response.choices) > 0:
+            content = response.choices[0].message.content
+
+            log.info(f"content: {content}")
+            
+            # 提取action
+            action_nl, grounded_action = extract_action(content)
+            
+            # 检查答案
+            is_correct = check_answer(grounded_action, answer)
+            
+            # 记录结果
+            result = {
+                "id": item_id,
+                "url": url,
+                "task": task_description,
+                "action_nl": action_nl,
+                "grounded_action": grounded_action,
+                "expected_answer": answer,
+                "is_correct": is_correct
+            }
+            
+            results.append(result)
+            
+            log.info(f"ID: {item_id}")
+            log.info(f"任务: {task_description}")
+            log.info(f"动作: {grounded_action}")
+            log.info(f"是否正确: {is_correct}")
+            log.info("-" * 50)
+        else:
+            log.error(f"API调用失败: {response}")
+    
+    # 保存结果
+    with open("temp_analysis/results.json", "w") as f:
+        json.dump(results, f, indent=2)
+    
+    # 计算正确率
+    correct_count = sum(1 for r in results if r["is_correct"])
+    total_count = len(results)
+    accuracy = correct_count / total_count if total_count > 0 else 0
+    
+    log.info(f"总计: {total_count}题，正确: {correct_count}题，正确率: {accuracy:.2%}")
+
+if __name__ == "__main__":
+    main()
+
+
+
+
+
+

misc/temp_analysis/test_run copy 3.py

@@ -0,0 +1,276 @@
+import json
+import os
+import re
+import logging
+import concurrent.futures
+from openai import OpenAI
+from dotenv import load_dotenv
+
+# 设置日志
+# 创建logger
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+# 创建文件处理器
+file_handler = logging.FileHandler('temp_analysis/test_run.log')
+file_handler.setLevel(logging.INFO)
+
+# 创建控制台处理器
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.INFO)
+
+# 创建格式器并添加到处理器
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+
+# 将处理器添加到logger
+log.addHandler(file_handler)
+log.addHandler(console_handler)
+
+# 加载环境变量
+load_dotenv()
+
+# 配置OpenAI客户端
+client = OpenAI(
+    api_key=os.getenv("OPENAI_API_KEY"),
+    base_url=os.getenv("OPENAI_API_BASE_URL")  # 如果使用其他兼容服务，可以设置基础URL
+)
+
+# modified version of the task proposer agent prompt from https://arxiv.org/pdf/2502.11357
+
+# System prompt
+cot_system_prompt = """
+What does this webpage show? Imagine you are a real user on this webpage. Given the webpage
+screenshot or ocr result and parsed HTML/accessibility tree and the task description, please provide 
+the first action towards completing that task.
+
+Do the following step by step:
+1. Given the webpage screenshot or ocr result and parsed HTML/accessibility tree, generate the first action
+towards completing that task (in natural language form).
+2. Given the webpage screenshot or ocr result, parsed HTML/accessibility tree, and the natural language
+action, generate the grounded version of that action.
+
+ACTION SPACE: Your action space is: ['click [element ID]', 'type [element ID] [content]',
+'select [element ID] [content of option to select]', 'scroll [up]', 'scroll [down]', and 'stop'].
+Action output should follow the syntax as given below:
+click [element ID]: This action clicks on an element with a specific ID on the webpage.
+type [element ID] [content]: Use this to type the content into the field with id. By default, the
+"Enter" key is pressed after typing. Both the content and the ID should be within square braces
+as per the syntax.
+select [element ID] [content of option to select]: Select an option from a dropdown menu. The
+content of the option to select should be within square braces. When you get (select an option)
+tags from the accessibility tree, you need to select the serial number (element_id) corresponding
+to the select tag, not the option, and select the most likely content corresponding to the option as
+input.
+scroll [down]: Scroll the page down.
+scroll [up]: Scroll the page up.
+
+IMPORTANT: 
+To be successful, it is important to STRICTLY follow the below rules:
+Action generation rules:
+1. You should generate a single atomic action at each step.
+2. The action should be an atomic action from the given vocabulary - click, type, select, scroll
+(up or down), or stop.
+3. The arguments to each action should be within square braces. For example, "click [127]",
+"type [43] [content to type]", "scroll [up]", "scroll [down]".
+4. The natural language form of action (corresponding to the field "action_in_natural_language")
+should be consistent with the grounded version of the action (corresponding to the field "grounded
+_action"). Do NOT add any additional information in the grounded action. For example, if a
+particular element ID is specified in the grounded action, a description of that element must be
+present in the natural language action.
+5. If the type action is selected, the natural language form of action ("action_in_natural_language") should always specify the actual text to be typed.
+6. You should issue a "stop" action if the current webpage asks to log in or for credit card
+information.
+7. To input text, there is NO need to click the textbox first, directly type content. After typing,
+the system automatically hits the 'ENTER' key.
+8. STRICTLY Avoid repeating the same action (click/type) if the webpage remains unchanged.
+You may have selected the wrong web element.
+9. Do NOT use quotation marks in the action generation.
+
+OUTPUT FORMAT: 
+Please give a short analysis of the screenshot, parsed
+HTML/accessibility tree, then put your answer within ``` ```, for example,
+"In summary, the proposed task and the corresponding action is: ```{
+"action_in_natural_language":<ACTION_IN_NATURAL_LANGUAGE>:str,
+"grounded_action": <ACTION>:str}"```
+"""
+
+# User prompt
+cot_user_prompt = """
+Website URL: {INIT_URL}
+Parsed HTML/Accessibility Tree: {A11Y_TREE}
+Screenshot ocr result: {SCREENSHOT}
+Task description: {TASK_DESCRIPTION}
+"""
+
+def call_api(messages):
+    """使用openai库调用API接口"""
+    try:
+        response = client.chat.completions.create(
+            messages=messages,
+            # model="gpt-4o-mini"
+            model="aiproxy/deepseek-reasoner"
+        )
+
+        return response
+    except Exception as e:
+        log.error(f"API调用出错: {e}")
+        return None
+
+def extract_action(response_text):
+    """从API响应中提取action_in_natural_language和grounded_action"""
+    # 使用正则表达式提取JSON部分
+    match = re.search(r'```\s*{(.+?)}```', response_text, re.DOTALL)
+    if match:
+        try:
+            # 构建完整的JSON字符串并解析
+            json_str = "{" + match.group(1) + "}"
+            action_data = json.loads(json_str)
+            return action_data.get("action_in_natural_language"), action_data.get("grounded_action")
+        except json.JSONDecodeError:
+            log.error(f"无法解析JSON: {match.group(1)}")
+    return None, None
+
+def check_answer(grounded_action, answer):
+    """检查grounded_action是否匹配answer"""
+    if not grounded_action or not grounded_action.startswith("click"):
+        return False
+    
+    # 提取element ID
+    match = re.search(r'click \[(\d+)\]', grounded_action)
+    if not match:
+        return False
+    
+    element_id = match.group(1)
+    # 将answer拆分为列表（逗号分隔）
+    answer_ids = [id.strip() for id in answer.split(",")]
+    
+    # 检查element ID是否在answer列表中
+    return element_id in answer_ids
+
+def process_item(item):
+    """处理单个测试项目"""
+    item_id = item["id"]
+    url = item["url"]
+    task_description = item["question"]
+    answer = item["answer"]
+    answer_text = item["answer_text"]
+    
+    log.info(f"处理ID: {item_id}, URL: {url}")
+    
+    # 读取a11y tree
+    try:
+        with open(f"axtrees/{item_id}.txt", "r") as f:
+            a11y_tree = f.read()
+    except FileNotFoundError:
+        log.error(f"找不到文件: axtrees/{item_id}.txt")
+        return None
+    
+    # 构建API请求
+    messages = [
+        {"role": "system", "content": cot_system_prompt},
+        {"role": "user", "content": cot_user_prompt.format(
+            INIT_URL=url,
+            A11Y_TREE=a11y_tree,
+            SCREENSHOT="",  # 这里没有提供截图OCR结果
+            TASK_DESCRIPTION=task_description
+        )}
+    ]
+
+    log.info(f"task_description: {task_description}")
+    log.info(f"answer: {answer}, answer_text: {answer_text}")
+    
+    # 调用API
+    response = call_api(messages)
+
+    if response and hasattr(response, 'choices') and len(response.choices) > 0:
+        content = response.choices[0].message.content
+        reasoning_content = response.choices[0].message.reasoning_content
+
+        log.info(f"reasoning_content: {reasoning_content}")
+        log.info(f"content: {content}")
+        
+        # 提取action
+        action_nl, grounded_action = extract_action(content)
+        
+        # 检查答案
+        is_correct = check_answer(grounded_action, answer)
+        
+        # 记录结果
+        result = {
+            "id": item_id,
+            "url": url,
+            "task": task_description,
+            "expected_answer": answer,
+            "thinking": reasoning_content,
+            "action_nl": action_nl,
+            "grounded_action": grounded_action,
+            "is_correct": is_correct
+        }
+        
+        log.info(f"ID: {item_id}")
+        log.info(f"任务: {task_description}")
+        log.info(f"动作: {grounded_action}")
+        log.info(f"是否正确: {is_correct}")
+        log.info("-" * 50)
+        
+        return result
+    else:
+        log.error(f"API调用失败: {response}")
+        return None
+
+def main():
+    # 读取exam.json
+    with open("temp_analysis/exam.json", "r") as f:
+        exam_data = json.load(f)
+    
+    results = []
+    total_items = len(exam_data)
+    completed = 0
+    success_count = 0
+    fail_count = 0
+    parallel = 4
+    
+    log.info(f"开始测试，总共 {total_items} 个任务")
+    
+    # 使用线程池并发处理
+    with concurrent.futures.ThreadPoolExecutor(max_workers=parallel) as executor:
+        # 提交所有任务
+        future_to_item = {executor.submit(process_item, item): item for item in exam_data}
+        
+        # 收集结果
+        for future in concurrent.futures.as_completed(future_to_item):
+            result = future.result()
+            completed += 1
+            
+            if result:
+                results.append(result)
+                if result["is_correct"]:
+                    success_count += 1
+                else:
+                    fail_count += 1
+            else:
+                fail_count += 1
+            
+            # 打印当前进度
+            progress = (completed / total_items) * 100
+            log.info(f"进度: {progress:.2f}% ({completed}/{total_items}) - 成功: {success_count}, 失败: {fail_count}")
+    
+    # 保存结果
+    with open("temp_analysis/results.json", "w") as f:
+        json.dump(results, f, indent=2)
+    
+    # 计算正确率
+    accuracy = success_count / total_items if total_items > 0 else 0
+    
+    log.info(f"测试完成! 总计: {total_items}题，正确: {success_count}题，错误: {fail_count}题，正确率: {accuracy:.2%}")
+
+if __name__ == "__main__":
+    main()
+
+
+
+
+
+

misc/temp_analysis/test_run copy 4.py

@@ -0,0 +1,364 @@
+import json
+import os
+import re
+import logging
+import concurrent.futures
+import argparse
+from openai import OpenAI
+from dotenv import load_dotenv
+
+# 设置日志
+# 创建logger
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+# 创建文件处理器
+file_handler = logging.FileHandler('temp_analysis/test_run.log')
+file_handler.setLevel(logging.INFO)
+
+# 创建控制台处理器
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.INFO)
+
+# 创建格式器并添加到处理器
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+
+# 将处理器添加到logger
+log.addHandler(file_handler)
+log.addHandler(console_handler)
+
+# 加载环境变量
+load_dotenv()
+
+# 配置OpenAI客户端
+client = OpenAI(
+    api_key=os.getenv("OPENAI_API_KEY"),
+    base_url=os.getenv("OPENAI_API_BASE_URL")  # 如果使用其他兼容服务，可以设置基础URL
+)
+
+# modified version of the task proposer agent prompt from https://arxiv.org/pdf/2502.11357
+
+# System prompt
+cot_system_prompt = """
+What does this webpage show? Imagine you are a real user on this webpage. Given the webpage
+screenshot or ocr result and parsed HTML/accessibility tree and the task description, please provide 
+the first action towards completing that task.
+
+Do the following step by step:
+1. Given the webpage screenshot or ocr result and parsed HTML/accessibility tree, generate the first action
+towards completing that task (in natural language form).
+2. Given the webpage screenshot or ocr result, parsed HTML/accessibility tree, and the natural language
+action, generate the grounded version of that action.
+
+ACTION SPACE: Your action space is: ['click [element ID]', 'type [element ID] [content]',
+'select [element ID] [content of option to select]', 'scroll [up]', 'scroll [down]', and 'stop'].
+Action output should follow the syntax as given below:
+click [element ID]: This action clicks on an element with a specific ID on the webpage.
+type [element ID] [content]: Use this to type the content into the field with id. By default, the
+"Enter" key is pressed after typing. Both the content and the ID should be within square braces
+as per the syntax.
+select [element ID] [content of option to select]: Select an option from a dropdown menu. The
+content of the option to select should be within square braces. When you get (select an option)
+tags from the accessibility tree, you need to select the serial number (element_id) corresponding
+to the select tag, not the option, and select the most likely content corresponding to the option as
+input.
+scroll [down]: Scroll the page down.
+scroll [up]: Scroll the page up.
+
+IMPORTANT: 
+To be successful, it is important to STRICTLY follow the below rules:
+Action generation rules:
+1. You should generate a single atomic action at each step.
+2. The action should be an atomic action from the given vocabulary - click, type, select, scroll
+(up or down), or stop.
+3. The arguments to each action should be within square braces. For example, "click [127]",
+"type [43] [content to type]", "scroll [up]", "scroll [down]".
+4. The natural language form of action (corresponding to the field "action_in_natural_language")
+should be consistent with the grounded version of the action (corresponding to the field "grounded
+_action"). Do NOT add any additional information in the grounded action. For example, if a
+particular element ID is specified in the grounded action, a description of that element must be
+present in the natural language action.
+5. If the type action is selected, the natural language form of action ("action_in_natural_language") should always specify the actual text to be typed.
+6. You should issue a "stop" action if the current webpage asks to log in or for credit card
+information.
+7. To input text, there is NO need to click the textbox first, directly type content. After typing,
+the system automatically hits the 'ENTER' key.
+8. STRICTLY Avoid repeating the same action (click/type) if the webpage remains unchanged.
+You may have selected the wrong web element.
+9. Do NOT use quotation marks in the action generation.
+
+OUTPUT FORMAT: 
+Please give a short analysis of the screenshot, parsed
+HTML/accessibility tree, then put your answer within ``` ```, for example,
+"In summary, the proposed task and the corresponding action is: ```{
+"action_in_natural_language":<ACTION_IN_NATURAL_LANGUAGE>:str,
+"grounded_action": <ACTION>:str}"```
+"""
+
+# User prompt
+cot_user_prompt = """
+Website URL: {INIT_URL}
+Parsed HTML/Accessibility Tree: {A11Y_TREE}
+Screenshot ocr result: {SCREENSHOT}
+Task description: {TASK_DESCRIPTION}
+"""
+
+def call_api(messages):
+    """使用openai库调用API接口"""
+    try:
+        response = client.chat.completions.create(
+            messages=messages,
+            # model="gpt-4o-mini"
+            model="aiproxy/deepseek-reasoner"
+        )
+
+        return response
+    except Exception as e:
+        log.error(f"API调用出错: {e}")
+        return None
+
+def extract_action(response_text):
+    """从API响应中提取action_in_natural_language和grounded_action"""
+    # 使用正则表达式提取JSON部分
+    match = re.search(r'```\s*{(.+?)}```', response_text, re.DOTALL)
+    if match:
+        try:
+            # 构建完整的JSON字符串并解析
+            json_str = "{" + match.group(1) + "}"
+            action_data = json.loads(json_str)
+            return action_data.get("action_in_natural_language"), action_data.get("grounded_action")
+        except json.JSONDecodeError:
+            log.error(f"无法解析JSON: {match.group(1)}")
+    return None, None
+
+def check_answer(grounded_action, answer):
+    """检查grounded_action是否匹配answer"""
+    if not grounded_action or not grounded_action.startswith("click"):
+        return False
+    
+    # 提取element ID
+    match = re.search(r'click \[(\d+)\]', grounded_action)
+    if not match:
+        return False
+    
+    element_id = match.group(1)
+    # 将answer拆分为列表（逗号分隔）
+    answer_ids = [id.strip() for id in answer.split(",")]
+    
+    # 检查element ID是否在answer列表中
+    return element_id in answer_ids
+
+def process_item(item):
+    """处理单个测试项目"""
+    item_id = item["id"]
+    url = item["url"]
+    task_description = item["question"]
+    answer = item["answer"]
+    answer_text = item["answer_text"]
+    
+    log.info(f"处理ID: {item_id}, URL: {url}")
+    
+    # 读取a11y tree
+    try:
+        with open(f"axtrees/{item_id}.txt", "r") as f:
+            a11y_tree = f.read()
+    except FileNotFoundError:
+        log.error(f"找不到文件: axtrees/{item_id}.txt")
+        return None
+    
+    # 构建API请求
+    messages = [
+        {"role": "system", "content": cot_system_prompt},
+        {"role": "user", "content": cot_user_prompt.format(
+            INIT_URL=url,
+            A11Y_TREE=a11y_tree,
+            SCREENSHOT="",  # 这里没有提供截图OCR结果
+            TASK_DESCRIPTION=task_description
+        )}
+    ]
+
+    log.info(f"task_description: {task_description}")
+    log.info(f"answer: {answer}, answer_text: {answer_text}")
+    
+    # 调用API
+    response = call_api(messages)
+
+    if response and hasattr(response, 'choices') and len(response.choices) > 0:
+        content = response.choices[0].message.content
+        reasoning_content = response.choices[0].message.reasoning_content
+
+        log.info(f"reasoning_content: {reasoning_content}")
+        log.info(f"content: {content}")
+        
+        # 提取action
+        action_nl, grounded_action = extract_action(content)
+        
+        # 检查答案
+        is_correct = check_answer(grounded_action, answer)
+        
+        # 记录结果
+        result = {
+            "id": item_id,
+            "url": url,
+            "task": task_description,
+            "expected_answer": answer,
+            "thinking": reasoning_content,
+            "action_nl": action_nl,
+            "grounded_action": grounded_action,
+            "is_correct": is_correct
+        }
+        
+        log.info(f"ID: {item_id}")
+        log.info(f"任务: {task_description}")
+        log.info(f"动作: {grounded_action}")
+        log.info(f"是否正确: {is_correct}")
+        log.info("-" * 50)
+        
+        return result
+    else:
+        log.error(f"API调用失败: {response}")
+        return None
+
+def main():
+    # 解析命令行参数
+    parser = argparse.ArgumentParser(description='运行测试或重跑失败的测试')
+    parser.add_argument('--rerun_failure', action='store_true', help='重跑失败的测试用例')
+    args = parser.parse_args()
+    
+    if args.rerun_failure:
+        # 重跑失败的测试
+        try:
+            with open("temp_analysis/results.json", "r") as f:
+                results = json.load(f)
+            
+            # 筛选出失败的测试用例
+            failed_results = [r for r in results if not r.get("is_correct", False)]
+            
+            if not failed_results:
+                log.info("没有找到失败的测试用例，无需重跑")
+                return
+            
+            # 读取exam.json以获取完整的测试数据
+            with open("temp_analysis/exam.json", "r") as f:
+                exam_data = json.load(f)
+            
+            # 创建ID到测试项的映射
+            exam_map = {item["id"]: item for item in exam_data}
+            
+            # 准备重跑的测试项
+            rerun_items = []
+            for result in failed_results:
+                item_id = result["id"]
+                if item_id in exam_map:
+                    rerun_items.append(exam_map[item_id])
+            
+            total_items = len(rerun_items)
+            log.info(f"开始重跑失败的测试，总共 {total_items} 个任务")
+            
+            # 创建ID到结果索引的映射，用于更新结果
+            result_indices = {r["id"]: i for i, r in enumerate(results)}
+            
+            completed = 0
+            success_count = 0
+            fail_count = 0
+            
+            # 使用线程池并发处理
+            with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
+                # 提交所有任务
+                future_to_item = {executor.submit(process_item, item): item for item in rerun_items}
+                
+                # 收集结果
+                for future in concurrent.futures.as_completed(future_to_item):
+                    item = future_to_item[future]
+                    result = future.result()
+                    completed += 1
+                    
+                    if result:
+                        # 更新结果
+                        if result["id"] in result_indices:
+                            results[result_indices[result["id"]]] = result
+                        else:
+                            results.append(result)
+                        
+                        if result["is_correct"]:
+                            success_count += 1
+                        else:
+                            fail_count += 1
+                    else:
+                        fail_count += 1
+                    
+                    # 打印当前进度
+                    progress = (completed / total_items) * 100
+                    log.info(f"重跑进度: {progress:.2f}% ({completed}/{total_items}) - 成功: {success_count}, 失败: {fail_count}")
+            
+            # 保存更新后的结果
+            with open("temp_analysis/results.json", "w") as f:
+                json.dump(results, f, indent=2)
+            
+            # 计算新的正确率
+            all_correct = sum(1 for r in results if r.get("is_correct", False))
+            all_total = len(results)
+            accuracy = all_correct / all_total if all_total > 0 else 0
+            
+            log.info(f"重跑完成! 重跑: {total_items}题，成功: {success_count}题，失败: {fail_count}题")
+            log.info(f"总计: {all_total}题，正确: {all_correct}题，正确率: {accuracy:.2%}")
+            
+        except FileNotFoundError:
+            log.error("找不到results.json文件，请先运行完整测试")
+            return
+    else:
+        # 运行完整测试
+        # 读取exam.json
+        with open("temp_analysis/exam.json", "r") as f:
+            exam_data = json.load(f)
+        
+        results = []
+        total_items = len(exam_data)
+        completed = 0
+        success_count = 0
+        fail_count = 0
+        
+        log.info(f"开始测试，总共 {total_items} 个任务")
+        
+        # 使用线程池并发处理
+        with concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="test_run") as executor:
+            # 提交所有任务
+            future_to_item = {executor.submit(process_item, item): item for item in exam_data}
+            
+            # 收集结果
+            for future in concurrent.futures.as_completed(future_to_item):
+                result = future.result()
+                completed += 1
+                
+                if result:
+                    results.append(result)
+                    if result["is_correct"]:
+                        success_count += 1
+                    else:
+                        fail_count += 1
+                else:
+                    fail_count += 1
+                
+                # 打印当前进度
+                progress = (completed / total_items) * 100
+                log.info(f"进度: {progress:.2f}% ({completed}/{total_items}) - 成功: {success_count}, 失败: {fail_count}")
+        
+        # 保存结果
+        with open("temp_analysis/results.json", "w") as f:
+            json.dump(results, f, indent=2)
+        
+        # 计算正确率
+        accuracy = success_count / total_items if total_items > 0 else 0
+        
+        log.info(f"测试完成! 总计: {total_items}题，正确: {success_count}题，错误: {fail_count}题，正确率: {accuracy:.2%}")
+
+if __name__ == "__main__":
+    main()
+
+
+
+
+
+

misc/temp_analysis/test_run copy.py

@@ -0,0 +1,84 @@
+
+# modified version of the task proposer agent prompt from https://arxiv.org/pdf/2502.11357
+
+# System prompt
+cot_system_prompt = """
+What does this webpage show? Imagine you are a real user on this webpage. Given the webpage
+screenshot or ocr result and parsed HTML/accessibility tree and the task description, please provide 
+the first action towards completing that task.
+
+Do the following step by step:
+1. Given the webpage screenshot or ocr result and parsed HTML/accessibility tree, generate the first action
+towards completing that task (in natural language form).
+2. Given the webpage screenshot or ocr result, parsed HTML/accessibility tree, and the natural language
+action, generate the grounded version of that action.
+
+ACTION SPACE: Your action space is: ['click [element ID]', 'type [element ID] [content]',
+'select [element ID] [content of option to select]', 'scroll [up]', 'scroll [down]', and 'stop'].
+Action output should follow the syntax as given below:
+click [element ID]: This action clicks on an element with a specific ID on the webpage.
+type [element ID] [content]: Use this to type the content into the field with id. By default, the
+"Enter" key is pressed after typing. Both the content and the ID should be within square braces
+as per the syntax.
+select [element ID] [content of option to select]: Select an option from a dropdown menu. The
+content of the option to select should be within square braces. When you get (select an option)
+tags from the accessibility tree, you need to select the serial number (element_id) corresponding
+to the select tag, not the option, and select the most likely content corresponding to the option as
+input.
+scroll [down]: Scroll the page down.
+scroll [up]: Scroll the page up.
+
+IMPORTANT: 
+To be successful, it is important to STRICTLY follow the below rules:
+Action generation rules:
+1. You should generate a single atomic action at each step.
+2. The action should be an atomic action from the given vocabulary - click, type, select, scroll
+(up or down), or stop.
+3. The arguments to each action should be within square braces. For example, "click [127]",
+"type [43] [content to type]", "scroll [up]", "scroll [down]".
+4. The natural language form of action (corresponding to the field "action_in_natural_language")
+should be consistent with the grounded version of the action (corresponding to the field "grounded
+_action"). Do NOT add any additional information in the grounded action. For example, if a
+particular element ID is specified in the grounded action, a description of that element must be
+present in the natural language action.
+5. If the type action is selected, the natural language form of action ("action_in_natural_language") should always specify the actual text to be typed.
+6. You should issue a “stop” action if the current webpage asks to log in or for credit card
+information.
+7. To input text, there is NO need to click the textbox first, directly type content. After typing,
+the system automatically hits the ‘ENTER’ key.
+8. STRICTLY Avoid repeating the same action (click/type) if the webpage remains unchanged.
+You may have selected the wrong web element.
+9. Do NOT use quotation marks in the action generation.
+
+OUTPUT FORMAT: 
+Please give a short analysis of the screenshot, parsed
+HTML/accessibility tree, then put your answer within ``` ```, for example,
+"In summary, the proposed task and the corresponding action is: ```{
+"action_in_natural_language":<ACTION_IN_NATURAL_LANGUAGE>:str,
+"grounded_action": <ACTION>:str}"```
+"""
+
+# User prompt
+cot_user_prompt = """
+Website URL: {INIT_URL}
+Parsed HTML/Accessibility Tree: {A11Y_TREE}
+Screenshot ocr result: {SCREENSHOT}
+Task description: {TASK_DESCRIPTION}
+"""
+
+# load json from path/processed_3_with_analysis_20250324163759.json
+with open('path/processed_3_with_analysis_20250324163759.json', 'r') as f:
+    data = json.load(f)
+
+# loop through each key in the json
+for key, value in data.items():
+    # print the key
+    url = key
+
+    shortestPathsMeta = value['shortestPathsMeta']
+    for shortestPathMeta in shortestPathsMeta:
+        chainTexts = shortestPathMeta['chainTexts']
+        for chainText in chainTexts:
+            print(chainText)
+
+

misc/temp_analysis/test_run_retry.py

@@ -0,0 +1,415 @@
+import json
+import os
+import re
+import logging
+import concurrent.futures
+import argparse
+import threading
+import datetime
+from openai import OpenAI
+from dotenv import load_dotenv
+
+# 设置日志
+# 创建logger
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+# 创建文件处理器
+file_handler = logging.FileHandler('temp_analysis/test_run.log')
+file_handler.setLevel(logging.INFO)
+
+# 创建控制台处理器
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.INFO)
+
+# 创建格式器并添加到处理器
+formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+
+# 将处理器添加到logger
+log.addHandler(file_handler)
+log.addHandler(console_handler)
+
+# 加载环境变量
+load_dotenv()
+
+# 配置OpenAI客户端
+client = OpenAI(
+    api_key=os.getenv("OPENAI_API_KEY"),
+    base_url=os.getenv("OPENAI_API_BASE_URL")  # 如果使用其他兼容服务，可以设置基础URL
+)
+
+# modified version of the task proposer agent prompt from https://arxiv.org/pdf/2502.11357
+
+# System prompt
+cot_system_prompt = """
+What does this webpage show? Imagine you are a real user on this webpage. Given the webpage
+screenshot or ocr result and parsed HTML/accessibility tree and the task description, please provide 
+the first action towards completing that task.
+
+Do the following step by step:
+1. Given the webpage screenshot or ocr result and parsed HTML/accessibility tree, generate the first action
+towards completing that task (in natural language form).
+2. Given the webpage screenshot or ocr result, parsed HTML/accessibility tree, and the natural language
+action, generate the grounded version of that action.
+
+ACTION SPACE: Your action space is: ['click [element ID]', 'type [element ID] [content]',
+'select [element ID] [content of option to select]', 'scroll [up]', 'scroll [down]', and 'stop'].
+Action output should follow the syntax as given below:
+click [element ID]: This action clicks on an element with a specific ID on the webpage.
+type [element ID] [content]: Use this to type the content into the field with id. By default, the
+"Enter" key is pressed after typing. Both the content and the ID should be within square braces
+as per the syntax.
+select [element ID] [content of option to select]: Select an option from a dropdown menu. The
+content of the option to select should be within square braces. When you get (select an option)
+tags from the accessibility tree, you need to select the serial number (element_id) corresponding
+to the select tag, not the option, and select the most likely content corresponding to the option as
+input.
+scroll [down]: Scroll the page down.
+scroll [up]: Scroll the page up.
+
+IMPORTANT: 
+To be successful, it is important to STRICTLY follow the below rules:
+Action generation rules:
+1. You should generate a single atomic action at each step.
+2. The action should be an atomic action from the given vocabulary - click, type, select, scroll
+(up or down), or stop.
+3. The arguments to each action should be within square braces. For example, "click [127]",
+"type [43] [content to type]", "scroll [up]", "scroll [down]".
+4. The natural language form of action (corresponding to the field "action_in_natural_language")
+should be consistent with the grounded version of the action (corresponding to the field "grounded
+_action"). Do NOT add any additional information in the grounded action. For example, if a
+particular element ID is specified in the grounded action, a description of that element must be
+present in the natural language action.
+5. If the type action is selected, the natural language form of action ("action_in_natural_language") should always specify the actual text to be typed.
+6. You should issue a "stop" action if the current webpage asks to log in or for credit card
+information.
+7. To input text, there is NO need to click the textbox first, directly type content. After typing,
+the system automatically hits the 'ENTER' key.
+8. STRICTLY Avoid repeating the same action (click/type) if the webpage remains unchanged.
+You may have selected the wrong web element.
+9. Do NOT use quotation marks in the action generation.
+
+OUTPUT FORMAT: 
+Please give a short analysis of the screenshot, parsed
+HTML/accessibility tree, then put your answer within ``` ```, for example,
+"In summary, the proposed task and the corresponding action is: ```{
+"action_in_natural_language":<ACTION_IN_NATURAL_LANGUAGE>:str,
+"grounded_action": <ACTION>:str}"```
+"""
+
+# User prompt
+cot_user_prompt = """
+Website URL: {INIT_URL}
+Parsed HTML/Accessibility Tree: {A11Y_TREE}
+Screenshot ocr result: {SCREENSHOT}
+Task description: {TASK_DESCRIPTION}
+"""
+
+# 创建文件锁
+file_locks = {
+    "temp_analysis/results.json": threading.Lock(),
+    "temp_analysis/results_success.json": threading.Lock(),
+    "temp_analysis/results_failure.json": threading.Lock()
+}
+
+def call_api(messages):
+    """使用openai库调用API接口"""
+    try:
+        response = client.chat.completions.create(
+            messages=messages,
+            # model="gpt-4o-mini"
+            model="aiproxy/deepseek-reasoner"
+        )
+
+        return response
+    except Exception as e:
+        log.error(f"API调用出错: {e}")
+        return None
+
+def extract_action(response_text):
+    """从API响应中提取action_in_natural_language和grounded_action"""
+    # 使用正则表达式提取JSON部分
+    match = re.search(r'```\s*{(.+?)}```', response_text, re.DOTALL)
+    if match:
+        try:
+            # 构建完整的JSON字符串并解析
+            json_str = "{" + match.group(1) + "}"
+            action_data = json.loads(json_str)
+            return action_data.get("action_in_natural_language"), action_data.get("grounded_action")
+        except json.JSONDecodeError:
+            log.error(f"无法解析JSON: {match.group(1)}")
+    return None, None
+
+def check_answer(grounded_action, answer):
+    """检查grounded_action是否匹配answer"""
+    if not grounded_action or not grounded_action.startswith("click"):
+        return False
+    
+    # 提取element ID
+    match = re.search(r'click \[(\d+)\]', grounded_action)
+    if not match:
+        return False
+    
+    element_id = match.group(1)
+    # 将answer拆分为列表（逗号分隔）
+    answer_ids = [id.strip() for id in answer.split(",")]
+    
+    # 检查element ID是否在answer列表中
+    return element_id in answer_ids
+
+def load_successful_items():
+    """加载已经成功的测试项目ID"""
+    successful_ids = set()
+    try:
+        if os.path.exists("temp_analysis/results_success.json"):
+            with open("temp_analysis/results_success.json", "r") as f:
+                try:
+                    success_data = json.load(f)
+                    successful_ids = {item["id"] for item in success_data if "id" in item}
+                except json.JSONDecodeError:
+                    log.warning("无法解析results_success.json，将视为空文件")
+    except Exception as e:
+        log.error(f"加载成功项目时出错: {e}")
+    
+    return successful_ids
+
+def process_item(item, successful_ids, max_retries=3):
+    """处理单个测试项目，失败时最多重试3次"""
+    item_id = item["id"]
+    
+    # 检查是否已经成功完成
+    if item_id in successful_ids:
+        log.info(f"ID: {item_id} 已经成功完成，跳过")
+        return None
+    
+    url = item["url"]
+    task_description = item["question"]
+    answer = item["answer"]
+    answer_text = item["answer_text"]
+    
+    log.info(f"处理ID: {item_id}, URL: {url}")
+    
+    # 读取a11y tree
+    try:
+        with open(f"axtrees/{item_id}.txt", "r") as f:
+            a11y_tree = f.read()
+    except FileNotFoundError:
+        log.error(f"找不到文件: axtrees/{item_id}.txt")
+        return None
+    
+    # 构建API请求
+    messages = [
+        {"role": "system", "content": cot_system_prompt},
+        {"role": "user", "content": cot_user_prompt.format(
+            INIT_URL=url,
+            A11Y_TREE=a11y_tree,
+            SCREENSHOT="",  # 这里没有提供截图OCR结果
+            TASK_DESCRIPTION=task_description
+        )}
+    ]
+
+    print(f"messages: #######\n {messages} \n######")
+
+    log.info(f"task_description: {task_description}")
+    log.info(f"answer: {answer}, answer_text: {answer_text}")
+    
+    # 尝试调用API，最多重试max_retries次
+    attempts_made = 0
+    all_attempts = []  # 记录所有尝试的结果
+    
+    for attempt in range(max_retries):
+        attempts_made = attempt + 1
+        timestamp = datetime.datetime.now().isoformat()
+
+        
+        # 调用API
+        response = call_api(messages)
+
+        if response and hasattr(response, 'choices') and len(response.choices) > 0:
+            content = response.choices[0].message.content
+            reasoning_content = response.choices[0].message.reasoning_content if hasattr(response.choices[0].message, 'reasoning_content') else ""
+
+            log.info(f"reasoning_content: {reasoning_content}")
+            log.info(f"content: {content}")
+            
+            # 提取action
+            action_nl, grounded_action = extract_action(content)
+            
+            # 检查答案
+            is_correct = check_answer(grounded_action, answer)
+            
+            # 记录本次尝试
+            attempt_result = {
+                "attempt": attempt + 1,
+                "timestamp": timestamp,
+                "action_nl": action_nl,
+                "grounded_action": grounded_action,
+                "is_correct": is_correct,
+                "thinking": reasoning_content
+            }
+            all_attempts.append(attempt_result)
+            
+            # 如果成功，直接返回结果
+            if is_correct:
+                log.info(f"ID: {item_id} 测试成功!")
+                break
+            else:
+                log.info(f"ID: {item_id} 测试失败，尝试 {attempt+1}/{max_retries}")
+                # 如果不是最后一次尝试，继续重试
+                if attempt < max_retries - 1:
+                    continue
+        else:
+            log.error(f"API调用失败: {response}")
+            # 记录失败的尝试
+            attempt_result = {
+                "attempt": attempt + 1,
+                "timestamp": timestamp,
+                "action_nl": None,
+                "grounded_action": None,
+                "is_correct": False,
+                "thinking": "API调用失败"
+            }
+            all_attempts.append(attempt_result)
+            
+            # 如果不是最后一次尝试，继续重试
+            if attempt < max_retries - 1:
+                log.info(f"ID: {item_id} API调用失败，尝试 {attempt+1}/{max_retries}")
+                continue
+            else:
+                reasoning_content = ""
+                content = ""
+                action_nl = None
+                grounded_action = None
+                is_correct = False
+    
+    # 获取最后一次尝试的结果（无论成功与否）
+    last_attempt = all_attempts[-1] if all_attempts else None
+    
+    # 记录结果
+    result = {
+        "id": item_id,
+        "url": url,
+        "task": task_description,
+        "expected_answer": answer,
+        "answer_text": answer_text,
+        "thinking": last_attempt.get("thinking", "") if last_attempt else "",
+        "action_nl": last_attempt.get("action_nl", None) if last_attempt else None,
+        "grounded_action": last_attempt.get("grounded_action", None) if last_attempt else None,
+        "is_correct": last_attempt.get("is_correct", False) if last_attempt else False,
+        "attempts": attempts_made,  # 添加重试次数记录
+        "timestamp": datetime.datetime.now().isoformat(),  # 添加时间戳
+        "all_attempts": all_attempts  # 记录所有尝试的详细信息
+    }
+    
+    log.info(f"ID: {item_id}")
+    log.info(f"任务: {task_description}")
+    log.info(f"动作: {last_attempt.get('grounded_action', None) if last_attempt else None}")
+    log.info(f"是否正确: {last_attempt.get('is_correct', False) if last_attempt else False}")
+    log.info(f"尝试次数: {attempts_made}")
+    log.info("-" * 50)
+    
+    return result
+
+def append_to_result_file(result, filename):
+    """将结果追加到指定的JSON文件中，使用锁防止并发冲突"""
+    try:
+        # 获取对应文件的锁
+        with file_locks.get(filename, threading.Lock()):
+            # 如果文件存在，读取现有内容
+            if os.path.exists(filename):
+                with open(filename, 'r') as f:
+                    try:
+                        data = json.load(f)
+                    except json.JSONDecodeError:
+                        # 如果文件为空或格式不正确，创建新列表
+                        data = []
+            else:
+                data = []
+            
+            # 追加新结果
+            data.append(result)
+            
+            # 写回文件
+            with open(filename, 'w') as f:
+                json.dump(data, f, indent=2)
+                
+            return True
+    except Exception as e:
+        log.error(f"写入结果到文件 {filename} 时出错: {e}")
+        return False
+
+def main():
+    # 读取exam.json
+    with open("temp_analysis/exam.json", "r") as f:
+        exam_data = json.load(f)
+    
+    # 加载已经成功的测试项目ID
+    successful_ids = load_successful_items()
+    log.info(f"已经成功完成的测试项目数: {len(successful_ids)}, 成功ID: {successful_ids}")
+    
+    results = []
+    total_items = len(exam_data)
+    completed = 0
+    success_count = 0
+    fail_count = 0
+    skip_count = 0
+    
+    log.info(f"开始测试，总共 {total_items} 个任务")
+    
+    # 确保结果文件存在并初始化为空列表（如果不存在的话）
+    for filename in ["temp_analysis/results.json", "temp_analysis/results_failure.json"]:
+        if not os.path.exists(filename):
+            with open(filename, 'w') as f:
+                json.dump([], f)
+    
+    # 使用线程池并发处理
+    with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+        # 提交所有任务
+        future_to_item = {executor.submit(process_item, item, successful_ids): item for item in exam_data}
+        
+        # 收集结果
+        for future in concurrent.futures.as_completed(future_to_item):
+            result = future.result()
+            completed += 1
+            
+            if result is None:
+                skip_count += 1
+                log.info(f"跳过一个已完成的测试项目")
+            elif result:
+                results.append(result)
+                
+                # 立即追加到结果文件
+                append_to_result_file(result, "temp_analysis/results.json")
+                
+                if result["is_correct"]:
+                    success_count += 1
+                    append_to_result_file(result, "temp_analysis/results_success.json")
+                else:
+                    fail_count += 1
+                    append_to_result_file(result, "temp_analysis/results_failure.json")
+            else:
+                fail_count += 1
+            
+            # 打印当前进度
+            progress = (completed / total_items) * 100
+            log.info(f"进度: {progress:.2f}% ({completed}/{total_items}) - 成功: {success_count}, 失败: {fail_count}, 跳过: {skip_count}")
+    
+    # 计算正确率
+    total_processed = success_count + fail_count
+    accuracy = success_count / total_processed if total_processed > 0 else 0
+    
+    log.info(f"测试完成! 总计: {total_items}题，成功: {success_count}题，失败: {fail_count}题，跳过: {skip_count}题，正确率: {accuracy:.2%}")
+    log.info(f"成功结果已保存到 results_success.json")
+    log.info(f"失败结果已保存到 results_failure.json")
+    log.info(f"全部结果已保存到 results.json")
+
+if __name__ == "__main__":
+    main()
+
+
+
+
+
+