yuyr/webrl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add plan and act ; gen task; run gen task

Browse Source
This commit is contained in:
yuyr 2025-04-28 18:21:20 +08:00
parent e50dd34bee
commit 498e9d83b8
32 changed files with 21624 additions and 1943 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@ -29,9 +29,16 @@ src/server/tasks/minecraft/vab_minecraft_src/jarvis/global_configs/envs/*
VAB-WebArena-Lite/config_files/wa/test_webarena
VAB-WebArena-Lite/config_files/wa/test_webarena_lite
VAB-WebArena-Lite/config_files/wa/test_webarena_gen
VAB-WebArena-Lite/log_files
VAB-WebArena-Lite/results
VAB-WebArena-Lite/debug_info
*/.env
*/.env.*
*/.auth/*

2
VAB-WebArena-Lite/.auth/gitlab.reddit_state.json
View File

@ -1 +1 @@
{"cookies": [{"name": "_cookie_check", "value": "1", "domain": "localhost", "path": "/login", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "REMEMBERME", "value": "QXBwXEVudGl0eVxVc2VyOlRXRnlkbVZzYzBkeVlXNTBUV0Z1TVRNMjoxNzc2OTA5MzI5OlltVTFZalF5Tldaa05qaG1Oamt6WWpJMllXRTVOVEE0TW1GaFl6ZGpaV0V4WVdFMk1HSmlNamsyTjJVd01qVXdaakkwTVRSaVlURTNNRGhpTVROa1lnPT0%3D", "domain": "localhost", "path": "/", "expires": 1776909329.082441, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "PHPSESSID", "value": "c002205f152b213deaedc0800c372f3f", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "known_sign_in", "value": "UlVrNWNuN1g3cDRWdjNiUmJZUVl3OGNZNmJGQmlhYll6WFN0ZDdXNFpaZnBWNk1VLzBFKyt0eDJWQ2F1VkhkUndNR1hjeHIybnNzYlY5RC8wbFVWejJrclpUSUZZR2g1T0ZJQTRVVlZMUGVjKzdiMmhWVk1yR2czb1pzYXAzQWstLW8vYnlFdTlMYmZHUXBTZENLRDVBNlE9PQ%3D%3D--4b1888d4ebbdb533cf162356c2630ea53dd95286", "domain": "localhost", "path": "/", "expires": 1746582930.098554, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "_gitlab_session", "value": "037c2d46106a9cea55da705dce5b5708", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": []}
{"cookies": [{"name": "_cookie_check", "value": "1", "domain": "localhost", "path": "/login", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "REMEMBERME", "value": "QXBwXEVudGl0eVxVc2VyOlRXRnlkbVZzYzBkeVlXNTBUV0Z1TVRNMjoxNzc3MzY0ODcyOk9UZ3pZekpqWmpKbU5qWXdOR05oWldZM1ptVTVObUl6TnpVNVlqWXpOVGc1TXpJek1UQTRaR0l6WkROak5EY3dZbVpsTlRBeFpqRm1ZalZrTnpNeVlRPT0%3D", "domain": "localhost", "path": "/", "expires": 1777364872.180308, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "PHPSESSID", "value": "1c8070255c5bf47e57bc59f69da7cdd2", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "known_sign_in", "value": "UFlTS2FscXZxR1BJRk5aMEs2MGJKQWR0THlBeTFLbjdOeGZiT2lVU3NZRFRVMmFvWVYrRm5YVHdOUzh1cTh2Q1VkZGdPY21KSjdhUlJRSTdVcWVPbXFQMEtSU1M0ZFE1QnllOVFxcE41MUtZdXZITC8vdWhKdDhPdk9ZaEh4TzQtLU9JcTIwbmY4SHFWYzNVUjRSaVhZRmc9PQ%3D%3D--5b400b19d3f378d0caafab985925805144621a92", "domain": "localhost", "path": "/", "expires": 1747038473.288167, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "_gitlab_session", "value": "b94fdfd98772f0eea1f7639feef9bb05", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": []}

2
VAB-WebArena-Lite/.auth/gitlab.shopping_admin_state.json
View File

@ -1 +1 @@
{"cookies": [{"name": "admin", "value": "1087c582d275f78834da637c08d2dd0c", "domain": "localhost", "path": "/admin", "expires": 1745733329.394644, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "known_sign_in", "value": "UW1ZRyt2cTd2VmltKzMzWGVDOWovVi9HUkwyU2lYQ3BteUdGNUFyNHpNNHpMVldsWEdTZnJ6TDM5ekpxaDFpSTNGTXAvWTBDOGtWRUp6YWlWbE5UdEJXZzM5a0FvUGQ2V0c5dHJib1J6b0VrNWhVQTgvOTZxTnVtd3NHMjBWQTktLXJub3VnR0RISHFkV1Y2QmtzRlg3Nnc9PQ%3D%3D--20fe4b8ba28a2bb065bc6691d0cc2733e25633c4", "domain": "localhost", "path": "/", "expires": 1746582930.401271, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "_gitlab_session", "value": "f98c1667e8eacdb4484308adbc9212fd", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": []}
{"cookies": [{"name": "admin", "value": "9323267b2c1a7ce0038c202f8e5f11e2", "domain": "localhost", "path": "/admin", "expires": 1746188874.136525, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "known_sign_in", "value": "dHZYdUU5bWdXdVpLZzBLdFhaZlFEdzFCK2w3S2UvWEVIa2RKYXJrSWhuSmFEZCtNSWREOGxUTFN2OWdPVTBkT3hXNDcrK1BpSGh0anRaZG04VERPRTRXNU5xdmpsZ2NDRUtYdjliTEtDLzZqUXdhSjFTSnRmUVpYczRYNVhadDQtLWhHTVN2dWE5cEE5YndIZERDWlM0Vnc9PQ%3D%3D--e5ab573d57a9787537878ab8511f3e6c8f2e14c0", "domain": "localhost", "path": "/", "expires": 1747038475.060972, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "_gitlab_session", "value": "afc8840b248522e4c22df9c09eea9621", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": []}

2
VAB-WebArena-Lite/.auth/gitlab.shopping_state.json
View File

@ -1 +1 @@
{"cookies": [{"name": "mage-cache-storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-cache-storage-section-invalidation", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-messages", "value": "", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Strict"}, {"name": "recently_viewed_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_viewed_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "product_data_storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "section_data_ids", "value": "{%22messages%22:null%2C%22customer%22:null%2C%22compare-products%22:null%2C%22last-ordered-items%22:null%2C%22cart%22:null%2C%22directory-data%22:null%2C%22captcha%22:null%2C%22instant-purchase%22:null%2C%22loggedAsCustomer%22:null%2C%22persistent%22:null%2C%22review%22:null%2C%22wishlist%22:null%2C%22recently_viewed_product%22:null%2C%22recently_compared_product%22:null%2C%22product_data_storage%22:null%2C%22paypal-billing-agreement%22:null}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "private_content_version", "value": "5a1361e59223072bebd2793394207496", "domain": "localhost", "path": "/", "expires": 1779933329.571442, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "PHPSESSID", "value": "177a7b68734a2083018b3fb119634917", "domain": "localhost", "path": "/", "expires": 1776909330.114376, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "X-Magento-Vary", "value": "9bf9a599123e6402b85cde67144717a08b817412", "domain": "localhost", "path": "/", "expires": 1776909330.114451, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "known_sign_in", "value": "VjEvQms0R1Y0WFJwNzhWcXJmWUQzTlM5aEZhS1FLd3pIaGcrRHZoUUMxb2RtK2gzWFVMK1N1YjA0QTJiV1NPL1NwQWYrc3dpRlU3U3Q1Q3p4T0RNbWd4UVl2Vk5pR2dSbVBFTUhrQUhQMnBJOXlxaGVsdG5ycXFkVDQ2dDBBejYtLWNYZ3pJREcxbkVXWjVsZE9LMnIvZkE9PQ%3D%3D--860cb1881ab774ae0647d031b6a58789ca2dc43f", "domain": "localhost", "path": "/", "expires": 1746582931.071908, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "_gitlab_session", "value": "23ff9045fd7ed259aa91a5056469be23", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": [{"origin": "http://localhost:28082", "localStorage": [{"name": "mage-cache-storage", "value": "{}"}, {"name": "product_data_storage", "value": "{}"}, {"name": "mage-cache-storage-section-invalidation", "value": "{\"messages\":true,\"customer\":true,\"compare-products\":true,\"last-ordered-items\":true,\"cart\":true,\"directory-data\":true,\"captcha\":true,\"instant-purchase\":true,\"loggedAsCustomer\":true,\"persistent\":true,\"review\":true,\"wishlist\":true,\"recently_viewed_product\":true,\"recently_compared_product\":true,\"product_data_storage\":true,\"paypal-billing-agreement\":true}"}, {"name": "mage-cache-timeout", "value": "\"2026-04-23T01:55:28.860Z\""}, {"name": "recently_compared_product_previous", "value": "{}"}, {"name": "recently_viewed_product", "value": "{}"}, {"name": "recently_compared_product", "value": "{}"}, {"name": "recently_viewed_product_previous", "value": "{}"}]}]}
{"cookies": [{"name": "mage-cache-storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-cache-storage-section-invalidation", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-messages", "value": "", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Strict"}, {"name": "recently_viewed_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_viewed_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "product_data_storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "section_data_ids", "value": "{%22messages%22:null%2C%22customer%22:null%2C%22compare-products%22:null%2C%22last-ordered-items%22:null%2C%22cart%22:null%2C%22directory-data%22:null%2C%22captcha%22:null%2C%22instant-purchase%22:null%2C%22loggedAsCustomer%22:null%2C%22persistent%22:null%2C%22review%22:null%2C%22wishlist%22:null%2C%22recently_viewed_product%22:null%2C%22recently_compared_product%22:null%2C%22product_data_storage%22:null%2C%22paypal-billing-agreement%22:null}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "private_content_version", "value": "91153949620a6ed8beb692662b9f0fa9", "domain": "localhost", "path": "/", "expires": 1780388873.087364, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "PHPSESSID", "value": "4049f550bd0f410e375b3d61a9682780", "domain": "localhost", "path": "/", "expires": 1777364873.634066, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "X-Magento-Vary", "value": "9bf9a599123e6402b85cde67144717a08b817412", "domain": "localhost", "path": "/", "expires": 1777364873.634145, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "known_sign_in", "value": "UmM0Y3p0VVhiUy8zUEJuM2RtbEpZak1hekhiNmpkem1wTHFYSUJwQlV3dE9nN1d1d3NzSGRTK1VXVHZtWjZDcnVjdC9PdDBtT0t1K3M1YVppWnBRWm1TN3FvQzc0WjBxMDNWMVJUa0p3c05DKzFBWW1YRUV4OW9vdndTMEUxdVctLURQOVlzWjRhNHRacldhZWx3MjN5T2c9PQ%3D%3D--d3fab3d9a66d2f95ccfc7e0c82898029b76f48fe", "domain": "localhost", "path": "/", "expires": 1747038474.540493, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "_gitlab_session", "value": "dca7bec6f0988552639100fc326c76dd", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": [{"origin": "http://localhost:28082", "localStorage": [{"name": "mage-cache-storage", "value": "{}"}, {"name": "product_data_storage", "value": "{}"}, {"name": "mage-cache-storage-section-invalidation", "value": "{\"messages\":true,\"customer\":true,\"compare-products\":true,\"last-ordered-items\":true,\"cart\":true,\"directory-data\":true,\"captcha\":true,\"instant-purchase\":true,\"loggedAsCustomer\":true,\"persistent\":true,\"review\":true,\"wishlist\":true,\"recently_viewed_product\":true,\"recently_compared_product\":true,\"product_data_storage\":true,\"paypal-billing-agreement\":true}"}, {"name": "mage-cache-timeout", "value": "\"2026-04-28T08:27:52.329Z\""}, {"name": "recently_compared_product_previous", "value": "{}"}, {"name": "recently_viewed_product", "value": "{}"}, {"name": "recently_compared_product", "value": "{}"}, {"name": "recently_viewed_product_previous", "value": "{}"}]}]}

2
VAB-WebArena-Lite/.auth/gitlab_state.json
View File

@ -1 +1 @@
{"cookies": [{"name": "known_sign_in", "value": "WHpKbmQyNkhlWWhHcGljQmZmVGE2aFl0WFh3VWxGMTRRYjBsa3dNbVJZLzIrYWM5RUpwa0IxYU85Q291WTE5cE9SWVcvdC9xTjNBTFp2ZHdCTGxaaEx6RXpDVm0zS1BCVldyODdMYlZuTWRHNFF0K0JQZXVqcGhPWWVkMWZWMmktLVNCc3F6VlFFQjBFSGN0TnlKUnd2MEE9PQ%3D%3D--752f2d15a69617f851d3140acb5500b2564af08f", "domain": "localhost", "path": "/", "expires": 1746582928.781901, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "_gitlab_session", "value": "85d65f4d47be2142eee2a21e8bbcec79", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": []}
{"cookies": [{"name": "known_sign_in", "value": "NEtLdkV3NXE0aUtodVRLUStQVEN2MmxIRU9mVGQrWStWUXl0cEdFeFNRV1hwV1VBODhRekQvYktMRE9HMHlqK2MzeWk1dEdqYjNBeGhFOGMweVowSWNCOUFJdi85aDh2dTlmOWJpTDh6d1gyTTRkZTFOS1FBSXB4Ni95dUdtcSstLXQrNzRlZmhkeWtIV29GNTBMWUt5VFE9PQ%3D%3D--c877605ff57e7912903f7c49fec4a75e62d340bb", "domain": "localhost", "path": "/", "expires": 1747038472.053481, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "_gitlab_session", "value": "b9c2a242f10712bf16389914a6d4c657", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": []}

2
VAB-WebArena-Lite/.auth/reddit_state.json
View File

@ -1 +1 @@
{"cookies": [{"name": "_cookie_check", "value": "1", "domain": "localhost", "path": "/login", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "REMEMBERME", "value": "QXBwXEVudGl0eVxVc2VyOlRXRnlkbVZzYzBkeVlXNTBUV0Z1TVRNMjoxNzc2OTA5MzI4OlpUQTRNV1F3WldNMU1HSmhOalUwTTJaaE1qWmxOVGs1TldNMlpUVTFNVEUyT1RRNFpqTXdaR1F3TVRRelpUTXhPVEl3TjJSa09USm1PREF4Wm1WaE5nPT0%3D", "domain": "localhost", "path": "/", "expires": 1776909328.956723, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "PHPSESSID", "value": "4a07cf057d5020f791451af6c2bc29e0", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": []}
{"cookies": [{"name": "_cookie_check", "value": "1", "domain": "localhost", "path": "/login", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "REMEMBERME", "value": "QXBwXEVudGl0eVxVc2VyOlRXRnlkbVZzYzBkeVlXNTBUV0Z1TVRNMjoxNzc3MzY0ODcyOk9UZ3pZekpqWmpKbU5qWXdOR05oWldZM1ptVTVObUl6TnpVNVlqWXpOVGc1TXpJek1UQTRaR0l6WkROak5EY3dZbVpsTlRBeFpqRm1ZalZrTnpNeVlRPT0%3D", "domain": "localhost", "path": "/", "expires": 1777364872.192796, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "PHPSESSID", "value": "79b96e107ca3dc36826ddd4ef27e8380", "domain": "localhost", "path": "/", "expires": -1, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": []}

2
VAB-WebArena-Lite/.auth/shopping.shopping_admin_state.json
View File

@ -1 +1 @@
{"cookies": [{"name": "admin", "value": "732e6b7c1a7ab1de0d1a4ab105b549b6", "domain": "localhost", "path": "/admin", "expires": 1745733331.781848, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "mage-cache-storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-cache-storage-section-invalidation", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-messages", "value": "", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Strict"}, {"name": "recently_viewed_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_viewed_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "product_data_storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "section_data_ids", "value": "{%22messages%22:null%2C%22customer%22:null%2C%22compare-products%22:null%2C%22last-ordered-items%22:null%2C%22cart%22:null%2C%22directory-data%22:null%2C%22captcha%22:null%2C%22instant-purchase%22:null%2C%22loggedAsCustomer%22:null%2C%22persistent%22:null%2C%22review%22:null%2C%22wishlist%22:null%2C%22recently_viewed_product%22:null%2C%22recently_compared_product%22:null%2C%22product_data_storage%22:null%2C%22paypal-billing-agreement%22:null}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "private_content_version", "value": "c694b83ea1426b503effa8c412f7ae6d", "domain": "localhost", "path": "/", "expires": 1779933329.584947, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "PHPSESSID", "value": "ac8e0a869234bb4b31a917ff6a016ce9", "domain": "localhost", "path": "/", "expires": 1776909330.149523, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "X-Magento-Vary", "value": "9bf9a599123e6402b85cde67144717a08b817412", "domain": "localhost", "path": "/", "expires": 1776909330.149613, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": [{"origin": "http://localhost:28082", "localStorage": [{"name": "mage-cache-storage", "value": "{}"}, {"name": "product_data_storage", "value": "{}"}, {"name": "mage-cache-storage-section-invalidation", "value": "{\"messages\":true,\"customer\":true,\"compare-products\":true,\"last-ordered-items\":true,\"cart\":true,\"directory-data\":true,\"captcha\":true,\"instant-purchase\":true,\"loggedAsCustomer\":true,\"persistent\":true,\"review\":true,\"wishlist\":true,\"recently_viewed_product\":true,\"recently_compared_product\":true,\"product_data_storage\":true,\"paypal-billing-agreement\":true}"}, {"name": "mage-cache-timeout", "value": "\"2026-04-23T01:55:28.879Z\""}, {"name": "recently_compared_product_previous", "value": "{}"}, {"name": "recently_viewed_product", "value": "{}"}, {"name": "recently_compared_product", "value": "{}"}, {"name": "recently_viewed_product_previous", "value": "{}"}]}]}
{"cookies": [{"name": "admin", "value": "743a462d8c7ba87e7f0bc8d10014bb14", "domain": "localhost", "path": "/admin", "expires": 1746188875.196865, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "mage-cache-storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-cache-storage-section-invalidation", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-messages", "value": "", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Strict"}, {"name": "recently_viewed_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_viewed_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "product_data_storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "section_data_ids", "value": "{%22messages%22:null%2C%22customer%22:null%2C%22compare-products%22:null%2C%22last-ordered-items%22:null%2C%22cart%22:null%2C%22directory-data%22:null%2C%22captcha%22:null%2C%22instant-purchase%22:null%2C%22loggedAsCustomer%22:null%2C%22persistent%22:null%2C%22review%22:null%2C%22wishlist%22:null%2C%22recently_viewed_product%22:null%2C%22recently_compared_product%22:null%2C%22product_data_storage%22:null%2C%22paypal-billing-agreement%22:null}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "private_content_version", "value": "5286c4b2846a04da47aa7ce906dbc1e3", "domain": "localhost", "path": "/", "expires": 1780388873.101019, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "PHPSESSID", "value": "8b5762bd0d55c2fe25ab877d9ca1c0fe", "domain": "localhost", "path": "/", "expires": 1777364873.628921, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "X-Magento-Vary", "value": "9bf9a599123e6402b85cde67144717a08b817412", "domain": "localhost", "path": "/", "expires": 1777364873.629015, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": [{"origin": "http://localhost:28082", "localStorage": [{"name": "mage-cache-storage", "value": "{}"}, {"name": "product_data_storage", "value": "{}"}, {"name": "mage-cache-storage-section-invalidation", "value": "{\"messages\":true,\"customer\":true,\"compare-products\":true,\"last-ordered-items\":true,\"cart\":true,\"directory-data\":true,\"captcha\":true,\"instant-purchase\":true,\"loggedAsCustomer\":true,\"persistent\":true,\"review\":true,\"wishlist\":true,\"recently_viewed_product\":true,\"recently_compared_product\":true,\"product_data_storage\":true,\"paypal-billing-agreement\":true}"}, {"name": "mage-cache-timeout", "value": "\"2026-04-28T08:27:52.378Z\""}, {"name": "recently_compared_product_previous", "value": "{}"}, {"name": "recently_viewed_product", "value": "{}"}, {"name": "recently_compared_product", "value": "{}"}, {"name": "recently_viewed_product_previous", "value": "{}"}]}]}

2
VAB-WebArena-Lite/.auth/shopping_admin_state.json
View File

@ -1 +1 @@
{"cookies": [{"name": "admin", "value": "753cf7bedc7f97223f77af0369ec4ccf", "domain": "localhost", "path": "/admin", "expires": 1745733329.396607, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": []}
{"cookies": [{"name": "admin", "value": "419103f3c04869358ff951ef01b0525e", "domain": "localhost", "path": "/admin", "expires": 1746188874.062332, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": []}

2
VAB-WebArena-Lite/.auth/shopping_state.json
View File

@ -1 +1 @@
{"cookies": [{"name": "mage-cache-storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-cache-storage-section-invalidation", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-messages", "value": "", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Strict"}, {"name": "recently_viewed_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_viewed_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "product_data_storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "section_data_ids", "value": "{%22messages%22:null%2C%22customer%22:null%2C%22compare-products%22:null%2C%22last-ordered-items%22:null%2C%22cart%22:null%2C%22directory-data%22:null%2C%22captcha%22:null%2C%22instant-purchase%22:null%2C%22loggedAsCustomer%22:null%2C%22persistent%22:null%2C%22review%22:null%2C%22wishlist%22:null%2C%22recently_viewed_product%22:null%2C%22recently_compared_product%22:null%2C%22product_data_storage%22:null%2C%22paypal-billing-agreement%22:null}", "domain": "localhost", "path": "/", "expires": 1776909328, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "private_content_version", "value": "db7c8357df43bdda4423b1d0be6cd1c0", "domain": "localhost", "path": "/", "expires": 1779933329.561383, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "PHPSESSID", "value": "9d34565b19e00a4310aba625db3a7fd7", "domain": "localhost", "path": "/", "expires": 1776909330.076342, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "X-Magento-Vary", "value": "9bf9a599123e6402b85cde67144717a08b817412", "domain": "localhost", "path": "/", "expires": 1776909330.076442, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": [{"origin": "http://localhost:28082", "localStorage": [{"name": "mage-cache-storage", "value": "{}"}, {"name": "product_data_storage", "value": "{}"}, {"name": "mage-cache-storage-section-invalidation", "value": "{\"messages\":true,\"customer\":true,\"compare-products\":true,\"last-ordered-items\":true,\"cart\":true,\"directory-data\":true,\"captcha\":true,\"instant-purchase\":true,\"loggedAsCustomer\":true,\"persistent\":true,\"review\":true,\"wishlist\":true,\"recently_viewed_product\":true,\"recently_compared_product\":true,\"product_data_storage\":true,\"paypal-billing-agreement\":true}"}, {"name": "mage-cache-timeout", "value": "\"2026-04-23T01:55:28.896Z\""}, {"name": "recently_compared_product_previous", "value": "{}"}, {"name": "recently_viewed_product", "value": "{}"}, {"name": "recently_compared_product", "value": "{}"}, {"name": "recently_viewed_product_previous", "value": "{}"}]}]}
{"cookies": [{"name": "mage-cache-storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-cache-storage-section-invalidation", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "mage-messages", "value": "", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Strict"}, {"name": "recently_viewed_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_viewed_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "recently_compared_product_previous", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "product_data_storage", "value": "{}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "section_data_ids", "value": "{%22messages%22:null%2C%22customer%22:null%2C%22compare-products%22:null%2C%22last-ordered-items%22:null%2C%22cart%22:null%2C%22directory-data%22:null%2C%22captcha%22:null%2C%22instant-purchase%22:null%2C%22loggedAsCustomer%22:null%2C%22persistent%22:null%2C%22review%22:null%2C%22wishlist%22:null%2C%22recently_viewed_product%22:null%2C%22recently_compared_product%22:null%2C%22product_data_storage%22:null%2C%22paypal-billing-agreement%22:null}", "domain": "localhost", "path": "/", "expires": 1777364872, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "private_content_version", "value": "fcd849367fdcd20799362fe284ca22e2", "domain": "localhost", "path": "/", "expires": 1780388873.078761, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "PHPSESSID", "value": "9448062b70e7d5ba339af29d1700080b", "domain": "localhost", "path": "/", "expires": 1777364873.561179, "httpOnly": true, "secure": false, "sameSite": "Lax"}, {"name": "X-Magento-Vary", "value": "9bf9a599123e6402b85cde67144717a08b817412", "domain": "localhost", "path": "/", "expires": 1777364873.561281, "httpOnly": true, "secure": false, "sameSite": "Lax"}], "origins": [{"origin": "http://localhost:28082", "localStorage": [{"name": "mage-cache-storage", "value": "{}"}, {"name": "product_data_storage", "value": "{}"}, {"name": "mage-cache-storage-section-invalidation", "value": "{\"messages\":true,\"customer\":true,\"compare-products\":true,\"last-ordered-items\":true,\"cart\":true,\"directory-data\":true,\"captcha\":true,\"instant-purchase\":true,\"loggedAsCustomer\":true,\"persistent\":true,\"review\":true,\"wishlist\":true,\"recently_viewed_product\":true,\"recently_compared_product\":true,\"product_data_storage\":true,\"paypal-billing-agreement\":true}"}, {"name": "mage-cache-timeout", "value": "\"2026-04-28T08:27:52.323Z\""}, {"name": "recently_compared_product_previous", "value": "{}"}, {"name": "recently_viewed_product", "value": "{}"}, {"name": "recently_compared_product", "value": "{}"}, {"name": "recently_viewed_product_previous", "value": "{}"}]}]}

63
VAB-WebArena-Lite/agent/prompts/prompt_constructor.py
View File

@ -556,6 +556,69 @@ class WebRLPromptConstructor(PromptConstructor):
def extract_action(self, response: str) -> str:
return response
class PlanAndActPromptConstructor(PromptConstructor):
"""The agent will direct predict the action"""
def __init__(
self,
instruction_path: str | Path,
lm_config: lm_config.LMConfig,
tokenizer: Tokenizer,
):
super().__init__(instruction_path, lm_config, tokenizer)
def construct(
self,
trajectory: Trajectory,
intent: str,
meta_data: dict[str, Any] = {},
) -> APIInput:
"""Construct prompt given the trajectory"""
state_info: StateInfo = trajectory[-1] # type: ignore[assignment]
obs = state_info["observation"][self.obs_modality]
max_obs_length = self.lm_config.gen_config["max_obs_length"]
if max_obs_length:
if self.lm_config.provider == "google":
print("NOTE: This is a Gemini model, so we use characters instead of tokens for max_obs_length.")
obs = obs[:max_obs_length]
else:
try:
obs = self.tokenizer.decode(self.tokenizer.encode(obs)[:max_obs_length]) # type: ignore[arg-type]
except:
print("NOTE: There is no available tokenizer, so we use characters instead of tokens for max_obs_length.")
obs = obs[:max_obs_length]
turn_num = len(meta_data["action_history"])
if turn_num == 1:
previous_action_str = []
else:
previous_action_str = meta_data["action_history"][1:]
index = turn_num - 1
history = ""
for i in range(index - 1, -1, -1):
if i == 0:
history = f"Round {i}\n\n<|eot_id|><|start_header_id|>user<|end_header_id|>\n{intent}\n\n<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n{previous_action_str[i]}\n\n" + history
else:
history = f"Round {i}\n\n<|eot_id|><|start_header_id|>user<|end_header_id|>\n** Simplified html **\n\n<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n{previous_action_str[i]}\n\n" + history
if len(history) + len(obs) > (16384 - 512):
obs = obs[:(16384 - 512)-len(history)]
current_turn = f"Round {index}\n\n<|eot_id|><|start_header_id|>user<|end_header_id|>\n{obs}\n\n<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n"
# Goal
prompt = f"Task Instruction: {intent}\n\n{history}{current_turn}"
return prompt
def extract_action(self, response: str) -> str:
return response
class WebRLChatPromptConstructor(PromptConstructor):
"""The agent will direct predict the action"""

4002
VAB-WebArena-Lite/config_files/wa/test_webarena_gen.json Normal file
View File

File diff suppressed because it is too large Load Diff

24
VAB-WebArena-Lite/config_files/wa/test_webarena_gen.json.old Normal file
View File

@ -0,0 +1,24 @@
[
{
"sites": [
"shopping_admin"
],
"task_id": 0,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "http://localhost:28083/admin",
"geolocation": null,
"intent_template": "",
"instantiation_dict": {
},
"intent": "Examine the details of the 10th order with the largest transaction volume from those that have a Pending status, and a base total exceeding 95.7.",
"require_reset": false,
"eval": {
"eval_types": [
"orm_match"
],
"orm_model": "aiproxy/deepseek-reasoner"
},
"intent_template_id": 0
}
]

4002
VAB-WebArena-Lite/config_files/wa/test_webarena_gen.raw.json Normal file
View File

File diff suppressed because it is too large Load Diff

24
VAB-WebArena-Lite/config_files/wa/test_webarena_gen.raw.json.old Normal file
View File

@ -0,0 +1,24 @@
[
{
"sites": [
"shopping_admin"
],
"task_id": 0,
"require_login": true,
"storage_state": "./.auth/shopping_admin_state.json",
"start_url": "__SHOPPING_ADMIN__",
"geolocation": null,
"intent_template": "",
"instantiation_dict": {
},
"intent": "Examine the details of the 10th order with the largest transaction volume from those that have a Pending status, and a base total exceeding 95.7.",
"require_reset": false,
"eval": {
"eval_types": [
"orm_match"
],
"orm_model": "aiproxy/deepseek-reasoner"
},
"intent_template_id": 0
}
]

1
VAB-WebArena-Lite/debug_info/all_element.json
View File

File diff suppressed because one or more lines are too long

BIN
VAB-WebArena-Lite/debug_info/marked.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 158 KiB

After

Width:  |  Height:  |  Size: 60 KiB

2
VAB-WebArena-Lite/debug_info/parsed.html
View File

File diff suppressed because one or more lines are too long

2314
VAB-WebArena-Lite/debug_info/raw.html
View File

File diff suppressed because one or more lines are too long

BIN
VAB-WebArena-Lite/debug_info/screenshot_raw.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 490 KiB

After

Width:  |  Height:  |  Size: 44 KiB

132
VAB-WebArena-Lite/evaluation_harness/evaluators.py
View File

@ -26,6 +26,7 @@ from evaluation_harness.helper_functions import (
get_query_text_lowercase,
gitlab_get_project_memeber_role,
llm_fuzzy_match,
llm_orm_match,
llm_ua_match,
reddit_get_latest_comment_content_by_username,
reddit_get_latest_comment_obj_by_username,
@ -627,6 +628,125 @@ class EvaluatorComb:
return score
# +++ 开始添加 OrmEvaluator 类 +++
@beartype
class OrmEvaluator(Evaluator):
"""
Evaluates task completion using an LLM based on intent, action history, and final HTML state.
Mirrors the logic from plannact/results/orm_test.py evaluate_task_completion.
"""
def __init__(self, eval_tag: str = "", model_name: str = "aiproxy/deepseek-reasoner"):
"""
Initializes the OrmEvaluator.
Args:
eval_tag: An optional tag for the evaluator.
model_name: The name of the OpenAI-compatible model to use for evaluation.
"""
super().__init__(eval_tag)
self.model = model_name
def __call__(
self,
trajectory: Trajectory,
config_file: Path | str,
page: Page | PseudoPage | None = None # page argument is not used by this evaluator
) -> float:
"""
Evaluates the trajectory using an LLM.
Args:
trajectory: The sequence of actions and states.
config_file: Path to the task configuration JSON file.
page: The final browser page state (not used by this evaluator).
Returns:
1.0 if the LLM evaluation is "YES", 0.0 otherwise.
"""
if not self.model:
print("OrmEvaluator Error: no model selected. Returning score 0.0.")
return 0.0
try:
with open(config_file, "r", encoding='utf-8') as f: # Added encoding
configs = json.load(f)
except FileNotFoundError:
print(f"OrmEvaluator Error: Config file not found at {config_file}. Returning score 0.0.")
return 0.0
except json.JSONDecodeError:
print(f"OrmEvaluator Error: Could not decode JSON from {config_file}. Returning score 0.0.")
return 0.0
except Exception as e:
print(f"OrmEvaluator Error: An unexpected error occurred while reading {config_file}: {e}. Returning score 0.0.")
return 0.0
# 1. Extract Intent
intent = configs.get("intent")
if not intent:
print(f"OrmEvaluator Warning: 'intent' not found in config file {config_file}.")
intent = "No intent provided." # Provide a default or handle as error
# 2. Extract Action History String
# Filter only Action objects and convert them to string representation
actions_list = [item["raw_prediction"] for item in trajectory[1::2] if is_bearable(item, Action)]
action_history_str = "\n".join(map(str, actions_list))
if not action_history_str:
action_history_str = "[No actions recorded in trajectory]"
# 3. Extract HTML of Last State
# The last state should be the second to last element (before the final STOP action, if any)
last_state = None
if len(trajectory) >= 2 and is_bearable(trajectory[-2], StateInfo):
last_state = trajectory[-2]
# Check the last element as a fallback? Less standard.
# elif len(trajectory) >= 1 and is_bearable(trajectory[-1], StateInfo):
# last_state = trajectory[-1] # Less likely, trajectory usually ends with Action
html_of_last_state = "<html><body>Error retrieving HTML.</body></html>" # Default error HTML
if last_state:
# Try accessing 'html' attribute first, then 'text' in 'obs' as fallback
html_of_last_state = last_state['observation'].get('text', "<html><body>HTML not found in last state obs.</body></html>")
else:
print("OrmEvaluator Warning: Could not find a StateInfo object as the second to last element in the trajectory.")
# 4. Call evaluation logic
print(f"OrmEvaluator: Evaluating Task Completion for config: {config_file}...")
evaluation_result, llm_raw_content = self.evaluate_with_llm(model=self.model, instruction=intent,
action_history=action_history_str,
html_of_last_state=html_of_last_state)
# 5. Determine score
is_complete = evaluation_result == "YES"
score = 1.0 if is_complete else 0.0
evaluation_status = "Unknown" # For logging
if evaluation_result == "YES":
evaluation_status = "YES"
elif evaluation_result == "NO":
evaluation_status = "NO"
elif evaluation_result is None:
evaluation_status = "Failed/Error/Unexpected"
print(f"OrmEvaluator: Evaluation Result: {evaluation_status}, Score: {score}")
# Optional: Log raw content for debugging
print(f"OrmEvaluator: LLM Raw Content:\n{llm_raw_content}")
return score
@staticmethod
@beartype
def evaluate_with_llm(model, instruction: str, action_history: str, html_of_last_state: str) -> Tuple[Optional[str], Optional[str]]:
return llm_orm_match(model, instruction, action_history, html_of_last_state)
# --- End OrmEvaluator ---
@beartype
def evaluator_router(
config_file: Path | str, captioning_fn=None
@ -636,7 +756,9 @@ def evaluator_router(
configs = json.load(f)
eval_types = configs["eval"]["eval_types"]
evaluators: list[Evaluator | EvaluatorPartial] = []
# Corrected type hint, EvaluatorPartial is not defined, assuming Evaluator is sufficient
# evaluators: list[Evaluator | EvaluatorPartial] = []
evaluators: list[Evaluator] = []
for eval_type in eval_types:
match eval_type:
case "string_match":
@ -646,7 +768,15 @@ def evaluator_router(
case "program_html":
evaluators.append(HTMLContentExactEvaluator())
case "page_image_query":
if captioning_fn is None:
raise ValueError("captioning_fn must be provided for page_image_query evaluator")
evaluators.append(PageImageEvaluator(captioning_fn))
# +++ 添加 orm_match case +++
case "orm_match":
# Allow overriding the model from config file if needed
orm_model = configs["eval"]["orm_model"]
evaluators.append(OrmEvaluator(model_name=orm_model))
# --- 结束添加 ---
case _:
raise ValueError(f"eval_type {eval_type} is not supported")

102
VAB-WebArena-Lite/evaluation_harness/helper_functions.py
View File

@ -4,7 +4,7 @@ from datetime import datetime, timezone
import os
from typing import Any, Union
from urllib.parse import urlparse
import re
import requests
from beartype import beartype
from beartype.typing import Dict, List
@ -652,3 +652,103 @@ def llm_ua_match(pred: str, reference: str, question: str) -> float:
else:
assert "same" in response
return 1.0
# --- Constants for OrmEvaluator ---
ORM_SYSTEM_PROMPT = """
You are an expert in evaluating the performance of a website navigation agent.
The agent is designed to help a human user navigate the website to complete a task.
Given the user's intent, the agent's action history, and the final state of the screen,
your goal is to decide whether the agent's execution is successful or not.
You must respond with YES or NO.
"""
ORM_USER_PROMPT_TEMPLATE = """
The User Intent: {instruction}
Action History: {action_history}
The Current Screenshot: {html_of_last_state}
"""
@beartype
def llm_orm_match(model: str, instruction: str, action_history: str, html_of_last_state: str):
"""
Calls the configured LLM API to evaluate task completion.
Args:
instruction: The user's intent.
action_history: String representation of the agent's actions.
html_of_last_state: HTML content of the final screen state.
Returns:
A tuple containing:
- The evaluation result ("YES", "NO", or None if error/unexpected).
- The raw content string returned by the LLM (or error message).
"""
if not all([instruction, action_history is not None, html_of_last_state]):
print("OrmEvaluator Error: Missing required data (instruction, action_history, or html_of_last_state) for LLM evaluation.")
return None, "Internal Error: Missing data for evaluation."
# Ensure HTML is not excessively long - truncate if necessary
# This is a simple truncation, might need more sophisticated handling
max_html_length = 30000 # Adjust as needed, balance context and token limits
if len(html_of_last_state) > max_html_length:
print(f"OrmEvaluator Warning: HTML content truncated from {len(html_of_last_state)} to {max_html_length} characters.")
html_of_last_state = html_of_last_state[:max_html_length] + "\n... [HTML truncated]"
user_prompt_content = ORM_USER_PROMPT_TEMPLATE.format(
instruction=instruction,
action_history=action_history,
html_of_last_state=html_of_last_state
)
# Using the specific prompt format from orm_test.py
prompt = f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{ORM_SYSTEM_PROMPT}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{user_prompt_content}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
raw_content = None
evaluation = None
try:
response = generate_from_openai_chat_completion(
model=model,
messages=[
# The entire formatted prompt goes into the user message content
{"role": "user", "content": prompt}
],
temperature=0,
max_tokens=768,
top_p=1.0,
context_length=0,
api_key=os.environ["OPENAI_API_KEY_FUZZY"],
base_url=os.environ["OPENAI_API_URL_FUZZY"],
).lower()
raw_content = response
# print(f"OrmEvaluator DEBUG: Raw LLM Response: {raw_content}") # Debugging line
# Process content: remove potential <think> tags and strip whitespace
processed_content = re.sub(r'<think>.*?</think>', '', raw_content, flags=re.DOTALL)
processed_content = processed_content.strip()
# Check if the response starts with YES or NO (case-insensitive)
result_match = re.match(r"^(YES|NO)", processed_content, re.IGNORECASE)
if result_match:
evaluation = result_match.group(1).upper()
else:
# Fallback: Check if YES or NO appears anywhere in the response
if "YES" in processed_content.upper():
evaluation = "YES"
print("OrmEvaluator Warning: 'YES' found but not at the start of the response.")
elif "NO" in processed_content.upper():
evaluation = "NO"
print("OrmEvaluator Warning: 'NO' found but not at the start of the response.")
else:
print(f"OrmEvaluator Warning: LLM response did not start with or contain YES/NO. Response: '{processed_content}'")
evaluation = None # Cannot determine outcome
except Exception as e:
print(f"OrmEvaluator Error: An error occurred during the API call to model '{model}': {e}")
raw_content = f"API Call Error: {e}"
evaluation = None
return evaluation, raw_content

142
VAB-WebArena-Lite/export.txt
View File

@ -1,25 +1,33 @@
1.0
0.0
1.0
0.0
1.0
1.0
0.0
0.0
1.0
0.0
0.0
1.0
0.0
0.0
0.0
1.0
1.0
1.0
0.0
1.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
1.0
1.0
0.0
1.0
0.0
1.0
0.0
1.0
0.0
0.0
@ -30,6 +38,17 @@
0.0
0.0
0.0
0.0
0.0
0.0
1.0
1.0
0.0
1.0
0.0
0.0
0.0
0.0
1.0
0.0
0.0
@ -37,24 +56,8 @@
0.0
0.0
1.0
1.0
1.0
0.0
0.0
1.0
0.0
0.0
1.0
0.0
0.0
1.0
0.0
0.0
0.0
0.0
0.0
1.0
0.0
0.0
1.0
0.0
@ -66,37 +69,8 @@
0.0
1.0
0.0
0.0
0.0
0.0
0.0
1.0
1.0
1.0
1.0
0.0
0.0
0.0
0.0
1.0
0.0
1.0
1.0
1.0
0.0
1.0
1.0
1.0
1.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
1.0
1.0
1.0
0.0
0.0
@ -107,30 +81,34 @@
0.0
1.0
0.0
1.0
0.0
1.0
0.0
1.0
0.0
1.0
0.0
0.0
0.0
0.0
1.0
1.0
0.0
0.0
1.0
1.0
0.0
0.0
0.0
0.0
0.0
1.0
1.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
@ -147,12 +125,6 @@
0.0
0.0
1.0
0.0
1.0
1.0
0.0
0.0
0.0
1.0
1.0
0.0
@ -160,6 +132,34 @@
0.0
0.0
0.0
0.0
0.0
0.0
1.0
0.0
0.0
0.0
0.0
0.0
0.0
1.0
0.0
1.0
0.0
1.0
0.0
0.0
0.0
0.0
0.0
0.0
1.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0
0.0

2
VAB-WebArena-Lite/scripts/generate_test_data.py
View File

@ -17,7 +17,7 @@ def main() -> None:
print(f"WIKIPEDIA: {WIKIPEDIA}")
print(f"MAP: {MAP}")
inp_paths = ["config_files/wa/test_webarena.raw.json", "config_files/wa/test_webarena_lite.raw.json"]
inp_paths = ["config_files/wa/test_webarena.raw.json", "config_files/wa/test_webarena_lite.raw.json", "config_files/wa/test_webarena_gen.raw.json"]
replace_map = {
"__REDDIT__": REDDIT,
"__SHOPPING__": SHOPPING,

142
plannact/doc/replication.md Normal file
View File

@ -0,0 +1,142 @@
好的，我们来系统地梳理一下复现 PLAN-AND-ACT 论文工作的完整过程和步骤，并分析每个步骤所需的 Prompt 及其在附录中的提供情况。
**复现目标：** 构建并评估一个由 PLANNER 和 EXECUTOR 组成的、具备动态 Replanning 能力的 LLM Agent，在 WebArena-Lite 基准上达到或接近论文报告的性能。
**核心组件：**
1. **PLANNER LLM:** 负责高级规划，将用户查询分解为结构化步骤。
2. **EXECUTOR LLM:** 负责低级执行，将计划步骤转化为具体环境（Web）动作。
3. **数据生成流水线:** 用于创建训练 PLANNER 和 EXECUTOR 的合成数据。
4. **动态 Replanning 机制:** 在执行过程中根据环境反馈更新计划。
**复现步骤详解：**
**阶段一：环境与模型准备 (Setup)**
1. **步骤 1.1: 获取基准测试环境**
* **描述:** 下载并设置 WebArena-Lite 基准测试环境。
* **所需:** WebArena-Lite 代码库和相关依赖。
* **Prompt 情况:** N/A (环境设置，非 Prompt 驱动)。
* **缺失分析:** 无，按基准测试说明操作。`map使用互联网版本，效果比论文明显差，数据暂排除`
2. **步骤 1.2: 获取基础 LLM**
* **描述:** 获取论文中使用的基础大语言模型。
* **所需:**
* LLaMA-3.3-70B-Instruct (用于 PLANNER 和 EXECUTOR 的微调基础)。
* GPT-4o (用于数据生成中的 Teacher 模型)。
* WebRL-Llama-3.1-70B (用于生成 Action Trajectory 的 Demonstrator Agent)。 `暂使用8b模型替代，准确率41%左右, 70B模型是49%`
* ORM-Llama-3.1-8B (用于过滤 Action Trajectory)。 `基本达到论文要求，但性能看r1-671b准确率，精确率和召回率都更好，可以替代`
* **Prompt 情况:** `已补全`。
* **缺失分析:** 需要能访问这些特定模型或找到性能相当的替代品。模型访问权限可能是个挑战。
**阶段二：合成数据生成 (Synthetic Data Generation)**
这是复现中最关键和复杂的部分，对应论文 Section 4 和 Figure 3。
1. **步骤 2.1: 初始用户查询生成 (Initial User Query Generation)**
* **描述:** 生成多样化的、适用于 Web 环境的初始合成用户查询。
* **方法:** 论文描述使用 Alpaca 风格，以 WebArena-Lite 训练集查询为种子，使用 LLM (GPT-4o) 生成新查询，并进行初步过滤。参考了 [25] 的方法。
* **Prompt 情况:** **缺失 (Likely Missing)**。附录主要关注 *Plan* 相关 Prompt。虽然描述了方法，但用于 *生成初始查询* 的具体 Prompt 文本未在附录中提供。
* **缺失获取方法:**
* **参考论文/方法:** 仔细研究论文 [25] 中可能包含的 Prompt 或方法描述。
* **自行构建:** 基于 Alpaca prompt 结构和 Sec 4.1 的描述，结合 WebArena-Lite 训练集中的查询示例，自行设计 Prompt，引导 GPT-4o 生成类似风格和主题的新查询。可能需要迭代调整 Prompt 以获得理想的多样性和相关性。
* **联系作者:** (成功率较低) 尝试联系论文作者询问。
2. **步骤 2.2: 动作轨迹生成 (Action Trajectory Generation)**
* **描述:** 使用 Demonstrator Agent (WebRL-Llama-3.1-70B) 尝试执行步骤 2.1 生成的查询，记录成功的 Action 序列。使用 ORM (ORM-Llama-3.1-8B) 过滤掉失败的轨迹。
* **方法:** Demonstrator Agent 接收查询和环境状态 (HTML)，输出动作。
* **Prompt 情况:** **部分提供**。
* Demonstrator Agent (作为执行者) 会使用类似 **Executor Prompt (Appendix A.4)** 的逻辑进行交互。
* ORM 的过滤逻辑/Prompt 未明确提供，但模型已指定。
* **缺失获取方法:** ORM 的使用可能需要参考 WebRL 论文 [25] 或根据其名称（Outcome-Supervised Reward Model）理解其功能，可能是一个分类模型或基于规则/启发式的过滤。
3. **步骤 2.3: 接地计划生成 (Grounded Plan Generation)**
* **描述:** 对于成功的 Action Trajectory，使用 Teacher LLM (GPT-4o) "反向工程" 出对应的、与 Trajectory Actions 绑定的高层结构化 Plan。
* **方法:** 输入 User Query, Initial HTML, Action Trajectory，输出结构化的 Plan (含 Reasoning, Description, Step, Actions 索引)。
* **Prompt 情况:** **已提供**。
* **Plan Data Annotator Prompt (Appendix A.5)** 提供了完整的系统提示和用户消息结构。
* **缺失分析:** 明确提到了需要 `{in_context_examples}` (Few-shot 示例)。这些具体的示例在附录中没有提供。
* **缺失获取方法:** 需要从 WebArena-Lite 训练数据中手动创建或挑选少量高质量的 (Query, Trajectory) -> Structured Plan 示例，作为 Prompt 的一部分。
4. **步骤 2.4: 合成计划扩展 (Synthetic Plan Expansion)**
* **描述:** 基于步骤 2.3 生成的 Query-Plan 对，生成更多、更多样化的合成 Query-Plan 对，以扩充 Planner 的训练数据。
* **方法:** 使用 Teacher LLM (GPT-4o)，以步骤 2.3 的数据作为种子/示例。
* **Prompt 情况:** **已提供**。
* **Synthetic Plan Generator Prompt (Appendix A.6)** 提供了系统提示和用户消息结构。
* **缺失分析:** 同样需要种子示例 (`{examples_str}`)，这些具体示例附录未提供。
* **缺失获取方法:** 使用步骤 2.3 生成的数据作为种子示例输入到 Prompt 中。
5. **步骤 2.5: 目标性计划增强 (Targeted Plan Augmentation)**
* **描述:** 针对模型在验证集上表现出的特定失败模式，生成额外的、有针对性的训练数据。
* **方法:**
* **失败分类:** 使用 LLM 根据 **Appendix A.7** 中的 Prompt (主系统提示 + 各网站失败类别定义) 对步骤 2.3/2.4 的数据进行分类，找出与失败模式相关的训练样本。
* **目标生成:** 使用 **Appendix A.8** 中的 Prompt (与 A.6 类似，但强调保持核心意图)，以分类出的相关样本作为种子，生成更多针对性数据。
* **Prompt 情况:** **已提供**。
* **Training Data Failure Classification Prompt (Appendix A.7)**。
* **Synthetic Plan Generation after Failure Analysis Prompt (Appendix A.8)**。
* **缺失分析:** 分类步骤需要实现，并将分类结果用于 A.8 的种子数据。
6. **步骤 2.6: Replanning 数据生成 (Replanning Data Generation)**
* **描述:** 为训练动态 Replanning 能力生成数据。
* **方法:** 使用 Teacher LLM (GPT-4o) 标注 *执行中途* 的状态：给定历史 Plan、当前 HTML、历史 Action 以及 *未来需要执行的 Action*，生成当前状态下 *应该* 更新成的 Plan (只规划未来步骤)。
* **Prompt 情况:** **已提供**。
* **Replanner Data Annotator Prompt (Appendix A.9)** 提供了系统提示和用户/助手消息格式。
* **缺失分析:** 需要 Action Trajectory 数据，并能模拟出“未来需要执行的 Action”作为 Prompt 输入。
**阶段三：模型训练 (Model Training - Finetuning)**
1. **步骤 3.1: 微调 EXECUTOR**
* **描述:** 使用基础模型 (LLaMA-3.3-70B-Instruct) 进行微调。
* **数据:** WebArena-Lite 原始训练数据 + 步骤 2.2 生成的合成 Action Trajectory。格式：(Plan Step, Current HTML, History) -> Action。
* **Prompt 情况:** N/A (训练过程)。训练数据的格式应与 **Executor Prompt (A.4)** 的推理格式对应。
* **缺失分析:** Finetuning 的具体超参数（学习率、batch size、epoch 数等）、数据格式化细节、优化器选择等未提供。
* **缺失获取方法:** 基于 LLaMA 微调的常规实践设置超参数，或参考相关 Agent 微调论文。
2. **步骤 3.2: 微调 PLANNER (静态)**
* **描述:** 使用基础模型 (LLaMA-3.3-70B-Instruct) 进行微调。
* **数据:** 步骤 2.3 (Grounded Plans) + 步骤 2.4 (Expanded Plans) + 步骤 2.5 (Targeted Plans)。格式：(User Query, Initial HTML) -> Structured Plan。
* **Prompt 情况:** N/A (训练过程)。训练数据格式应与 **Planner Prompt (A.3)** / **Plan Annotator Output (A.5)** 的推理格式对应。
* **缺失分析:** 同步骤 3.1，Finetuning 细节缺失。
* **缺失获取方法:** 同步骤 3.1。
3. **步骤 3.3: 微调 PLANNER (动态 Replanning - 使用 LoRA)**
* **描述:** 在步骤 3.2 微调好的 Planner 基础上，使用 LoRA 进一步微调以具备 Replanning 能力。
* **数据:** 步骤 2.6 生成的 Replanning 数据。格式遵循 **Appendix A.9/A.10** 的用户/助手消息历史。
* **Prompt 情况:** N/A (训练过程)。
* **缺失分析:** LoRA 的具体配置（rank, alpha 等）、Finetuning 超参数、数据格式化细节缺失。
* **缺失获取方法:** 参考 LoRA 微调的最佳实践和相关论文。
**阶段四：推理与评估 (Inference and Evaluation)**
1. **步骤 4.1: 静态 PLAN-AND-ACT 推理**
* **描述:** 使用微调后的静态 PLANNER (步骤 3.2) 和 EXECUTOR (步骤 3.1) 在 WebArena-Lite 测试集上运行。Planner 只在开始时调用一次。
* **Prompt 情况:** **已提供**。
* **Planner Prompt (A.3)** 用于生成初始计划。
* **Executor Prompt (A.4)** 用于根据计划和 HTML 执行动作。
2. **步骤 4.2: 动态 PLAN-AND-ACT 推理**
* **描述:** 使用具备 Replanning 能力的 PLANNER (步骤 3.3) 和 EXECUTOR (步骤 3.1) 在 WebArena-Lite 测试集上运行。Planner 在每次 Executor 执行动作后都可能被调用以更新计划。
* **Prompt 情况:** **已提供**。
* **Planner Prompt (A.3)** 用于生成 *初始* 计划。
* **Replanner Prompt (A.10)** 用于在后续步骤中根据历史和当前 HTML 更新计划。
* **Executor Prompt (A.4)** 用于执行动作。
3. **步骤 4.3: 运行基线模型**
* **描述:** 运行论文中提到的基线模型（Zero-shot LLaMA, ReAct Finetuned LLaMA, WebRL）进行比较。
* **Prompt 情况:** ReAct 需要其标准 Prompt (如论文 [42] 所述)，其他基线按其各自方法运行。
4. **步骤 4.4: 计算成功率**
* **描述:** 使用 WebArena-Lite 提供的评估脚本计算任务成功率。
* **Prompt 情况:** N/A (评估脚本)。
**总结：**
* **已提供的 Prompt:** 覆盖了 PLANNER 推理、EXECUTOR 推理、从 Trajectory 生成 Plan、扩展 Plan、目标性增强 Plan、生成 Replanning 训练数据、Replanning 推理等核心环节的 Prompt 结构和指南。这对于理解系统的工作方式和数据生成逻辑至关重要。
* **可能缺失的 Prompt/信息:**
* **初始用户查询生成 Prompt (最可能缺失)。**
* **具体的 Few-Shot 示例 (普遍缺失)。**
* **ORM 过滤细节。**
* **所有 Finetuning 过程的超参数和具体设置。**
* **获取方法:** 主要依靠对论文描述、引用文献的研读、基于 Alapca/LoRA 等公开方法的实践经验，以及在复现过程中进行合理的实验和调整。手动构建高质量的 Few-Shot 示例是必要的。
总而言之，附录提供了非常丰富的 Prompt 信息，使得复现具有可行性，但达到与原文完全一致的结果需要补充一些关键细节，特别是数据生成的第一步和模型微调的具体参数。

261
plannact/doc/working_plan.md Normal file
View File

@ -0,0 +1,261 @@
# WebArena 工作推进计划
## 1. 总体计划
1. 运行webarena-lite，评测webrl， gpt-4o模型， deepseek-r1模型效果，并分析原因。
2. 复现Plan-and-Act 数据合成及模型训练管线；
3. 分析Plan-and-Act的问题，思考改进方向；
## 2. 进展
### 2.1 第一次评测
Webarena-Lite测试集总共165条，
使用VAB中缺省`temperature=0.0`
- webr1-llama-3.1-8b: SR 18.3%; finished 161
- gpt-4o: SR 15.2%; finished 164
- deepseek-r1: SR 19.3%; finished 98
分类分析：
shopping_admin: webrl 29%, gpt-4o 11.43%, deepseek-r1 19.05%
站点: shopping_admin
webrl:
通过数量: 10
失败数量: 24
通过率: 29.41%
通过的case IDs: ['110', '113', '115', '125', '145', '17', '21', '40', '50', '88']
失败的case IDs: ['0', '101', '106', '107', '108', '116', '126', '13', '149', '150', '160', '2', '26', '27', '30', '32', '33', '47', '48', '51', '59', '70', '8', '80']
gpt4o:
通过数量: 4
失败数量: 31
通过率: 11.43%
通过的case IDs: ['17', '21', '40', '88']
失败的case IDs: ['0', '101', '106', '107', '108', '110', '113', '114', '115', '116', '125', '126', '13', '145', '149', '150', '160', '2', '26', '27', '30', '32', '33', '47', '48', '50', '51', '59', '70', '8', '80']
r1:
通过数量: 4
失败数量: 17
通过率: 19.05%
通过的case IDs: ['125', '21', '40', '8']
失败的case IDs: ['113', '115', '126', '13', '145', '149', '160', '17', '2', '26', '27', '30', '32', '33', '47', '50', '51']
### 2.2 测试orm
基于上周使用webrl进行评测的160条测试轨迹，使用orm和gpt-4o， r1判断是否完成目标，
prompt使用的webrl论文附录Figure 17提供的提示词。
分析：
- orm的准确率只有42，精确率很低才15，意味着着判断为完成的轨迹大部分是假阳。
- orm对模板敏感，使用新prompt模板后，正确率提高到81,精确率达到40，召回36；
- gpt-4o 准确率高达86，精确率56，判断成功的轨迹中有一半是对的。
- r1的准确率高达88，精确率 66 意味着假阳性降低到1/3，但是召回率48，和gpt-4o（52）基本持平。
初步结论：
- 当前设定下, temperature值为0.0
- orm配合正确的prompt，accuracy基本达到论文中的水平，实测81%，论文test set 80, rollout 79。
- 在当前prompt的条件下，r1 能够取得最高的正确率和精确率，假阳1/3，召回一半成功的轨迹。
- 可能优化方向：优化prompt，提供更多上下文信息，譬如中间历史网页，网页url等等。
gpt-4o
```
{
"accuracy": 0.8625,
"precision": 0.5652173913043478,
"recall": 0.52,
"f1_score": 0.5416666666666666,
"total_samples_analyzed": 160
}
```
orm
```
{
"accuracy": 0.425,
"precision": 0.15463917525773196,
"recall": 0.6,
"f1_score": 0.2459016393442623,
"total_samples_analyzed": 160
}
使用新prompt模板后
Accuracy: 0.8187
Precision (for Positive class 'True'): 0.4091
Recall (for Positive class 'True'): 0.3600
F1 Score (for Positive class 'True'): 0.3830
Total Samples Analyzed: 160
```
deepseek-r1
```
{
"accuracy": 0.88125,
"precision": 0.6666666666666666,
"recall": 0.48,
"f1_score": 0.5581395348837209,
"total_samples_analyzed": 160
}
```
### 2.3 2025-4-27 按照completion/温度1测试webrl
先说结论：
- 除去map的case，只考虑shopping/shopping_admin/gitlab三类任务，
平均SR达到`41.04%`，**基本达到论文水平**。
- map部分的case成功率只有15.4%，4/26，与论文差别较大。怀疑是因为
我测试用的是对接了官网的openstreetmap，数据已经发生了变化。
执行代码
```bash
python score.py results/webrl_chat_completion/python score.py result/
```
输出：
```
--------
src file: results/webrl_chat_completion/
successed: 60 / 164 (812)
partial accuracy: 36.59
overall accuracy: 36.36
--------
--- Category Statistics ---
Category: ('shopping_admin',)
Total Cases: 35
Success Rate: 45.7% (16/35)
Successful old_task_ids: [4, 15, 43, 65, 77, 95, 127, 157, 348, 374, 458, 488, 497, 538, 678, 710]
Failed old_task_ids: [109, 115, 123, 131, 196, 202, 211, 215, 247, 288, 423, 454, 464, 470, 491, 505, 548, 704, 768]
Category: ('map',)
Total Cases: 26
Success Rate: 15.4% (4/26)
Successful old_task_ids: [71, 93, 155, 369]
Failed old_task_ids: [7, 20, 33, 37, 56, 58, 75, 82, 88, 98, 139, 220, 221, 236, 250, 254, 287, 367, 381, 382, 383, 762]
Category: ('shopping',)
Total Cases: 45
Success Rate: 37.8% (17/45)
Successful old_task_ids: [23, 96, 190, 227, 240, 276, 300, 313, 354, 361, 368, 376, 466, 516, 521, 693, 798]
Failed old_task_ids: [48, 117, 118, 125, 144, 149, 162, 167, 225, 235, 270, 283, 285, 321, 324, 333, 335, 384, 386, 387, 431, 440, 506, 509, 514, 528, 574, 657]
Category: ('reddit',)
Total Cases: 19
Success Rate: 36.8% (7/19)
Successful old_task_ids: [27, 69, 401, 404, 582, 599, 714]
Failed old_task_ids: [601, 605, 612, 619, 626, 631, 641, 645, 652, 720, 729, 733]
Category: ('gitlab',)
Total Cases: 30
Success Rate: 50.0% (15/30)
Successful old_task_ids: [44, 156, 169, 205, 258, 259, 311, 318, 357, 415, 476, 534, 745, 809, 811]
Failed old_task_ids: [103, 135, 173, 182, 296, 349, 392, 419, 448, 485, 524, 567, 577, 668, 748]
Category: ('map', 'wikipedia')
Total Cases: 1
Success Rate: 0.0% (0/1)
Successful old_task_ids: []
Failed old_task_ids: [97]
Category: ('wikipedia', 'map')
Total Cases: 3
Success Rate: 33.3% (1/3)
Successful old_task_ids: [741]
Failed old_task_ids: [268, 426]
Category: ('gitlab', 'reddit')
Total Cases: 2
Success Rate: 0.0% (0/2)
Successful old_task_ids: []
Failed old_task_ids: [566, 791]
Category: ('shopping', 'reddit')
Total Cases: 1
Success Rate: 0.0% (0/1)
Successful old_task_ids: []
Failed old_task_ids: [673]
Category: ('reddit', 'gitlab')
Total Cases: 2
Success Rate: 0.0% (0/2)
Successful old_task_ids: []
Failed old_task_ids: [682, 686]
Category: ('map', 'shopping_admin')
Total Cases: 1
Success Rate: 0.0% (0/1)
Successful old_task_ids: []
Failed old_task_ids: [760]
--- Overall Accuracy without Map ---
Accuracy (excluding map tasks): 41.04% (55/134)
```
### 2.4 基于completion的测试评估orm模型
结论：
- r1 依然最强，正确率保持83，精确率接近80，召回71；
- orm 模型正确率`72`左右，精确率`63~66`，质量还可以, **正确率和论文79~80稍有差距**。
- gpt-4o 和orm表现差不多。
```json
{
"overall": {
"accuracy": 0.7212121212121212,
"precision": 0.6296296296296297,
"recall": 0.5666666666666667,
"f1_score": 0.5964912280701754,
"total_samples_analyzed": 165
},
"without_map": {
"accuracy": 0.7089552238805971,
"precision": 0.6666666666666666,
"recall": 0.5818181818181818,
"f1_score": 0.6213592233009708,
"total_samples_analyzed": 134
}
}
```
gpt-4o作为鉴别：
```json
{
"overall": {
"accuracy": 0.7393939393939394,
"precision": 0.6164383561643836,
"recall": 0.75,
"f1_score": 0.6766917293233082,
"total_samples_analyzed": 165
},
"without_map": {
"accuracy": 0.753731343283582,
"precision": 0.6774193548387096,
"recall": 0.7636363636363637,
"f1_score": 0.717948717948718,
"total_samples_analyzed": 134
}
}
```
deepseek-r作为鉴别：
```json
{
"overall": {
"accuracy": 0.8303030303030303,
"precision": 0.7962962962962963,
"recall": 0.7166666666666667,
"f1_score": 0.7543859649122807,
"total_samples_analyzed": 165
},
"without_map": {
"accuracy": 0.835820895522388,
"precision": 0.8367346938775511,
"recall": 0.7454545454545455,
"f1_score": 0.7884615384615384,
"total_samples_analyzed": 134
}
}
```
### 2.5 复现Plan-and-Act query生成
按照plan-and-act论文的方法，使用了WebRL的`gen_task.py`去进行采样和生成：
- 将训练集按网站分类，每一类随机选10个sample，给gpt-4o去参考和生成，提取variable，复用variable
- `TODO`：缺少过滤，要过滤一眼看出来不可行的任务；
- 将生成的task组织成类似WebArena任务raw config_file格式
### 2.6 生成轨迹
- 将生成的task，使用gen_test_data脚本，填充上URL宏和拆分，能够给vab框架执行
- 增加了OrmEvaluator。与原先测试集evaluation不同，因为没有参考答案，这里直接
使用orm来判断轨迹是否完成了预定的意图。
- 使用`deepseek-r1`作为orm，对轨迹进行判断
- `TODO`：还需要修改一下run.sh脚本，现在固定164个任务，生成了200个任务后面的没有执行。

2
plannact/gen_task/.env Normal file
View File

@ -0,0 +1,2 @@
OPENAI_API_KEY="sk-15Yc25Jl2ydulX2GEe531752EaF24b7298F641393f4f4266"
OPENAI_BASE_URL='https://aiproxy.lmzgc.cn:8080/v1'

7863
plannact/gen_task/WebArena-Lite_info.json Normal file
View File

File diff suppressed because one or more lines are too long

225
plannact/gen_task/gen_task.py Normal file
View File

@ -0,0 +1,225 @@
import json
import torch
import textwrap
from openai import OpenAI
import random
import re
import os
from dotenv import load_dotenv
import logging
# Setup logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
load_dotenv()
random.seed(42)
client = OpenAI(
base_url = os.getenv("OPENAI_BASE_URL"),
api_key = os.getenv("OPENAI_API_KEY")
)
# version 2 from paper
PROMPT = """You are a smart task creator for a website intelligent assistant. Your goal is to generate clear and practical tasks that the assistant can assist people with when they use {web} in their daily lives. These tasks should encompass a wide range of possible instructions and questions that may arise when using {web} website.
Your need to draw inspiration from the #Given Task# to create new tasks. These new tasks should belong to the same domain as the #Given Task# but be more diverse. The difficulty level of the #Created Task# should be similar to that of the #Given Task#. The #Created Task# must be reasonable, understandable and realistic. '#Given Task#', '#Created Task#', 'given task' and 'created task' are not allowed to appear in #Created Task#.
**Guidelines:**
- **Format each task** clearly using backticks (`) for each command description.
- Use a variety of phrasing styles to avoid repetitive expressions.
- Use variable names that match those in the provided task examples, such as place names, usernames, and product names. Avoid inventing entirely new variable names.
- Maintain the same or similar difficulty level as the #Given Task#. Tasks can be slightly more or less challenging but should stay within a reasonable range.
#Given Task#
{task_examples}
#Created Task#
"""
STORAGE_STATE_MAP = {
"shopping_admin": "./.auth/shopping_admin_state.json",
"shopping": "./.auth/shopping_state.json",
"map": None,
"reddit": "./.auth/reddit_state.json",
"gitlab": "./.auth/gitlab_state.json",
"wikipedia": None, # Assuming wikipedia doesn't need login state
# Add other sites if necessary based on your input file
}
def call_gpt(model='gpt-3.5-turbo', temperature=0, top_p=0, prompt=''):
try:
response = client.chat.completions.create(
model=model,
messages= [{"role": "user", "content": prompt}],
temperature=temperature,
top_p=top_p
)
response_content=response.choices[0].message.content
return response_content
except Exception as e:
logging.error(f"Error calling OpenAI API: {e}")
return None
def get_next_task_id(filepath):
"""Reads the JSON file and returns the next available task_id."""
next_id = 0
if os.path.exists(filepath):
try:
with open(filepath, 'r') as f:
# Handle potentially empty file
content = f.read()
if not content.strip():
return 0
existing_tasks = json.loads(content)
if existing_tasks:
max_id = max(task.get('task_id', -1) for task in existing_tasks)
next_id = max_id + 1
except json.JSONDecodeError:
logging.error(f"Error decoding JSON from {filepath}. Starting task_id from 0.")
return 0
except Exception as e:
logging.error(f"Error reading {filepath}: {e}. Starting task_id from 0.")
return 0
return next_id
def save_tasks(filepath, new_tasks):
"""Appends new tasks to the JSON file."""
all_tasks = []
if os.path.exists(filepath):
try:
with open(filepath, 'r') as f:
content = f.read()
if content.strip():
all_tasks = json.loads(content)
if not isinstance(all_tasks, list):
logging.error(f"Existing content in {filepath} is not a list. Overwriting.")
all_tasks = []
except json.JSONDecodeError:
logging.error(f"Error decoding JSON from {filepath}. Overwriting with new tasks.")
all_tasks = []
except Exception as e:
logging.error(f"Error reading {filepath}: {e}. Overwriting with new tasks.")
all_tasks = []
all_tasks.extend(new_tasks)
try:
with open(filepath, 'w') as fp:
json.dump(all_tasks, fp, indent=4)
logging.info(f"Successfully saved {len(new_tasks)} new tasks to {filepath}")
except Exception as e:
logging.error(f"Error writing tasks to {filepath}: {e}")
if __name__ == '__main__':
input_path = "WebArena-Lite_info.json" # Use the provided example file path
output_path = 'test_webarena_gen.raw.json'
try:
with open(input_path) as fp:
all_tasks_data = json.load(fp)
except FileNotFoundError:
logging.error(f"Input file not found: {input_path}")
exit(1)
except json.JSONDecodeError:
logging.error(f"Error decoding JSON from input file: {input_path}")
exit(1)
grouped_tasks = {}
start_url_map = {}
for item in all_tasks_data:
# Handle cases where 'sites' might be empty or None
if item.get('sites'):
# Prioritize single site, handle multiple if necessary
web = item['sites'][0] # Assume the first site is the primary one
task = item.get('intent', '') # Use get for safer access
if task: # Only add if intent exists
if web not in grouped_tasks:
grouped_tasks[web] = []
grouped_tasks[web].append(task)
# Store the start_url for this web type if not already stored
if web not in start_url_map and item.get('start_url'):
start_url_map[web] = item['start_url']
webs = list(grouped_tasks.keys())
current_task_id = get_next_task_id(output_path)
logging.info(f"Starting task generation with next task_id: {current_task_id}")
all_new_tasks_batch = [] # Collect all tasks generated in this run
for web in webs:
if not grouped_tasks[web]:
logging.warning(f"No tasks found for website: {web}. Skipping.")
continue
seed_num = 10
generation_turns = 2 # Set desired number of generation turns
start_url = start_url_map.get(web, f"__{web.upper()}__") # Fallback if URL not found
storage_state = STORAGE_STATE_MAP.get(web) # Get storage state, defaults to None if not in map
logging.info(f"Generating tasks for website: {web}")
tasks_for_web = grouped_tasks[web]
num_tasks_available = len(tasks_for_web)
num_seeds_to_sample = min(seed_num, num_tasks_available)
if num_seeds_to_sample == 0:
logging.warning(f"Not enough tasks available for {web} to generate seeds. Skipping.")
continue
for turn in range(generation_turns):
logging.info(f"--- Turn {turn + 1}/{generation_turns} for {web} ---")
seed_tasks = random.sample(tasks_for_web, num_seeds_to_sample)
task_examples = ""
for task in seed_tasks:
task_examples += f'- `{task.strip()}`\n'
task_examples = task_examples.strip()
prompt = PROMPT.format(web=web, task_examples=task_examples)
response = call_gpt(model='gpt-4o', prompt=prompt, temperature=0.8, top_p=0.9) # Adjust temp/top_p for diversity
if not response:
logging.warning(f"No response from GPT for {web}, turn {turn+1}. Skipping turn.")
continue
extracted_content = re.findall(r'`([^`]+)`', response)
logging.info(f"GPT generated {len(extracted_content)} potential tasks for {web}, turn {turn+1}")
generated_tasks_in_turn = []
for task_intent in extracted_content:
if not task_intent.strip(): # Skip empty tasks
continue
new_task_item = {
"sites": [web],
"task_id": current_task_id,
"require_login": True, # Default based on examples
"storage_state": storage_state,
"start_url": start_url,
"geolocation": None, # Default based on examples
"intent_template": "",
"instantiation_dict": {}, # Default based on examples
"intent": task_intent.strip(),
"require_reset": False, # Default based on examples
"eval": { # Default based on user request
"eval_types": ["orm_match"],
"orm_model": "aiproxy/deepseek-reasoner"
}
}
generated_tasks_in_turn.append(new_task_item)
logging.info(f"Formatted task {current_task_id}: {task_intent.strip()} for {web}")
current_task_id += 1
all_new_tasks_batch.extend(generated_tasks_in_turn)
# Save all generated tasks at the end of the script run
if all_new_tasks_batch:
save_tasks(output_path, all_new_tasks_batch)
else:
logging.info("No new tasks were generated in this run.")
logging.info("Task generation process finished.")

100
plannact/gen_task/generated_tasks.json Normal file
View File

@ -0,0 +1,100 @@
{"task": "Add a comment 'Looking great!' to the merge request linked to the navigation redesign in the ux-enhancements project.", "web": "gitlab"}
{"task": "List the last names of the top 3 contributors in the Kotlin DSL repository, sorted by commit frequency.", "web": "gitlab"}
{"task": "Can you retrieve and display the web URL of the new branch created in the data-analysis-project repository?", "web": "gitlab"}
{"task": "Please generate a new access token for my Gitlab account with the name 'Project Automation Token'.", "web": "gitlab"}
{"task": "Identify the GitLab usernames of the leading 2 contributors in the react-native repository, ordered by their number of commits.", "web": "gitlab"}
{"task": "Start a new private project analytics_dashboard_v2 and include yjlou, Rashida Malouda, and Jakub Klinkovsk\u00fd as members.", "web": "gitlab"}
{"task": "What are the usernames of the top 3 committers in the Booking System repo, ranked by their commit counts?", "web": "gitlab"}
{"task": "Add a new issue with the title 'Performance Optimization' to the jetstream/website project.", "web": "gitlab"}
{"task": "Post a message saying 'Code review needed ASAP' on the new merge request related to the mobile app optimization in the performance-improvements project.", "web": "gitlab"}
{"task": "Create a new milestone titled 'Q4 Release' in the health_tracker_rails_max_java project.", "web": "gitlab"}
{"task": "Generate an access token for the private API of the TimeTracking project.", "web": "gitlab"}
{"task": "Clone CoolWidgets into a new repository under your account.", "web": "gitlab"}
{"task": "Open the oldest unresolved issue containing the keyword \"layout bug\" and modify its status to 'in progress'.", "web": "gitlab"}
{"task": "Identify the top contributor by commit count to the SpaceX-Launches project.", "web": "gitlab"}
{"task": "Add the comment 'Insight required' on the pull request related to CSS best practices within the responsive_web_design project.", "web": "gitlab"}
{"task": "Examine the newest issue regarding 'login failures' to certify that all authentication methods are operational.", "web": "gitlab"}
{"task": "List the last names and total commits of the top 2 contributors in the ClimateTracking repository, sorted by the number of commits.", "web": "gitlab"}
{"task": "Develop a new marketing strategy plan titled marketing_plan_auto_gen using the static site template and assign it to TeamSquad, asia_ace, and Harper7.", "web": "gitlab"}
{"task": "Determine the top contributor by commit count to the NewPlatform initiative project.", "web": "gitlab"}
{"task": "Go to the performance improvement merge request under the database-optimization project needing your approval. Ensure the submitter has addressed all comments, and confirm with a 'Good job'. If they haven't, politely nudge them using an '@' mention.", "web": "gitlab"}
{"task": "Generate a list of transactions for Emily Daniels with total costs under 150.5, organized by descending order of purchase date.", "web": "shopping_admin"}
{"task": "Extract the pending feedback for the item 'Dynamic Duo Miter Tee'.", "web": "shopping_admin"}
{"task": "Execute the process 'Modify specifications' for the product 'Stellar Short Sleeve Tee'.", "web": "shopping_admin"}
{"task": "Create a new user account for Ava Thompson using the email ava.thompson@sample.com.", "web": "shopping_admin"}
{"task": "Identify the customer name and order ID for the oldest pending transaction.", "web": "shopping_admin"}
{"task": "Examine the details of the 10th order with the largest transaction volume from those that have a Pending status, and a base total exceeding 95.7.", "web": "shopping_admin"}
{"task": "Can you find out how many reviews have the nickname 'Tina' and are marked as 'excellent'?", "web": "shopping_admin"}
{"task": "Calculate the total number of items classified as 'Virtual Product' that are currently active.", "web": "shopping_admin"}
{"task": "Count the entries where reviews have the highest rating and have an authorized status.", "web": "shopping_admin"}
{"task": "Since 60 units of Argus All-Weather Tank have arrived, please refresh the stock levels.", "web": "shopping_admin"}
{"task": "List all transactions from the top-selling electronics category for the period June 1, 2022, to November 30, 2022, sorted by monthly sales.", "web": "shopping_admin"}
{"task": "Identify all customer comments that were left without a 'star rating' for review.", "web": "shopping_admin"}
{"task": "Collate details of discounts applied to products in the top-rated items from March 11, 2023, to March 13, 2023, analyzed annually.", "web": "shopping_admin"}
{"task": "Show me the top-3 products by revenue in May 2023.", "web": "shopping_admin"}
{"task": "Initiate the 'Subscription Removal' process for users located in ZIP 10001, within New York state.", "web": "shopping_admin"}
{"task": "Who ranks as the customer with the fourth highest number of purchase completions on record?", "web": "shopping_admin"}
{"task": "Turn all Orion Running Sneakers status to 'discontinued'.", "web": "shopping_admin"}
{"task": "Extract 'Sales Volume' metrics from the summary of best-selling gadgets between November 1st and 30th, 2022.", "web": "shopping_admin"}
{"task": "Now that 200 of Chiron Running Shorts -45-Blue are stocked, please adjust the inventory accordingly.", "web": "shopping_admin"}
{"task": "Perform the 'Deactivate' function on listings modified since '03/19/2023', targeting IDs ranging from 10 to 19.", "web": "shopping_admin"}
{"task": "Search for a set of kitchen knives priced between $50 and $150 for your upcoming cooking project.", "web": "shopping"}
{"task": "Add the Philips Sonicare electric toothbrush, model HX6877/21, with smart sensors and a pressure alert, to your wishlist.", "web": "shopping"}
{"task": "Track down order number 256 and initiate a return process for the item.", "web": "shopping"}
{"task": "Browse for \"water bottles\" that cost between $5 and $30.", "web": "shopping"}
{"task": "Summarize user comments regarding the performance and comfort of the running shoes offered.", "web": "shopping"}
{"task": "Conduct a detailed investigation of customer satisfaction concerning the ergonomic office chairs.", "web": "shopping"}
{"task": "Write a review for the \"Gourmet Popcorn Gift Basket,\" highlighting issues with the packaging and flavor compared to your expectations.", "web": "shopping"}
{"task": "Organize the Electronics items from the highest to the lowest rating based on customer reviews.", "web": "shopping"}
{"task": "Search for \"laptop accessories\" and analyze the first five products for user ratings and price comparisons.", "web": "shopping"}
{"task": "List all orders completed from January 1, 2023, to March 31, 2023, and calculate the total spending.", "web": "shopping"}
{"task": "Help me contact customer support to initiate a return for the LEFYXO Desk Organizer, as it arrived with a cracked compartment. You'll need to use my order details and request a replacement or refund.", "web": "shopping"}
{"task": "Locate \"Blenders\" that are priced between $50 and $300.", "web": "shopping"}
{"task": "Select the 'French Vanilla' flavor and the 'Ground Coffee' option for the Starbucks brand, then proceed to add them to your cart.", "web": "shopping"}
{"task": "What is the current status of the order placed on 3/15/23?", "web": "shopping"}
{"task": "Compute the total amount spent on orders made from August 1, 2022, to August 31, 2022, and share the result.", "web": "shopping"}
{"task": "Find and click on the fifth product in the list to view its detailed overview.", "web": "shopping"}
{"task": "Show the items available under the \"Outdoor Furniture\" section.", "web": "shopping"}
{"task": "Navigate to the \"Laptop Accessories\" category and select the third item to view its specific details.", "web": "shopping"}
{"task": "Provide a summary of all orders made between January 20, 2023, and April 20, 2023, including the cumulative total cost.", "web": "shopping"}
{"task": "Identify the date of order number 244 once it is found.", "web": "shopping"}
{"task": "In the subreddit Technology, locate three recent posts discussing sustainable tech innovations and summarize each post in one sentence.", "web": "reddit"}
{"task": "Join the Cooking subreddit and start a thread titled 'Evolving Flavors: Your Favorite Summer Recipes with a Twist' to explore how culinary aficionados modify classic recipes for the warmer months.", "web": "reddit"}
{"task": "In the subreddit Gaming, find a post among the top 6 discussing VR game development and offer your perspective in the comments on the future of this technology.", "web": "reddit"}
{"task": "Participate in the Photography subreddit by posting a commendation on the 3rd most-upvoted artwork shared by user LensFlareExpert.", "web": "reddit"}
{"task": "Among the top 5 trending posts in LosAngeles forum, pinpoint any discussions about new local coffee shops and extract the names of these establishments.", "web": "reddit"}
{"task": "Identify a community centered around digital art and initiate a conversation titled 'Digital Versatility: Honoring the Art of Beeple' to engage artists in a dialogue about Beeple's influence on digital art forms.", "web": "reddit"}
{"task": "Like the latest post discussing AI-generated art in the Art subreddit.", "web": "reddit"}
{"task": "Give a thumbs up to all submissions by user DataDrivenDebate in the DataScience subreddit.", "web": "reddit"}
{"task": "On the hiking subreddit, locate and upvote the most recent post recommending trails in the White Mountains.", "web": "reddit"}
{"task": "Find a subreddit where cinephiles often browse, and initiate a discussion thread titled 'Reel Innovations: The Cinematic Revolution of Christopher Nolan' to examine with others Nolan's impact on the film industry.", "web": "reddit"}
{"task": "Write a comment in the r/movies subreddit about the impact of soundtrack on a film's emotional depth.", "web": "reddit"}
{"task": "Join the 'Art' subreddit and sort the posts by 'Top of this month'.", "web": "reddit"}
{"task": "Comment on the top post in the r/AskReddit about users' most unconventional travel experiences.", "web": "reddit"}
{"task": "Among the recent 10 posts in the 'futurology' subforum, identify the discussions related to space exploration initiatives and list their highlights.", "web": "reddit"}
{"task": "Add the line 'Emma Stone embraces the unpredictable nature of acting, emphasizing spontaneity and authenticity in her craft' to the description of my post titled 'The Unpredictable Journey of Emma Stone in Hollywood'.", "web": "reddit"}
{"task": "Upvote the post by reeb123 in the 'worldnews' subreddit about climate change policies in Europe.", "web": "reddit"}
{"task": "Upvote the thread by cats4life on r/Aww about the unique friendship between a dog and a cat going viral.", "web": "reddit"}
{"task": "In the Berlin subreddit, among the latest 5 highlighted posts, commend the post by franz88 discussing local street food markets.", "web": "reddit"}
{"task": "Create a new thread in the Boston subreddit asking for recommendations on day trips within two hours of the city, titled \"Day Trips from Boston Suggestions?\"", "web": "reddit"}
{"task": "Update 'FutureTech' bio to read 'Exploring Tomorrow's Innovations, Today.'", "web": "reddit"}
{"task": "I'm considering a visit to Marineland of Florida and need to organize my trip. Can you inform me of the travel time by car from Historic Oakwood to Marineland of Florida, as well as the time it would take by public transportation?", "web": "map"}
{"task": "Could you assist me in determining the nearest coffee shop from Mountain Farm Museum? I'm hoping to grab a cup of coffee and need some direction.", "web": "map"}
{"task": "Is there an airport in close proximity to West Point Military Academy that offers international flights? I need to plan my itinerary and would appreciate your assistance.", "web": "map"}
{"task": "I'm currently at Balboa Park, and I'm curious about the nearest Art Galleries. Can you indicate the closest one for me?", "web": "map"}
{"task": "Can you ascertain how much time it will take to bike from Ford's Theatre to Mount Vernon, and then how long a car journey would be from Mount Vernon to the National Baseball Hall of Fame?", "web": "map"}
{"task": "Starting from Walnut Street Theatre and heading toward Liberty Bell Center, which method of travel\u2014walking or taking a bike\u2014would get me there faster?", "web": "map"}
{"task": "I'm at the Florida Aquarium and want to explore a museum in the New York Public Library vicinity. Can you point out the closest museum to the library?", "web": "map"}
{"task": "Can you identify the nearest restaurant to Graceland Mansion? I'd like to find a place to eat close by.", "web": "map"}
{"task": "Could you guide me on how to get from Blue Ridge Parkway to The Henry Ford by train, and also if driving by car would be faster?", "web": "map"}
{"task": "Could you help me calculate the driving distance from Fallingwater to Albright-Knox Art Gallery, and also how far it is to walk from Albright-Knox Art Gallery to Napoleon House?", "web": "map"}
{"task": "Can you determine the travel time for driving from Central Park to Brooklyn Botanic Garden, and then calculate the walking time from Brooklyn Botanic Garden to Brooklyn Museum?", "web": "map"}
{"task": "I need to find the quickest way to get to the Public Library from Madison Square Garden, either by walking or taking a bus. Could you provide the estimated times for both options?", "web": "map"}
{"task": "If I plan to travel from Fisherman's Wharf to Golden Gate Bridge, can you tell me the travel times for both cycling and driving routes?", "web": "map"}
{"task": "While staying in Empire State Building, I wish to visit the nearest Art Gallery. Could you tell me the distance and estimated time it will take to drive there?", "web": "map"}
{"task": "Is it possible to walk from Times Square to Metropolitan Museum of Art and then drive from there to Central Park in less than 90 minutes?", "web": "map"}
{"task": "I'm currently at the Space Needle and plan to reach the closest Lighthouse. Could you provide the distance and driving time needed for this trip?", "web": "map"}
{"task": "If I leave the Guggenheim Museum and head to Statue of Liberty National Monument, can you calculate the time it will take by ferry and bus?", "web": "map"}
{"task": "Could you find out if it's possible to bike from Griffith Observatory to Universal Studios Hollywood and then drive from there to LAX airport in under two hours?", "web": "map"}
{"task": "Can you tell me how long it would take to walk from Lincoln Memorial to National Gallery of Art and then drive from there to the Smithsonian Institution?", "web": "map"}
{"task": "I'm currently at Frank Lloyd Wright's Fallingwater and considering driving to Gettysburg National Military Park. Could you provide the travel distance and estimated driving time for this route?", "web": "map"}

4002
plannact/gen_task/test_webarena_gen.raw.json Normal file
View File

File diff suppressed because it is too large Load Diff

130
plannact/run.sh Executable file
View File

@ -0,0 +1,130 @@
#!/bin/bash
cur_dir=`pwd`
vab_home="$cur_dir/../"
webarenalite_home="${vab_home}/VAB-WebArena-Lite"
# Parse command-line arguments
if [ "$1" = "test" ]; then
result_dir="${cur_dir}/results/webrl_chat_completion"
test_config_base_dir="${webarenalite_home}/config_files/wa/test_webarena_lite"
elif [ "$1" = "gen" ]; then
result_dir="${cur_dir}/results/webrl_gen"
test_config_base_dir="${webarenalite_home}/config_files/wa/test_webarena_gen"
else
echo "Usage: $0 [test | gen]"
echo "Error: You must specify either test or gen."
exit 1
fi
# Ensure no other arguments are passed
if [ "$#" -ne 1 ]; then
echo "Usage: $0 [test | gen]"
echo "Error: You must specify exactly one argument: test or gen."
exit 1
fi
DATASET='webarena' # TODO: select from ['webarena', 'visualwebarena']
provider='openai' # TODO: select from ['openai', 'finetune', ...]
model='webrl-llama-3.1-8b' # TODO: assign model name, which is used for action generation
# model='aiproxy/deepseek-reasoner'
planner_ip='' # TODO: ip address of the model you are deploying (only if you are deploying your own model using e.g. vllm)
instruction_path="${webarenalite_home}/agent/prompts/jsons/p_webrl.json" # e.g., agent/prompts/jsons/p_cot_id_actree_2s.json
temperature=0.0
proxy_url='socks5://98.152.200.61:8081'
SERVER='localhost' # TODO: your server address
MAP_SERVER='https://www.openstreetmap.org' # TODO: the server address for MAP tasks
OPENAI_API_KEY='none' # TODO: if you test OpenAI APIs
OPENAI_API_URL='http://192.168.16.116:18080/v1' # TODO: if you test OpenAI APIs
# OPENAI_API_KEY="sk-uG8JLGz3wlXTAmiC8337A02f5d2946F2Ba3dE64427B90c2f"
# OPENAI_API_URL="https://aiproxy.lmzgc.cn:8080/v1/"
OPENAI_API_KEY_FUZZY="sk-uG8JLGz3wlXTAmiC8337A02f5d2946F2Ba3dE64427B90c2f"
OPENAI_API_URL_FUZZY="https://aiproxy.lmzgc.cn:8080/v1/"
OPENAI_ORGANIZATION=''
CONDA_ENV_NAME='vab' # TODO: the name of your conda environment for testing WebArena
ENV_VARIABLES="export DATASET=${DATASET}; export SHOPPING='http://${SERVER}:28082';export SHOPPING_ADMIN='http://${SERVER}:28083/admin';export REDDIT='http://${SERVER}:28080';export GITLAB='http://${SERVER}:28084';export MAP='${MAP_SERVER}';export WIKIPEDIA='http://${SERVER}:28081/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing';export HOMEPAGE='http://${SERVER}:20080';export OPENAI_API_KEY=${OPENAI_API_KEY};export OPENAI_API_URL=${OPENAI_API_URL};export OPENAI_ORGANIZATION=${OPENAI_ORGANIZATION};export OPENAI_API_KEY_FUZZY=${OPENAI_API_KEY_FUZZY};export OPENAI_API_URL_FUZZY=${OPENAI_API_URL_FUZZY}"
echo $ENV_VARIABLES
# 进入目录
cd ${webarenalite_home}
# get the number of tmux panes
num_panes=$(tmux list-panes | wc -l)
# calculate how many panes need to be created
let "panes_to_create = 7 - num_panes"
# let "panes_to_create = 1 - num_panes"
# array of tmux commands to create each pane
tmux_commands=(
'tmux split-window -h'
'tmux split-window -v'
'tmux select-pane -t 0; tmux split-window -v'
'tmux split-window -v'
'tmux select-pane -t 3; tmux split-window -v'
'tmux select-pane -t 5; tmux split-window -v'
)
# create panes up to 7
for ((i=0; i<$panes_to_create; i++)); do
eval ${tmux_commands[$i]}
done
#!/bin/bash
# Function to run a job
run_job() {
tmux select-pane -t $1
COMMAND="conda activate vab; python run.py \
--instruction_path ${instruction_path} \
--test_start_idx $2 \
--test_end_idx $3 \
--result_dir ${result_dir} \
--test_config_base_dir ${test_config_base_dir} \
--provider ${provider} \
--mode completion \
--model ${model} \
--stop_token \"<|eot_id|>\" \
--max_obs_length 0 \
--max_tokens 2048 \
--viewport_width 1280 \
--viewport_height 720 \
--proxy_url ${proxy_url} \
--action_set_tag webrl_id --observation_type webrl"
tmux send-keys "tmux set mouse on; conda activate ${CONDA_ENV_NAME}; ${ENV_VARIABLES}; until ${COMMAND}; do echo 'crashed' >&2; sleep 1; done" C-m
sleep 3
}
TOLERANCE=2
run_batch() {
args=("$@") # save all arguments in an array
num_jobs=${#args[@]} # get number of arguments
for ((i=1; i<$num_jobs; i++)); do
run_job $i ${args[i-1]} ${args[i]}
done
# Wait for all jobs to finish
while tmux list-panes -F "#{pane_pid} #{pane_current_command}" | grep -q python; do
sleep 100 # wait for 10 seconds before checking again
done
# Run checker
while ! python ${webarenalite_home}/scripts/check_error_runs.py ${result_dir} --delete_errors --tolerance ${TOLERANCE}; do
echo "Check failed, rerunning jobs..."
for ((i=1; i<$num_jobs; i++)); do
run_job $i ${args[i-1]} ${args[i]}
done
# Wait for all jobs to finish
while tmux list-panes -F "#{pane_pid} #{pane_current_command}" | grep -q python; do
sleep 100 # wait for 10 seconds before checking again
done
done
}
run_batch 0 28 56 84 112 140 165

9
plannact/score.sh Normal file
View File

@ -0,0 +1,9 @@
#!/bin/bash
cur_dir=`pwd`
vab_home="$cur_dir/.."
webarenalite_home="${vab_home}/VAB-WebArena-Lite"
cd ${webarenalite_home}
python score.py ${cur_dir}/$1