init

browser_env/__init__.py

@@ -0,0 +1,78 @@
+import asyncio
+
+from .actions import (
+    Action,
+    ActionParsingError,
+    ActionTypes,
+    action2create_function,
+    action2str,
+    create_check_action,
+    create_click_action,
+    create_focus_and_click_action,
+    create_focus_and_type_action,
+    create_go_back_action,
+    create_go_forward_action,
+    create_goto_url_action,
+    create_hover_action,
+    create_id_based_action,
+    create_id_based_actions,
+    create_key_press_action,
+    create_keyboard_type_action,
+    create_mouse_click_action,
+    create_mouse_hover_action,
+    create_new_tab_action,
+    create_none_action,
+    create_page_close_action,
+    create_page_focus_action,
+    create_playwright_action,
+    create_random_action,
+    create_scroll_action,
+    create_select_option_action,
+    create_stop_action,
+    create_type_action,
+    is_equivalent,
+)
+from .async_envs import AsyncScriptBrowserEnv
+from .envs import ScriptBrowserEnv
+from .processors import ObservationMetadata
+from .trajectory import Trajectory
+from .utils import DetachedPage, StateInfo
+
+__all__ = [
+    "ScriptBrowserEnv",
+    "AsyncScriptBrowserEnv",
+    "DetachedPage",
+    "StateInfo",
+    "ObservationMetadata",
+    "Action",
+    "ActionTypes",
+    "action2str",
+    "create_random_action",
+    "create_focus_and_click_action",
+    "create_focus_and_type_action",
+    "is_equivalent",
+    "create_mouse_click_action",
+    "create_mouse_hover_action",
+    "create_none_action",
+    "create_keyboard_type_action",
+    "create_page_focus_action",
+    "create_new_tab_action",
+    "create_go_back_action",
+    "create_go_forward_action",
+    "create_goto_url_action",
+    "create_page_close_action",
+    "action2create_function",
+    "create_playwright_action",
+    "create_id_based_action",
+    "create_id_based_actions",
+    "create_scroll_action",
+    "create_key_press_action",
+    "create_check_action",
+    "create_click_action",
+    "create_type_action",
+    "create_hover_action",
+    "create_select_option_action",
+    "create_stop_action",
+    "ActionParsingError",
+    "Trajectory",
+]

browser_env/async_envs.py

@@ -0,0 +1,153 @@
+import asyncio
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import numpy.typing as npt
+from gymnasium import Env
+from gymnasium.spaces import Box, Text
+from playwright.async_api import Page, ViewportSize, async_playwright
+
+from .actions import Action, aexecute_action, get_action_space
+from .utils import DetachedPage, png_bytes_to_numpy
+
+
+class AsyncScriptBrowserEnv(Env[npt.NDArray[np.uint8], Action]):
+    """
+    The goal of this environment is to produce a prototype of a browser environment.
+    In the end, we want to support a fully configurable browser environment with wide
+    range of action spaces and observation spaces, both structured and unstructured.
+    But in this prototype, we just support action space specified by Playwright script,
+    and observation space is the html content of the page.
+    """
+
+    def __init__(
+        self,
+        max_page_length: int = 2048,
+        headless: bool = True,
+        slow_mo: int = 0,
+        timeout: int = 30000,
+        viewport_size: ViewportSize = {"width": 1280, "height": 720},
+    ):
+        self.observation_space = Box(
+            0,
+            255,
+            (viewport_size["height"], viewport_size["width"], 4),
+            np.uint8,
+        )
+        # TODO: make Space[Action] = ActionSpace
+        self.action_space = get_action_space()  # type: ignore[assignment]
+        self.headless = headless
+        self.slow_mo = slow_mo
+        self.reset_finished = False
+        self.timeout = timeout
+        self.viewport_size = viewport_size
+
+    async def setup(self, config_file: Path | None = None) -> None:
+        self.context_manager = async_playwright()
+        self.playwright = await self.context_manager.__aenter__()
+        self.browser = await self.playwright.chromium.launch(
+            headless=self.headless, slow_mo=self.slow_mo
+        )
+        if config_file:
+            with open(config_file, "r") as f:
+                instance_config = json.load(f)
+        else:
+            instance_config = {}
+
+        storage_state = instance_config.get("storage_state", None)
+        start_url = instance_config.get("start_url", None)
+        geolocation = instance_config.get("geolocation", None)
+
+        self.context = await self.browser.new_context(
+            viewport=self.viewport_size,
+            storage_state=storage_state,
+            geolocation=geolocation,
+            device_scale_factor=1,
+        )
+        self.page = await self.context.new_page()
+        if start_url:
+            await self.page.goto(start_url)
+
+    async def areset(
+        self,
+        *,
+        seed: int | None = None,
+        options: dict[str, str] | None = None,
+    ) -> tuple[npt.NDArray[np.uint8], dict[str, object]]:
+        """
+        Reset the environment.
+        :param options: options for the environment. The options are:
+            - storage_state: the path to the storage state file
+        """
+        super().reset(seed=seed, options=options)
+        if self.reset_finished:
+            await self.context_manager.__aexit__()
+        if options is not None and "config_file" in options:
+            config_file = Path(options["config_file"])
+            if config_file.exists():
+                await self.setup(config_file=config_file)
+            else:
+                raise ValueError(f"Config state {config_file} does not exist.")
+        else:
+            await self.setup()
+        self.reset_finished = True
+        content = await self.page.content()
+        screenshot = png_bytes_to_numpy(await self.page.screenshot())
+        return (
+            screenshot,
+            {"page": DetachedPage(self.page.url, content)},
+        )
+
+    def reset(
+        self,
+        *,
+        seed: int | None = None,
+        options: dict[str, str] | None = None,
+    ) -> tuple[npt.NDArray[np.uint8], dict[str, object]]:
+        return asyncio.run(self.areset(seed=seed, options=options))
+
+    async def aclose(self) -> None:
+        if self.reset_finished:
+            await self.context_manager.__aexit__()
+
+    def close(self) -> None:
+        asyncio.run(self.aclose())
+
+    async def astep(
+        self, action: Action
+    ) -> tuple[npt.NDArray[np.uint8], float, bool, bool, dict[str, object]]:
+        if not self.reset_finished:
+            raise RuntimeError("Call reset first before calling step.")
+        success = False
+        fail_error = ""
+        try:
+            self.page = await aexecute_action(action, self.page, self.context)
+            success = True
+        except Exception as e:
+            fail_error = str(e)
+
+        try:
+            content = await self.page.content()
+            screenshot = png_bytes_to_numpy(await self.page.screenshot())
+        except:
+            await self.page.wait_for_load_state("load")
+            content = await self.page.content()
+            screenshot = png_bytes_to_numpy(await self.page.screenshot())
+
+        return (
+            screenshot,
+            float(success),
+            False,
+            False,
+            {
+                "page": DetachedPage(self.page.url, content),
+                "fail_error": fail_error,
+            },
+        )
+
+    def step(
+        self, action: Action
+    ) -> tuple[npt.NDArray[np.uint8], float, bool, bool, dict[str, object]]:
+        return asyncio.run(self.astep(action), debug=True)

browser_env/auto_login.py

@@ -0,0 +1,207 @@
+"""Script to automatically login each website"""
+import argparse
+import glob
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor
+from itertools import combinations
+from pathlib import Path
+
+from playwright.sync_api import sync_playwright
+
+import sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from browser_env.env_config import (
+    ACCOUNTS,
+    GITLAB,
+    REDDIT,
+    SHOPPING,
+    SHOPPING_ADMIN,
+)
+
+HEADLESS = True
+SLOW_MO = 0
+
+
+SITES = ["gitlab", "shopping", "shopping_admin", "reddit"]
+URLS = [
+    f"{GITLAB}/-/profile",
+    f"{SHOPPING}/wishlist/",
+    f"{SHOPPING_ADMIN}/dashboard",
+    f"{REDDIT}/user/{ACCOUNTS['reddit']['username']}/account",
+]
+EXACT_MATCH = [True, True, True, True]
+KEYWORDS = ["", "", "Dashboard", "Delete"]
+
+
+def is_expired(
+    storage_state: Path, url: str, keyword: str, url_exact: bool = True
+) -> bool:
+    """Test whether the cookie is expired"""
+    if not storage_state.exists():
+        return True
+
+    context_manager = sync_playwright()
+    playwright = context_manager.__enter__()
+    browser = playwright.chromium.launch(headless=True, slow_mo=SLOW_MO)
+    context = browser.new_context(storage_state=storage_state)
+    page = context.new_page()
+    page.goto(url)
+    time.sleep(1)
+    d_url = page.url
+    content = page.content()
+    context_manager.__exit__()
+    if keyword:
+        return keyword not in content
+    else:
+        if url_exact:
+            return d_url != url
+        else:
+            return url not in d_url
+
+
+def renew_comb(comb: list[str], auth_folder: str = "./.auth") -> None:
+    for c in comb:
+        context_manager = sync_playwright()
+        playwright = context_manager.__enter__()
+        browser = playwright.chromium.launch(headless=HEADLESS)
+        context = browser.new_context()
+        page = context.new_page()
+
+        if c == "shopping":
+            username = ACCOUNTS["shopping"]["username"]
+            password = ACCOUNTS["shopping"]["password"]
+            page.goto(f"{SHOPPING}/customer/account/login/")
+            page.get_by_label("Email", exact=True).fill(username)
+            page.get_by_label("Password", exact=True).fill(password)
+            page.get_by_role("button", name="Sign In").click()
+
+        if c == "reddit":
+            username = ACCOUNTS["reddit"]["username"]
+            password = ACCOUNTS["reddit"]["password"]
+            page.goto(f"{REDDIT}/login")
+            page.get_by_label("Username").fill(username)
+            page.get_by_label("Password").fill(password)
+            page.get_by_role("button", name="Log in").click()
+
+        if c == "shopping_admin":
+            username = ACCOUNTS["shopping_admin"]["username"]
+            password = ACCOUNTS["shopping_admin"]["password"]
+            page.goto(f"{SHOPPING_ADMIN}")
+            page.get_by_placeholder("user name").fill(username)
+            page.get_by_placeholder("password").fill(password)
+            page.get_by_role("button", name="Sign in").click()
+
+        if c == "gitlab":
+            username = ACCOUNTS["gitlab"]["username"]
+            password = ACCOUNTS["gitlab"]["password"]
+            page.goto(f"{GITLAB}/users/sign_in")
+            page.screenshot(path="debug.png")
+            page.get_by_test_id("username-field").click()
+            page.get_by_test_id("username-field").fill(username)
+            page.get_by_test_id("username-field").press("Tab")
+            page.get_by_test_id("password-field").fill(password)
+            page.get_by_test_id("sign-in-button").click()
+
+        context.storage_state(path=f"{auth_folder}/{c}_state.json")
+
+        context_manager.__exit__()
+    context_manager = sync_playwright()
+    playwright = context_manager.__enter__()
+    browser = playwright.chromium.launch(headless=HEADLESS)
+    context = browser.new_context()
+    page = context.new_page()
+
+    if "shopping" in comb:
+        username = ACCOUNTS["shopping"]["username"]
+        password = ACCOUNTS["shopping"]["password"]
+        page.goto(f"{SHOPPING}/customer/account/login/")
+        page.get_by_label("Email", exact=True).fill(username)
+        page.get_by_label("Password", exact=True).fill(password)
+        page.get_by_role("button", name="Sign In").click()
+
+    if "reddit" in comb:
+        username = ACCOUNTS["reddit"]["username"]
+        password = ACCOUNTS["reddit"]["password"]
+        page.goto(f"{REDDIT}/login")
+        page.get_by_label("Username").fill(username)
+        page.get_by_label("Password").fill(password)
+        page.get_by_role("button", name="Log in").click()
+
+    if "shopping_admin" in comb:
+        username = ACCOUNTS["shopping_admin"]["username"]
+        password = ACCOUNTS["shopping_admin"]["password"]
+        page.goto(f"{SHOPPING_ADMIN}")
+        page.get_by_placeholder("user name").fill(username)
+        page.get_by_placeholder("password").fill(password)
+        page.get_by_role("button", name="Sign in").click()
+
+    if "gitlab" in comb:
+        username = ACCOUNTS["gitlab"]["username"]
+        password = ACCOUNTS["gitlab"]["password"]
+        page.goto(f"{GITLAB}/users/sign_in")
+        page.get_by_test_id("username-field").click()
+        page.get_by_test_id("username-field").fill(username)
+        page.get_by_test_id("username-field").press("Tab")
+        page.get_by_test_id("password-field").fill(password)
+        page.get_by_test_id("sign-in-button").click()
+
+    context.storage_state(path=f"{auth_folder}/{'.'.join(comb)}_state.json")
+
+    context_manager.__exit__()
+
+
+def get_site_comb_from_filepath(file_path: str) -> list[str]:
+    comb = os.path.basename(file_path).rsplit("_", 1)[0].split(".")
+    return comb
+
+
+def main(auth_folder: str = "./.auth") -> None:
+    pairs = list(combinations(SITES, 2))
+
+    max_workers = 8
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        for pair in pairs:
+            # TODO[shuyanzh] auth don't work on these two sites
+            if "reddit" in pair and (
+                "shopping" in pair or "shopping_admin" in pair
+            ):
+                continue
+            executor.submit(
+                renew_comb, list(sorted(pair)), auth_folder=auth_folder
+            )
+
+        for site in SITES:
+            executor.submit(renew_comb, [site], auth_folder=auth_folder)
+
+    futures = []
+    cookie_files = list(glob.glob(f"{auth_folder}/*.json"))
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        for c_file in cookie_files:
+            comb = get_site_comb_from_filepath(c_file)
+            for cur_site in comb:
+                url = URLS[SITES.index(cur_site)]
+                keyword = KEYWORDS[SITES.index(cur_site)]
+                match = EXACT_MATCH[SITES.index(cur_site)]
+                future = executor.submit(
+                    is_expired, Path(c_file), url, keyword, match
+                )
+                futures.append(future)
+
+    for i, future in enumerate(futures):
+        assert not future.result(), f"Cookie {cookie_files[i]} expired."
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--site_list", nargs="+", default=["all"])
+    parser.add_argument("--auth_folder", type=str, default="./.auth")
+    args = parser.parse_args()
+    if not args.site_list:
+        main()
+    else:
+        if "all" in args.site_list:
+            main(auth_folder=args.auth_folder)
+        else:
+            renew_comb(args.site_list, auth_folder=args.auth_folder)

browser_env/constants.py

@@ -0,0 +1,295 @@
+from typing import Literal
+
+ROLES = (
+    "alert",
+    "alertdialog",
+    "application",
+    "article",
+    "banner",
+    "blockquote",
+    "button",
+    "caption",
+    "cell",
+    "checkbox",
+    "code",
+    "columnheader",
+    "combobox",
+    "complementary",
+    "contentinfo",
+    "definition",
+    "deletion",
+    "dialog",
+    "directory",
+    "document",
+    "emphasis",
+    "feed",
+    "figure",
+    "form",
+    "generic",
+    "grid",
+    "gridcell",
+    "group",
+    "heading",
+    "img",
+    "insertion",
+    "link",
+    "list",
+    "listbox",
+    "listitem",
+    "log",
+    "main",
+    "marquee",
+    "math",
+    "meter",
+    "menu",
+    "menubar",
+    "menuitem",
+    "menuitemcheckbox",
+    "menuitemradio",
+    "navigation",
+    "none",
+    "note",
+    "option",
+    "paragraph",
+    "presentation",
+    "progressbar",
+    "radio",
+    "radiogroup",
+    "region",
+    "row",
+    "rowgroup",
+    "rowheader",
+    "scrollbar",
+    "search",
+    "searchbox",
+    "separator",
+    "slider",
+    "spinbutton",
+    "status",
+    "strong",
+    "subscript",
+    "superscript",
+    "switch",
+    "tab",
+    "table",
+    "tablist",
+    "tabpanel",
+    "term",
+    "textbox",
+    "time",
+    "timer",
+    "toolbar",
+    "tooltip",
+    "tree",
+    "treegrid",
+    "treeitem",
+)
+
+SPECIAL_LOCATORS = (
+    "alt_text",
+    "label",
+    "placeholder",
+)
+
+ASCII_CHARSET = "".join(chr(x) for x in range(32, 128))
+FREQ_UNICODE_CHARSET = "".join(chr(x) for x in range(129, 110000))
+UTTERANCE_MAX_LENGTH = 8192
+ATTRIBUTE_MAX_LENGTH = 256
+TEXT_MAX_LENGTH = 256
+TYPING_MAX_LENGTH = 64
+URL_MAX_LENGTH = 256
+MAX_ELEMENT_INDEX_IN_VIEWPORT = 10
+MAX_ELEMENT_ID = 1000
+MAX_ANSWER_LENGTH = 512
+
+MIN_REF = -1000000
+MAX_REF = 1000000
+
+WINDOW_WIDTH = 500
+WINDOW_HEIGHT = 240
+TASK_WIDTH = 160
+TASK_HEIGHT = 210
+
+FLIGHT_WINDOW_WIDTH = 600
+FLIGHT_WINDOW_HEIGHT = 700
+FLIGHT_TASK_WIDTH = 375
+FLIGHT_TASK_HEIGHT = 667
+MAX_PAGE_NUMBER = 10
+
+SPECIAL_KEYS = (
+    "Enter",
+    "Tab",
+    "Control",
+    "Shift",
+    "Meta",
+    "Backspace",
+    "Delete",
+    "Escape",
+    "ArrowUp",
+    "ArrowDown",
+    "ArrowLeft",
+    "ArrowRight",
+    "PageDown",
+    "PageUp",
+    "Meta+a",
+)
+
+SPECIAL_KEY_MAPPINGS = {
+    "backquote": "Backquote",
+    "minus": "Minus",
+    "equal": "Equal",
+    "backslash": "Backslash",
+    "backspace": "Backspace",
+    "meta": "Meta",
+    "tab": "Tab",
+    "delete": "Delete",
+    "escape": "Escape",
+    "arrowdown": "ArrowDown",
+    "end": "End",
+    "enter": "Enter",
+    "home": "Home",
+    "insert": "Insert",
+    "pagedown": "PageDown",
+    "pageup": "PageUp",
+    "arrowright": "ArrowRight",
+    "arrowup": "ArrowUp",
+    "f1": "F1",
+    "f2": "F2",
+    "f3": "F3",
+    "f4": "F4",
+    "f5": "F5",
+    "f6": "F6",
+    "f7": "F7",
+    "f8": "F8",
+    "f9": "F9",
+    "f10": "F10",
+    "f11": "F11",
+    "f12": "F12",
+}
+
+RolesType = Literal[
+    "alert",
+    "alertdialog",
+    "application",
+    "article",
+    "banner",
+    "blockquote",
+    "button",
+    "caption",
+    "cell",
+    "checkbox",
+    "code",
+    "columnheader",
+    "combobox",
+    "complementary",
+    "contentinfo",
+    "definition",
+    "deletion",
+    "dialog",
+    "directory",
+    "document",
+    "emphasis",
+    "feed",
+    "figure",
+    "form",
+    "generic",
+    "grid",
+    "gridcell",
+    "group",
+    "heading",
+    "img",
+    "insertion",
+    "link",
+    "list",
+    "listbox",
+    "listitem",
+    "log",
+    "main",
+    "marquee",
+    "math",
+    "meter",
+    "menu",
+    "menubar",
+    "menuitem",
+    "menuitemcheckbox",
+    "menuitemradio",
+    "navigation",
+    "none",
+    "note",
+    "option",
+    "paragraph",
+    "presentation",
+    "progressbar",
+    "radio",
+    "radiogroup",
+    "region",
+    "row",
+    "rowgroup",
+    "rowheader",
+    "scrollbar",
+    "search",
+    "searchbox",
+    "separator",
+    "slider",
+    "spinbutton",
+    "status",
+    "strong",
+    "subscript",
+    "superscript",
+    "switch",
+    "tab",
+    "table",
+    "tablist",
+    "tabpanel",
+    "term",
+    "textbox",
+    "time",
+    "timer",
+    "toolbar",
+    "tooltip",
+    "tree",
+    "treegrid",
+    "treeitem",
+    "alt_text",
+    "label",
+    "placeholder",
+]
+
+MAX_VANILLA_STR_LENGTH = 1000
+
+PLAYWRIGHT_LOCATORS = (
+    "get_by_role",
+    "get_by_text",
+    "get_by_label",
+    "get_by_placeholder",
+    "get_by_alt_text",
+    "get_by_title",
+    "get_by_test_id",
+    "filter",
+    "frame_locator",
+    "locator",
+)
+
+PLAYWRIGHT_ACTIONS = (
+    "fill",
+    "check",
+    "select_option",
+    "click",
+    "hover",
+    "dclick",
+    "type",
+    "focus",
+    "goto",
+    "press",
+    "scroll",
+)
+
+IGNORED_ACTREE_PROPERTIES = (
+    "focusable",
+    "editable",
+    "readonly",
+    "level",
+    "settable",
+    "multiline",
+    "invalid",
+)

browser_env/env_config.py

@@ -0,0 +1,51 @@
+# websites domain
+import os
+
+REDDIT = os.environ.get("REDDIT", "http://localhost:9999")
+SHOPPING = os.environ.get("SHOPPING", "http://localhost:7770")
+SHOPPING_ADMIN = os.environ.get("SHOPPING_ADMIN", "http://localhost:7780/admin")
+GITLAB = os.environ.get("GITLAB", "http://localhost:8023")
+WIKIPEDIA = os.environ.get("WIKIPEDIA", "http://localhost:8888/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing")
+MAP = os.environ.get("MAP", "http://localhost:3000")
+HOMEPAGE = os.environ.get("HOMEPAGE", "http://localhost:4399")
+
+assert (
+    REDDIT
+    and SHOPPING
+    and SHOPPING_ADMIN
+    and GITLAB
+    and WIKIPEDIA
+    and MAP
+    and HOMEPAGE
+), (
+    f"Please setup the URLs to each site. Current: \n"
+    + f"Reddit: {REDDIT}\n"
+    + f"Shopping: {SHOPPING}\n"
+    + f"Shopping Admin: {SHOPPING_ADMIN}\n"
+    + f"Gitlab: {GITLAB}\n"
+    + f"Wikipedia: {WIKIPEDIA}\n"
+    + f"Map: {MAP}\n"
+    + f"Homepage: {HOMEPAGE}\n"
+)
+
+
+ACCOUNTS = {
+    "reddit": {"username": "MarvelsGrantMan136", "password": "test1234"},
+    "gitlab": {"username": "byteblaze", "password": "hello1234"},
+    "shopping": {
+        "username": "emma.lopez@gmail.com",
+        "password": "Password.123",
+    },
+    "shopping_admin": {"username": "admin", "password": "admin1234"},
+    "shopping_site_admin": {"username": "admin", "password": "admin1234"},
+}
+
+URL_MAPPINGS = {
+    REDDIT: "http://reddit.com",
+    SHOPPING: "http://onestopmarket.com",
+    SHOPPING_ADMIN: "http://luma.com/admin",
+    GITLAB: "http://gitlab.com",
+    WIKIPEDIA: "http://wikipedia.org",
+    MAP: "http://openstreetmap.org",
+    HOMEPAGE: "http://homepage.com",
+}

browser_env/envs.py

@@ -0,0 +1,334 @@
+import json
+import re
+import time
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Union
+
+import numpy as np
+import numpy.typing as npt
+from beartype import beartype
+from beartype.door import is_bearable
+from gymnasium import Env
+from gymnasium.spaces import Box, Text
+from playwright.sync_api import (
+    CDPSession,
+    Page,
+    Playwright,
+    ViewportSize,
+    expect,
+    sync_playwright,
+)
+
+from .actions import Action, execute_action, get_action_space
+from .processors import ObservationHandler, ObservationMetadata
+from .utils import (
+    AccessibilityTree,
+    DetachedPage,
+    Observation,
+    png_bytes_to_numpy,
+)
+
+import base64
+from .scripts import *
+
+@dataclass
+class PlaywrightScript:
+    function: str  # goto, get_by_role
+    destination: str  # https://www.google.com/, combobox
+    name: str | None = None  # Search, Avatar 2009
+    operation: str | None = None  # click, fill, press
+    value: str | None = None  # avatar movie, Enter
+
+
+def parse_action(action: str) -> PlaywrightScript:
+    splitted = action.strip().split(" ")
+    assert len(splitted) >= 2
+    match splitted[:2]:
+        case ["goto", url]:
+            assert len(splitted) == 2
+            return PlaywrightScript("goto", url)
+        case ["get_by_role", destination]:
+            assert len(splitted) >= 4
+            match splitted[2:]:
+                case [name, operation]:
+                    return PlaywrightScript(
+                        "get_by_role", destination, name, operation
+                    )
+                case [name, operation, value]:
+                    return PlaywrightScript(
+                        "get_by_role", destination, name, operation, value
+                    )
+                case _:
+                    raise ValueError("Invalid action")
+        case _:
+            raise ValueError(f"Invalid action {action}")
+
+
+class ScriptBrowserEnv(Env[dict[str, Observation], Action]):
+    """
+    The goal of this environment is to produce a prototype of a browser environment.
+    In the end, we want to support a fully configurable browser environment with wide
+    range of action spaces and observation spaces, both structured and unstructured.
+    But in this prototype, we just support action space specified by Playwright script,
+    and observation space is the html content of the page.
+    """
+
+    @beartype
+    def __init__(
+        self,
+        max_page_length: int = 8192,
+        headless: bool = True,
+        slow_mo: int = 0,
+        observation_type: str = "html",
+        current_viewport_only: bool = False,
+        viewport_size: ViewportSize = {"width": 1280, "height": 720},
+        save_trace_enabled: bool = False,
+        sleep_after_execution: float = 5.0,
+        global_config = None,
+    ):
+        # TODO: make Space[Action] = ActionSpace
+        self.action_space = get_action_space()  # type: ignore[assignment]
+        self.headless = headless
+        self.slow_mo = slow_mo
+        self.current_viewport_only = current_viewport_only
+        self.reset_finished = False
+        self.viewport_size = viewport_size
+        self.save_trace_enabled = save_trace_enabled
+        self.sleep_after_execution = sleep_after_execution
+        self.global_config = global_config
+
+        match observation_type:
+            case "html" | "accessibility_tree":
+                self.text_observation_type = observation_type
+                self.image_observation_type = ""
+                self.main_observation_type = "text"
+            case "image":
+                self.image_observation_type = observation_type
+                self.text_observation_type = ""  # type: ignore[assignment]
+                self.main_observation_type = "image"
+            case _:
+                raise ValueError(
+                    f"Unsupported observation type: {observation_type}"
+                )
+
+        self.observation_handler = ObservationHandler(
+            self.main_observation_type,
+            self.text_observation_type,
+            self.image_observation_type,
+            self.current_viewport_only,
+            self.viewport_size,
+        )
+
+        self.observation_space = (
+            self.observation_handler.get_observation_space()
+        )
+
+    @beartype
+    def setup(self, config_file: Path | None = None) -> None:
+        def handle_dialog(dialog):
+            self.page.dialog_message = dialog.message
+            dialog.dismiss()
+        self.context_manager = sync_playwright()
+        self.playwright = self.context_manager.__enter__()
+        self.browser = self.playwright.chromium.launch(
+            headless=self.headless, slow_mo=self.slow_mo
+        )
+
+        if config_file:
+            with open(config_file, "r") as f:
+                instance_config = json.load(f)
+        else:
+            instance_config = {}
+
+        storage_state = instance_config.get("storage_state", None)
+        start_url = instance_config.get("start_url", None)
+        geolocation = instance_config.get("geolocation", None)
+
+        self.context = self.browser.new_context(
+            viewport=self.viewport_size,
+            storage_state=storage_state,
+            geolocation=geolocation,
+            device_scale_factor=1,
+        )
+        if self.save_trace_enabled:
+            self.context.tracing.start(screenshots=True, snapshots=True)
+        if start_url:
+            start_urls = start_url.split(" |AND| ")
+            for url in start_urls:
+                page = self.context.new_page()
+                page.on("dialog", handle_dialog)
+                client = page.context.new_cdp_session(
+                    page
+                )  # talk to chrome devtools
+                if self.text_observation_type == "accessibility_tree":
+                    client.send("Accessibility.enable")
+                page.client = client  # type: ignore # TODO[shuyanzh], fix this hackey client
+                page.goto(url)
+            # set the first page as the current page
+            self.page = self.context.pages[0]
+            self.page.bring_to_front()
+        else:
+            self.page = self.context.new_page()
+            page.on("dialog", handle_dialog)
+            client = self.page.context.new_cdp_session(self.page)
+            if self.text_observation_type == "accessibility_tree":
+                client.send("Accessibility.enable")
+            self.page.client = client  # type: ignore
+
+    def get_page_client(self, page: Page) -> CDPSession:
+        return page.client  # type: ignore
+
+    def _get_obs(self) -> dict[str, Observation]:
+        obs = self.observation_handler.get_observation(
+            self.page, self.get_page_client(self.page)
+        )
+        return obs
+
+    def _get_obs_metadata(self) -> dict[str, ObservationMetadata]:
+        metadata = self.observation_handler.get_observation_metadata()
+        return metadata
+
+    @beartype
+    def reset(
+        self,
+        *,
+        seed: int | None = None,
+        options: dict[str, str] | None = None,
+    ) -> tuple[dict[str, Observation], dict[str, Any]]:
+        """
+        Reset the environment.
+        :param options: options for the environment. The current supported options are:
+            - "storage_state": the storage state of the browser. It is a file path to a json file.
+        """
+        super().reset(seed=seed, options=options)
+        if self.reset_finished:
+            self.context_manager.__exit__()
+
+        if options is not None and "config_file" in options:
+            config_file = Path(options["config_file"])
+            if config_file.exists():
+                self.setup(config_file=config_file)
+            else:
+                raise ValueError(f"Config file {config_file} does not exist.")
+        else:
+            self.setup()
+        self.reset_finished = True
+
+        if self.sleep_after_execution > 0:
+            time.sleep(self.sleep_after_execution)
+            
+        images = self.modify_page()
+
+        observation = self._get_obs()
+        observation_metadata = self._get_obs_metadata()
+        info = {
+            "page": DetachedPage(self.page.url, ""),
+            "fail_error": "",
+            "observation_metadata": observation_metadata,
+            "images": images,
+        } 
+
+        return (observation, info)
+
+    def save_trace(self, trace_path: str | Path) -> None:
+        if self.save_trace_enabled:
+            self.context.tracing.stop(path=trace_path)
+
+    def close(self) -> None:
+        if self.reset_finished:
+            self.context_manager.__exit__()
+
+    def step(
+        self, action: Action
+    ) -> tuple[dict[str, Observation], float, bool, bool, dict[str, Any]]:
+        if not self.reset_finished:
+            raise RuntimeError("Call reset first before calling step.")
+
+        success = False
+        fail_error = ""
+        try:
+            self.page = execute_action(
+                action,
+                self.page,
+                self.context,
+                self.observation_handler.action_processor,
+            )
+            success = True
+        except Exception as e:
+            fail_error = str(e)
+            raise e
+
+        # hard sleep TODO[shuyanzh] suboptimal, may need to check network
+        if self.sleep_after_execution > 0:
+            time.sleep(self.sleep_after_execution)
+
+        images = self.modify_page()
+        
+        observation = self._get_obs()
+        observation_metadata = self._get_obs_metadata()
+
+        info = {
+            "page": DetachedPage(self.page.url, self.page.content()),
+            "fail_error": fail_error,
+            "observation_metadata": observation_metadata,
+            "images": images,
+        }
+        
+        msg = (
+            observation,
+            float(success),  # reward
+            False,  # terminated
+            False,  # truncated
+            info,
+        )
+        return msg
+
+    def modify_page(self):
+        self.page.wait_for_timeout(500)
+        try:
+            self.page.evaluate(remove_id_script)
+        except:
+            pass
+        
+        suffix = getattr(self.global_config, "logname", "")
+        if suffix:
+            img_bytes = self.page.screenshot(path=f"output/screenshot-{suffix}.png", full_page=True)
+        else:
+            img_bytes = self.page.screenshot(path="output/screenshot_raw.png")
+        raw_image = base64.b64encode(img_bytes).decode()
+        
+        self.page.evaluate(mix_marker_script)
+        self.page.wait_for_timeout(100)
+        
+        # get all clickable elements
+        start_id = 0
+        elem_items, start_id = self.page.evaluate(get_rect_script, {
+            "selector": ".possible-clickable-element",
+            "startIndex": start_id
+        })
+        
+        # get ocr items
+        ocr_items = []
+        # ocr_items = page.evaluate(canva_handler_script)
+        # svg_items, _ = page.evaluate(get_rect_script, {"selector": "svg", "startIndex": -1})
+        # ocr_items = ocr_items + svg_items
+        # ocr_items, start_id = get_canva_images(ocr_items, img_bytes, start_id)
+        
+        items = elem_items + ocr_items
+        
+        # mark our own labels and get the images
+        items = self.page.evaluate(label_marker_script, items)
+        if suffix:
+            img_bytes = self.page.screenshot(path=f"output/marked-{suffix}.png", full_page=True)
+        else:
+            img_bytes = self.page.screenshot(path="output/marked.png")
+        marked_image = base64.b64encode(img_bytes).decode()
+        
+        self.page.evaluate(remove_label_mark_script)
+        
+        return {
+            "raw_image": raw_image,
+            "marked_image": marked_image,
+        }

browser_env/helper_functions.py

@@ -0,0 +1,307 @@
+import base64
+import io
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from PIL import Image
+
+from agent.prompts import *
+from browser_env import (
+    Action,
+    ActionTypes,
+    ObservationMetadata,
+    StateInfo,
+    action2str,
+)
+
+HTML_TEMPLATE = """
+<!DOCTYPE html>
+<head>
+    <style>
+        pre {{
+            white-space: pre-wrap;
+            word-wrap: break-word;
+        }}
+    </style>
+</head>
+<html>
+    <body>
+     {body}
+    </body>
+</html>
+"""
+
+
+def get_render_action(
+    action: Action,
+    observation_metadata: dict[str, ObservationMetadata],
+    action_set_tag: str,
+) -> str:
+    """Parse the predicted actions for rendering purpose. More comprehensive information"""
+    match action_set_tag:
+        case "id_html_tree":
+            text_meta_data = observation_metadata["text"]
+            if action["element_id"] in text_meta_data["obs_nodes_info"]:
+                node_content = text_meta_data["obs_nodes_info"][
+                    action["element_id"]
+                ]["text"]
+            else:
+                node_content = "No match found"
+
+            action_str = f"<div class='raw_parsed_prediction' style='background-color:grey'><pre>{action['raw_prediction']}</pre></div>"
+            action_str += f"<div class='action_object' style='background-color:grey'><pre>{repr(action)}</pre></div>"
+            action_str += f"<div class='parsed_action' style='background-color:yellow'><pre>{action2str(action, action_set_tag, node_content)}</pre></div>"
+
+        case "id_html_nasc_tree":
+            text_meta_data = observation_metadata["text"]
+            if action["element_id"] in text_meta_data["obs_nodes_info"]:
+                node_content = text_meta_data["obs_nodes_info"][
+                    action["element_id"]
+                ]["text"]
+            else:
+                node_content = "No match found"
+
+            action_str = f"<div class='raw_parsed_prediction' style='background-color:grey'><pre>{action['raw_prediction']}</pre></div>"
+            action_str += f"<div class='action_object' style='background-color:grey'><pre>{repr(action)}</pre></div>"
+            action_str += f"<div class='parsed_action' style='background-color:yellow'><pre>{action2str(action, action_set_tag, node_content)}</pre></div>"
+
+        case "id_accessibility_tree":
+            text_meta_data = observation_metadata["text"]
+            if action["element_id"] in text_meta_data["obs_nodes_info"]:
+                node_content = text_meta_data["obs_nodes_info"][
+                    action["element_id"]
+                ]["text"]
+            else:
+                node_content = "No match found"
+
+            action_str = f"<div class='raw_parsed_prediction' style='background-color:grey'><pre>{action['raw_prediction']}</pre></div>"
+            action_str += f"<div class='action_object' style='background-color:grey'><pre>{repr(action)}</pre></div>"
+            action_str += f"<div class='parsed_action' style='background-color:yellow'><pre>{action2str(action, action_set_tag, node_content)}</pre></div>"
+
+        case "playwright":
+            action_str = action["pw_code"]
+        case _:
+            raise ValueError(f"Unknown action type {action['action_type']}")
+    return action_str
+
+
+def get_action_description(
+    action: Action,
+    observation_metadata: dict[str, ObservationMetadata],
+    action_set_tag: str,
+    prompt_constructor: PromptConstructor | None,
+) -> str:
+    """Generate the text version of the predicted actions to store in action history for prompt use.
+    May contain hint information to recover from the failures"""
+    
+    match action_set_tag:
+        case "id_html_tree":
+            # old_op_prompt = "Website: %s; Thinking process: %s; Html segment: %s; Operation: %s; Result: %s"
+            op_prompt = "Html segment: %s; Operation: %s;"
+            text_meta_data = observation_metadata["text"]
+            node_info = text_meta_data["obs_nodes_info"]
+            result = 'Operation Success'
+            
+            if action["action_type"] in [
+                ActionTypes.CLICK,
+                ActionTypes.HOVER,
+                ActionTypes.TYPE,
+            ]:
+                action_name = str(action["action_type"]).split(".")[1].lower()
+                if action["element_id"] in node_info:
+                    node_content = node_info[action["element_id"]]["text"]
+                    node_content = " ".join(node_content.split()[1:])
+                    action["label"] = node_info[action["element_id"]]["label"]
+                    action_str = action2str(
+                        action, action_set_tag, node_content
+                    )
+                else:
+                    action_str = "None"
+                    result = f"Cannot find the corresponding tag. Attempt to perfom \"{action_name}\" on element \"[{action['element_id']}]\" but no matching element found. Please check the observation more carefully."
+            else:
+                if (
+                    action["action_type"] == ActionTypes.NONE
+                    and prompt_constructor is not None
+                ):
+                    text = action["answer"]
+                    if text is not None and text.count("#Record#") > 0:
+                        action_str = text
+                    else:
+                        action_str = "None"
+                        result = f'Operation invalid. The format was incorrect. Ensure that the action is wrapped inside a pair of # and seperate arguments within spaces as follows: #action# arg1 arg2 ....'
+                else:
+                    action_str = action2str(action, action_set_tag, "")
+                    
+            # action_str = op_prompt % (
+            #     prompt_constructor.state["url"],
+            #     prompt_constructor.state["intention"],
+            #     prompt_constructor.state["segment"],
+            #     action_str,
+            #     result,
+            # )
+            
+            action_str = op_prompt % (
+                prompt_constructor.state["segment"],
+                action_str,
+            )
+        case "id_html_nasc_tree":
+            op_prompt = "%s #HTML Segment: %s"
+            text_meta_data = observation_metadata["text"]
+            node_info = text_meta_data["obs_nodes_info"]
+            result = 'Operation Success'
+            
+            if action["action_type"] in [
+                ActionTypes.CLICK,
+                ActionTypes.HOVER,
+                ActionTypes.TYPE,
+            ]:
+                action_name = str(action["action_type"]).split(".")[1].lower()
+                if action["element_id"] in node_info:
+                    node_content = node_info[action["element_id"]]["text"]
+                    node_content = " ".join(node_content.split()[1:])
+                    action["label"] = node_info[action["element_id"]]["label"]
+                    action_str = action2str(
+                        action, action_set_tag, node_content
+                    )
+                else:
+                    action_str = "None"
+                    result = f"Cannot find the corresponding tag. Attempt to perfom \"{action_name}\" on element \"[{action['element_id']}]\" but no matching element found. Please check the observation more carefully."
+            else:
+                if (
+                    action["action_type"] == ActionTypes.NONE
+                    and prompt_constructor is not None
+                ):
+                    text = action["answer"]
+                    if text is not None and text.count("record") > 0:
+                        action_str = text
+                    else:
+                        action_str = "None"
+                        result = f'Operation invalid. The format was incorrect. Ensure that the action is wrapped inside a pair of # and seperate arguments within spaces as follows: #action# arg1 arg2 ....'
+                else:
+                    action_str = action2str(action, action_set_tag, "")
+                    
+            action_str = op_prompt % (
+                action_str,
+                prompt_constructor.state["segment"],
+            )
+            
+        case "id_accessibility_tree":
+            text_meta_data = observation_metadata["text"]
+            if action["action_type"] in [
+                ActionTypes.CLICK,
+                ActionTypes.HOVER,
+                ActionTypes.TYPE,
+            ]:
+                action_name = str(action["action_type"]).split(".")[1].lower()
+                if action["element_id"] in text_meta_data["obs_nodes_info"]:
+                    node_content = text_meta_data["obs_nodes_info"][
+                        action["element_id"]
+                    ]["text"]
+                    node_content = " ".join(node_content.split()[1:])
+                    action_str = action2str(
+                        action, action_set_tag, node_content
+                    )
+                else:
+                    action_str = f"Attempt to perfom \"{action_name}\" on element \"[{action['element_id']}]\" but no matching element found. Please check the observation more carefully."
+            else:
+                if (
+                    action["action_type"] == ActionTypes.NONE
+                    and prompt_constructor is not None
+                ):
+                    action_splitter = prompt_constructor.instruction[
+                        "meta_data"
+                    ]["action_splitter"]
+                    action_str = f'The previous prediction you issued was "{action["raw_prediction"]}". However, the format was incorrect. Ensure that the action is wrapped inside a pair of {action_splitter} and enclose arguments within [] as follows: {action_splitter}action [arg] ...{action_splitter}.'
+                else:
+                    action_str = action2str(action, action_set_tag, "")
+
+        case "playwright":
+            action_str = action["pw_code"]
+
+        case _:
+            raise ValueError(f"Unknown action type {action['action_type']}")
+
+    return action_str
+
+
+class RenderHelper(object):
+    """Helper class to render text and image observations and meta data in the trajectory"""
+
+    def __init__(
+        self, config_file: str, result_dir: str, action_set_tag: str
+    ) -> None:
+        with open(config_file, "r") as f:
+            _config = json.load(f)
+            _config_str = ""
+            for k, v in _config.items():
+                _config_str += f"{k}: {v}\n"
+            _config_str = f"<pre>{_config_str}</pre>\n"
+            task_id = _config["task_id"]
+
+        self.action_set_tag = action_set_tag
+
+        self.render_file = open(
+            Path(result_dir) / f"render_{task_id}.html", "a+"
+        )
+        self.render_file.truncate(0)
+        # write init template
+        self.render_file.write(HTML_TEMPLATE.format(body=f"{_config_str}"))
+        self.render_file.read()
+        self.render_file.flush()
+
+    def render(
+        self,
+        action: Action,
+        state_info: StateInfo,
+        meta_data: dict[str, Any],
+        render_screenshot: bool = False,
+    ) -> None:
+        """Render the trajectory"""
+        # text observation
+        observation = state_info["observation"]
+        text_obs = observation["text"]
+        info = state_info["info"]
+        new_content = f"<h2>New Page</h2>\n"
+        new_content += f"<h3 class='url'><a href={state_info['info']['page'].url}>URL: {state_info['info']['page'].url}</a></h3>\n"
+        new_content += f"<div class='state_obv'><pre>{text_obs}</pre><div>\n"
+
+        if render_screenshot:
+            # image observation
+            img_obs = observation["image"]
+            image = Image.fromarray(img_obs)
+            byte_io = io.BytesIO()
+            image.save(byte_io, format="PNG")
+            byte_io.seek(0)
+            image_bytes = base64.b64encode(byte_io.read())
+            image_str = image_bytes.decode("utf-8")
+            new_content += f"<img src='data:image/png;base64,{image_str}' style='width:50vw; height:auto;'/>\n"
+
+        # meta data
+        new_content += f"<div class='prev_action' style='background-color:pink'>{meta_data['action_history'][-1]}</div>\n"
+
+        # action
+        action_str = get_render_action(
+            action,
+            info["observation_metadata"],
+            action_set_tag=self.action_set_tag,
+        )
+        # with yellow background
+        action_str = f"<div class='predict_action'>{action_str}</div>"
+        new_content += f"{action_str}\n"
+
+        # add new content
+        self.render_file.seek(0)
+        html = self.render_file.read()
+        html_body = re.findall(r"<body>(.*?)</body>", html, re.DOTALL)[0]
+        html_body += new_content
+
+        html = HTML_TEMPLATE.format(body=html_body)
+        self.render_file.seek(0)
+        self.render_file.truncate()
+        self.render_file.write(html)
+        self.render_file.flush()
+
+    def close(self) -> None:
+        self.render_file.close()

browser_env/html_tools/__init__.py

@@ -0,0 +1,6 @@
+from .identifier import IdentifierTool
+from .prompt import HtmlPrompt
+from .html_parser import HtmlParser
+
+from .utils import print_html_object
+from .configs import basic_attrs, mind2web_keep_attrs

browser_env/html_tools/configs/__init__.py

@@ -0,0 +1,3 @@
+from .html_prompt import prompts
+from .config import basic_attrs, mind2web_keep_attrs, miniwob_attrs
+from .config import config_meta

browser_env/html_tools/configs/config.py

@@ -0,0 +1,49 @@
+basic_attrs = [
+    'title',
+    'value',
+    'placeholder',
+    'selected',
+]
+    
+mind2web_keep_attrs = [
+    'alt',
+    'aria_description',
+    'aria_label',
+    'aria_role',
+    'input_checked',
+    'input_value',
+    'label',
+    'name',
+    'option_selected',
+    'placeholder',
+    'role',
+    'text_value',
+    'title',
+    'type',
+    'value',
+]
+
+miniwob_attrs = [
+    'id',
+    'type',
+    'value',
+]
+
+config_meta = """
+======= Configs =======
+Columns:
+  - id:        {id_attr}
+  - label:     {label_attr}
+Position:      {use_position}
+  - window:    {window_size}
+  - rect_dict: {rect}
+Keep:
+  - parents:   {parent_chain}
+  - attrs:     {keep_attrs}
+  - elems:     {keep_elem}
+  - obs_elem:  {obs_elem}
+Generator:
+  - prompt:    {prompt_name}
+  - label:     {identifier_name}
+========================
+"""

browser_env/html_tools/configs/html_prompt.py

@@ -0,0 +1,22 @@
+refine_prompt = {
+    'dom': '<{tag}{label}|{attr}{content}{subtree} >',
+    'label': '[{label}]',
+    'attr': '{attr}',
+    'attr_splitter': '; ',
+    'subtree_splitter': ' ',
+}
+
+xml_prompt = {
+    'dom': '<{tag}{label}{attr}>{content}{subtree} </{tag}>',
+    'label': ' id="{label}"',
+    'attr': '{key}="{attr}"',
+    'attr_splitter': ' ',
+    'subtree_splitter': ' ',
+}
+
+prompts = {
+    'refine': refine_prompt,
+    'xml': xml_prompt,
+    'new_data': refine_prompt, 
+}
+    

browser_env/html_tools/html_parser.py

@@ -0,0 +1,426 @@
+from lxml import html
+import time, copy, random
+import json, re, os
+
+from .identifier import IdentifierTool
+from .prompt import HtmlPrompt
+from .configs import config_meta
+from .utils import get_xpath_top_down, rect2tuple
+
+class HtmlParser():
+    def __init__(self, ctx: str, args: dict[str]={}) -> None:
+        stt = time.time()
+        self.dom_tree = self.ctx2tree(ctx)
+        # tool related
+        self.bids2label = {}
+        self.bids2xpath = {}
+        self.used_labels = {}
+        
+        # parse args
+        self.parse_args(args)
+        self.init_time = time.time() - stt
+        
+    def parse_args(self, args: dict[str]={}) -> None:
+        def attr_check(attr, type_model='str'):
+            if attr is None:
+                return False
+            attr_type = type(attr)
+            if attr_type != type(type_model):
+                return False
+            if attr_type == type('str') and len(attr) == 0:
+                return False
+            return True
+        
+        args = {} if args is None else args
+
+        # [Position] use_pos: False -> use full page, otherwise use window_size
+        dataset = args.get('dataset', '')
+        use_position = args.get('use_position', False)
+        window_size = args.get('window_size', None)
+        rect = args.get('rect_dict', None)
+        if use_position:
+            if not attr_check(window_size, ()):
+                raise ValueError('window_size must be set when use_position is True')
+            if not attr_check(rect, {}):
+                raise ValueError('rect_dict must be set when use_position is True')
+        
+        if not attr_check(rect, {}):
+            rect = {}
+        
+        # [Label] for vimium is temp_clickable_label, otherwise keep all of it
+        label_attr = args.get('label_attr', '')
+        get_new_label = args.get('regenerate_label', False)
+        label_method = args.get('label_generator', None)
+        regen_label = not attr_check(label_method)
+        
+        # [id] for mind2web is backend_node_id, for normal website use our method
+        id_attr = args.get('id_attr', '')
+        regen_id = not attr_check(id_attr)
+        
+        if regen_id:
+            id_attr = 'temp_id'
+
+        # [attributes] 
+        keep_attrs = args.get('attr_list', [])
+        if not attr_check(keep_attrs, []):
+            keep_attrs = []
+        
+        # [Tags] for clickable elem, keep: must keep, obs: keep if follow specific rule
+        parent_chain = args.get('parent_chain', False)
+        keep_elem = args.get('keep_elem', [])
+        obs_elem = args.get('obs_elem', [])
+        
+        # sanity check
+        self.set_args(use_position, window_size, rect, label_attr, id_attr, keep_attrs, keep_elem, obs_elem, parent_chain, get_new_label, dataset)
+        
+        # [Prompt]
+        prompt = args.get('prompt', None)
+        self.prompt = HtmlPrompt(prompt)
+
+        # traverse and get special data
+        if regen_id or regen_label:
+            self.mark_id()
+        
+        if get_new_label:
+            self.used_labels = {}
+            
+        self.identifier = IdentifierTool(label_method, self.used_labels)
+          
+    def set_args(self, use_position: bool=False, window_size: tuple=(), rect_dict: dict[str]={}, label_attr: str='', 
+                 id_attr: str='', keep_attrs: list[str]=[], keep_elem: list[str]=[], obs_elem: list[str]=[], 
+                 parent_chain: bool=False, get_new_label: bool=False, dataset: str='') -> None:
+        
+        self.use_position = use_position
+        self.window_size = window_size
+        self.rect = rect_dict
+        self.label_attr = label_attr
+        self.id_attr = id_attr
+        self.keep_attrs = keep_attrs
+        self.keep = keep_elem
+        self.obs = obs_elem
+        self.parent_chain = parent_chain
+        self.get_new_label = get_new_label
+        self.dataset = dataset
+        
+    def get_config(self):
+        config = {
+            'id_attr': self.id_attr,
+            'keep_attrs': self.keep_attrs[:5],
+            'label_attr': self.label_attr,
+            'use_position': self.use_position,
+            'window_size': self.window_size,
+            'rect': dict(list(self.rect.items())[:3]),
+            'keep_elem': self.keep[:5],
+            'obs_elem': self.obs[:5],
+            'parent_chain': self.parent_chain,
+            'prompt_name': self.prompt.name,
+            'identifier_name': self.identifier.name
+        }
+        
+        return config, config_meta.format(**config)
+    
+    def update_rect_dict(self, rect_dict: dict[str]={}) -> None:
+        self.rect = rect_dict
+    
+    @staticmethod
+    def ctx2tree(ctx: str) -> html.HtmlElement:
+        # remove useless tags, eg. style and script
+        ctx = re.sub('<!--[\W\w]*?-->', '', ctx)
+        ctx = re.sub('<style[\W\w]*?>[\W\w]*?</style>', '', ctx)
+        ctx = re.sub('<script[\W\w]*?>[\W\w]*?</script>', '', ctx)
+        ctx = '' if ctx is None else re.sub(r'\s+', ' ', ctx).strip()
+        dom_tree = html.fromstring(ctx)
+        return dom_tree
+
+    @staticmethod
+    def get_root(tree: html.HtmlElement) -> html.HtmlElement:
+        node = tree.xpath('//*')[0]
+        while True:
+            parent = node.getparent()
+            if parent is None:
+                break
+            node = parent
+        return node
+    
+    def get_node_by_bid(self, tree: html.HtmlElement, bid: str) -> html.HtmlElement:
+        nodes = tree.xpath(f'//*[@{self.id_attr}="{bid}"]')
+        if len(nodes) == 0:
+            return None
+        return nodes[0]
+    
+    def id_label_converter(self, label: str) -> str:
+        return self.bids2label.get(label, '')
+    
+    def id_xpath_converter(self, label: str) -> str:
+        return self.bids2xpath.get(label, '')
+        
+    def mark_id(self) -> None:
+        root = self.get_root(self.dom_tree)
+        _, i2xpath, used_labels = get_xpath_top_down(root, self.id_attr, self.label_attr)
+        self.used_labels = used_labels        
+        self.bids2xpath = i2xpath
+    
+    def parse(self, root: html.HtmlElement, keep: list[str], obs: list[str], parent_chain: bool=False, get_new_label: bool=False) -> dict[str]:
+        def get_text(str: str) -> str:
+            return '' if str is None else str.strip()[:500]
+        
+        def check_attr(attr: str, node: html.HtmlElement) -> bool:
+            tag = node.tag
+            if (
+                ( attr == 'role' and node.attrib.get(attr, '') in ['presentation', 'none', 'link'] )
+                or ( attr == 'type' and node.attrib.get(attr, '') == 'hidden' )
+                # or ( attr == 'value' and tag in ['option'] )
+                ):
+                return False
+            return True
+        
+        def is_visible(node: html.HtmlElement, bid: str) -> bool:
+            if self.dataset == 'mind2web':
+                bound = node.attrib.get('bounding_box_rect', None)
+                self.rect[bid] = rect2tuple(bound)
+
+            if not self.use_position:
+                return True
+            
+            rect = self.rect.get(bid, None)
+            if rect is None:
+                return False
+            
+            if self.window_size is None:
+                return True
+            
+            # get window size
+            wx, wy, ww, wh = self.window_size
+            x, y, w, h = rect
+            if x + w < wx or x > wx + ww or y + h < wy or y > wy + wh:
+                return False
+
+            return True
+        
+        def _dfs(node: html.HtmlElement, keep: list[str]=[], obs: list[str]=[], 
+                 parent_chain: bool=False, get_new_label: bool=False, par_keep: bool=False) -> (str, dict[str]):
+            # basic information
+            bid = node.attrib.get(self.id_attr, '')
+            tag = node.tag
+            label = node.attrib.get(self.label_attr, '')
+            
+            # element which is keeped equivalent to visible
+            visible = is_visible(node, bid)
+            in_keep_list = bid in keep
+            in_obs_list = (bid in obs or len(label) > 0) and visible
+            keep_element = in_keep_list or in_obs_list or visible or par_keep
+            
+            # mark label
+            bids2label, labeled_elems = {}, []
+            have_label = False
+            if in_keep_list or in_obs_list:
+                if label is None or len(label) == 0 or get_new_label:
+                    label = self.identifier.generate()
+                    node.attrib[self.label_attr] = label
+                bids2label[bid] = label
+                bids2label[label] = bid
+                have_label = True
+            
+            # get text or alt_text of current element
+            text = get_text(node.text)
+            
+            classes = {}
+            # keep attributes if needed
+            keep_all_attrs = len(self.keep_attrs) == 0
+            keep_attrs = node.attrib.keys() if keep_all_attrs else self.keep_attrs
+            
+            # traverse attributes
+            for attr in keep_attrs:
+                if attr not in node.attrib or not check_attr(attr, node):
+                    continue
+                if attr in [self.id_attr, self.label_attr]:
+                    continue
+                val = get_text(node.attrib[attr])
+                if len(val) > 0 or keep_all_attrs:
+                    classes[attr] = val
+
+            have_text = len(text) > 0 or len(classes) > 0
+            
+            parts = []
+            clickable_count = 0
+            children = node.getchildren()
+            for child in children:
+                cres, cmsg = _dfs(child, keep, obs, parent_chain, get_new_label)
+                clickable_count += 1 if cmsg.get('have_clickable', False) else 0
+                bids2label.update(cmsg.get('bids2label', {}))
+                labeled_elems.extend(cmsg.get('label_element', []))
+                if len(cres) != 0:
+                    parts.append(cres)
+
+            dom = self.prompt.subtree_constructor(parts)
+            
+            # remove <text|> if all children are text
+            keep_as_all_text = (dom.count('<') == dom.count('<text|')) and dom.count('<') > 0
+            if keep_as_all_text:
+                matches = re.findall(r'<text\| ([^>]+) >', dom)
+                dom = self.prompt.subtree_constructor(matches)
+            
+            keep_element = keep_element and (clickable_count > 1 or have_text or have_label or keep_as_all_text)
+            keep_as_parent = len(dom) > 0 and parent_chain
+            if in_keep_list or keep_element or keep_as_parent:
+                dom = self.prompt.prompt_constructor(tag, label, text, dom, classes)
+            
+            if have_label:
+                labeled_elems.append(bid)
+            
+            control_msg = {
+                'have_clickable': bool(clickable_count or have_text),
+                'bids2label': bids2label,
+                'label_element': labeled_elems,
+            }
+            
+            return dom, control_msg
+        
+        dom, cmsg = _dfs(root, keep, obs, parent_chain, get_new_label)
+        return dom, cmsg
+        
+    def parse_tree(self) -> dict[str]:
+        # start from here
+        stt = time.time()
+        root = self.get_root(self.dom_tree)
+        dom, cmsg = self.parse(root, self.keep, self.obs, self.parent_chain, self.get_new_label)
+        self.bids2label = cmsg.get('bids2label', {})
+        self.keep = list(set(self.keep + cmsg.get('label_element', [])))
+        
+        obj = {
+            'html': dom,
+            'parse_time': time.time() - stt
+        }
+
+        return obj
+
+    # From mind2web, https://github.com/OSU-NLP-Group/Mind2Web/blob/main/src/data_utils/dom_utils.py
+    def get_keep_elements(self, tree: html.HtmlElement, keep: list[str], max_depth: int, max_children: int, 
+                            max_sibling: int, dfs_count: int=1, keep_parent: bool=False) -> list[str]:
+        def get_anscendants(node: html.HtmlElement, max_depth: int, current_depth: int=0) -> list[str]:
+            if current_depth > max_depth:
+                return []
+
+            anscendants = []
+            parent = node.getparent()
+            if parent is not None:
+                anscendants.append(parent)
+                anscendants.extend(get_anscendants(parent, max_depth, current_depth + 1))
+
+            return anscendants
+        
+        def get_descendants(node: html.HtmlElement, max_depth: int, current_depth: int=0) -> list[str]:
+            if current_depth > max_depth:
+                return []
+
+            descendants = []
+            for child in node:
+                descendants.append(child)
+                descendants.extend(get_descendants(child, max_depth, current_depth + 1))
+
+            return descendants
+        
+        to_keep = set(copy.deepcopy(keep))
+        nodes_to_keep = set()
+        
+        for _ in range(max(1, dfs_count)):
+            for bid in to_keep:
+                candidate_node = self.get_node_by_bid(tree, bid)
+                if candidate_node is None:
+                    continue
+                
+                nodes_to_keep.add(candidate_node.attrib[self.id_attr])
+                # get all ancestors or with max depth
+                nodes_to_keep.update([x.attrib.get(self.id_attr, '') for x in get_anscendants(candidate_node, max_depth)])
+                
+                # get descendants with max depth
+                nodes_to_keep.update([x.attrib.get(self.id_attr, '') for x in get_descendants(candidate_node, max_depth)][:max_children])
+                # get siblings within range
+                parent = candidate_node.getparent()
+                if parent is None:
+                    continue
+                
+                siblings = [x for x in parent.getchildren() if x.tag != 'text']
+                if candidate_node not in siblings:
+                    continue
+                
+                idx_in_sibling = siblings.index(candidate_node)
+                nodes_to_keep.update([x.attrib.get(self.id_attr, '') 
+                                    for x in siblings[max(0, idx_in_sibling - max_sibling) : idx_in_sibling + max_sibling + 1]])
+            
+            max_children = int(max_children * 0.5)
+            max_depth = int(max_depth * 0.5)
+            max_sibling = int(max_sibling * 0.7)
+            
+            to_keep = copy.deepcopy(nodes_to_keep)
+            
+        if keep_parent:
+            for bid in keep:
+                candidate_node = self.get_node_by_bid(tree, bid)
+                if candidate_node is None:
+                    continue
+                nodes_to_keep.update([x.attrib.get(self.id_attr, '') for x in candidate_node.xpath("ancestor::*")])
+
+        return list(nodes_to_keep)
+    
+    def prune(self, tree: html.HtmlElement, nodes_to_keep: list[str]) -> html.HtmlElement:
+        # remove nodes not in nodes_to_keep
+        for node in tree.xpath('//*')[::-1]:
+            if node.tag != 'text':
+                is_keep = node.attrib.get(self.id_attr, '') in nodes_to_keep
+                is_candidate = node.attrib.get(self.id_attr, '') in self.keep
+            else:
+                is_keep = (node.getparent().attrib.get(self.id_attr, '') in nodes_to_keep)
+                is_candidate = (node.getparent().attrib.get(self.id_attr, '') in self.keep)
+            
+            if not is_keep and node.getparent() is not None:
+                # insert all children into parent
+                for child in node.getchildren():
+                    node.addprevious(child)
+                node.getparent().remove(node)
+            else:
+                # if not is_candidate or node.tag == 'text':
+                #     node.attrib.pop(self.id_attr, None)
+                if (
+                    len(node.attrib) == 0
+                    and not any([x.tag == 'text' for x in node.getchildren()])
+                    and node.getparent() is not None
+                    and node.tag != "text"
+                    and len(node.getchildren()) <= 1
+                ):
+                    # insert all children into parent
+                    for child in node.getchildren():
+                        node.addprevious(child)
+                    node.getparent().remove(node)
+        
+        return tree
+                    
+    def prune_tree(self, dfs_count: int=1, max_depth: int=3, max_children: int=30, 
+                   max_sibling: int=3, keep_parent: bool=False) -> None:
+        # clone the tree
+        new_tree = copy.deepcopy(self.dom_tree)
+        nodes_to_keep = self.get_keep_elements(new_tree, self.keep, max_depth, max_children, max_sibling, dfs_count, keep_parent)
+        new_tree = self.prune(new_tree, nodes_to_keep)
+        
+        self.dom_tree = new_tree
+    
+    def get_segment(self, bid: str) -> str:
+        # clone the tree
+        new_tree = copy.deepcopy(self.dom_tree)
+        nodes_to_keep = self.get_keep_elements(new_tree, [bid], 0, 2, 1)
+        new_tree = self.prune(new_tree, nodes_to_keep)
+        dom, _ = self.parse(new_tree, self.keep, [], False)
+        return dom
+    
+    def get_rect_data(self, bids: list[str]) -> list[dict[str]]:
+        res = []
+        for bid in bids:
+            label = self.bids2label.get(bid, '')
+            rect = self.rect.get(bid, None)
+            res.append({
+                'bid': bid,
+                'label': label,
+                'rect': rect
+            })
+        return res
+        

browser_env/html_tools/identifier.py

@@ -0,0 +1,64 @@
+import secrets
+
+class IdentifierTool:
+    def __init__(self, method: str='order', existing_labels: dict[str]={}) -> None:
+        self.methods = {
+            'order': self.get_identifier_in_order,
+            'random': self.get_random_identifier,
+        }
+        
+        if method is None:
+            method = 'order'
+            
+        self.func = self.methods.get(method, None)
+        self.name = method
+        if self.func is None:
+            raise ValueError(f'Invalid method for identifier: {method}')
+        
+        self.reset(existing_labels)
+    
+    def reset(self, exists: dict[str]={}) -> None:
+        self.identifier = -1
+        self.exists = {} if exists is None else exists
+        
+    def get_identifier_in_order(self) -> str:
+        def id2str(id: int) -> str:
+            if id < 26:
+                return chr(id + 65)
+            id -= 26
+            c0 = id // 676
+            c1 = (id // 26) % 26
+            c2 = id % 26
+            label = f'{chr(c1 + 65)}{chr(c2 + 65)}'
+            return label if c0 == 0 else f'{chr(c0 + 64)}{label}'
+        
+        self.identifier += 1
+        label = id2str(self.identifier)
+        
+        while label in self.exists:
+            self.identifier += 1
+            label = id2str(self.identifier)
+        
+        self.exists[label] = True
+        return label
+    
+    def get_random_identifier(self) -> str:
+        secret_generator = secrets.SystemRandom()
+        
+        def get_random_label(n: int=2) -> str:
+            tmp = ''
+            for _ in range(n):
+                tmp += chr(secret_generator.randint(65, 90))
+            return tmp
+
+        wc = 3 if len(self.exists) > 280 else 2
+
+        label = get_random_label(wc)
+        while label in self.exists:
+            label = get_random_label(wc)
+        
+        self.exists[label] = True
+        return label
+        
+    def generate(self):
+        return self.func()

browser_env/html_tools/prompt.py

@@ -0,0 +1,97 @@
+from .configs import prompts
+
+class HtmlPrompt:    
+    def __init__(self, prompt: str='') -> None:
+        prompt = self.extract(prompt, 'xml')
+        if prompt not in prompts:
+            raise Exception('Unknown prompt: ' + prompt)
+        
+        constructors = {
+            'refine': self.normal_prompt_constructor,
+            'xml': self.normal_prompt_constructor,
+            'new_data': self.new_data_prompt_constructor,
+        }
+
+        self.name = prompt
+        self.prompt = prompts[prompt]
+        self.constructor = constructors[prompt]
+
+    @staticmethod
+    def extract(data, default=''):
+        return data if data is not None else default
+    
+    def subtree_constructor(self, subtree: list[str]=[]) -> str:
+        return self.prompt['subtree_splitter'].join(subtree)
+
+    def normal_prompt_constructor(self, tag: str='', label: str='', content: str='', subtree_str: str='', class_dict: dict[str]={}) -> str:
+        def add_prefix(data, prefix):
+            return prefix + data if len(data) > 0 else ''
+        
+        tag = self.extract(tag)
+        label = self.extract(label)
+        content = self.extract(content)
+        subtree_str = self.extract(subtree_str, '')
+        class_dict = self.extract(class_dict, {})
+        
+        label_str = ''
+        if len(label) > 0:
+            label_str = self.prompt['label'].format(label=label)
+        
+        classes = []
+        values = set()
+        for key, val in class_dict.items():
+            if val in values:
+                continue
+            values.add(val)
+            classes.append(self.prompt['attr'].format(key=key, attr=val))
+        classes_str = self.prompt['attr_splitter'].join(classes)
+        
+        content_splitter = ' ' if len(classes_str) == 0 else self.prompt['attr_splitter']
+        classes_str = add_prefix(classes_str, ' ')
+        content_str = add_prefix(content, content_splitter)
+        subtree_str = add_prefix(subtree_str, ' ')
+
+        return self.prompt['dom'].format(tag=tag, label=label_str, attr=classes_str, content=content_str, subtree=subtree_str)
+    
+    def new_data_prompt_constructor(self, tag: str='', label: str='', content: str='', subtree_str: str='', class_dict: dict[str]={}) -> str:
+        def add_prefix(data, prefix):
+            return prefix + data if len(data) > 0 else ''
+        
+        tag = self.extract(tag)
+        label = self.extract(label)
+        content = self.extract(content)
+        subtree_str = self.extract(subtree_str, '')
+        class_dict = self.extract(class_dict, {})
+        
+        label_str = ''
+        if len(label) > 0:
+            label_str = self.prompt['label'].format(label=label)
+        
+        classes = []
+        values = set()
+        
+        message = []
+        for key, val in class_dict.items():
+            if val == '':
+                message.append(key)
+                continue
+            if val in values:
+                continue
+            values.add(val)
+            classes.append(self.prompt['attr'].format(key=key, attr=val))
+        
+        if len(message) > 0:
+            message_str = ' '.join(message)
+            classes.append(self.prompt['attr'].format(key='message', attr=message_str))
+            
+        classes_str = self.prompt['attr_splitter'].join(classes)
+        
+        content_splitter = ' ' if len(classes_str) == 0 else self.prompt['attr_splitter']
+        classes_str = add_prefix(classes_str, ' ')
+        content_str = add_prefix(content, content_splitter)
+        subtree_str = add_prefix(subtree_str, ' ')
+
+        return self.prompt['dom'].format(tag=tag, label=label_str, attr=classes_str, content=content_str, subtree=subtree_str)
+
+    def prompt_constructor(self, tag: str='', label: str='', content: str='', subtree_str: str='', class_dict: dict[str]={}) -> str:
+        return self.constructor(tag, label, content, subtree_str, class_dict)

browser_env/html_tools/utils.py

@@ -0,0 +1,101 @@
+from lxml import html
+def get_xpath_top_down(element: html.HtmlElement, id_column: str='temp_id', label_column: str='temp_clickable_label', path: str='', order: int=0, 
+                        in_svg: bool=False, temp_id: int=0) -> tuple[int, dict[str, str], dict[str]]:
+    used_labels, i2xpath = {}, {}
+    # path
+    tag = element.tag.lower()
+    in_svg = in_svg or (tag == 'svg')
+    
+    if not in_svg and 'id' in element.attrib:
+        node_id = element.attrib['id']
+        path = f'//*[@id="{node_id}"]'
+    else:
+        suffix = f'[{order}]' if order > 0 else ''
+        prefix = f'*[name()="{tag}"]' if in_svg else tag
+        path = path + '/' + prefix + suffix
+    
+    # add temp id
+    element.attrib[id_column] = str(temp_id)
+    ori_label = element.attrib.get(label_column, '')
+    if ori_label != '':
+        used_labels[ori_label] = True
+    
+    bid = str(temp_id)
+    i2xpath[bid] = path
+    i2xpath[path] = bid
+    i2xpath[f'xpath/{path}'] = bid
+    i2xpath[f'xpath=/{path}'] = bid
+    
+    temp_id += 1
+    
+    # traverse node
+    children = element.getchildren()
+    tag_dict = {}
+    id_list = []
+    for child in children:
+        ctag = child.tag.lower()
+        if ctag not in tag_dict:
+            tag_dict[ctag] = 0
+        tag_dict[ctag] += 1
+        id_list.append(tag_dict[ctag])
+    
+    for cid, child in zip(id_list, children):
+        ctag = child.tag.lower()
+        cod = cid if tag_dict[ctag] > 1 else 0
+        temp_id, i2x, ulabels = get_xpath_top_down(child, id_column, label_column, path, cod, in_svg, temp_id)
+        i2xpath.update(i2x)
+        used_labels.update(ulabels)
+    
+    return temp_id, i2xpath, used_labels
+        
+def print_html_object(obj: str='') -> str:
+    tab_cnt = 0
+    result, content, sep = '', '', ''
+    last_is_left, last_is_right = False, False
+    for ch in obj:
+        if ch == '<':
+            result += '\n'
+            if len(content.strip()) > 0:
+                result += sep + content.strip() + '\n'
+            result += sep + '<'
+            
+            tab_cnt += 1
+            sep = '  ' * tab_cnt
+            
+            content = ''
+            last_is_right = False
+            last_is_left = True
+        elif ch == '>':
+            if last_is_left:
+                result += content
+            else:
+                if last_is_right:
+                    result += '\n'
+                if len(content.strip()) > 0:
+                    result += sep + content.strip() + '\n'
+            
+            tab_cnt -= 1
+            sep = '  ' * tab_cnt
+            
+            if not last_is_left:
+                result += sep
+            
+            result += '>'
+            content = ''
+            
+            last_is_right = True
+            last_is_left = False
+        else:
+            content += ch
+    
+    return result
+
+def rect2tuple(rect: str) -> tuple[int, int, int, int]:
+    if rect is None or type(rect) != type('str'):
+        return None
+    rect = rect.strip()
+    if rect.count(',') != 3:
+        return None
+    rect = rect.split(',')
+    rect = [float(r) for r in rect]
+    return tuple(rect)

browser_env/scripts/__init__.py

@@ -0,0 +1,43 @@
+import os
+from pathlib import Path
+rootdir = Path(__file__).parent
+
+# marker, gpt-4v-act style
+with open(os.path.join(rootdir, 'local_marker.js'), 'r') as f:
+    local_marker_script = f.read()
+    
+with open(os.path.join(rootdir, 'mix_marker.js'), 'r') as f:
+    mix_marker_script = f.read()
+
+with open(os.path.join(rootdir, 'get_data.js'), 'r') as f:
+    get_rect_script = f.read()
+
+# canva handler
+with open(os.path.join(rootdir, 'canva_handler.js'), 'r') as f:
+    canva_handler_script = f.read()
+
+# draw label on page
+with open(os.path.join(rootdir, 'label_marker.js'), 'r') as f:
+    label_marker_script = f.read()
+    
+# get text from page
+with open(os.path.join(rootdir, 'get_text.js'), 'r') as f:
+    get_text_script = f.read()
+
+# remove label draw on page
+remove_label_mark_script = """
+    () => {
+        document.querySelectorAll(".our-dom-marker").forEach(item => {
+            document.body.removeChild(item);
+        });
+    }
+"""
+
+remove_id_script = """
+    () => {
+        Array.from(document.getElementsByClassName('possible-clickable-element')).forEach((element) => {
+            element.classList.remove('possible-clickable-element');
+            element.removeAttribute('data-testid');
+        });
+    }
+"""

browser_env/scripts/canva_handler.js

@@ -0,0 +1,44 @@
+() => {
+    var items = Array.prototype.slice.call(
+        document.querySelectorAll("canvas")
+    );
+
+    var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
+    var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+    
+    items = items.map(element => {
+        // var img = element.toDataURL("image/png");
+        var bb = element.getClientRects();
+        var rect = {
+            left: -1,
+            top: -1,
+            right: -1,
+            bottom: -1,
+            width: 0,
+            height: 0,
+        };
+        if (bb.length > 0) {
+            bb = bb[0];
+            rect = {
+                left: Math.max(0, bb.left),
+                top: Math.max(0, bb.top),
+                right: Math.min(vw, bb.right),
+                bottom: Math.min(vh, bb.bottom)
+            };
+            rect = {
+                ...rect,
+                width: rect.right - rect.left,
+                height: rect.bottom - rect.top
+            };
+        }
+        
+        return {
+            rects: rect,
+            tag: element.tagName.toLowerCase?.() || "",
+            text: element.textContent.trim().replace(/\s{2,}/g, ' '),
+            // img: img
+        };
+    });
+
+    return items;
+}

browser_env/scripts/get_data.js

@@ -0,0 +1,73 @@
+(packet) => {
+    function int2str(index) {
+        var str = "";
+        while (index >= 0) {
+            str = String.fromCharCode(65 + index % 26) + str;
+            index = Math.floor(index / 26) - 1;
+        }
+        return str;
+    };
+
+    selector = packet.selector
+    index = packet.startIndex
+    var items = Array.prototype.slice.call(
+        document.querySelectorAll(selector)
+    );
+
+    var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
+    var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+    
+    items = items.filter(
+        x => !items.some(y => x.contains(y) && !(x == y))
+    ).map(element => {
+        var bb = element.getClientRects();
+        var rect = {
+            left: 0,
+            top: 0,
+            right: 0,
+            bottom: 0,
+            width: 0,
+            height: 0
+        };
+        var keep = false;
+        var text = "", id = -1;
+        if (bb.length > 0) {
+            bb = bb[0];
+            rect = {
+                left: Math.max(0, bb.left),
+                top: Math.max(0, bb.top),
+                right: Math.min(vw, bb.right),
+                bottom: Math.min(vh, bb.bottom)
+            };
+            rect = {
+                ...rect,
+                width: rect.right - rect.left,
+                height: rect.bottom - rect.top
+            };
+            if (rect.width > 0 || rect.height > 0) {
+                keep = true;
+                if (index >= 0) { 
+                    id = int2str(index++);
+                    element.setAttribute("data-testid", id);
+                }
+                var childNodes = element.childNodes;
+                
+                for (var i = 0; i < childNodes.length; i++) {
+                    if (childNodes[i].nodeType == Node.TEXT_NODE) {
+                        text += childNodes[i].textContent;
+                    }
+                }
+            }
+        }
+        
+        return {
+            keep: true,
+            id,
+            rects: rect,
+            tag: element.tagName.toLowerCase?.() || "",
+            text,//: element.innerText?.trim().replace(/\s{2,}/g, " ") || ""
+        };
+    }).filter(x => x.keep);
+
+    return [items, index];
+}

browser_env/scripts/get_text.js

@@ -0,0 +1,48 @@
+() => {
+    var items = Array.prototype.slice.call(
+        document.querySelectorAll("*")
+    );
+
+    var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
+    var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+    const ignoreTags = ["script", "html"];
+    items = items.map(element => {
+        const tag = element.tagName.toLowerCase?.() || "";
+        var bb = element.getClientRects();
+        var keep = false;
+        var text = '';
+
+        const domId = element.getAttribute('data-testid');
+        var id = domId? parseInt(domId): "-";
+
+        if (bb.length > 0) {
+            bb = bb[0];
+            var width = Math.min(vw, bb.right) - Math.max(0, bb.left);
+            var height = Math.min(vh, bb.bottom) - Math.max(0, bb.top);
+
+            if (width > 0 && height > 0) {
+                keep = true;
+                var childNodes = element.childNodes;
+                
+                for (var i = 0; i < childNodes.length; i++) {
+                    if (childNodes[i].nodeType == Node.TEXT_NODE) {
+                        text += childNodes[i].textContent;
+                    }
+                }
+            }
+        }
+        
+        text = text.trim().replace(/\s{2,}/g, ' ');
+        if (ignoreTags.includes(tag)) keep = false;
+        if (id == "-" && text.length == 0) keep = false;
+        
+        return {
+            keep,
+            tag,
+            id,
+            text, //:element.innerText?.trim().replace(/\s{2,}/g, " ") || ""
+        };
+    }).filter(x => x.keep);
+ 
+    return items;
+}

browser_env/scripts/label_marker.js

@@ -0,0 +1,65 @@
+(items) => {
+    function getRandomColor() {
+        var letters = '0123456789ABCDEF';
+        var color = '#';
+        for (var i = 0; i < 6; i++) {
+            color += letters[Math.floor(Math.random() * 16)];
+        }
+        return color;
+    }
+
+    items.filter(
+        item => item.id != ""
+    ).forEach((item) => {
+        const bbox = item.rects;
+        const id_string = `dom-marker-id-${index}`;
+        
+        index = item.id;
+
+        outerElement = document.createElement("div");
+        outerElement.classList.add("our-dom-marker");
+        // var borderColor = getRandomColor();
+        var borderColor = "#FFFF00";
+        outerElement.style.outline = `2px dashed ${borderColor}`; 
+        outerElement.style.position = "fixed";
+        outerElement.style.left = bbox.left - 2 + "px";
+        outerElement.style.top = bbox.top - 2 + "px";
+        outerElement.style.width = bbox.width + 4 + "px";
+        outerElement.style.height = bbox.height + 4 + "px";
+        outerElement.style.pointerEvents = "none";
+        outerElement.style.boxSizing = "border-box";
+        outerElement.style.zIndex = 2147483647;
+
+        innerElement = document.createElement("div");
+        innerElement.classList.add("our-dom-marker");
+        innerElement.style.outline = `2px dashed #222288`;
+        innerElement.style.position = "fixed";
+        innerElement.style.left = bbox.left + "px";
+        innerElement.style.top = bbox.top + "px";
+        innerElement.style.width = bbox.width + "px";
+        innerElement.style.height = bbox.height + "px";
+        innerElement.style.pointerEvents = "none";
+        innerElement.style.boxSizing = "border-box";
+        innerElement.style.zIndex = 2147483647;
+    
+        // Add floating label at the corner
+        var label = document.createElement("span");
+        var topPosition = 25;
+        if (bbox.top < 25) topPosition = bbox.top;
+        label.textContent = index;
+        label.style.position = "absolute";
+        label.style.top = `-${topPosition}px`;
+        label.style.left = "0px";
+        label.style.background = borderColor;
+        label.style.color = "black";
+        label.style.padding = "2px 4px";
+        label.style.fontSize = "16px";
+        label.style.borderRadius = "2px";
+        label.style.fontWeight = "bold";
+        outerElement.appendChild(label);
+    
+        document.body.appendChild(outerElement);
+        document.body.appendChild(innerElement);
+    })
+    return items;
+}

browser_env/scripts/local_marker.js

@@ -0,0 +1,46 @@
+() => {
+    var items = Array.prototype.slice.call(
+        document.querySelectorAll('*')
+    ).map((element) => {
+        var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
+        var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+
+        var rects = [...element.getClientRects()].filter(bb => {
+            var center_x = bb.left + bb.width / 2;
+            var center_y = bb.top + bb.height / 2;
+            var elAtCenter = document.elementFromPoint(center_x, center_y);
+
+            return elAtCenter === element || element.contains(elAtCenter) 
+        }).map(bb => {
+            const rect = {
+                left: Math.max(0, bb.left),
+                top: Math.max(0, bb.top),
+                right: Math.min(vw, bb.right),
+                bottom: Math.min(vh, bb.bottom)
+            };
+            return {
+                ...rect,
+                width: rect.right - rect.left,
+                height: rect.bottom - rect.top
+            }
+        });
+
+        var area = rects.reduce((acc, rect) => acc + rect.width * rect.height, 0);
+        
+        return {
+            element: element,
+            include: window.getComputedStyle(element).cursor == "pointer",
+            area,
+            rects,
+            text: element.textContent.trim().replace(/\s{2,}/g, ' '),
+        };
+    }).filter(item =>
+        item.include && (item.area >= 20)
+    )
+  
+    items = items.filter(x => !items.some(y => x.element.contains(y.element) && !(x == y)))
+
+    items.forEach(item => {
+        item.element.classList.add('possible-clickable-element');
+    });
+}

browser_env/scripts/mix_marker.js

@@ -0,0 +1,148 @@
+() => {
+    var items = Array.prototype.slice.call(
+        document.querySelectorAll('*')
+    ).map(function(element) {
+        var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
+        var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
+        
+        var rects = [...element.getClientRects()].filter(bb => {
+            var center_x = bb.left + bb.width / 2;
+            var center_y = bb.top + bb.height / 2;
+            var elAtCenter = document.elementFromPoint(center_x, center_y);
+        
+            if (!elAtCenter) return false;
+            return elAtCenter === element || element.contains(elAtCenter) 
+        }).map(bb => {
+            const rect = {
+                left: Math.max(0, bb.left),
+                top: Math.max(0, bb.top),
+                right: Math.min(vw, bb.right),
+                bottom: Math.min(vh, bb.bottom)
+            };
+            return {
+                ...rect,
+                width: rect.right - rect.left,
+                height: rect.bottom - rect.top
+            }
+        });
+        // var rects = [];
+        var area = rects.reduce((acc, rect) => acc + rect.width * rect.height, 0);
+    
+        const tagName = element.tagName.toLowerCase?.() || "";
+        let isClickable = ((element.onclick != null) || window.getComputedStyle(element).cursor == "pointer");
+        
+        // Insert area elements that provide click functionality to an img.
+        if (tagName === "img") {
+            let mapName = element.getAttribute("usemap");
+            if (mapName) {
+                const imgClientRects = element.getClientRects();
+                mapName = mapName.replace(/^#/, "").replace('"', '\\"');
+                const map = document.querySelector(`map[name=\"${mapName}\"]`);
+                if (map && (imgClientRects.length > 0)) isClickable = true;
+            }
+        }
+    
+        if (!isClickable) {
+            const role = element.getAttribute("role");
+            const clickableRoles = [
+                "button",
+                "tab",
+                "link",
+                "checkbox",
+                "menuitem",
+                "menuitemcheckbox",
+                "menuitemradio",
+                "radio",
+            ];
+            if (role != null && clickableRoles.includes(role.toLowerCase())) {
+                isClickable = true;
+            } else {
+                const contentEditable = element.getAttribute("contentEditable");
+                if (
+                contentEditable != null &&
+                ["", "contenteditable", "true"].includes(contentEditable.toLowerCase())
+                ) {
+                isClickable = true;
+                }
+            }
+        }
+    
+        // Check for jsaction event listeners on the element.
+        if (!isClickable && element.hasAttribute("jsaction")) {
+            const jsactionRules = element.getAttribute("jsaction").split(";");
+            for (let jsactionRule of jsactionRules) {
+                const ruleSplit = jsactionRule.trim().split(":");
+                if ((ruleSplit.length >= 1) && (ruleSplit.length <= 2)) {
+                const [eventType, namespace, actionName] = ruleSplit.length === 1
+                    ? ["click", ...ruleSplit[0].trim().split("."), "_"]
+                    : [ruleSplit[0], ...ruleSplit[1].trim().split("."), "_"];
+                if (!isClickable) {
+                    isClickable = (eventType === "click") && (namespace !== "none") && (actionName !== "_");
+                }
+                }
+            }
+        }
+    
+        if (!isClickable) {
+            const clickableTags = [
+                "input",
+                "textarea",
+                "select",
+                "button",
+                "a",
+                "iframe",
+                "video",
+                "object",
+                "embed",
+                "details"
+            ];
+        isClickable = clickableTags.includes(tagName);
+            }
+        
+        if (!isClickable) {
+            if (tagName === "label")
+                isClickable = (element.control != null) && !element.control.disabled;
+            else if (tagName === "img")
+                isClickable = ["zoom-in", "zoom-out"].includes(element.style.cursor);
+        }
+
+        // An element with a class name containing the text "button" might be clickable. However, real
+        // clickables are often wrapped in elements with such class names. So, when we find clickables
+        // based only on their class name, we mark them as unreliable.
+        const className = element.getAttribute("class");
+        if (!isClickable && className && className.toLowerCase().includes("button")) {
+            isClickable = true;
+        }
+    
+        // Elements with tabindex are sometimes useful, but usually not. We can treat them as second
+        // class citizens when it improves UX, so take special note of them.
+        const tabIndexValue = element.getAttribute("tabindex");
+        const tabIndex = tabIndexValue ? parseInt(tabIndexValue) : -1;
+        if (!isClickable && !(tabIndex < 0) && !isNaN(tabIndex)) {
+            isClickable = true;
+        }
+    
+        const idValue = element.getAttribute("id");
+        const id = idValue ? idValue.toLowerCase() : "";
+        if (isClickable && area == 0) {
+            const textValue = element.textContent.trim().replace(/\s{2,}/g, ' ');
+            clickable_msg = `${tagName}[id=${id}] ${isClickable} (${area}) ${textValue}`
+        }
+    
+        return {
+            element: element,
+            include: isClickable,
+            area,
+            rects,
+            text: element.textContent.trim().replace(/\s{2,}/g, ' ')
+        };
+    }).filter(item =>
+        item.include && (item.area >= 1)
+    );
+
+    items = items.filter(x => !items.some(y => x.element.contains(y.element) && !(x == y)))
+
+    items.forEach(item => {
+        item.element.classList.add('possible-clickable-element');
+    });
+}

browser_env/trajectory.py

@@ -0,0 +1,6 @@
+from typing import Union
+
+from .actions import Action
+from .utils import StateInfo
+
+Trajectory = list[Union[StateInfo, Action]]

browser_env/utils.py

@@ -0,0 +1,80 @@
+from dataclasses import dataclass
+from io import BytesIO
+from typing import Any, Dict, TypedDict, Union
+
+import numpy as np
+import numpy.typing as npt
+from PIL import Image
+
+
+@dataclass
+class DetachedPage:
+    url: str
+    content: str  # html
+
+
+def png_bytes_to_numpy(png: bytes) -> npt.NDArray[np.uint8]:
+    """Convert png bytes to numpy array
+
+    Example:
+
+    >>> fig = go.Figure(go.Scatter(x=[1], y=[1]))
+    >>> plt.imshow(png_bytes_to_numpy(fig.to_image('png')))
+    """
+    return np.array(Image.open(BytesIO(png)))
+
+
+class AccessibilityTreeNode(TypedDict):
+    nodeId: str
+    ignored: bool
+    role: dict[str, Any]
+    chromeRole: dict[str, Any]
+    name: dict[str, Any]
+    properties: list[dict[str, Any]]
+    childIds: list[str]
+    parentId: str
+    backendDOMNodeId: str
+    frameId: str
+    bound: list[float] | None
+    union_bound: list[float] | None
+    offsetrect_bound: list[float] | None
+
+
+class DOMNode(TypedDict):
+    nodeId: str
+    nodeType: str
+    nodeName: str
+    nodeValue: str
+    attributes: str
+    backendNodeId: str
+    parentId: str
+    childIds: list[str]
+    cursor: int
+    union_bound: list[float] | None
+
+
+class BrowserConfig(TypedDict):
+    win_top_bound: float
+    win_left_bound: float
+    win_width: float
+    win_height: float
+    win_right_bound: float
+    win_lower_bound: float
+    device_pixel_ratio: float
+
+
+class BrowserInfo(TypedDict):
+    DOMTree: dict[str, Any]
+    config: BrowserConfig
+
+
+AccessibilityTree = list[AccessibilityTreeNode]
+DOMTree = list[DOMNode]
+
+
+Observation = str | npt.NDArray[np.uint8]
+
+
+class StateInfo(TypedDict):
+    observation: dict[str, Observation]
+    info: Dict[str, Any]