192 lines
8.8 KiB
JavaScript
192 lines
8.8 KiB
JavaScript
/**
|
|
* Go through all DOM elements in the frame (including shadowDOMs), give them unique browsergym
|
|
* identifiers (bid), and store custom data in the aria-roledescription attribute.
|
|
*/
|
|
var { innerWidth: windowWidth, innerHeight: windowHeight } = window;
|
|
var scrollX = window.scrollX || document.documentElement.scrollLeft;
|
|
var scrollY = window.scrollY || document.documentElement.scrollTop;
|
|
|
|
([parent_bid, bid_attr_name, iframe_position, super_iframe_offset]) => {
|
|
|
|
// standard html tags
|
|
// https://www.w3schools.com/tags/
|
|
const html_tags = [
|
|
"a", "abbr", "acronym", "address", "applet", "area", "article", "aside", "audio",
|
|
"b", "base", "basefont", "bdi", "bdo", "big", "blockquote", "body", "br", "button",
|
|
"canvas", "caption", "center", "cite", "code", "col", "colgroup", "data", "datalist",
|
|
"dd", "del", "details", "dfn", "dialog", "dir", "div", "dl", "dt", "em", "embed",
|
|
"fieldset", "figcaption", "figure", "font", "footer", "form", "frame", "frameset",
|
|
"h1", "h2", "h3", "h4", "h5", "h6", "head", "header", "hgroup", "hr", "html", "i",
|
|
"iframe", "img", "input", "ins", "kbd", "label", "legend", "li", "link", "main",
|
|
"map", "mark", "menu", "meta", "meter", "nav", "noframes", "noscript", "object",
|
|
"ol", "optgroup", "option", "output", "p", "param", "picture", "pre", "progress",
|
|
"q", "rp", "rt", "ruby", "s", "samp", "script", "search", "section", "select",
|
|
"small", "source", "span", "strike", "strong", "style", "sub", "summary", "sup",
|
|
"svg", "table", "tbody", "td", "template", "textarea", "tfoot", "th", "thead",
|
|
"time", "title", "tr", "track", "tt", "u", "ul", "var", "video", "wbr"
|
|
];
|
|
|
|
if (super_iframe_offset == null) {
|
|
|
|
iframe_offset = { x: scrollX, y: scrollY, right: windowWidth, bottom: windowHeight };
|
|
}
|
|
else {
|
|
[super_x, super_y, super_right, super_bottom] = [super_iframe_offset["x"], super_iframe_offset["y"], super_iframe_offset["right"], super_iframe_offset["bottom"]];
|
|
|
|
x = Math.max(-iframe_position.x, 0);
|
|
y = Math.max(-iframe_position.y, 0);
|
|
right = Math.min(...[super_right, windowWidth, super_right - iframe_position.x]);
|
|
bottom = Math.min(...[super_bottom, windowHeight, super_bottom - iframe_position.y]);
|
|
iframe_offset = { x: x, y: y, right: right, bottom: bottom };
|
|
}
|
|
|
|
let browsergym_first_visit = false;
|
|
// if no yet set, set the frame (local) element counter to 0
|
|
if (!("browsergym_frame_elem_counter" in window)) {
|
|
window.browsergym_frame_elem_counter = 0;
|
|
browsergym_first_visit = true;
|
|
}
|
|
|
|
// get all DOM elements in the current frame (does not include elements in shadowDOMs)
|
|
let elements = Array.from(document.querySelectorAll('*'));
|
|
i = 0;
|
|
while (i < elements.length) {
|
|
const elem = elements[i];
|
|
// add shadowDOM elements to the elements array, in such a way that order is preserved
|
|
// TODO: do we really need the order preserved?
|
|
if (elem.shadowRoot !== null) {
|
|
elements = new Array(
|
|
...Array.prototype.slice.call(elements, 0, i + 1),
|
|
...Array.from(elem.shadowRoot.querySelectorAll("*")),
|
|
...Array.prototype.slice.call(elements, i + 1)
|
|
);
|
|
}
|
|
i++;
|
|
// we will mark only standard HTML tags
|
|
if (!elem.tagName || !html_tags.includes(elem.tagName.toLowerCase())) {
|
|
// console.log(`Skipping element ${elem.outerHTML}`)
|
|
continue; // stop and move on to the next element
|
|
}
|
|
// console.log(`Processing element ${elem.outerHTML}`)
|
|
// write dynamic element values to the DOM
|
|
if (typeof elem.value !== 'undefined') {
|
|
elem.setAttribute("value", elem.value);
|
|
}
|
|
// write dynamic checked properties to the DOM
|
|
if (typeof elem.checked !== 'undefined') {
|
|
if (elem.checked === true) {
|
|
elem.setAttribute("checked", "");
|
|
}
|
|
else {
|
|
elem.removeAttribute("checked");
|
|
}
|
|
}
|
|
// add the element global id to a custom HTML attribute
|
|
// https://playwright.dev/docs/locators#locate-by-test-id
|
|
// recover the element id if it has one already, else compute a new element id
|
|
let elem_global_bid;
|
|
if (elem.hasAttribute(bid_attr_name)) {
|
|
// throw an error if the attribute is already set while this is the first visit of the page
|
|
if (browsergym_first_visit) {
|
|
throw new Error(`Attribute ${bid_attr_name} already used in element ${elem.outerHTML}`);
|
|
}
|
|
elem_global_bid = elem.getAttribute(bid_attr_name);
|
|
}
|
|
else {
|
|
let elem_local_id = window.browsergym_frame_elem_counter++;
|
|
if (parent_bid == "") {
|
|
elem_global_bid = `${elem_local_id}`;
|
|
}
|
|
else {
|
|
elem_global_bid = `${parent_bid}-${elem_local_id}`;
|
|
}
|
|
elem.setAttribute(bid_attr_name, `${elem_global_bid}`);
|
|
}
|
|
// Hack: store custom data inside the aria-roledescription attribute (will be available in DOM and AXTree)
|
|
// - elem_global_bid: global element identifier (unique over multiple frames)
|
|
// TODO: add more data if needed (x, y coordinates, bounding box, is_visible, is_clickable etc.)
|
|
|
|
let [rect, is_in_viewport] = getElementPositionInfo(elem, iframe_offset, iframe_position);
|
|
let left = (rect.left + iframe_position.x).toString();
|
|
let top = (rect.top + iframe_position.y ).toString();
|
|
let right = (rect.right + iframe_position.x ).toString();
|
|
let bottom = (rect.bottom + iframe_position.y).toString();
|
|
let center_x = ((rect.left + rect.right) / 2 + iframe_position.x).toString();
|
|
let center_y = ((rect.top + rect.bottom) / 2 + iframe_position.y).toString();
|
|
|
|
elem.setAttribute("browsergym_center", `(${center_x}, ${center_y})`);
|
|
elem.setAttribute("browsergym_bounding_box", `(${left}, ${top}, ${right}, ${bottom})`);
|
|
elem.setAttribute("browsergym_is_in_viewport", `${is_in_viewport}`);
|
|
|
|
let original_content = "";
|
|
if (elem.hasAttribute("aria-roledescription")) {
|
|
original_content = elem.getAttribute("aria-roledescription");
|
|
}
|
|
let new_content = `${elem_global_bid}_${left}_${top}_${center_x}_${center_y}_${right}_${bottom}_${is_in_viewport}_${original_content}`
|
|
elem.setAttribute("aria-roledescription", new_content);
|
|
|
|
}
|
|
return iframe_offset;
|
|
|
|
}
|
|
function getElementPositionInfo(element, iframe_offset, iframe_position) {
|
|
var rect = element.getBoundingClientRect();
|
|
let x = (rect.left + rect.right) / 2 ;
|
|
let y = (rect.top + rect.bottom) / 2 ;
|
|
//loop over element ancestors (parent) and refine iframe offset to be the most precise possible
|
|
var parent = element.parentElement;
|
|
parent_iframe_offset = { x: 0, y: 0, right: windowWidth, bottom: windowHeight };
|
|
while (parent !== null) {
|
|
var parent_rect = parent.getBoundingClientRect();
|
|
parent_iframe_offset["x"] = Math.max(parent_rect.left , parent_iframe_offset["x"] );
|
|
parent_iframe_offset["y"] = Math.max(parent_rect.top , parent_iframe_offset["y"] );
|
|
parent_iframe_offset["right"] = Math.min(parent_rect.right , parent_iframe_offset["right"] );
|
|
parent_iframe_offset["bottom"] = Math.min(parent_rect.bottom , parent_iframe_offset["bottom"] );
|
|
parent = parent.parentElement;
|
|
}
|
|
|
|
var is_in_viewport = (
|
|
x >= iframe_offset["x"] &&
|
|
y >= iframe_offset["y"] &&
|
|
x <= iframe_offset["right"] &&
|
|
y <= iframe_offset["bottom"]
|
|
);
|
|
//this features is broken for the moment
|
|
var NotBehindParent = (
|
|
x >= parent_iframe_offset["x"] &&
|
|
y >= parent_iframe_offset["y"] &&
|
|
x <= parent_iframe_offset["right"] &&
|
|
y <= parent_iframe_offset["bottom"]
|
|
);
|
|
|
|
var isVisible = (typeof element.offsetWidth === 'undefined' || typeof element.offsetHeight === 'undefined') || (element.offsetWidth > 0 && element.offsetHeight > 0);
|
|
|
|
// Return true if the element is both in the viewport and has non-zero dimensions
|
|
return [rect, (is_in_viewport && isVisible && IsInFront(element))? 1 : 0];
|
|
}
|
|
|
|
|
|
function IsInFront(element){
|
|
var rect = element.getBoundingClientRect();
|
|
var x = (rect.left + rect.right) / 2 ;
|
|
var y = (rect.top + rect.bottom) / 2 ;
|
|
var newElement = elementFromPoint(x, y); //return the element in the foreground at position (x,y)
|
|
if(newElement){
|
|
if(newElement === element)
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function elementFromPoint(x, y) {
|
|
let node = document.elementFromPoint(x, y);
|
|
|
|
let child = node?.shadowRoot?.elementFromPoint(x, y);
|
|
|
|
while (child && child !== node) {
|
|
node = child;
|
|
child = node?.shadowRoot?.elementFromPoint(x, y);
|
|
}
|
|
|
|
return child || node;
|
|
} |