init

2025-04-27 09:56:14 +08:00 · 2025-04-27 09:56:14 +08:00 · a924233573
commit a924233573
37 changed files with 3724 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,16 @@
+
+openstreetmap-website/
+
+openstreetmap-templates/
+
+osm/
+
+venv_reset/
+
+tmpfs/
+
+wiki/
+
+venv/
+
+*.tar.gz
--- a/00_vars.sh
+++ b/00_vars.sh
@ -0,0 +1,45 @@
+#!/bin/bash
+
+# PUBLIC_HOSTNAME=$(curl -s ifconfig.me)
+PUBLIC_HOSTNAME="localhost"
+
+# Change ports as desired
+REDDIT_PORT=28080
+# WIKIPEDIA_PORT=28081
+SHOPPING_PORT=28082
+SHOPPING_ADMIN_PORT=28083
+GITLAB_PORT=28084
+# MAP_PORT=28085
+HOMEPAGE_PORT=20080
+RESET_PORT=20081
+
+# Original webarena ports
+# SHOPPING_PORT=7770
+# SHOPPING_ADMIN_PORT=7780
+# REDDIT_PORT=9999
+# GITLAB_PORT=8023
+# WIKIPEDIA_PORT=8888
+# MAP_PORT=3000
+# HOMEPAGE_PORT=4399
+
+SHOPPING_URL="http://${PUBLIC_HOSTNAME}:${SHOPPING_PORT}"
+SHOPPING_ADMIN_URL="http://${PUBLIC_HOSTNAME}:${SHOPPING_ADMIN_PORT}/admin"
+REDDIT_URL="http://${PUBLIC_HOSTNAME}:${REDDIT_PORT}/forums/all"
+GITLAB_URL="http://${PUBLIC_HOSTNAME}:${GITLAB_PORT}/explore"
+# WIKIPEDIA_URL="http://${PUBLIC_HOSTNAME}:${WIKIPEDIA_PORT}/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing"
+# MAP_URL="http://${PUBLIC_HOSTNAME}:${MAP_PORT}"
+MAP_URL="https://www.openstreetmap.org/"
+
+# download the archives from the webarena instructions
+# https://github.com/web-arena-x/webarena/tree/main/environment_docker
+# Download the additional openstreetmap docker files from Zenodo (see README)
+#  - shopping_final_0712.tar
+#  - shopping_admin_final_0719.tar
+#  - postmill-populated-exposed-withimg.tar
+#  - gitlab-populated-final-port8023.tar
+#  - openstreetmap-website-db.tar.gz
+#  - openstreetmap-website-web.tar.gz
+#  - openstreetmap-website.tar.gz
+#  - wikipedia_en_all_maxi_2022-05.zim
+
+ARCHIVES_LOCATION="/home/yuyr/webarena_docker_tar"
--- a/01_docker_load_images.sh
+++ b/01_docker_load_images.sh
@ -0,0 +1,61 @@
+#!/bin/bash
+
+# stop if any error occur
+set -e
+
+source ./00_vars.sh
+
+assert() {
+  if ! "$@"; then
+    echo "Assertion failed: $@" >&2
+    exit 1
+  fi
+}
+
+load_docker_image() {
+  local IMAGE_NAME="$1"
+  local INPUT_FILE="$2"
+
+  if ! docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "^${IMAGE_NAME}:"; then
+    echo "Loading Docker image ${IMAGE_NAME} from ${INPUT_FILE}"
+    docker load --input "${INPUT_FILE}"
+  else
+    echo "Docker image ${IMAGE_NAME} is already loaded."
+  fi
+}
+
+# make sure all required files are here
+assert [ -f ${ARCHIVES_LOCATION}/shopping_final_0712.tar ]
+assert [ -f ${ARCHIVES_LOCATION}/shopping_admin_final_0719.tar ]
+assert [ -f ${ARCHIVES_LOCATION}/postmill-populated-exposed-withimg.tar ]
+assert [ -f ${ARCHIVES_LOCATION}/gitlab-populated-final-port8023.tar ]
+# assert [ -f ${ARCHIVES_LOCATION}/openstreetmap-website-db.tar.gz ]
+# assert [ -f ${ARCHIVES_LOCATION}/openstreetmap-website-web.tar.gz ]
+# assert [ -f ${ARCHIVES_LOCATION}/openstreetmap-website.tar.gz ]
+# assert [ -f ${ARCHIVES_LOCATION}/wikipedia_en_all_maxi_2022-05.zim ]
+
+# load docker images (if needed)
+load_docker_image "shopping_final_0712" "${ARCHIVES_LOCATION}/shopping_final_0712.tar"
+load_docker_image "shopping_admin_final_0719" ${ARCHIVES_LOCATION}/shopping_admin_final_0719.tar
+load_docker_image "postmill-populated-exposed-withimg" "${ARCHIVES_LOCATION}/postmill-populated-exposed-withimg.tar"
+load_docker_image "gitlab-populated-final-port8023" "${ARCHIVES_LOCATION}/gitlab-populated-final-port8023.tar"
+# load_docker_image "openstreetmap-website-db" "${ARCHIVES_LOCATION}/openstreetmap-website-db.tar.gz"
+#load_docker_image "openstreetmap-website-web" "${ARCHIVES_LOCATION}/openstreetmap-website-web.tar.gz"
+
+# extract openstreetmap archive locally (if needed)
+# if [ ! -d ./openstreetmap-website ]; then
+#   echo "Extracting openstreemap archive..."
+#   tar -xzf ${ARCHIVES_LOCATION}/openstreetmap-website.tar.gz
+# else
+#   echo "Openstreemap archive already extracted."
+# fi
+
+# copy wikipedia archive to local folder (if needed)
+# WIKIPEDIA_ARCHIVE=wikipedia_en_all_maxi_2022-05.zim
+# if [ ! -f ./wiki/${WIKIPEDIA_ARCHIVE} ]; then
+#   echo "Moving wikipedia archive..."
+#   mkdir -p ./wiki
+#   cp ${ARCHIVES_LOCATION}/${WIKIPEDIA_ARCHIVE} ./wiki
+# else
+#   echo "Wikipedia archive already present."
+# fi
--- a/02_docker_remove_containers.sh
+++ b/02_docker_remove_containers.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+
+docker stop shopping_admin forum gitlab shopping  || true
+docker rm shopping_admin forum gitlab shopping  || true
+
+# docker stop shopping_admin forum gitlab shopping wikipedia openstreetmap-website-db-1 openstreetmap-website-web-1 || true
+#docker rm shopping_admin forum gitlab shopping wikipedia openstreetmap-website-db-1 openstreetmap-website-web-1 || true
+
--- a/03_docker_create_containers.sh
+++ b/03_docker_create_containers.sh
@ -0,0 +1,55 @@
+#!/bin/bash
+
+# stop if any error occur
+set -e
+
+source ./00_vars.sh
+
+WORKING_DIR=$(pwd)
+
+docker create --name shopping -p $SHOPPING_PORT:80 shopping_final_0712
+docker create --name shopping_admin -p $SHOPPING_ADMIN_PORT:80 shopping_admin_final_0719
+docker create --name forum -p $REDDIT_PORT:80 postmill-populated-exposed-withimg
+docker create --name gitlab --sysctl net.ipv6.conf.all.disable_ipv6=0 --sysctl net.ipv6.conf.default.disable_ipv6=0 \
+              -p $GITLAB_PORT:$GITLAB_PORT gitlab-populated-final-port8023 /opt/gitlab/embedded/bin/runsvdir-start --env GITLAB_PORT=$GITLAB_PORT
+# docker create --name wikipedia --volume=${WORKING_DIR}/wiki_patch/start.sh:/usr/local/bin/start.sh \
+#                 --volume=${WORKING_DIR}/wiki/:/data --network=host ghcr.io/kiwix/kiwix-serve:3.3.0  wikipedia_en_all_maxi_2022-05.zim $WIKIPEDIA_PORT
+
+# 
+# # openstreetmap docker set up
+# cd openstreetmap-website/
+
+# # tile server URL (use default openstreetmap server)
+# OSM_TILE_SERVER_URL="http://tile-server-run/tile/{z}/{x}/{y}.png"
+# # geocoding server URL (use default openstreetmap server)
+# OSM_GEOCODING_SERVER_URL="http://geocoding:8080/"
+# # routing server URLs (use default openstreetmap server)
+# OSM_ROUTING_SERVER_URL="http://nginx"
+# OSM_CAR_SUFFIX="/routed-car"
+# OSM_BIKE_SUFFIX="/routed-bike"
+# OSM_FOOT_SUFFIX="/routed-foot"
+# # original WebArena config (CMU server with different ports for each vehicule type)
+# # OSM_ROUTING_SERVER_URL="http://metis.lti.cs.cmu.edu"
+# # OSM_CAR_SUFFIX=":5000"
+# # OSM_BIKE_SUFFIX=":5001"
+# # OSM_FOOT_SUFFIX=":5002"
+
+# # copy template files to be set up
+# cp ../openstreetmap-templates/docker-compose.yml ./docker-compose.yml
+# cp ../openstreetmap-templates/nginx.conf ./nginx.conf
+# cp ../openstreetmap-templates/leaflet.osm.js ./vendor/assets/leaflet/leaflet.osm.js
+# cp ../openstreetmap-templates/fossgis_osrm.js ./app/assets/javascripts/index/directions/fossgis_osrm.js
+
+# # set up web server port
+# sed -i "s|MAP_PORT|${MAP_PORT}|g" docker-compose.yml
+# # set up tile server URL
+# sed -i "s|url: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png'|url: '${OSM_TILE_SERVER_URL}'|g" ./vendor/assets/leaflet/leaflet.osm.js
+# # set up geocoding server URL
+# sed -i "s|nominatim_url:.*|nominatim_url: \"$OSM_GEOCODING_SERVER_URL\"|g" ./config/settings.yml
+# # set up routing server URLs
+# sed -i "s|fossgis_osrm_url:.*|fossgis_osrm_url: \"$OSM_ROUTING_SERVER_URL\"|g" ./config/settings.yml
+# sed -i "s|__OSMCarSuffix__|${OSM_CAR_SUFFIX}|g" ./app/assets/javascripts/index/directions/fossgis_osrm.js
+# sed -i "s|__OSMBikeSuffix__|${OSM_BIKE_SUFFIX}|g" ./app/assets/javascripts/index/directions/fossgis_osrm.js
+# sed -i "s|__OSMFootSuffix__|${OSM_FOOT_SUFFIX}|g" ./app/assets/javascripts/index/directions/fossgis_osrm.js
+
+# docker compose create
--- a/04_docker_start_containers.sh
+++ b/04_docker_start_containers.sh
@ -0,0 +1,20 @@
+#!/bin/bash
+
+# stop if any error occur
+set -e
+
+docker start gitlab
+docker start shopping
+docker start shopping_admin
+docker start forum
+# docker start kiwix33
+# docker start wikipedia
+
+# cd openstreetmap-website/
+# docker compose start
+
+# 尤其是等gitlab，这个服务启动很慢，要5分钟以上
+echo -n -e "Waiting 300 seconds for all services to start..."
+sleep 300
+echo -n -e " done\n"
+
--- a/05_docker_patch_containers.sh
+++ b/05_docker_patch_containers.sh
@ -0,0 +1,48 @@
+#!/bin/bash
+
+# stop if any error occur
+set -e
+
+source ./00_vars.sh
+
+# reddit - make server more responsive
+docker exec forum sed -i \
+  -e 's/^pm.max_children = .*/pm.max_children = 32/' \
+  -e 's/^pm.start_servers = .*/pm.start_servers = 10/' \
+  -e 's/^pm.min_spare_servers = .*/pm.min_spare_servers = 5/' \
+  -e 's/^pm.max_spare_servers = .*/pm.max_spare_servers = 20/' \
+  -e 's/^;pm.max_requests = .*/pm.max_requests = 500/' \
+  /usr/local/etc/php-fpm.d/www.conf
+docker exec forum supervisorctl restart php-fpm
+
+echo "Patching shopping container"
+# shopping + shopping admin
+docker exec shopping /var/www/magento2/bin/magento setup:store-config:set --base-url="http://$PUBLIC_HOSTNAME:$SHOPPING_PORT" # no trailing /
+docker exec shopping mysql -u magentouser -pMyPassword magentodb -e  "UPDATE core_config_data SET value='http://$PUBLIC_HOSTNAME:$SHOPPING_PORT/' WHERE path = 'web/secure/base_url';"
+# remove the requirement to reset password
+echo "Removing the requirement to reset password"
+docker exec shopping_admin php /var/www/magento2/bin/magento config:set admin/security/password_is_forced 0
+docker exec shopping_admin php /var/www/magento2/bin/magento config:set admin/security/password_lifetime 0
+docker exec shopping /var/www/magento2/bin/magento cache:flush
+
+echo "Patching shopping admin container"
+docker exec shopping_admin /var/www/magento2/bin/magento setup:store-config:set --base-url="http://$PUBLIC_HOSTNAME:$SHOPPING_ADMIN_PORT"
+docker exec shopping_admin mysql -u magentouser -pMyPassword magentodb -e  "UPDATE core_config_data SET value='http://$PUBLIC_HOSTNAME:$SHOPPING_ADMIN_PORT/' WHERE path = 'web/secure/base_url';"
+docker exec shopping_admin /var/www/magento2/bin/magento cache:flush
+
+echo "Patching gitlab container"
+# gitlab
+docker exec gitlab sed -i "s|^external_url.*|external_url 'http://$PUBLIC_HOSTNAME:$GITLAB_PORT'|" /etc/gitlab/gitlab.rb
+docker exec gitlab bash -c "printf '\n\npuma[\"worker_processes\"] = 4' >> /etc/gitlab/gitlab.rb"  # bugfix https://github.com/ServiceNow/BrowserGym/issues/285
+# 添加 IPv6 支持
+docker exec gitlab bash -c "echo '
+# Enable IPv6
+nginx[\"listen_addresses\"] = [\"[::]\", \"0.0.0.0\"]
+nginx[\"listen_port\"] = $GITLAB_PORT
+' >> /etc/gitlab/gitlab.rb"
+docker exec gitlab gitlab-ctl reconfigure
+
+
+# echo "Patching openstreetmap container"
+# maps
+# docker exec openstreetmap-website-web-1 bin/rails db:migrate RAILS_ENV=development
--- a/06_serve_homepage.sh
+++ b/06_serve_homepage.sh
@ -0,0 +1,21 @@
+#!/bin/bash
+
+source ./00_vars.sh
+
+# install flask in a venv
+# apt install python3-venv -y
+python3 -m venv venv
+source venv/bin/activate
+pip install flask
+
+
+cd webarena-homepage
+cp templates/index.backup templates/index.html
+sed -i "s|SHOPPING_URL|${SHOPPING_URL}|g" templates/index.html
+sed -i "s|SHOPPING_ADMIN_URL|${SHOPPING_ADMIN_URL}|g" templates/index.html
+sed -i "s|GITLAB_URL|${GITLAB_URL}|g" templates/index.html
+sed -i "s|REDDIT_URL|${REDDIT_URL}|g" templates/index.html
+sed -i "s|MAP_URL|${MAP_URL}|g" templates/index.html
+sed -i "s|WIKIPEDIA_URL|${WIKIPEDIA_URL}|g" templates/index.html
+
+flask run --host=0.0.0.0 --port=$HOMEPAGE_PORT
--- a/07_serve_reset.sh
+++ b/07_serve_reset.sh
@ -0,0 +1,14 @@
+#!/bin/bash
+
+# stop if any error occur
+set -e
+
+source ./00_vars.sh
+
+# install flask in a venv
+# apt install python3-venv -y
+python3 -m venv venv_reset
+source venv_reset/bin/activate
+
+cd reset_server/
+python server.py --port ${RESET_PORT} 2>&1 | tee -a server.log
--- a/README.md
+++ b/README.md
@ -0,0 +1,15 @@
+
+# 简化部署WebArena-Lite环境
+1. 只管理shopping, shopping_admin, gitlab, reddit四个网站。
+2. Wiki在webarena-lite中移除，所以先不用部署
+3. openstreetmap部署非常复杂，建议使用官网 https://opensteetmap.org
+
+# 部署方法
+1. 保证环境已经加载了四个网站的docker镜像，如果还没加载，使用`01_docker_load_images.sh`
+2. 修改`00_vars.sh`，配置每个网页的端口
+3. 拉起/重置环境，执行`sh reset.sh`
+
+# 注意事项
+1. 当前配置了访问url为`localhost`，所以只能同一台服务器上的程序能够访问，不能跨机器；
+2. 可以使用vscode的端口映射功能映射到笔记本上，打开四个网站进行浏览。
+
--- a/README_original.md
+++ b/README_original.md
@ -0,0 +1,75 @@
+> :warning: **This is not an official WebArena repo. For the official instructions refer to [WebArena](https://github.com/web-arena-x/webarena/tree/main/environment_docker)**
+
+# webarena-setup
+
+Setup scripts for webarena.
+
+## Get the files
+
+Download the necessary docker images from the [official webarena repo](https://github.com/web-arena-x/webarena/tree/main/environment_docker). You'll need the following files:
+- shopping_final_0712.tar
+- shopping_admin_final_0719.tar
+- postmill-populated-exposed-withimg.tar
+- gitlab-populated-final-port8023.tar
+- wikipedia_en_all_maxi_2022-05.zim
+
+Download the additional openstreetmap docker files from Zenodo:
+```sh
+wget https://zenodo.org/records/12636845/files/openstreetmap-website-db.tar.gz
+wget https://zenodo.org/records/12636845/files/openstreetmap-website-web.tar.gz
+wget https://zenodo.org/records/12636845/files/openstreetmap-website.tar.gz
+```
+
+(if you're planning to setup multiple instances, should download these files once in a shared location)
+
+## Configure the scripts
+
+Edit `00_vars.sh` with your ports and hostname (or ip address). You can also change `ARCHIVES_LOCATION="./"` to where your files are if you put them at a different location (like a shared folder).
+
+## Extract / load the image files (very long, do it just once)
+
+Untar the openstreemap docker folder:
+```sh
+tar -xzf ./openstreetmap-website.tar.gz
+```
+
+Create a wiki folder and move (or copy) the wikipedia file to it
+```sh
+mkdir wiki
+mv ./wikipedia_en_all_maxi_2022-05.zim wiki/
+```
+
+Load the docker image files
+```sh
+sudo bash 01_docker_load_images.sh
+```
+
+Once these three steps are completed, you can get rid of the downloaded files (or unmount your shared folder) as these won't be needed any more.
+
+## Run the server
+
+Easiest way is to start a tmux or screen session, then run scripts 02 to 06 in order. The last script serves the homepage and should stay up.
+```bash
+sudo bash 02_docker_remove_containers.sh
+sudo bash 03_docker_create_containers.sh
+sudo bash 04_docker_start_containers.sh
+sudo bash 05_docker_patch_containers.sh
+sudo bash 06_serve_homepage.sh
+```
+
+Optional: to start the reset server (automated full instance resets) run the following in a side tmux or screen terminal
+```bash
+sudo bash 07_serve_reset.sh
+```
+
+Then you can trigger a full instance reset by accessing `http://${PUBLIC_HOSTNAME}:${RESET_PORT}/reset` check the instance status via `http://${PUBLIC_HOSTNAME}:${RESET_PORT}/reset`.
+
+## Safety check
+
+Go to the homepage and click each link to make sure the websites are operational (some might take ~10 secs to load the first time).
+
+## Reset
+
+After an agent is evaluated on WebArena, a reset is required before another agent can be evaluated.
+
+Run scripts 02 to 06 again, or query the reset URL `http://${PUBLIC_HOSTNAME}:${RESET_PORT}/reset` if the reset server is running.
--- a/download_openstreemap.sh
+++ b/download_openstreemap.sh
@ -0,0 +1,3 @@
+wget https://zenodo.org/records/12636845/files/openstreetmap-website-db.tar.gz
+wget https://zenodo.org/records/12636845/files/openstreetmap-website-web.tar.gz
+wget https://zenodo.org/records/12636845/files/openstreetmap-website.tar.gz
--- a/reset.sh
+++ b/reset.sh
@ -0,0 +1,10 @@
+#!/bin/bash
+
+# stop if any error occur
+set -e
+
+bash 02_docker_remove_containers.sh
+bash 03_docker_create_containers.sh
+bash 04_docker_start_containers.sh
+bash 05_docker_patch_containers.sh
+
--- a/reset_server/fail_message
+++ b/reset_server/fail_message
--- a/reset_server/reset.sh
+++ b/reset_server/reset.sh
@ -0,0 +1,11 @@
+#!/bin/bash
+
+# stop if any error occur
+set -e
+
+cd ..
+bash 02_docker_remove_containers.sh
+bash 03_docker_create_containers.sh
+bash 04_docker_start_containers.sh
+bash 05_docker_patch_containers.sh
+
--- a/reset_server/server.log
+++ b/reset_server/server.log
--- a/reset_server/server.py
+++ b/reset_server/server.py
@ -0,0 +1,126 @@
+import argparse
+import http.server
+import logging
+import os
+import pathlib
+import socketserver
+import subprocess
+import sys
+import threading
+
+# setup logging
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler(sys.stdout)
+handler.setFormatter(logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s]  %(message)s"))
+logger.setLevel(logging.INFO)
+logger.addHandler(handler)
+
+# setup files config
+lock_file_path = "reset.lock"
+fail_file_path = "fail_message"
+
+def write_fail_message(message: str):
+    with open(fail_file_path, 'w') as f:
+        f.write(message)
+
+def read_fail_message():
+    with open(fail_file_path, 'r') as f:
+        fail_message = f.read()
+    return fail_message
+
+def reset_ongoing():
+    return os.path.exists(lock_file_path)
+
+def initiate_reset():
+
+    # Attempt to acquire lock (create lock file atomically)
+    try:
+        fd = os.open(lock_file_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+        with os.fdopen(fd, 'w') as file:
+            file.write('')  # empty file
+    except FileExistsError:
+        return False
+
+    # Execute reset (and then release lock) in a separate thread
+    def reset_fun():
+        try:
+            # Execute the reset script
+            subprocess.run(['bash', 'reset.sh'], check=True)
+            logger.info("Reset successful!")
+            write_fail_message("")
+        except subprocess.CalledProcessError as e:
+            logger.info("Reset failed :(")
+            write_fail_message(str(e))
+
+        # Release lock (remove lock file)
+        pathlib.Path(lock_file_path).unlink(missing_ok=True)
+
+    thread = threading.Thread(target=reset_fun)
+    thread.start()
+
+    return True
+
+
+class CustomHandler(http.server.SimpleHTTPRequestHandler):
+    def do_GET(self):
+        parsed_path = self.path
+        logger.info(f"{parsed_path} request received")
+        match parsed_path:
+            case '/reset':
+                if initiate_reset():
+                    logger.info("Running reset script...")
+                    self.send_response(200)  # OK
+                    self.send_header('Content-type', 'text/html')
+                    self.end_headers()
+                    self.wfile.write(f'Reset initiated, check status <a href="/status">here</a>'.encode())
+                else:
+                    logger.warning("Reset already running, ignoring request.")
+                    self.send_response(418)  # I'm a teapot
+                    self.send_header('Content-type', 'text/html')
+                    self.end_headers()
+                    self.wfile.write(f'Reset already running, check status <a href="/status">here</a>'.encode())
+            case "/status":
+                fail_message = read_fail_message()
+                if reset_ongoing():
+                    logger.info("Returning ongoing status")
+                    self.send_response(200)  # OK
+                    self.send_header('Content-type', 'text/html')
+                    self.end_headers()
+                    self.wfile.write(f'Reset ongoing'.encode())
+                elif fail_message:
+                    logger.error("Returning error status")
+                    self.send_response(500)  # Internal Server Error
+                    self.send_header('Content-type', 'text/html')
+                    self.end_headers()
+                    self.wfile.write(f'Error executing reset script:<p>{fail_message}</p>'.encode())
+                else:
+                    logger.info("Returning ready status")
+                    self.send_response(200)  # OK
+                    self.send_header('Content-type', 'text/html')
+                    self.end_headers()
+                    self.wfile.write(f'Ready for duty!'.encode())
+            case _:
+                logger.info("Wrong request")
+                self.send_response(404)  # Not Found
+                self.send_header('Content-type', 'text/html')
+                self.end_headers()
+                self.wfile.write(f'Endpoint not found'.encode())
+
+
+# Parse command-line arguments
+parser = argparse.ArgumentParser(description='Start a simple HTTP server to execute a reset script.')
+parser.add_argument('--port', type=int, help='Port number the server will listen to')
+args = parser.parse_args()
+
+# Clear fail and lock files
+write_fail_message("")
+if reset_ongoing():
+    os.remove(lock_file_path)
+
+# Run the server
+with http.server.ThreadingHTTPServer(('', args.port), CustomHandler) as httpd:
+    logger.info(f'Serving on port {args.port}...')
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        httpd.server_close()
--- a/serve_homepage.log
+++ b/serve_homepage.log
@ -0,0 +1,16 @@
+Requirement already satisfied: flask in ./venv/lib/python3.11/site-packages (3.1.0)
+Requirement already satisfied: Werkzeug>=3.1 in ./venv/lib/python3.11/site-packages (from flask) (3.1.3)
+Requirement already satisfied: Jinja2>=3.1.2 in ./venv/lib/python3.11/site-packages (from flask) (3.1.6)
+Requirement already satisfied: itsdangerous>=2.2 in ./venv/lib/python3.11/site-packages (from flask) (2.2.0)
+Requirement already satisfied: click>=8.1.3 in ./venv/lib/python3.11/site-packages (from flask) (8.1.8)
+Requirement already satisfied: blinker>=1.9 in ./venv/lib/python3.11/site-packages (from flask) (1.9.0)
+Requirement already satisfied: MarkupSafe>=2.0 in ./venv/lib/python3.11/site-packages (from Jinja2>=3.1.2->flask) (3.0.2)
+
+[notice] A new release of pip is available: 24.0 -> 25.0.1
+[notice] To update, run: pip install --upgrade pip
+ * Debug mode: off
+[31m[1mWARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead.[0m
+ * Running on all addresses (0.0.0.0)
+ * Running on http://127.0.0.1:20080
+ * Running on http://192.168.16.114:20080
+[33mPress CTRL+C to quit[0m
--- a/service.md
+++ b/service.md
@ -0,0 +1,22 @@
+# WebArena实例服务
+
+## 1. 背景
+
+操作智能体项目在数据采集、模型训练、模型推理评估等环节，需要频繁的与WebArena实例服务进行交互。交互过程会对WebArena实例进行状态的修改，因此需要对WebArena实例服务进行实例化。
+通过统一个管理服务实现WebArena实例的自动创建、销毁、状态查询等操作。
+
+## 2. 架构
+
+服务包含以下几个部分：
+- 前端控制台：提供网页界面，可以实现服务器管理、WebArena实例的创建、销毁、状态查询等操作。
+- 后端服务：提供WebArena实例的创建、销毁、状态查询等操作。
+
+## 3. 使用流程
+
+### 3.1 创建WebArena实例
+
+### 3.2 销毁WebArena实例
+
+### 3.3 查询WebArena实例状态
+
+### 3.4 查询WebArena实例日志
--- a/webarena-homepage/pycache/app.cpython-311.pyc
+++ b/webarena-homepage/pycache/app.cpython-311.pyc
--- a/webarena-homepage/app.py
+++ b/webarena-homepage/app.py
@ -0,0 +1,27 @@
+from flask import Flask, render_template
+
+app = Flask(__name__)
+
+
+@app.route("/")
+def index() -> str:
+    return render_template("index.html")
+
+
+@app.route("/scratchpad.html")
+def scratchpad() -> str:
+    return render_template("scratchpad.html")
+
+
+@app.route("/calculator.html")
+def calculator() -> str:
+    return render_template("calculator.html")
+
+
+@app.route("/password.html")
+def password() -> str:
+    return render_template("password.html")
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=4399)
--- a/webarena-homepage/static/figures/calculator.png
+++ b/webarena-homepage/static/figures/calculator.png
--- a/webarena-homepage/static/figures/cms.png
+++ b/webarena-homepage/static/figures/cms.png
--- a/webarena-homepage/static/figures/gitlab.png
+++ b/webarena-homepage/static/figures/gitlab.png
--- a/webarena-homepage/static/figures/manual1.png
+++ b/webarena-homepage/static/figures/manual1.png
--- a/webarena-homepage/static/figures/manual2.png
+++ b/webarena-homepage/static/figures/manual2.png
--- a/webarena-homepage/static/figures/map.png
+++ b/webarena-homepage/static/figures/map.png
--- a/webarena-homepage/static/figures/onestopshop.png
+++ b/webarena-homepage/static/figures/onestopshop.png
--- a/webarena-homepage/static/figures/password.png
+++ b/webarena-homepage/static/figures/password.png
--- a/webarena-homepage/static/figures/reddit.png
+++ b/webarena-homepage/static/figures/reddit.png
--- a/webarena-homepage/static/figures/scratchpad.png
+++ b/webarena-homepage/static/figures/scratchpad.png
--- a/webarena-homepage/static/figures/wikipedia.png
+++ b/webarena-homepage/static/figures/wikipedia.png
--- a/webarena-homepage/templates/calculator.html
+++ b/webarena-homepage/templates/calculator.html
@ -0,0 +1,109 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Calculator</title>
+    <style>
+        body {
+            display: flex;
+            justify-content: center;
+            align-items: center;
+            height: 100vh;
+            background-color: #f2f2f2;
+            font-family: Arial, sans-serif;
+        }
+        #calculator {
+            border: 1px solid #333;
+            padding: 30px;
+            border-radius: 10px;
+            width: 400px;
+            background-color: #fff;
+            box-shadow: 0px 0px 10px rgba(0,0,0,0.1);
+            box-sizing: border-box;
+            display: flex;
+            flex-direction: column;
+            align-items: flex-end;
+        }
+        #calculator h1 {
+            text-align: center;
+            margin-bottom: 20px;
+            align-self: center;
+        }
+        #calculator p {
+            text-align: center;
+            margin-bottom: 20px;
+            color: #666;
+            align-self: center;
+        }
+        #inputExpression {
+            width: 100%;
+            padding: 15px;
+            font-size: 18px;
+            border-radius: 5px;
+            border: 1px solid #ddd;
+            box-sizing: border-box;
+            margin-bottom: 10px;
+        }
+        #result {
+            margin-top: 20px;
+            font-size: 20px;
+            text-align: center;
+            color: #333;
+            width: 100%;
+            align-self: center;
+        }
+        button {
+            padding: 10px 20px;
+            margin-top: 10px;
+            font-size: 18px;
+            border: none;
+            border-radius: 5px;
+            cursor: pointer;
+            align-self: flex-end;
+        }
+        #calculate {
+            background-color: #4CAF50;
+            color: white;
+        }
+        #clear {
+            background-color: #f44336;
+            color: white;
+        }
+    </style>
+</head>
+<body>
+    <div id="calculator">
+        <h1>Calculator</h1>
+        <p>Enter the expression and get the results</p>
+        <input type="text" id="inputExpression" placeholder="Enter expression" />
+        <button id="calculate">=</button>
+        <button id="clear">Clear</button>
+        <div id="result">Result: <span id="calculationResult"></span></div>
+    </div>
+
+    <script>
+        document.getElementById('calculate').addEventListener('click', function() {
+            try {
+                const result = eval(document.getElementById('inputExpression').value);
+                if (!isNaN(result)) {
+                    document.getElementById('calculationResult').textContent = result;
+                } else {
+                    document.getElementById('calculationResult').textContent = 'Invalid expression';
+                }
+            } catch {
+                document.getElementById('calculationResult').textContent = 'Invalid expression';
+            }
+        });
+
+        document.getElementById('clear').addEventListener('click', function() {
+            document.getElementById('inputExpression').value = '';
+            document.getElementById('calculationResult').textContent = '';
+        });
+
+        document.getElementById('inputExpression').addEventListener('keypress', function(e) {
+            if (e.key === 'Enter') {
+                document.getElementById('calculate').click();
+            }
+        });
+    </script>
+</body>
+</html>
--- a/webarena-homepage/templates/index.backup
+++ b/webarena-homepage/templates/index.backup
@ -0,0 +1,157 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Homepage</title>
+    <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500&display=swap" rel="stylesheet">
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            background-color: white;
+            margin: 0;
+            padding: 0;
+            line-height: 1.6;
+        }
+
+        #container {
+            width: 90%;
+            margin: auto;
+            overflow: hidden;
+        }
+
+        #header {
+            background-color: #f4f7f9;
+            color: #232f34;
+            padding-top: 30px;
+            min-height: 70px;
+            border-bottom: #b1c1c6 3px solid;
+            font-family: 'Roboto', sans-serif;
+        }
+
+        #header h1 {
+            padding: 5px;
+            margin: 0;
+            text-align: center;
+            font-weight: 500;
+        }
+
+        .card {
+            border: 1px solid #ddd;
+            border-radius: 5px;
+            width: 200px; /* Changed width */
+            height: 200px; /* Added height */
+            text-align: center;
+            margin: 10px;
+            padding: 10px;
+            box-shadow: 2px 2px 5px rgba(0,0,0,0.1);
+            display: inline-block;
+            overflow: auto; /* To handle content that might overflow */
+        }
+
+        .card img {
+            width: 35%;
+        }
+
+        .card h2 {
+            font-size: 15px;
+        }
+
+        .card p {
+            font-size: 14px;
+        }
+
+        .card a {
+            color: #35424a;
+            text-decoration: none;
+        }
+    </style>
+</head>
+<body>
+    <div id="container">
+        <div id="header">
+            <h1>Welcome to WebArena</h1>
+        </div>
+
+        <div class="card" role="region" aria-label="OneStopShop">
+         <img src="{{ url_for('static', filename='figures/onestopshop.png') }}"  alt="Logo for OneStopShop">
+         <a href="SHOPPING_URL">
+            <h2 id="appName">OneStopShop</h2>
+         </a>
+         <p id="appDescription">An online shopping site</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Merchant Admin Portal">
+            <img src="{{ url_for('static', filename='figures/cms.png') }}"  alt="Logo for CMS">
+            <a href="SHOPPING_ADMIN_URL">
+                <h2 id="appName">Merchant Admin Portal</h2>
+            </a>
+            <p id="appDescription">An admin portal to manage E-commerce business (u: admin, p: admin1234)</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Reddit">
+            <img src="{{ url_for('static', filename='figures/reddit.png') }}"  alt="Logo for Reddit">
+            <a href="REDDIT_URL">
+                <h2 id="appName">Reddit</h2>
+            </a>
+            <p id="appDescription">A social news aggregation and discussion website</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Gitlab">
+            <img src="{{ url_for('static', filename='figures/gitlab.png') }}"  alt="Logo for Gitlab">
+            <a href="GITLAB_URL">
+                <h2 id="appName">Gitlab</h2>
+            </a>
+            <p id="appDescription">a DevOps software</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Map">
+            <img src="{{ url_for('static', filename='figures/map.png') }}"  alt="Logo for Map">
+            <a href="MAP_URL">
+                <h2 id="appName">OpenStreetMap</h2>
+            </a>
+            <p id="appDescription">North east US map</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Calculator">
+            <img src="{{ url_for('static', filename='figures/calculator.png') }}"  alt="Logo for Calculator">
+            <a href="calculator.html">
+                <h2 id="appName">Calculator</h2>
+            </a>
+            <p id="appDescription">A calculator</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Scratchpad">
+            <img src="{{ url_for('static', filename='figures/scratchpad.png') }}"  alt="Logo for Scratchpad">
+            <a href="scratchpad.html">
+                <h2 id="appName">Scratchpad</h2>
+            </a>
+            <p id="appDescription">A scratchpad for taking notes</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Wikipedia">
+            <img src="{{ url_for('static', filename='figures/wikipedia.png') }}"  alt="Logo for Wikipedia">
+            <a href="WIKIPEDIA_URL">
+                <h2 id="appName">Wikipedia</h2>
+            </a>
+            <p id="appDescription">An online encyclopedia</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Gitlab Manual">
+            <img src="{{ url_for('static', filename='figures/manual1.png') }}"  alt="Logo for Gitlab Manual">
+            <a href="https://docs.gitlab.com/">
+                <h2 id="appName">Gitlab Documentation</h2>
+            </a>
+            <p id="appDescription">Documentation for GitLab</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Admin Manual">
+            <img src="{{ url_for('static', filename='figures/manual2.png') }}"  alt="Logo for Admin Manual">
+            <a href="https://experienceleague.adobe.com/docs/commerce-admin/user-guides/home.html?lang=en">
+                <h2 id="appName">Admin Portal Manual</h2>
+            </a>
+            <p id="appDescription">Manual on using the admin portal</p>
+        </div>
+        <!-- Repeat the above card structure for each app -->
+
+    </div>
+</body>
+</html>
--- a/webarena-homepage/templates/index.html
+++ b/webarena-homepage/templates/index.html
@ -0,0 +1,157 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Homepage</title>
+    <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500&display=swap" rel="stylesheet">
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            background-color: white;
+            margin: 0;
+            padding: 0;
+            line-height: 1.6;
+        }
+
+        #container {
+            width: 90%;
+            margin: auto;
+            overflow: hidden;
+        }
+
+        #header {
+            background-color: #f4f7f9;
+            color: #232f34;
+            padding-top: 30px;
+            min-height: 70px;
+            border-bottom: #b1c1c6 3px solid;
+            font-family: 'Roboto', sans-serif;
+        }
+
+        #header h1 {
+            padding: 5px;
+            margin: 0;
+            text-align: center;
+            font-weight: 500;
+        }
+
+        .card {
+            border: 1px solid #ddd;
+            border-radius: 5px;
+            width: 200px; /* Changed width */
+            height: 200px; /* Added height */
+            text-align: center;
+            margin: 10px;
+            padding: 10px;
+            box-shadow: 2px 2px 5px rgba(0,0,0,0.1);
+            display: inline-block;
+            overflow: auto; /* To handle content that might overflow */
+        }
+
+        .card img {
+            width: 35%;
+        }
+
+        .card h2 {
+            font-size: 15px;
+        }
+
+        .card p {
+            font-size: 14px;
+        }
+
+        .card a {
+            color: #35424a;
+            text-decoration: none;
+        }
+    </style>
+</head>
+<body>
+    <div id="container">
+        <div id="header">
+            <h1>Welcome to WebArena</h1>
+        </div>
+
+        <div class="card" role="region" aria-label="OneStopShop">
+         <img src="{{ url_for('static', filename='figures/onestopshop.png') }}"  alt="Logo for OneStopShop">
+         <a href="http://localhost:28082">
+            <h2 id="appName">OneStopShop</h2>
+         </a>
+         <p id="appDescription">An online shopping site</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Merchant Admin Portal">
+            <img src="{{ url_for('static', filename='figures/cms.png') }}"  alt="Logo for CMS">
+            <a href="http://localhost:28083/admin">
+                <h2 id="appName">Merchant Admin Portal</h2>
+            </a>
+            <p id="appDescription">An admin portal to manage E-commerce business (u: admin, p: admin1234)</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Reddit">
+            <img src="{{ url_for('static', filename='figures/reddit.png') }}"  alt="Logo for Reddit">
+            <a href="http://localhost:28080/forums/all">
+                <h2 id="appName">Reddit</h2>
+            </a>
+            <p id="appDescription">A social news aggregation and discussion website</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Gitlab">
+            <img src="{{ url_for('static', filename='figures/gitlab.png') }}"  alt="Logo for Gitlab">
+            <a href="http://localhost:28084/explore">
+                <h2 id="appName">Gitlab</h2>
+            </a>
+            <p id="appDescription">a DevOps software</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Map">
+            <img src="{{ url_for('static', filename='figures/map.png') }}"  alt="Logo for Map">
+            <a href="https://www.openstreetmap.org/">
+                <h2 id="appName">OpenStreetMap</h2>
+            </a>
+            <p id="appDescription">North east US map</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Calculator">
+            <img src="{{ url_for('static', filename='figures/calculator.png') }}"  alt="Logo for Calculator">
+            <a href="calculator.html">
+                <h2 id="appName">Calculator</h2>
+            </a>
+            <p id="appDescription">A calculator</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Scratchpad">
+            <img src="{{ url_for('static', filename='figures/scratchpad.png') }}"  alt="Logo for Scratchpad">
+            <a href="scratchpad.html">
+                <h2 id="appName">Scratchpad</h2>
+            </a>
+            <p id="appDescription">A scratchpad for taking notes</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Wikipedia">
+            <img src="{{ url_for('static', filename='figures/wikipedia.png') }}"  alt="Logo for Wikipedia">
+            <a href="http://localhost:28081/wikipedia_en_all_maxi_2022-05/A/User:The_other_Kiwix_guy/Landing">
+                <h2 id="appName">Wikipedia</h2>
+            </a>
+            <p id="appDescription">An online encyclopedia</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Gitlab Manual">
+            <img src="{{ url_for('static', filename='figures/manual1.png') }}"  alt="Logo for Gitlab Manual">
+            <a href="https://docs.gitlab.com/">
+                <h2 id="appName">Gitlab Documentation</h2>
+            </a>
+            <p id="appDescription">Documentation for GitLab</p>
+        </div>
+
+        <div class="card" role="region" aria-label="Admin Manual">
+            <img src="{{ url_for('static', filename='figures/manual2.png') }}"  alt="Logo for Admin Manual">
+            <a href="https://experienceleague.adobe.com/docs/commerce-admin/user-guides/home.html?lang=en">
+                <h2 id="appName">Admin Portal Manual</h2>
+            </a>
+            <p id="appDescription">Manual on using the admin portal</p>
+        </div>
+        <!-- Repeat the above card structure for each app -->
+
+    </div>
+</body>
+</html>
--- a/webarena-homepage/templates/scratchpad.html
+++ b/webarena-homepage/templates/scratchpad.html
@ -0,0 +1,122 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>Note Taking App</title>
+  <style>
+    body {
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+      justify-content: flex-start;
+      min-height: 100vh;
+      margin: 0;
+      font-family: Arial, sans-serif;
+    }
+
+    h1 {
+      text-align: center;
+    }
+
+    #note-creation {
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+      margin-bottom: 20px;
+    }
+
+    #note-form {
+      display: flex;
+      flex-direction: column;
+      align-items: center;
+      width: 300px;
+    }
+
+    #note-input {
+      width: 100%;
+      min-height: 300px;
+      max-height: 900px;
+      min-width: 600px;
+      max-width: 600px;
+      padding: 10px;
+      box-sizing: border-box;
+      border-radius: 4px;
+      border: 1px solid #ddd;
+      overflow-y: auto;
+      resize: none;
+    }
+
+    #note-form button {
+      padding: 10px 20px;
+      margin-top: 10px;
+      color: white;
+      background-color: #007bff;
+      border: none;
+      border-radius: 4px;
+      cursor: pointer;
+      text-align: center;
+      text-decoration: none;
+    }
+
+    #note-form button:hover {
+      background-color: #0056b3;
+    }
+
+    #notes-display {
+      display: flex;
+      flex-direction: column;
+      align-items: left;
+      width: 600px;
+    }
+
+    .note {
+      margin: 1em 0;
+      padding: 1em;
+      border: 1px solid #ddd;
+      border-radius: 4px;
+      background: #f9f9f9;
+      white-space: pre-wrap;
+    }
+  </style>
+</head>
+<body>
+  <header>
+    <h1>My Notes</h1>
+  </header>
+
+  <main>
+    <section id="note-creation">
+      <form id="note-form">
+        <textarea id="note-input" placeholder="Type your note here..."></textarea>
+        <button type="submit">Add Note</button>
+      </form>
+    </section>
+
+    <h2>History</h2>
+
+    <section id="notes-display">
+      <!-- Notes will be dynamically added here -->
+    </section>
+  </main>
+
+  <script>
+    const form = document.querySelector("#note-form");
+    const noteInput = document.querySelector("#note-input");
+    const notesDisplay = document.querySelector("#notes-display");
+
+    form.addEventListener("submit", (event) => {
+      event.preventDefault();
+
+      const note = document.createElement("div");
+      note.classList.add("note");
+      note.textContent = noteInput.value;
+
+      note.innerHTML = noteInput.value.replace(/\n/g, '<br>');
+      note.tabIndex = 0;
+
+      notesDisplay.prepend(note);
+      noteInput.value = '';
+    });
+  </script>
+</body>
+</html>
--- a/wiki_patch/start.sh
+++ b/wiki_patch/start.sh
@ -0,0 +1,27 @@
+#!/bin/sh
+
+# path: /usr/local/bin/start.sh
+
+# Download if necessary a file
+if [ ! -z "$DOWNLOAD" ]
+then
+    ZIM=`basename $DOWNLOAD`
+    wget $DOWNLOAD -O "$ZIM"
+
+    # Set arguments
+    if [ "$#" -eq "0" ]
+    then
+        set -- "$@" $ZIM
+    fi
+fi
+
+CMD="/usr/local/bin/kiwix-serve -v --port=$2 $1"
+echo $CMD
+$CMD
+
+# If error, print the content of /data
+if [ $? -ne 0 ]
+then
+    echo "Here is the content of /data:"
+    find /data -type f
+fi