fix tmp failure isolation

scripts/soak/run_soak.sh

@@ -482,11 +482,22 @@ isolate_state_after_failure() {
   mkdir -p "$invalid_dir"
   move_if_exists "$DB_DIR" "$invalid_dir"
   move_if_exists "$META_DIR" "$invalid_dir"
-  move_if_exists "$TMP_DIR" "$invalid_dir"
+  if [[ -e "$TMP_DIR" ]]; then
+    INVALID_TMP_PATH="$TMP_DIR"
+    TMP_CLEANUP_STATUS="pending"
+    TMP_CLEANUP_REASON="discard_after_failure"
+    rm -rf "$TMP_DIR"
+    mkdir -p "$TMP_DIR"
+    TMP_CLEANUP_STATUS="discarded_recreated"
+  else
+    INVALID_TMP_PATH=""
+    TMP_CLEANUP_STATUS="not_present"
+    TMP_CLEANUP_REASON="discard_after_failure"
+    mkdir -p "$TMP_DIR"
+  fi
   mkdir -p "$DB_DIR" "$META_DIR" "$TMP_DIR"
   INVALID_DB_PATH="$invalid_dir/$(basename "$DB_DIR")"
   INVALID_STATE_PATH="$invalid_dir/$(basename "$META_DIR")"
-  INVALID_TMP_PATH="$invalid_dir/$(basename "$TMP_DIR")"
 }
 
 periodic_snapshot_delta_scan() {
@@ -610,12 +621,15 @@ write_run_meta() {
   local periodic_snapshot_forced="${20}"
   local reset_db_staging_path="${21}"
   local reset_db_cleanup_status="${22}"
+  local tmp_cleanup_status="${23}"
+  local tmp_cleanup_reason="${24}"
   python3 - "$output_path" "$status" "$run_index" "$run_id" "$sync_mode" "$snapshot_reason" \
     "$previous_run_id" "$previous_run_success_value" "$started_at" "$completed_at" \
     "$invalid_db_path" "$invalid_state_path" "$invalid_tmp_path" "$daemon_exit_code" \
     "$package_root" "$env_file" "$periodic_snapshot_reset_enabled" \
     "$periodic_snapshot_max_deltas" "$periodic_snapshot_delta_count" \
-    "$periodic_snapshot_forced" "$reset_db_staging_path" "$reset_db_cleanup_status" <<'PY'
+    "$periodic_snapshot_forced" "$reset_db_staging_path" "$reset_db_cleanup_status" \
+    "$tmp_cleanup_status" "$tmp_cleanup_reason" <<'PY'
 import json
 import sys
 
@@ -658,6 +672,8 @@ def bool_value(value):
     periodic_snapshot_forced,
     reset_db_staging_path,
     reset_db_cleanup_status,
+    tmp_cleanup_status,
+    tmp_cleanup_reason,
 ) = sys.argv[1:]
 
 data = {
@@ -682,6 +698,8 @@ data = {
     "periodic_snapshot_forced": bool_value(periodic_snapshot_forced),
     "reset_db_staging_path": nullable(reset_db_staging_path),
     "reset_db_cleanup_status": nullable(reset_db_cleanup_status),
+    "tmp_cleanup_status": nullable(tmp_cleanup_status),
+    "tmp_cleanup_reason": nullable(tmp_cleanup_reason),
 }
 with open(output_path, "w", encoding="utf-8") as handle:
     json.dump(data, handle, indent=2, sort_keys=True)
@@ -931,7 +949,8 @@ run_one_round() {
     "$snapshot_reason" "$previous_run_id" "$previous_success_value" "$started_at" "" \
     "$INVALID_DB_PATH" "$INVALID_STATE_PATH" "$INVALID_TMP_PATH" "" "$PACKAGE_ROOT" "$ENV_FILE" \
     "$RUN_META_PERIODIC_ENABLED" "$RUN_META_PERIODIC_MAX_DELTAS" "$RUN_META_PERIODIC_DELTA_COUNT" \
-    "$RUN_META_PERIODIC_FORCED" "$RUN_META_RESET_DB_STAGING_PATH" "$RUN_META_RESET_DB_CLEANUP_STATUS"
+    "$RUN_META_PERIODIC_FORCED" "$RUN_META_RESET_DB_STAGING_PATH" "$RUN_META_RESET_DB_CLEANUP_STATUS" \
+    "$RUN_META_TMP_CLEANUP_STATUS" "$RUN_META_TMP_CLEANUP_REASON"
 
   prepare_live_ta_inputs_for_run "$run_id" "$sync_mode"
   build_child_args
@@ -985,7 +1004,8 @@ run_one_round() {
     "$snapshot_reason" "$previous_run_id" "$previous_success_value" "$started_at" "$completed_at" \
     "$INVALID_DB_PATH" "$INVALID_STATE_PATH" "$INVALID_TMP_PATH" "$daemon_exit_code" "$PACKAGE_ROOT" "$ENV_FILE" \
     "$RUN_META_PERIODIC_ENABLED" "$RUN_META_PERIODIC_MAX_DELTAS" "$RUN_META_PERIODIC_DELTA_COUNT" \
-    "$RUN_META_PERIODIC_FORCED" "$RUN_META_RESET_DB_STAGING_PATH" "$RUN_META_RESET_DB_CLEANUP_STATUS"
+    "$RUN_META_PERIODIC_FORCED" "$RUN_META_RESET_DB_STAGING_PATH" "$RUN_META_RESET_DB_CLEANUP_STATUS" \
+    "$RUN_META_TMP_CLEANUP_STATUS" "$RUN_META_TMP_CLEANUP_REASON"
   printf '%s\n' "$run_id" > "$META_DIR/last-run-id"
   if is_true "$CLEAN_TMP_AFTER_RUN"; then
     rm -rf "$daemon_state_root"
@@ -1063,6 +1083,8 @@ main() {
     INVALID_DB_PATH=""
     INVALID_STATE_PATH=""
     INVALID_TMP_PATH=""
+    TMP_CLEANUP_STATUS=""
+    TMP_CLEANUP_REASON=""
     PERIODIC_SCAN_STATUS=""
     PERIODIC_SCAN_DELTA_COUNT=""
     PERIODIC_SCAN_SNAPSHOT_RUN_ID=""
@@ -1079,6 +1101,8 @@ main() {
     RUN_META_PERIODIC_FORCED="false"
     RUN_META_RESET_DB_STAGING_PATH=""
     RUN_META_RESET_DB_CLEANUP_STATUS=""
+    RUN_META_TMP_CLEANUP_STATUS=""
+    RUN_META_TMP_CLEANUP_REASON=""
     local previous_run_id=""
     local previous_success_value=""
     local sync_mode="snapshot"
@@ -1113,6 +1137,8 @@ main() {
         previous_success_value="false"
         if is_true "$FAILURE_SNAPSHOT_RESET"; then
           isolate_state_after_failure "$previous_run_id"
+          RUN_META_TMP_CLEANUP_STATUS="$TMP_CLEANUP_STATUS"
+          RUN_META_TMP_CLEANUP_REASON="$TMP_CLEANUP_REASON"
           sync_mode="snapshot"
           snapshot_reason="previous_run_failed"
         else
@@ -1123,6 +1149,8 @@ main() {
       sync_mode="snapshot"
       if db_state_exists; then
         isolate_state_after_failure "no_previous_run"
+        RUN_META_TMP_CLEANUP_STATUS="$TMP_CLEANUP_STATUS"
+        RUN_META_TMP_CLEANUP_REASON="$TMP_CLEANUP_REASON"
         snapshot_reason="no_successful_previous_run"
       else
         snapshot_reason="first_run"