preprocessing/run_preprocess.sh
| 1 | #!/usr/bin/env bash |
| 2 | # Rebuild the training inputs of one task from the raw mp4/parquet files: |
| 3 | # raw/ (download.py --what raw) -> <OUT>/<tree>_latents/{tac,visual,wrist}/<ep>.pt |
| 4 | # -> <OUT>/<tree>_actions/<ep>/{sensor_actions_with_dforce_fps15.pt, |
| 5 | # sensor_pose7d_force_fps15.pt, view_actions_fps15.pt, ...} |
| 6 | # bash preprocessing/run_preprocess.sh rope [OUT] [episode ...] |
| 7 | # Default OUT = $ACTS_DATA_ROOT/rebuilt (the released trees in $ACTS_DATA_ROOT are not touched). |
| 8 | # Point training at the result with data.latent_root=... data.action_root=... overrides. |
| 9 | # Recipe (identical to the released data): 30 -> 15 fps (stride 2), 8+8-frame windows, stride 1, |
| 10 | # 256x256 scale+center-crop; tactile = Wan2.2 VAE + tactile LoRA, view_middle + wrist = base VAE. |
| 11 | # Encoding needs 1 GPU (~10-40 min per episode group); action building is CPU-only. |
| 12 | set -euo pipefail |
| 13 | HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" |
| 14 | source "${ACTS_REPO:-$HERE}/env.sh" |
| 15 | TASK="${1:?usage: run_preprocess.sh <umi_all|rope|pusht> [OUT] [episode ...]}" |
| 16 | OUT="${2:-$ACTS_DATA_ROOT/rebuilt}"; shift $(( $# >= 2 ? 2 : 1 )) |
| 17 | declare -A TREE=([umi_all]=mixed_mb [rope]=mixed_rope [pusht]=mixed_pushT) |
| 18 | T="${TREE[$TASK]}"; RAW="$ACTS_DATA_ROOT/raw" |
| 19 | VAE="${WAN22_VAE_PTH:-$ACTS_CKPT_ROOT/Wan2.2_VAE.pth}"; LORA="${VTM_TACTILE_LORA:-$ACTS_CKPT_ROOT/vae/lora_last.pt}" |
| 20 | LAT="$OUT/${T}_latents"; ACT="$OUT/${T}_actions"; MAN="$OUT/episodes_${TASK}.jsonl" |
| 21 | mkdir -p "$LAT" "$ACT" |
| 22 | |
| 23 | echo "=== [1/3] manifest" |
| 24 | "$PYTHON_BIN" "$ACTS_REPO/preprocessing/make_manifest.py" --raw-root "$RAW" --task "$TASK" --out "$MAN" ${@:+--episodes "$@"} |
| 25 | EPS=$("$PYTHON_BIN" -c "import json,sys; print(' '.join(json.loads(l)['episode'] for l in open(sys.argv[1]) if l.strip()))" "$MAN") |
| 26 | |
| 27 | echo "=== [2/3] Wan2.2 latents (GPU)" |
| 28 | COMMON=(--vae-pth "$VAE" --fps 15 --src-fps 30 --history 8 --prediction 8 --stride 1 --resize 256 |
| 29 | --manifest "$MAN" --output-root "$LAT" --episodes $EPS) |
| 30 | cd "$VTM_WAN22_DIR" |
| 31 | "$PYTHON_BIN" -u encode_tactile_wan22.py --modality tactile --adapter "$LORA" "${COMMON[@]}" |
| 32 | "$PYTHON_BIN" -u encode_tactile_wan22.py --modality visual --adapter none "${COMMON[@]}" |
| 33 | "$PYTHON_BIN" -u encode_tactile_wan22.py --modality wrist --adapter none "${COMMON[@]}" |
| 34 | |
| 35 | echo "=== [3/3] actions (CPU)" |
| 36 | export ACTS_PARQUET_ROOT="$RAW/parquet" |
| 37 | P="$ACTS_REPO/preprocessing" |
| 38 | export PYTHONPATH="$P:$P/data_preprocessing:$PYTHONPATH" |
| 39 | for ep in $EPS; do mkdir -p "$ACT/$ep"; done |
| 40 | AARGS=(--parquet-root "$RAW/parquet" --videos-root "$ACT" --latent-dir "$LAT/tac" --target-fps 15 --episodes $EPS) |
| 41 | # order matters: the two force twins first, pose7d cross-checks its valid mask against them |
| 42 | "$PYTHON_BIN" -u "$P/vm_diffusion/data/build_force_actions.py" --force-term delta "${AARGS[@]}" |
| 43 | "$PYTHON_BIN" -u "$P/vm_diffusion/data/build_force_actions.py" --force-term absolute "${AARGS[@]}" |
| 44 | "$PYTHON_BIN" -u "$P/vm_diffusion/data/build_pose7d_force.py" "${AARGS[@]}" |
| 45 | # camera-frame view actions (gel centre / normal / up points + projections), 15 fps |
| 46 | "$PYTHON_BIN" -u "$P/data_preprocessing/build_view_actions.py" --videos-root "$ACT" --episodes $EPS \ |
| 47 | --stride 2 --out-name view_actions_fps15.pt |
| 48 | echo "PREPROCESS DONE -> $LAT $ACT" |
| 49 | echo "train on it: bash scripts/train.sh configs/nano_action_causal_${TASK}_v1.yaml data.latent_root=$LAT data.action_root=$ACT" |
| 50 | |