-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtrain.sh
More file actions
executable file
·52 lines (40 loc) · 1.11 KB
/
train.sh
File metadata and controls
executable file
·52 lines (40 loc) · 1.11 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
#!/bin/bash
RUN_CONFIG_PATH="run_config.json"
ENV_ID="Freeway"
SEEDS=(0 1 2)
AVAILABLE_GPUS=(1 2 3)
AUTHOR="LR"
run_training() {
local seed=$1
local gpu_id=$2
echo "Starting evaluation for modification: $modif on GPU $gpu_id"
CUDA_VISIBLE_DEVICES=$gpu_id python continual_training_example.py \
--env_id "$ENV_ID" \
--seed "$seed" \
--config "$RUN_CONFIG_PATH" \
--track \
--capture_video \
--author "$AUTHOR" \
echo "Completed evaluation for seed: $seed on GPU $gpu_id"
}
export -f run_training
export RUN_CONFIG_PATH AUTHOR ENV_ID
num_seeds=${#SEEDS[@]}
num_gpus=${#AVAILABLE_GPUS[@]}
pids=()
for i in "${!SEEDS[@]}"; do
seed=${SEEDS[$i]}
gpu_idx=$((i % num_gpus))
gpu_id=${AVAILABLE_GPUS[$gpu_idx]}
if [ ${#pids[@]} -ge $num_gpus ]; then
wait ${pids[0]}
pids=("${pids[@]:1}") # Remove first element
fi
run_training "$seed" "$gpu_id" &
pids+=($!)
echo "Launched seed $seed on GPU $gpu_id (PID: $!)"
done
for pid in "${pids[@]}"; do
wait $pid
done
echo "All evaluations completed!"