pretrain_path='logs/pretrain_20241111_112514_66c75731-c017-4e25-8afa-bf8372a98938/state_step014241.pt'
epochs=2
weight_decay=0.01
prefix='perturbed'

torchrun --standalone --nproc_per_node=1 train_gpt2.py --input_folder bioS_single/SFT_addition_40 --save_every 125 --val_loss_every 125 --load_checkpoint logs/perturbed_SFT_20241111_204348_40afb390-7348-4bc0-8acc-6308f8cfa700/state_step000250.pt --wandb_run_name second_stage_SFT_addition_40 --warmup_ratio 0.1 --warmdown_ratio 0.8 --sequence_length 512 --device_batch_size 16 --num_epochs 2 --learning_rate 0.0001 --batch_size 96 --weight_decay 0.01

torchrun --standalone --nproc_per_node=1 train_gpt2.py --input_folder bioS_single/SFT_additional --save_every 125 --val_loss_every 125 --load_checkpoint logs/perturbed_SFT_20241111_204348_40afb390-7348-4bc0-8acc-6308f8cfa700/state_step000250.pt --wandb_run_name second_stage_SFT_addition_all --warmup_ratio 0.1 --warmdown_ratio 0.8 --sequence_length 512 --device_batch_size 16 --num_epochs 5 --learning_rate 0.0001 --batch_size 96 --weight_decay 0.01

torchrun --standalone --nproc_per_node=1 train_gpt2.py --input_folder bioS_single/SFT --save_every 125 --val_loss_every 125 --load_checkpoint logs/perturbed_SFT_addition_20_20241111_214756_9b350459-12d6-456d-953f-3f88160e0402/state_step001220.pt --wandb_run_name second_stage_SFT_addition_revert --warmup_ratio 0.1 --warmdown_ratio 0.8 --sequence_length 512 --device_batch_size 16 --num_epochs 2 --learning_rate 0.0001 --batch_size 96 --weight_decay 0.01
# torchrun --standalone --nproc_per_node=1 train_gpt2.py --input_folder bioS_single/SFT --save_every 125 --load_checkpoint $pretrain_path --wandb_run_name ${prefix}_SFT --warmup_ratio 0.1 --warmdown_ratio 0.8 --sequence_length 512 --device_batch_size 16 --num_epochs $epochs --learning_rate 0.0001 --batch_size 96 --weight_decay $weight_decay
# torchrun --standalone --nproc_per_node=1 train_gpt2.py --input_folder bioS_single/SFT_addition_20 --save_every 125 --load_checkpoint $pretrain_path --wandb_run_name ${prefix}_SFT_addition_20 --warmup_ratio 0.1 --warmdown_ratio 0.8 --sequence_length 512 --device_batch_size 16 --num_epochs $epochs --learning_rate 0.0001 --batch_size 96 --weight_decay $weight_decay
torchrun --standalone --nproc_per_node=1 train_gpt2.py --input_folder bioS_single/SFT_addition_40 --save_every 125 --load_checkpoint $pretrain_path --wandb_run_name ${prefix}_SFT_addition_40 --warmup_ratio 0.1 --warmdown_ratio 0.8 --sequence_length 512 --device_batch_size 16 --num_epochs $epochs --learning_rate 0.0001 --batch_size 96 --weight_decay $weight_decay
torchrun --standalone --nproc_per_node=1 train_gpt2.py --input_folder bioS_single/SFT_addition_10 --save_every 125 --load_checkpoint $pretrain_path --wandb_run_name ${prefix}_SFT_addition_10 --warmup_ratio 0.1 --warmdown_ratio 0.8 --sequence_length 512 --device_batch_size 16 --num_epochs $epochs --learning_rate 0.0001 --batch_size 96 --weight_decay $weight_decay
torchrun --standalone --nproc_per_node=1 train_gpt2.py --input_folder bioS_single/SFT_addition_5 --save_every 125 --load_checkpoint $pretrain_path --wandb_run_name ${prefix}_SFT_addition_5 --warmup_ratio 0.1 --warmdown_ratio 0.8 --sequence_length 512 --device_batch_size 16 --num_epochs $epochs --learning_rate 0.0001 --batch_size 96 --weight_decay $weight_decay


