#! /bin/bash


CUDA_VISIBLE_DEVICES=2
# MODEL CANDIDATES
MODEL_NAME="YOUR_MODEL_NAME"
TENSOR_PARALLEL_SIZE=1 # length of A
HOST="YOUR_HOST"
PORT="YOUR_PORT"

CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES python -m vllm.entrypoints.openai.api_server \
    --model $MODEL_NAME \
    --tokenizer $MODEL_NAME \
    --tensor-parallel-size $TENSOR_PARALLEL_SIZE \
    --trust-remote-code \
    --seed 42 \
    --host $HOST \
    --port $PORT