merve/inkling / slurm /submit_inkling_generate.sbatch
merve's picture
download
raw
5.15 kB
#!/bin/bash
#SBATCH --job-name=inkling-generate
#SBATCH --nodes=4
#SBATCH --gpus-per-node=8
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=88
#SBATCH --mem=0
#SBATCH --exclusive
#SBATCH --time=01:00:00
#SBATCH --output=inkling-generate-%j.out
# Cluster-specific scheduling (partition / qos / account) is intentionally NOT hardcoded.
# Pass it at submit time, e.g.
# sbatch --partition=<your_gpu_partition> --qos=<your_qos> submit_inkling_generate.sbatch
# or add your own #SBATCH lines here. The resource lines above assume 8-GPU nodes; adjust
# --nodes / --gpus-per-node / --cpus-per-task for your hardware and the model's size.
# Multimodal (image / text / audio) generation with the Inkling model pulled from the Hub
# (default thinkingmachines/Inkling), sharded across all nodes x GPUs with tp_plan="auto".
# Pairs with the generate_inkling.py sitting next to this file.
#
# Usage:
# sbatch --partition=... submit_inkling_generate.sbatch
# PROMPT="What is in this image?" IMAGE=/path/cat.png sbatch --partition=... submit_inkling_generate.sbatch
# PROMPT="Transcribe this." AUDIO=/path/clip_16k.wav sbatch --partition=... submit_inkling_generate.sbatch
#
# Configure via environment (all optional):
# MODEL_PATH Hub repo id or local folder (default thinkingmachines/Inkling)
# PROCESSOR_PATH processor location if separate from MODEL_PATH
# PROMPT the text question
# IMAGE image path(s)/URL(s), space-separated for several
# AUDIO 16 kHz mono PCM .wav path(s), space-separated for several
# MAX_NEW_TOKENS generation length (default 512)
# THINKING_EFFORT optional reasoning effort 0.0-1.0
# HF_TOKEN token for a private/gated repo (passed through only if set)
# HF_HOME model/dataset cache dir (default $HOME/.cache/huggingface)
# ENV_ACTIVATE path to a venv/conda activate script to source (default: none)
# CUDA_MODULE environment-modules CUDA to load, e.g. cuda/12.8 (default: none)
# FI_PROVIDER set to "efa" on AWS EFA fabrics; leave unset otherwise
# TRANSFORMERS_SPEC pip-install this Transformers before running (default transformers==5.14.0,
# the release that includes Inkling); set to empty to use the env as-is
# TRANSFORMERS_SRC local checkout whose src/ to prepend to PYTHONPATH instead of pip
set -euo pipefail
# ======================= EDIT / OVERRIDE FOR YOUR CLUSTER ================================
HF_HOME=${HF_HOME:-$HOME/.cache/huggingface} # small home dirs: point at scratch/shared FS
ENV_ACTIVATE=${ENV_ACTIVATE:-} # e.g. /path/to/venv/bin/activate
CUDA_MODULE=${CUDA_MODULE:-} # e.g. cuda/12.8 (only if your site uses `module`)
# =========================================================================================
HERE=${SLURM_SUBMIT_DIR:-$PWD}
SCRIPT=${SCRIPT:-${HERE}/generate_inkling.py}
MODEL_PATH=${MODEL_PATH:-thinkingmachines/Inkling}
PROCESSOR_PATH=${PROCESSOR_PATH:-${MODEL_PATH}}
PROMPT=${PROMPT:-"Explain tensor parallelism in simple terms."}
IMAGE=${IMAGE:-}
AUDIO=${AUDIO:-}
MAX_NEW_TOKENS=${MAX_NEW_TOKENS:-512}
THINKING_EFFORT=${THINKING_EFFORT:-}
TRANSFORMERS_SPEC=${TRANSFORMERS_SPEC:-transformers==5.14.0} # Inkling ships in this release
TRANSFORMERS_SRC=${TRANSFORMERS_SRC:-}
NNODES=${SLURM_NNODES:-4}
GPUS_PER_NODE=${GPUS_PER_NODE:-8}
MASTER_PORT=${MASTER_PORT:-29500}
if [[ -n "$CUDA_MODULE" ]] && command -v module >/dev/null 2>&1; then module load "$CUDA_MODULE"; fi
if [[ -n "$ENV_ACTIVATE" ]]; then source "$ENV_ACTIVATE"; fi
if [[ -n "$TRANSFORMERS_SPEC" ]]; then
echo "[INFO] pip install -U ${TRANSFORMERS_SPEC}"
pip install -U "$TRANSFORMERS_SPEC"
fi
if [[ -n "$TRANSFORMERS_SRC" ]]; then
export PYTHONPATH="${TRANSFORMERS_SRC}/src:${PYTHONPATH:-}"
fi
python -c "import transformers; print('[INFO] transformers', transformers.__version__, transformers.__file__)"
export HF_HOME
export PYTHONUNBUFFERED=1
[[ -n "${FI_PROVIDER:-}" ]] && export FI_PROVIDER
HEAD_NODE=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n1)
HEAD_IP=$(srun --nodes=1 --ntasks=1 -w "$HEAD_NODE" hostname -I | awk '{print $1}')
echo "Nodes = $SLURM_JOB_NODELIST"
echo "Head node = $HEAD_NODE ($HEAD_IP)"
echo "Script = $SCRIPT"
echo "Model = $MODEL_PATH"
echo "Processor = $PROCESSOR_PATH"
echo "HF_HOME = $HF_HOME"
echo "Prompt = $PROMPT"
echo "Image(s) = ${IMAGE:-<none>}"
echo "Audio(s) = ${AUDIO:-<none>}"
# Assemble the script flags: repeat --image / --audio for each space-separated entry.
SCRIPT_ARGS=(--model-path "$MODEL_PATH" --processor-path "$PROCESSOR_PATH"
--prompt "$PROMPT" --max-new-tokens "$MAX_NEW_TOKENS")
for img in $IMAGE; do SCRIPT_ARGS+=(--image "$img"); done
for aud in $AUDIO; do SCRIPT_ARGS+=(--audio "$aud"); done
[[ -n "$THINKING_EFFORT" ]] && SCRIPT_ARGS+=(--thinking-effort "$THINKING_EFFORT")
srun --ntasks-per-node=1 torchrun \
--nnodes="$NNODES" --nproc-per-node="$GPUS_PER_NODE" \
--rdzv-backend=c10d --rdzv-endpoint="${HEAD_IP}:${MASTER_PORT}" --rdzv-id="$SLURM_JOB_ID" \
"$SCRIPT" "${SCRIPT_ARGS[@]}"

Xet Storage Details

Size:
5.15 kB
·
Xet hash:
632eedd91e3731980ad25735c3c6e9b223bd3a52f0ac46c56da7656ffe64c25e

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.