| #SBATCH --job-name=inkling-generate | |
| #SBATCH --nodes=4 | |
| #SBATCH --gpus-per-node=8 | |
| #SBATCH --ntasks-per-node=1 | |
| #SBATCH --cpus-per-task=88 | |
| #SBATCH --mem=0 | |
| #SBATCH --exclusive | |
| #SBATCH --time=01:00:00 | |
| #SBATCH --output=inkling-generate-%j.out | |
| # Cluster-specific scheduling (partition / qos / account) is intentionally NOT hardcoded. | |
| # Pass it at submit time, e.g. | |
| # sbatch --partition=<your_gpu_partition> --qos=<your_qos> submit_inkling_generate.sbatch | |
| # or add your own #SBATCH lines here. The resource lines above assume 8-GPU nodes; adjust | |
| # --nodes / --gpus-per-node / --cpus-per-task for your hardware and the model's size. | |
| # Multimodal (image / text / audio) generation with the Inkling model pulled from the Hub | |
| # (default thinkingmachines/Inkling), sharded across all nodes x GPUs with tp_plan="auto". | |
| # Pairs with the generate_inkling.py sitting next to this file. | |
| # | |
| # Usage: | |
| # sbatch --partition=... submit_inkling_generate.sbatch | |
| # PROMPT="What is in this image?" IMAGE=/path/cat.png sbatch --partition=... submit_inkling_generate.sbatch | |
| # PROMPT="Transcribe this." AUDIO=/path/clip_16k.wav sbatch --partition=... submit_inkling_generate.sbatch | |
| # | |
| # Configure via environment (all optional): | |
| # MODEL_PATH Hub repo id or local folder (default thinkingmachines/Inkling) | |
| # PROCESSOR_PATH processor location if separate from MODEL_PATH | |
| # PROMPT the text question | |
| # IMAGE image path(s)/URL(s), space-separated for several | |
| # AUDIO 16 kHz mono PCM .wav path(s), space-separated for several | |
| # MAX_NEW_TOKENS generation length (default 512) | |
| # THINKING_EFFORT optional reasoning effort 0.0-1.0 | |
| # HF_TOKEN token for a private/gated repo (passed through only if set) | |
| # HF_HOME model/dataset cache dir (default $HOME/.cache/huggingface) | |
| # ENV_ACTIVATE path to a venv/conda activate script to source (default: none) | |
| # CUDA_MODULE environment-modules CUDA to load, e.g. cuda/12.8 (default: none) | |
| # FI_PROVIDER set to "efa" on AWS EFA fabrics; leave unset otherwise | |
| # TRANSFORMERS_SPEC pip-install this Transformers before running (default transformers==5.14.0, | |
| # the release that includes Inkling); set to empty to use the env as-is | |
| # TRANSFORMERS_SRC local checkout whose src/ to prepend to PYTHONPATH instead of pip | |
| set -euo pipefail | |
| # ======================= EDIT / OVERRIDE FOR YOUR CLUSTER ================================ | |
| HF_HOME=${HF_HOME:-$HOME/.cache/huggingface} # small home dirs: point at scratch/shared FS | |
| ENV_ACTIVATE=${ENV_ACTIVATE:-} # e.g. /path/to/venv/bin/activate | |
| CUDA_MODULE=${CUDA_MODULE:-} # e.g. cuda/12.8 (only if your site uses `module`) | |
| # ========================================================================================= | |
| HERE=${SLURM_SUBMIT_DIR:-$PWD} | |
| SCRIPT=${SCRIPT:-${HERE}/generate_inkling.py} | |
| MODEL_PATH=${MODEL_PATH:-thinkingmachines/Inkling} | |
| PROCESSOR_PATH=${PROCESSOR_PATH:-${MODEL_PATH}} | |
| PROMPT=${PROMPT:-"Explain tensor parallelism in simple terms."} | |
| IMAGE=${IMAGE:-} | |
| AUDIO=${AUDIO:-} | |
| MAX_NEW_TOKENS=${MAX_NEW_TOKENS:-512} | |
| THINKING_EFFORT=${THINKING_EFFORT:-} | |
| TRANSFORMERS_SPEC=${TRANSFORMERS_SPEC:-transformers==5.14.0} # Inkling ships in this release | |
| TRANSFORMERS_SRC=${TRANSFORMERS_SRC:-} | |
| NNODES=${SLURM_NNODES:-4} | |
| GPUS_PER_NODE=${GPUS_PER_NODE:-8} | |
| MASTER_PORT=${MASTER_PORT:-29500} | |
| if [[ -n "$CUDA_MODULE" ]] && command -v module >/dev/null 2>&1; then module load "$CUDA_MODULE"; fi | |
| if [[ -n "$ENV_ACTIVATE" ]]; then source "$ENV_ACTIVATE"; fi | |
| if [[ -n "$TRANSFORMERS_SPEC" ]]; then | |
| echo "[INFO] pip install -U ${TRANSFORMERS_SPEC}" | |
| pip install -U "$TRANSFORMERS_SPEC" | |
| fi | |
| if [[ -n "$TRANSFORMERS_SRC" ]]; then | |
| export PYTHONPATH="${TRANSFORMERS_SRC}/src:${PYTHONPATH:-}" | |
| fi | |
| python -c "import transformers; print('[INFO] transformers', transformers.__version__, transformers.__file__)" | |
| export HF_HOME | |
| export PYTHONUNBUFFERED=1 | |
| [[ -n "${FI_PROVIDER:-}" ]] && export FI_PROVIDER | |
| HEAD_NODE=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n1) | |
| HEAD_IP=$(srun --nodes=1 --ntasks=1 -w "$HEAD_NODE" hostname -I | awk '{print $1}') | |
| echo "Nodes = $SLURM_JOB_NODELIST" | |
| echo "Head node = $HEAD_NODE ($HEAD_IP)" | |
| echo "Script = $SCRIPT" | |
| echo "Model = $MODEL_PATH" | |
| echo "Processor = $PROCESSOR_PATH" | |
| echo "HF_HOME = $HF_HOME" | |
| echo "Prompt = $PROMPT" | |
| echo "Image(s) = ${IMAGE:-<none>}" | |
| echo "Audio(s) = ${AUDIO:-<none>}" | |
| # Assemble the script flags: repeat --image / --audio for each space-separated entry. | |
| SCRIPT_ARGS=(--model-path "$MODEL_PATH" --processor-path "$PROCESSOR_PATH" | |
| --prompt "$PROMPT" --max-new-tokens "$MAX_NEW_TOKENS") | |
| for img in $IMAGE; do SCRIPT_ARGS+=(--image "$img"); done | |
| for aud in $AUDIO; do SCRIPT_ARGS+=(--audio "$aud"); done | |
| [[ -n "$THINKING_EFFORT" ]] && SCRIPT_ARGS+=(--thinking-effort "$THINKING_EFFORT") | |
| srun --ntasks-per-node=1 torchrun \ | |
| --nnodes="$NNODES" --nproc-per-node="$GPUS_PER_NODE" \ | |
| --rdzv-backend=c10d --rdzv-endpoint="${HEAD_IP}:${MASTER_PORT}" --rdzv-id="$SLURM_JOB_ID" \ | |
| "$SCRIPT" "${SCRIPT_ARGS[@]}" | |
Xet Storage Details
- Size:
- 5.15 kB
- Xet hash:
- 632eedd91e3731980ad25735c3c6e9b223bd3a52f0ac46c56da7656ffe64c25e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.