DeepSpeed Tutorial
conda create -n Torch python=3.10
conda activate Torch
pip install torch==2.0.0 torchvision==0.15.1 torchaudio==2.0.1
# on CentOS 7
sudo yum install -y centos-release-scl-rh
sudo yum update
sudo yum install -y devtoolset-9
source /opt/rh/devtoolset-9/enable
# on debian
sudo apt update
sudo apt install build-essential manpages-dev
git clone https://github.com/microsoft/DeepSpeed.git
cd DeepSpeed
git checkout v0.9.5
# !!! below is single command
TORCH_CUDA_ARCH_LIST="6.1;7.5;8.6;8.9" \
DS_BUILD_CCL_COMM=1 \
DS_BUILD_CPU_ADAM=1 \
DS_BUILD_CPU_ADAGRAD=1 \
DS_BUILD_FUSED_ADAM=1 \
DS_BUILD_FUSED_LAMB=1 \
DS_BUILD_UTILS=1 \
python setup.py build_ext -j24 bdist_wheel
pip install ./dist/deepspeed-0.9.5+fc9e1ee-*-linux_x86_64.whl
some precompiled wheels:
OS | torch version | cuda version | download link |
---|---|---|---|
CentOS 7 x86_64 | 1.13.1 | cuda11.8 | wheel |
CentOS 7 x86_64 | 2.0.1 | cuda11.8 | wheel |
Ubuntu 20.04 x86_64 | 1.13.1 | cuda11.8 | wheel |
Ubuntu 20.04 x86_64 | 2.0.1 | cuda11.8 | wheel |
git clone https://github.com/OvJat/DeepSpeedTutorial.git
cd DeepSpeedTutorial
# editor hostfile
./run_dist.sh &
disown
# check logging
tail -f ./logs/run_dist.sh.log
E-MAIL: [email protected]