nvbandwidth测速GPU间带宽

一、环境信息

4台Ubuntu22.04.4 LTS服务器，每台服务器上有8个NVIDIA A800-SXM4-80GB

二、安装nvbandwidth

参考：https://github.com/NVIDIA/nvbandwidth

root@deepseek2:/opt/code_repos# git clone https://github.com/NVIDIA/nvbandwidth.git
root@deepseek2:/opt/code_repos# cd nvbandwidth

#安装依赖
root@deepseek2:/opt/code_repos/nvbandwidth# ./debian_install.sh

#安装nvidia-imex（Imex服务是NVIDIA Internode内存交换服务）
root@deepseek2:/opt/code_repos/nvbandwidth# apt install nvidia-imex
root@deepseek2:/opt/code_repos/nvbandwidth# systemctl enable nvidia-imex
root@deepseek2:/opt/code_repos/nvbandwidth# systemctl start nvidia-imex.service 

#安装 NUMA 开发库：libnuma-dev
root@deepseek2:/opt/installPkgs/mpich-4.2.3# apt install libnuma-dev

#安装MPI（https://www.mpich.org/static/downloads/4.2.3/）
root@deepseek2:/opt/installPkgs# wget https://www.mpich.org/static/downloads/4.2.3/mpich-4.2.3.tar.gz
#检测系统环境并生成适合当前系统的构建配置，此操作将生成Makefile
root@deepseek2:/opt/installPkgs/mpich-4.2.3# ./configure --prefix=/usr/local/mpich-4.2.3
#编译
root@deepseek2:/opt/installPkgs/mpich-4.2.3# make -j$(nproc)
#安装
root@deepseek2:/opt/installPkgs/mpich-4.2.3# make install

#将mpirun可执行文件路径添加到PATH中
root@deepseek2:/opt/installPkgs/mpich-4.2.3# vi /root/.bashrc
#在文件最后添加如下内容
export PATH=/usr/local/mpich-4.2.3/bin:$PATH
#重新加载上述文件
root@deepseek2:/opt/installPkgs/mpich-4.2.3# source /root/.bashrc

root@deepseek2:/opt/installPkgs/mpich-4.2.3# cd examples/
#mpi接口对hellow.c进行编译
root@deepseek2:/opt/installPkgs/mpich-4.2.3/examples# mpicc  hellow.c -o hellow
#运行hellow文件“mpirun -np N ./hellow”，将其中的N换成自定义进程数比如3
root@deepseek2:/opt/installPkgs/mpich-4.2.3/examples# mpirun -np 3 ./hellow
Hello world from process 0 of 3
Hello world from process 1 of 3
Hello world from process 2 of 3

root@deepseek2:/opt/code_repos/nvbandwidth# cmake -DMULTINODE=1 .
root@deepseek2:/opt/code_repos/nvbandwidth# make