CUBLAS Library Dr. Bo Yuan E-mail: yuanb@sz.tsinghua.edu.cn What is CUBLAS Library? • BLAS – Basic Linear Algebra Subprogram – A library to perform basic linear algebra – Divided into three levels – Such as MKL BLAS,CUBLAS, C++ AMP BLAS…… • CUBLAS – An high level implementation of BLAS on top of the NVIDIA CUDA runtime – Single GPU or Multiple GPUs – Support CUDA Stream 2 Three Levels Of BLAS Level 1 This level contains vector operations of the form y x y Level 2 This level contains matrix-vector operations of the form y Ax y Level 3 This level contains matrix-matrix operations of the form C AB C 3 Why we need CUBLAS? • CUBLAS – Full support for all 152 standard BLAS routines – Support single-precision, double-precision, complex and double complex number data types – Support for CUDA steams – Fortran bindings – Support for multiple GPUs and concurrent kernels – Very efficient 4 Why we need CUBLAS? 5 Getting Started • Basic preparation – Install CUDA Toolkit – Include cublas_v2.h – Link cublas.lib • Some basic tips – Every CUBLAS function needs a handle – The CUBLAS function must be written between cublasCreate() and cublasDestory() – Every CUBLAS function returns a cublasStatus_t to report the state of execution. – Column-major storage • References – http://cudazone.nvidia.cn/cublas/ – CUDA Toolkit 5.0 CUBLAS Library.pdf 6 CUBLAS Data Types • cublasHandle_t • cublasStatus_t • • • • • • • • CUBLAS_STATUS_SUCCESS CUBLAS_STATUS_NOT_INITIALIZED CUBLAS_STATUS_ALLOC_FAILED CUBLAS_STATUS_INVALID_VALUE CUBLAS_STATUS_ARCH_MISMATCH CUBLAS_STATUS_MAPPING_ERROR CUBLAS_STATUS_EXECUTION_FAILED CUBLAS_STATUS_INTERNAL_ERROR 7 CUBLAS Data Types • cublasOperation_t 8 CUBLAS Datatypes • cublasFillMode_t • cublasSideMode_t 9 CUBLAS Data Types • cublasPointerMode_t • cublasAtomicsMode_t 10 Example Code #include <stdio.h> #include <stdlib.h> #include <math.h> #include <cuda_runtime.h> #include "cublas_v2.h" //调用CUBLAS必须包含的头文件 #define M 6 #define N 5 #define IDX2F(i,j,ld) ((((j)-1)*(ld))+((i)-1)) //按列访问数组下标 static __inline__ void modify(cublasHandle_t handle,float* m,int ldm,int n,int p,int q,float alpha,float beta) { cublasSscal(handle,n-p+1,&alpha,&m[IDX2F(p,q,ldm)],ldm); cublasSscal(handle,ldm-p+1,&beta,&m[IDX2F(p,q,ldm)],1); } 11 Example Code int main(void){ cudaError_t cudaStat; cublasStatus_t stat; cublasHandle_t handle; int i,j; float* devPtrA; float* a=0; a=(float*)malloc(M*N*sizeof(*a)); //在host上开辟数组空间 if (!a) { printf("host memory allocation failed"); return EXIT_FAILURE; } 12 Example Code for (j=1;j<=N;j++) //数组初始化 { for (i=1;i<=M;i++) { a[IDX2F(i,j,M)]=(float)((i-1)*M+j); } } cudaStat = cudaMalloc((void**)&devPtrA,M*N*sizeof(*a)); //在device上开辟内存空间 if (cudaStat != cudaSuccess) { printf("device memory allocation failed"); return EXIT_FAILURE; } stat = cublasCreate(&handle); //初始化CUBLAS环境 13 Example Code if (stat != cudaSuccess) { printf("CUBLAS initialization failed\n"); return EXIT_FAILURE; } stat = cublasSetMatrix(M,N,sizeof(*a),a,M,devPtrA,M); //把数据从host拷贝到device if (stat != CUBLAS_STATUS_SUCCESS) { printf("data download failed"); cudaFree(devPtrA); cublasDestroy(handle); return EXIT_FAILURE; } modify(handle,devPtrA,M,N,2,3,16.0f,12.0f); stat = cublasGetMatrix(M,N,sizeof(*a),devPtrA,M,a,M); //把数据从device拷贝到host 14 Example Code if (stat != CUBLAS_STATUS_SUCCESS) { printf("data upload failed"); cudaFree(devPtrA); cublasDestroy(handle); return EXIT_FAILURE; } cudaFree(devPtrA); //释放指针 cublasDestroy(handle); //关闭CULBAS环境 for (j=1;j<=N;j++) { for (i=1;i<=M;i++) { printf("%7.0f",a[IDX2F(i,j,M)]); } } return EXIT_SUCCESS; } 15 Matrix Multiply • Use level-3 function C op( A)op( B) C • Function Introduce • cublasStatus_t cublasSgemm(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, int k, const float *alpha, const float *A, int lda, const float *B, int ldb, const float *beta, float *C, int ldc) 16 Matrix Multiply 17 Matrix Multiply int MatrixMulbyCUBLAS(float *A,float *B,int HA, int WB,int WA,float *C){ float *d_A,*d_B,*d_C; CUDA_SAFE_CALL(cudaMalloc((void **)&d_A,WA*HA*sizeof(float))); CUDA_SAFE_CALL(cudaMalloc((void **)&d_B,WB*WA*sizeof(float))); CUDA_SAFE_CALL(cudaMalloc((void **)&d_C,WB*HA*sizeof(float))); CUDA_SAFE_CALL(cudaMemcpy(d_A,A,WA*HA*sizeof(float),cudaMemcpyH ostToDevice)); CUDA_SAFE_CALL(cudaMemcpy(d_B,B,WB*WA*sizeof(float),cudaMemcpyH ostToDevice)); cublasStatus_t status; cublasHandle_t handle; status=cublasCreate(&handle); if (status!=CUBLAS_STATUS_SUCCESS) { printf("CUBLAS initialization error\n"); return EXIT_FAILURE; } 18 Matrix Multiply int devID; cudaDeviceProp props; CUDA_SAFE_CALL(cudaGetDevice(&devID)); CUDA_SAFE_CALL(cudaGetDeviceProperties(&props,devID)); printf("Device %d: \"%s\" with Compute %d.%d capability\n", devID, props.name, props.major, props.minor); const float alpha=1.0f; const float beta=0.0f; cublasSgemm(handle,CUBLAS_OP_N,CUBLAS_OP_N,WB,HA,WA,&alpha,d_B, WB,d_A,WA,&beta,d_C,WB); //level 3 function CUDA_SAFE_CALL(cudaMemcpy(C,d_C,WB*HA*sizeof(float),cudaMemcpyD eviceToHost)); cublasDestroy(handle); cudaFree(d_A); cudaFree(d_B); cudaFree(d_C); return 0; } 19 The Rusult 20 Some New Features • The handle to the CUBLAS library context is initialized using the cublasCreate function and is explicitly passed to every subsequent library function call. This allows the user to have more control over the library setup when using multiple host threads and multiple GPUs. • The scalars a and b can be passed by reference on the host or the device, instead of only being allowed to be passed by value on the host. This change allows library functions to execute asynchronously using streams even when a and b are generated by a previous kernel. 21 Some New Features • When a library routine returns a scalar result, it can be returned by reference on the host or the device, instead of only being allowed to be returned by value only on the host. This change allows library routines to be called asynchronously when the scalar result is generated and returned by reference on the device resulting in maximum parallelism. 22 Stream • Stream – Concurrent Execution between Host and Device • Overlap of Data Transfer and Kernel Execution – With devices of compute capability 1.1 or higher – Hidden Data Transfer Time • Rules – Functions in a same stream execute sequentially – Functions in different streams execute concurrently • References – http://cudazone.nvidia.cn/ – CUDA C Programming Guide.pdf 23 Parallelism with Streams • Create and set stream to be used by each CUBLAS routine – Users must call function cudaStreamCreate() to create different streams . – Users must call function cublasSetStream() to set a stream to be used by each individual CUBLAS routine. • Use asynchronous transfer function – cudaMemcpyAsync() 24 Parallelism with Streams start=clock(); for (int i = 0; i < nstreams; i++) { cudaMemcpy(d_A,A,WA*HA*sizeof(float),cudaMemcpyHostToDevice); cudaMemcpy(d_B,B,WB*WA*sizeof(float),cudaMemcpyHostToDevice); cublasSgemm(handle,CUBLAS_OP_N,CUBLAS_OP_N,WB,HA,WA,&alpha,d_B,WB, d_A,WA,&beta,d_C,WB); cudaMemcpy(C,d_C,WB*HA*sizeof(float),cudaMemcpyDeviceToHost); } end=clock(); printf(“GPU Without Stream time: %.2f秒.\n", (double)(endstart)/CLOCKS_PER_SEC); 25 Parallelism with Streams start=clock(); for (int i = 0; i < nstreams; i++) { cudaMemcpyAsync(d_A,A,WA*HA*sizeof(float),cudaMemcpyHostToDevice, streams[i]); cudaMemcpyAsync(d_B,B,WB*WA*sizeof(float),cudaMemcpyHostToDevice, streams[i]); cublasSetStream(handle,streams[i]); cublasSgemm(handle,CUBLAS_OP_N,CUBLAS_OP_N,WB,HA,WA,&alpha,d_B,WB ,d_A,WA,&beta,d_C,WB); cudaMemcpyAsync(C,d_C,WB*HA*sizeof(float),cudaMemcpyDeviceToHost) ; } end=clock(); 26 printf("GPU With Stream time: %.2f秒.\n", (double)(endstart)/CLOCKS_PER_SEC); The Result 27 Review • What is core functionality of BLAS and CUBLAS? • What is the advantage of CUBLAS? • What is the importance of handle in CUBLAS? • How to perform matrix multiplication using CUBLAS? • How is a matrix stored in CUBLAS? • How to use CUBLAS with stream techniques? • What can we do using CUBLAS in our research? 28