Windows CXX11-ABI libtorch debug builds for CUDA 11.1 fail with unresolved external symbol

🐛 Bug

To Reproduce

After the addition of CUDA 11.1 to our nightly matrix we’ve observed that windows debug builds for CUDA 11.1 of libtorch compiled with the CXX11-ABI are failing with this error:

   Creating library lib\torch_cuda.lib and object lib\torch_cuda.exp
RNN.cpp.obj : error LNK2019: unresolved external symbol "__declspec(dllimport) public: void __cdecl at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::lstm_cudnn_stub>::set_cuda_dispatch_ptr(void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool))" (__imp_?set_cuda_dispatch_ptr@?$DispatchStub@P6AXAEAVTensor@at@@00AEBV12@V?$ArrayRef@VTensor@at@@@c10@@2_N_JN333@ZUlstm_cudnn_stub@native@2@@native@at@@QEAAXP6AXAEAVTensor@3@00AEBV43@V?$ArrayRef@VTensor@at@@@c10@@2_N_JN333@Z@Z) referenced in function "public: __cdecl at::native::`anonymous namespace'::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::lstm_cudnn_stub>::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::lstm_cudnn_stub>(struct at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::lstm_cudnn_stub> &,void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool))" (??0?$RegisterCUDADispatch@P6AXAEAVTensor@at@@00AEBV12@V?$ArrayRef@VTensor@at@@@c10@@2_N_JN333@ZUlstm_cudnn_stub@native@2@@?A0x5f01b69b@native@at@@QEAA@AEAU?$DispatchStub@P6AXAEAVTensor@at@@00AEBV12@V?$ArrayRef@VTensor@at@@@c10@@2_N_JN333@ZUlstm_cudnn_stub@native@2@@23@P6AXAEAVTensor@3@11AEBV53@V?$ArrayRef@VTensor@at@@@c10@@3_N_JN444@Z@Z)
RNN.cpp.obj : error LNK2019: unresolved external symbol "__declspec(dllimport) public: void __cdecl at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::gru_cudnn_stub>::set_cuda_dispatch_ptr(void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool))" (__imp_?set_cuda_dispatch_ptr@?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@ZUgru_cudnn_stub@native@2@@native@at@@QEAAXP6AXAEAVTensor@3@0AEBV43@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@Z@Z) referenced in function "public: __cdecl at::native::`anonymous namespace'::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::gru_cudnn_stub>::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::gru_cudnn_stub>(struct at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::gru_cudnn_stub> &,void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool))" (??0?$RegisterCUDADispatch@P6AXAEAVTensor@at@@0AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@ZUgru_cudnn_stub@native@2@@?A0x5f01b69b@native@at@@QEAA@AEAU?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@ZUgru_cudnn_stub@native@2@@23@P6AXAEAVTensor@3@1AEBV53@2V?$ArrayRef@VTensor@at@@@c10@@_N_JN444@Z@Z)
RNN.cpp.obj : error LNK2019: unresolved external symbol "__declspec(dllimport) public: void __cdecl at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::rnn_tanh_cudnn_stub>::set_cuda_dispatch_ptr(void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool))" (__imp_?set_cuda_dispatch_ptr@?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@ZUrnn_tanh_cudnn_stub@native@2@@native@at@@QEAAXP6AXAEAVTensor@3@0AEBV43@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@Z@Z) referenced in function "public: __cdecl at::native::`anonymous namespace'::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::rnn_tanh_cudnn_stub>::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::rnn_tanh_cudnn_stub>(struct at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::rnn_tanh_cudnn_stub> &,void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool))" (??0?$RegisterCUDADispatch@P6AXAEAVTensor@at@@0AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@ZUrnn_tanh_cudnn_stub@native@2@@?A0x5f01b69b@native@at@@QEAA@AEAU?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@ZUrnn_tanh_cudnn_stub@native@2@@23@P6AXAEAVTensor@3@1AEBV53@2V?$ArrayRef@VTensor@at@@@c10@@_N_JN444@Z@Z)
RNN.cpp.obj : error LNK2019: unresolved external symbol "__declspec(dllimport) public: void __cdecl at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::rnn_relu_cudnn_stub>::set_cuda_dispatch_ptr(void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool))" (__imp_?set_cuda_dispatch_ptr@?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@ZUrnn_relu_cudnn_stub@native@2@@native@at@@QEAAXP6AXAEAVTensor@3@0AEBV43@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@Z@Z) referenced in function "public: __cdecl at::native::`anonymous namespace'::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::rnn_relu_cudnn_stub>::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::rnn_relu_cudnn_stub>(struct at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool),struct at::native::rnn_relu_cudnn_stub> &,void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool,bool))" (??0?$RegisterCUDADispatch@P6AXAEAVTensor@at@@0AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@ZUrnn_relu_cudnn_stub@native@2@@?A0x5f01b69b@native@at@@QEAA@AEAU?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@_N_JN333@ZUrnn_relu_cudnn_stub@native@2@@23@P6AXAEAVTensor@3@1AEBV53@2V?$ArrayRef@VTensor@at@@@c10@@_N_JN444@Z@Z)
RNN.cpp.obj : error LNK2019: unresolved external symbol "__declspec(dllimport) public: void __cdecl at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::lstm_packed_cudnn_stub>::set_cuda_dispatch_ptr(void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool))" (__imp_?set_cuda_dispatch_ptr@?$DispatchStub@P6AXAEAVTensor@at@@00AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@2_N_JN33@ZUlstm_packed_cudnn_stub@native@2@@native@at@@QEAAXP6AXAEAVTensor@3@00AEBV43@1V?$ArrayRef@VTensor@at@@@c10@@2_N_JN33@Z@Z) referenced in function "public: __cdecl at::native::`anonymous namespace'::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::lstm_packed_cudnn_stub>::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::lstm_packed_cudnn_stub>(struct at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::lstm_packed_cudnn_stub> &,void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool))" (??0?$RegisterCUDADispatch@P6AXAEAVTensor@at@@00AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@2_N_JN33@ZUlstm_packed_cudnn_stub@native@2@@?A0x5f01b69b@native@at@@QEAA@AEAU?$DispatchStub@P6AXAEAVTensor@at@@00AEBV12@1V?$ArrayRef@VTensor@at@@@c10@@2_N_JN33@ZUlstm_packed_cudnn_stub@native@2@@23@P6AXAEAVTensor@3@11AEBV53@2V?$ArrayRef@VTensor@at@@@c10@@3_N_JN44@Z@Z)
RNN.cpp.obj : error LNK2019: unresolved external symbol "__declspec(dllimport) public: void __cdecl at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::gru_packed_cudnn_stub>::set_cuda_dispatch_ptr(void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool))" (__imp_?set_cuda_dispatch_ptr@?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@ZUgru_packed_cudnn_stub@native@2@@native@at@@QEAAXP6AXAEAVTensor@3@0AEBV43@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@Z@Z) referenced in function "public: __cdecl at::native::`anonymous namespace'::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::gru_packed_cudnn_stub>::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::gru_packed_cudnn_stub>(struct at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::gru_packed_cudnn_stub> &,void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool))" (??0?$RegisterCUDADispatch@P6AXAEAVTensor@at@@0AEBV12@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@ZUgru_packed_cudnn_stub@native@2@@?A0x5f01b69b@native@at@@QEAA@AEAU?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@ZUgru_packed_cudnn_stub@native@2@@23@P6AXAEAVTensor@3@1AEBV53@22V?$ArrayRef@VTensor@at@@@c10@@_N_JN44@Z@Z)
RNN.cpp.obj : error LNK2019: unresolved external symbol "__declspec(dllimport) public: void __cdecl at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::rnn_tanh_packed_cudnn_stub>::set_cuda_dispatch_ptr(void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool))" (__imp_?set_cuda_dispatch_ptr@?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@ZUrnn_tanh_packed_cudnn_stub@native@2@@native@at@@QEAAXP6AXAEAVTensor@3@0AEBV43@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@Z@Z) referenced in function "public: __cdecl at::native::`anonymous namespace'::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::rnn_tanh_packed_cudnn_stub>::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::rnn_tanh_packed_cudnn_stub>(struct at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::rnn_tanh_packed_cudnn_stub> &,void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool))" (??0?$RegisterCUDADispatch@P6AXAEAVTensor@at@@0AEBV12@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@ZUrnn_tanh_packed_cudnn_stub@native@2@@?A0x5f01b69b@native@at@@QEAA@AEAU?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@ZUrnn_tanh_packed_cudnn_stub@native@2@@23@P6AXAEAVTensor@3@1AEBV53@22V?$ArrayRef@VTensor@at@@@c10@@_N_JN44@Z@Z)
RNN.cpp.obj : error LNK2019: unresolved external symbol "__declspec(dllimport) public: void __cdecl at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::rnn_relu_packed_cudnn_stub>::set_cuda_dispatch_ptr(void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool))" (__imp_?set_cuda_dispatch_ptr@?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@ZUrnn_relu_packed_cudnn_stub@native@2@@native@at@@QEAAXP6AXAEAVTensor@3@0AEBV43@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@Z@Z) referenced in function "public: __cdecl at::native::`anonymous namespace'::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::rnn_relu_packed_cudnn_stub>::RegisterCUDADispatch<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::rnn_relu_packed_cudnn_stub>(struct at::native::DispatchStub<void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool),struct at::native::rnn_relu_packed_cudnn_stub> &,void (__cdecl*)(class at::Tensor &,class at::Tensor &,class at::Tensor const &,class at::Tensor const &,class at::Tensor const &,class c10::ArrayRef<class at::Tensor>,bool,__int64,double,bool,bool))" (??0?$RegisterCUDADispatch@P6AXAEAVTensor@at@@0AEBV12@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@ZUrnn_relu_packed_cudnn_stub@native@2@@?A0x5f01b69b@native@at@@QEAA@AEAU?$DispatchStub@P6AXAEAVTensor@at@@0AEBV12@11V?$ArrayRef@VTensor@at@@@c10@@_N_JN33@ZUrnn_relu_packed_cudnn_stub@native@2@@23@P6AXAEAVTensor@3@1AEBV53@22V?$ArrayRef@VTensor@at@@@c10@@_N_JN44@Z@Z)
bin\torch_cuda.dll : fatal error LNK1120: 8 unresolved externals

CircleCI logs

This is true for all builds of this type, see workflow: https://app.circleci.com/pipelines/github/pytorch/pytorch/274955/workflows/a33e017e-0508-4367-b2b1-88704fd8525f

cc @malfet @seemethere @walterddr @ngimel @peterjc123 @maxluk @nbcsm @guyang3532 @gunandrose4u @mszhanyi @skyline75489

1 possible answer(s) on “Windows CXX11-ABI libtorch debug builds for CUDA 11.1 fail with unresolved external symbol