How to stop cuda register resource from removing with host-device mapping of an OpenGL shader storage buffer?

I'm trying to take an OpenGL storage buffer, load data into it from the CPU (persistently mapping it), taking that data and processing it with cuda, then using that same data again in further OpenGL processing:

#include <glad/gl.h> #include <GLFW/glfw3.h> #include <cuda_gl_interop.h> #include <fmt/format.h> #include <vector> #include <iostream> #define TEST_RETURN_NAME_CASE(e) case e: return #e std::string debugTypeToString(GLenum type){ switch(type){ TEST_RETURN_NAME_CASE(GL_DEBUG_TYPE_ERROR); TEST_RETURN_NAME_CASE(GL_DEBUG_TYPE_DEPRECATED_BEHAVIOR); TEST_RETURN_NAME_CASE(GL_DEBUG_TYPE_UNDEFINED_BEHAVIOR); TEST_RETURN_NAME_CASE(GL_DEBUG_TYPE_PORTABILITY); TEST_RETURN_NAME_CASE(GL_DEBUG_TYPE_PERFORMANCE); TEST_RETURN_NAME_CASE(GL_DEBUG_TYPE_MARKER); TEST_RETURN_NAME_CASE(GL_DEBUG_TYPE_PUSH_GROUP); TEST_RETURN_NAME_CASE(GL_DEBUG_TYPE_POP_GROUP); TEST_RETURN_NAME_CASE(GL_DEBUG_TYPE_OTHER); default: return "UNKNOWN ERROR TYPE"; } } std::string debugSourceToString(GLenum source){ switch(source){ TEST_RETURN_NAME_CASE(GL_DEBUG_SOURCE_API); TEST_RETURN_NAME_CASE(GL_DEBUG_SOURCE_WINDOW_SYSTEM); TEST_RETURN_NAME_CASE(GL_DEBUG_SOURCE_SHADER_COMPILER); TEST_RETURN_NAME_CASE(GL_DEBUG_SOURCE_THIRD_PARTY); TEST_RETURN_NAME_CASE(GL_DEBUG_SOURCE_APPLICATION); TEST_RETURN_NAME_CASE(GL_DEBUG_SOURCE_OTHER); default: return "UNKNOWN SOURCE TYPE"; } } std::string debugSeverityToString(GLenum severity){ switch(severity){ TEST_RETURN_NAME_CASE(GL_DEBUG_SEVERITY_HIGH); TEST_RETURN_NAME_CASE(GL_DEBUG_SEVERITY_MEDIUM); TEST_RETURN_NAME_CASE(GL_DEBUG_SEVERITY_LOW); TEST_RETURN_NAME_CASE(GL_DEBUG_SEVERITY_NOTIFICATION); default: return "UNKNOWN SEVERITY TYPE"; } } void debug_message_callback(GLenum source, GLenum type, GLuint id, GLenum severity, GLsizei length, const GLchar *message, const void *userParam){ std::cerr << "GL DEBUG CALLBACK: " << "\n\tsource: " << debugSourceToString(source) << "\n\ttype: " << debugTypeToString(type) << "\n\tid: " << id << "\n\tseverity: " << debugSeverityToString(severity) << "\n\tlength: " << length << "\n\tmessage: " << message << "\n"; if (type == GL_DEBUG_TYPE_ERROR) { std::terminate(); } } static const char* glsl_version = "#version 460"; int main(){ glfwSetErrorCallback([](int error_code, const char* description) { std::cout << "GLFW Error " << error_code << ": " << description << "\n"; }); int inited = glfwInit(); if (!inited) { std::cout << "Couldn't init glfw" << std::endl; return 1; } glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4); glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 6); glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE); auto window = glfwCreateWindow(100, 100, "test", nullptr, nullptr); glfwMakeContextCurrent(window); int version = gladLoadGL(glfwGetProcAddress); fmt::print("Glad Version GL {}.{}\n", GLAD_VERSION_MAJOR(version), GLAD_VERSION_MINOR(version)); if (!version) { throw std::runtime_error("Failed to initialize GLAD"); } bool enable_debug_level_notification = GL_FALSE; bool enable_performance_level_notification = GL_FALSE; glEnable(GL_DEBUG_OUTPUT); glDebugMessageControl(GL_DONT_CARE, GL_DONT_CARE, GL_DEBUG_SEVERITY_NOTIFICATION, 0, nullptr, enable_debug_level_notification); glDebugMessageControl(GL_DONT_CARE, GL_DEBUG_TYPE_PERFORMANCE, GL_DONT_CARE, 0, nullptr, enable_performance_level_notification); glDebugMessageCallback(debug_message_callback, nullptr); auto data_size = 512*512; std::vector<std::uint16_t> data(data_size, 100); auto data_size_bytes = data_size * sizeof(std::uint16_t); GLuint buffer_handle; glGenBuffers(1, &buffer_handle); glBindBuffer(GL_SHADER_STORAGE_BUFFER, buffer_handle); glBufferStorage(GL_SHADER_STORAGE_BUFFER, data_size_bytes, data.data(), GL_MAP_PERSISTENT_BIT | GL_MAP_WRITE_BIT); auto mapped_ptr = glMapBufferRange(GL_SHADER_STORAGE_BUFFER, 0, data_size_bytes, GL_MAP_PERSISTENT_BIT | GL_MAP_WRITE_BIT | GL_MAP_FLUSH_EXPLICIT_BIT); glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0); //If this is done before mapping the buffer, also get errors. cudaGraphicsResource_t cuda_gl_resource = nullptr; if (auto result = cudaGraphicsGLRegisterBuffer(&cuda_gl_resource, buffer_handle, cudaGraphicsRegisterFlagsReadOnly); result != cudaSuccess) { throw std::runtime_error(fmt::format("cudaGraphicsGLRegisterBuffer error:\n\t{}", cudaGetErrorString(result))); } std::memcpy(mapped_ptr, data.data(), data_size_bytes); { void* device_data = nullptr; if (auto result = cudaGraphicsResourceSetMapFlags(cuda_gl_resource, cudaGraphicsMapFlagsReadOnly); result != cudaSuccess) { throw std::runtime_error(fmt::format("cudaGraphicsResourceSetMapFlags error:\n\t{}", cudaGetErrorString(result))); } if (auto result = cudaGraphicsMapResources(1, &cuda_gl_resource, 0); result != cudaSuccess) { throw std::runtime_error(fmt::format("cudaGraphicsMapResources error:\n\t{}", cudaGetErrorString(result))); } if (auto result = cudaGraphicsResourceGetMappedPointer(&device_data, &data_size_bytes, cuda_gl_resource); result != cudaSuccess) { throw std::runtime_error(fmt::format("cudaGraphicsResourceGetMappedPointer error:\n\t{}", cudaGetErrorString(result))); } //normally would do something with device data, but error is reproducable with out doing anything if (auto result = cudaGraphicsUnmapResources(1, &cuda_gl_resource, 0); result != cudaSuccess) { throw std::runtime_error(fmt::format("cudaGraphicsUnmapResources error:\n\t{}", cudaGetErrorString(result))); } } auto result = cudaGraphicsUnregisterResource(cuda_gl_resource); if (result != cudaSuccess) { throw std::runtime_error( fmt::format("cudaGraphicsGLRegisterBuffer error:\n\t{}", cudaGetErrorString(result)) ); } glBindBuffer(GL_SHADER_STORAGE_BUFFER, buffer_handle); glUnmapBuffer(GL_SHADER_STORAGE_BUFFER); glDeleteBuffers(1, &buffer_handle); glfwDestroyWindow(window); glfwTerminate(); return 0; }

I've run into four issues with trying to use cuda interop while using this object as a mapped resource:

I can't explicitly flush with out running into errors.

I can't map the buffer (opengl mapping) with out running into errors unless I register the resource after the mapping.

I can't un-map the opengl buffer with out running into errors, even if I unregister the resource first.

The error I get (with virtually all the issues I stated) is:

GL DEBUG CALLBACK: source: GL_DEBUG_SOURCE_API type: GL_DEBUG_TYPE_ERROR id: 1282 severity: GL_DEBUG_SEVERITY_HIGH length: 79 message: GL_INVALID_OPERATION error generated. Buffer is unbound or is already unmapped.

Note the only mildly relevant post on SO that I can find is :

CUDA OpenGL interop, resource mapping messes up buffer

Which is not relevant because all the flags I use are read-only from the CUDA side, and the actual issue I have is different.