Copy framebuffer of 3D View into custom frame buffer

Question

General question: Is there a reliable way to directly access the OpenGL framebuffer of a View3D for copying?

(Please see the end of the post for more detailed questions)

I need a very fast way to copy the pixel data of a SpaceView3D into an OpenGL texture. The blender internal GPUOffscreen.draw_view3d functionality unfortunately takes 10 to 50 ms per call, which is too slow as can be seen in this short example:

import bpy
import bgl
import gpu
import time

WIDTH = 512
HEIGHT =512

offscreen = gpu.types.GPUOffScreen(WIDTH, HEIGHT)

def draw():
    context = bpy.context
    scene = context.scene

    view_matrix = scene.camera.matrix_world.inverted()

    projection_matrix = scene.camera.calc_matrix_camera(
        context.evaluated_depsgraph_get(), x=WIDTH, y=HEIGHT)

    start_time = time.time()
    
    offscreen.draw_view3d(    # this takes 10 to 50 ms (on a Macbook Pro 2016)
        scene,
        context.view_layer,
        context.space_data,
        context.region,
        view_matrix,
        projection_matrix)

    print("draw_view3d call took ", round((time.time() - start_time) * 1000, 2), " ms")


bpy.types.SpaceView3D.draw_handler_add(draw, (), 'WINDOW', 'POST_PIXEL')

What I achieved so far

The following code accesses the currently bound OpenGL draw buffer (Update note: see first comment below) and reads its pixel to a bgl.Buffer object:

import bpy
import bgl

# Draw function which copies data from the 3D View
def draw(self, context):

    if self.modal_redraw == True:

        # get currently bound framebuffer
        bgl.glGetIntegerv(bgl.GL_DRAW_FRAMEBUFFER_BINDING, self.framebuffer)
        #print("Framebuffer: ", framebuffer)

        # get information on current viewport 
        bgl.glGetIntegerv(bgl.GL_VIEWPORT, self.viewport_info)
        #print("Viewport info: ", viewport_info)

        self.width = self.viewport_info[2]
        self.height = self.viewport_info[3]

        # Write copied data to image
        ######################################################
        # resize image obect to fit the current 3D View size
        self.framebuffer_image.scale(self.width, self.height)
        self.pixelBuffer = bgl.Buffer(bgl.GL_FLOAT, self.width * self.height * 4)

        # obtain pixels from the framebuffer
        bgl.glBindBuffer(bgl.GL_FRAMEBUFFER, self.framebuffer[0]) # POST EDIT 08th Aug 2020: This functions fails for some reason, so it doesn't play a role - but why ... ?
        bgl.glReadPixels(0, 0, self.width, self.height, bgl.GL_RGBA, bgl.GL_FLOAT, self.pixelBuffer)

        # write all pixels into the blender image
        self.framebuffer_image.pixels.foreach_set(self.pixelBuffer)

        # reset draw variable:
        # This is here to prevent excessive redrawing
        self.modal_redraw = False


# Modal operator for controlled redrawing of the image object
# NOTE: This code is only for a more conveniant testing of the draw function
#       If you want to stop the test, press 'ESC'

class ModalFramebufferCopy(bpy.types.Operator):
    bl_idname = "view3d.modal_framebuffer_copy"
    bl_label = "Draw 3D View Framebuffer"

    def __init__(self):
        print("Start example code")

        # init variables
        self.width = 32
        self.height = 32
        self.modal_redraw = False
        self.image_name = "framebuffer_copy"
        self.framebuffer = bgl.Buffer(bgl.GL_INT, 1)
        self.viewport_info = bgl.Buffer(bgl.GL_INT, 4)
        self.pixelBuffer = bgl.Buffer(bgl.GL_INT, self.width * self.height * 4)

        # create or update image object to which the framebuffer
        # data will be copied
        if not self.image_name in bpy.data.images:
            self.framebuffer_image = bpy.data.images.new(self.image_name , 32, 32)
        else:
            self.framebuffer_image = bpy.data.images[self.image_name ]


    # 
    def __del__(self):
        print("End example code")


    # modal operator for controlled redraw of the image
    def modal(self, context, event):
        # stop the execution of this example code if 'ESC' is pressed
        if event.type in {'ESC'}:
            bpy.types.SpaceView3D.draw_handler_remove(self._handle_3d, 'WINDOW')
            print("Removing draw handler")
            return {'CANCELLED'}

        else:

            # set draw variable to update:
            # This is here to prevent excessive redrawing
            self.modal_redraw = True

        return {'PASS_THROUGH'}
        #return {'RUNNING_MODAL'}


    def invoke(self, context, event):
        print("Invoking modal operator")

        # Add the region OpenGL drawing callback
        # draw in view space with 'POST_VIEW' and 'PRE_VIEW'
        self._handle_3d = bpy.types.SpaceView3D.draw_handler_add(draw, (self, context), 'WINDOW', 'PRE_VIEW') # this draws the viewport objects alone (without grid)
        #bpy.types.SpaceView3D.draw_handler_add(draw, (self, context), 'WINDOW', 'POST_VIEW') # this draws the grid alone (without objects)

        context.window_manager.modal_handler_add(self)
        return {'RUNNING_MODAL'}


def register():
    bpy.utils.register_class(ModalFramebufferCopy)


def unregister():
    bpy.utils.unregister_class(ModalFramebufferCopy)


if __name__ == "__main__":
    register()

    # Invoke modal operator for the example code
    bpy.ops.view3d.modal_framebuffer_copy('INVOKE_DEFAULT')

This basically works, but neither the 'PRE_VIEW' nor the 'POST_VIEW' flag of the draw_handler_add() give the complete result. The following screenshot shows the copied image (upper left area) and the viewport (lower left area) it was copied from via the code above.

For clarification

In the end, I don't want to copy the data into a bpy.types.image object. Instead, I want to copy several SpaceView3D frame buffers into one texture and draw it into another framebuffer for applying another shader. The way my code above approaches this is only, because the blender internal approach GPUOffscreen.draw_view3d is too slow.

My open questions

1. Is there an alternative to GPUOffscreen.draw_view3d that is faster (i.e., < 1 ms)?
2. Is there a more reliable (maybe blender internal) way to access the 3D View framebuffer?
3. How can I access the final framebuffer drawn and used by blender (the one where all controls, etc. are implemented)?
4. The framebuffer I currently access seems to be the object pixel data before any "Color management" settings are applied. Therefore my copied image is much darker than the original 3D View (see attached image). If nobody has an answer to question 3, maybe someone can hint me at how to transform the copied framebuffer data to the correct color space?

Answer 1

There is currently no alternative to GPUOffscreen.draw_view3d. For security reasons the internal frame buffer isn't accessible via python. Scripts/Add-ons could change the internal behavior without the user knowing what happens. When using stereo views there are multiple framebuffers to be aware off. Currently if you want access to the framebuffer you need to make changes directly in the draw manager.

The framebuffer of the 3d viewport has no color management applied. Only during drawing to the screen the color management is applied. External render engine can use the bind_display_space_shader to use the color management shader.

Answer 2

Due to BGL deprecation (https://developer.blender.org/T80730), it's safer to use the new pyGPU API whenever possible. I slightly adapted the code provided by @reg.cs and tested it on blender 3.0. It seems that there is no obvious color space issue, if the image format is set to FLOAT.

import bpy
import gpu

# Draw function which copies data from the 3D View
def draw(self, context):

    if self.modal_redraw == True:

        # get currently bound framebuffer
        self.framebuffer = gpu.state.active_framebuffer_get()

        # get information on current viewport 
        self.viewport_info = gpu.state.viewport_get()
        self.width = self.viewport_info[2]
        self.height = self.viewport_info[3]

        # Write copied data to image
        ######################################################
        # resize image obect to fit the current 3D View size
        self.framebuffer_image.scale(self.width, self.height)
        
        # obtain pixels from the framebuffer
        self.pixelBuffer = self.framebuffer.read_color(0, 0, self.width, self.height, 4, 0, 'FLOAT')
        
        # write all pixels into the blender image
        self.pixelBuffer.dimensions = self.width * self.height * 4
        self.framebuffer_image.pixels.foreach_set(self.pixelBuffer)

        # reset draw variable:
        # This is here to prevent excessive redrawing
        self.modal_redraw = False


# Modal operator for controlled redrawing of the image object
# NOTE: This code is only for a more conveniant testing of the draw function
#       If you want to stop the test, press 'ESC'

class ModalFramebufferCopy(bpy.types.Operator):
    bl_idname = "view3d.modal_framebuffer_copy"
    bl_label = "Draw 3D View Framebuffer"

    def __init__(self):
        print("Start example code")

        # init variables
        self.width = 32
        self.height = 32
        self.modal_redraw = False
        self.image_name = "color_buffer_copy"
        self.framebuffer = None
        self.viewport_info = None
        self.pixelBuffer = None

        # create or update image object to which the framebuffer
        # data will be copied
        if not self.image_name in bpy.data.images:
            self.framebuffer_image = bpy.data.images.new(self.image_name , 32, 32, float_buffer=True)
        else:
            self.framebuffer_image = bpy.data.images[self.image_name ]


    # 
    def __del__(self):
        print("End example code")


    # modal operator for controlled redraw of the image
    def modal(self, context, event):
        # stop the execution of this example code if 'ESC' is pressed
        if event.type in {'ESC'}:
            bpy.types.SpaceView3D.draw_handler_remove(self._handle_3d, 'WINDOW')
            print("Removing draw handler")
            return {'CANCELLED'}

        else:

            # set draw variable to update:
            # This is here to prevent excessive redrawing
            self.modal_redraw = True

        return {'PASS_THROUGH'}
        #return {'RUNNING_MODAL'}


    def invoke(self, context, event):
        print("Invoking modal operator")

        # Add the region OpenGL drawing callback
        # draw in view space with 'POST_VIEW' and 'PRE_VIEW'
        self._handle_3d = bpy.types.SpaceView3D.draw_handler_add(draw, (self, context), 'WINDOW', 'PRE_VIEW') # this draws the viewport objects alone (without grid)
        # self._handle_3d = bpy.types.SpaceView3D.draw_handler_add(draw, (self, context), 'WINDOW', 'POST_VIEW') # this draws the grid alone (without objects)

        context.window_manager.modal_handler_add(self)
        return {'RUNNING_MODAL'}


def register():
    bpy.utils.register_class(ModalFramebufferCopy)


def unregister():
    bpy.utils.unregister_class(ModalFramebufferCopy)


if __name__ == "__main__":
    register()

    # Invoke modal operator for the example code
    bpy.ops.view3d.modal_framebuffer_copy('INVOKE_DEFAULT')

Here is the result.