Wsrtjtyk

I'm need to send data from GL_TEXTURE_EXTERNAL_OES to simple GL_TEXTURE_2D (Render image from Android player to Unity texture) and currently do it through read pixels from buffer with attached source texture. This process work correctly on my OnePlus 5 phone, but have some glitches with image on phones like xiaomi note 4, mi a2 and etc (like image is very green), and also there is perfomance issues becouse of this process works every frame and than more pixels to read, than worser perfomance (even my phone has low fps at 4k resolution). Any idea how to optimize this process or do it in some other way?

Thanks and best regards!

GLuint FramebufferName;

glGenFramebuffers(1, &FramebufferName);

glBindFramebuffer(GL_FRAMEBUFFER, FramebufferName);



glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_EXTERNAL_OES, g_ExtTexturePointer, 0);



if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)

{

    LOGD("%s", "Error: Could not setup frame buffer.");

}



unsigned char* data = new unsigned char[g_SourceWidth * g_SourceHeight * 4];

glReadPixels(0, 0, g_SourceWidth, g_SourceHeight, GL_RGBA, GL_UNSIGNED_BYTE, data);



glBindTexture(GL_TEXTURE_2D, g_TexturePointer);

glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, g_SourceWidth, g_SourceHeight, 0, GL_RGBA, GL_UNSIGNED_BYTE, data);



glDeleteFramebuffers(1, &FramebufferName);

glBindFramebuffer(GL_FRAMEBUFFER, 0);

glBindTexture(GL_TEXTURE_2D, 0);



delete data;

UPDATE.
Function which contain this code and function which calls it from Unity side

static void UNITY_INTERFACE_API OnRenderEvent(int eventID) { ... }



extern "C" UnityRenderingEvent UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API UMDGetRenderEventFunc()

{

    return OnRenderEvent;

}

Which called from Unity Update function like this:

[DllImport("RenderingPlugin")]

static extern IntPtr UMDGetRenderEventFunc();



IEnumerator UpdateVideoTexture()

{

    while (true)

    {

        ...

        androidPlugin.UpdateSurfaceTexture();

        GL.IssuePluginEvent(UMDGetRenderEventFunc, 1);

    } 

}

And Android plugin do this on its side (surfaceTexture its texture which contain this external texture on which ExoPlayer render video)

public void exportUpdateSurfaceTexture() {

    synchronized (this) {

        if (this.mIsStopped) {

            return;

        }

        surfaceTexture.updateTexImage();

    }

}

On the C++ side:

You're creating and destroying pixel data every frame when you do new unsigned char[g_SourceWidth * g_SourceHeight * 4]; and delete data and that's expensive depending on the Texture size. Create the texture data once then re-use it.

One way to do this is to have static variables on the C++ side hold the texture information then a function to initialize those variables::

static void* pixelData = nullptr;

static int _x;

static int _y;

static int _width;

static int _height;



void initPixelData(void* buffer, int x, int y, int width, int height) {

    pixelData = buffer;

    _x = x;

    _y = y;

    _width = width;

    _height = height;

}

Then your capture function should be re-written to remove new unsigned char[g_SourceWidth * g_SourceHeight * 4]; and delete data but use the static variables.

static void UNITY_INTERFACE_API OnRenderEvent(int eventID)

{

    if (pixelData == nullptr) {

        //Debug::Log("Pointer is null", Color::Red);

        return;

    }



    GLuint FramebufferName;

    glGenFramebuffers(1, &FramebufferName);

    glBindFramebuffer(GL_FRAMEBUFFER, FramebufferName);



    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_EXTERNAL_OES, g_ExtTexturePointer, 0);



    if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)

    {

        LOGD("%s", "Error: Could not setup frame buffer.");

    }



    glReadPixels(_x, _y, _width, _height, GL_RGBA, GL_UNSIGNED_BYTE, pixelData);



    glBindTexture(GL_TEXTURE_2D, g_TexturePointer);

    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, _width, _height, 0, GL_RGBA, GL_UNSIGNED_BYTE, pixelData);



    glDeleteFramebuffers(1, &FramebufferName);

    glBindFramebuffer(GL_FRAMEBUFFER, 0);

    glBindTexture(GL_TEXTURE_2D, 0);

}



extern "C" UnityRenderingEvent UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API

UMDGetRenderEventFunc()

{

    return OnRenderEvent;

}

On the C# side:

[DllImport("RenderingPlugin", CallingConvention = CallingConvention.Cdecl)]

public static extern void initPixelData(IntPtr buffer, int x, int y, int width, int height);



[DllImport("RenderingPlugin", CallingConvention = CallingConvention.StdCall)]

private static extern IntPtr UMDGetRenderEventFunc();

Create the Texture information, pin it and send the pointer to C++:

int width = 500;

int height = 500;



//Where Pixel data will be saved

byte screenData;

//Where handle that pins the Pixel data will stay

GCHandle pinHandler;



//Used to test the color

public RawImage rawImageColor;

private Texture2D texture;



// Use this for initialization

void Awake()

{

    Resolution res = Screen.currentResolution;

    width = res.width;

    height = res.height;



    //Allocate array to be used

    screenData = new byte[width * height * 4];

    texture = new Texture2D(width, height, TextureFormat.RGBA32, false, false);



    //Pin the Array so that it doesn't move around

    pinHandler = GCHandle.Alloc(screenData, GCHandleType.Pinned);



    //Register the screenshot and pass the array that will receive the pixels

    IntPtr arrayPtr = pinHandler.AddrOfPinnedObject();



    initPixelData(arrayPtr, 0, 0, width, height);



    StartCoroutine(UpdateVideoTexture());

}

Then to update the texture, see the sample below. Note that there are two methods to update the texture as shown on the code below. If you run into issues with Method1, comment out the two lines which uses texture.LoadRawTextureData and texture.Apply and un-comment the Method2 code which uses the ByteArrayToColor, texture.SetPixels and texture.Apply function:

IEnumerator UpdateVideoTexture()

{

    while (true)

    {

        //Take screenshot of the screen

        GL.IssuePluginEvent(UMDGetRenderEventFunc(), 1);



        //Update Texture Method1

        texture.LoadRawTextureData(screenData);

        texture.Apply();



        //Update Texture Method2. Use this if the Method1 above crashes

        /*

        ByteArrayToColor();

        texture.SetPixels(colors);

        texture.Apply();

        */



        //Test it by assigning the texture to a raw image

        rawImageColor.texture = texture;



        //Wait for a frame

        yield return null;

    }

}



Color colors = null;



void ByteArrayToColor()

{

    if (colors == null)

    {

        colors = new Color[screenData.Length / 4];

    }



    for (int i = 0; i < screenData.Length; i += 4)

    {

        colors[i / 4] = new Color(screenData[i],

            screenData[i + 1],

            screenData[i + 2],

            screenData[i + 3]);

    }

}

Unpin the array when done or when the script is about to be destroyed:

void OnDisable()

{

    //Unpin the array when disabled

    pinHandler.Free();

}

Calling glReadPixels is always going to be slow; CPUs are not good at bulk data transfer.

Ideally you'd managed to convince Unity to accept an external image handle, and do the whole process zero copy, but failing that I would use a GPU render-to-texture and use a shader to transfer from the external image to the RGB surface.