This Question have no Text answers yet!

How to use multi-renderers of OpenGL ES 2.0 on android

By : THK
Source: Stackoverflow.com
Question!

I'm making live-filtering camera app on android. Although I'm not an experienced user of OpenGL ES 2.0, I could implement live-filtering camera preview due to many information from internet. Now, I'm trying to store the live-filtered image, but, "Fatal signal 11 (SIGSEGV) at 0xa53e7688 (code=1)" error was occurred.

This is my plan.

byte array from PictureCallback >> convert to Bitmap >> process by OpenGL ES Renderer >> save as file from returned Bitmap

And... Here is my code.

public class CameraRenderer implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {
    private static final String vertexShaderCode
            = "attribute vec4 vPosition;\n"
            + "attribute vec2 vTexCoord;\n"
            + "varying vec2 texCoord;\n"
            + "void main() {\n"
            + "   gl_Position = vPosition;\n"
            + "   texCoord = vTexCoord;\n"
            + "}";
    private static final String fragmentShaderCodePreview
            = "#extension GL_OES_EGL_image_external : require\n"
            + "precision mediump float;\n"
            + "uniform samplerExternalOES sTexture;\n"
            + "uniform float brightness;\n"
            + "uniform float contrast;\n"
            + "uniform float saturation;\n"
            + "varying vec2 texCoord;\n"
            + "void main() {\n"
            + "   vec4 tex = texture2D(sTexture, texCoord);\n"
            + "   vec3 pBrightness = clamp(tex.rgb + brightness, 0.0, 1.0);\n"
            + "   float scale = contrast + 1.0;\n"
            + "   float translate = -0.5 * scale + 0.5;\n"
            + "   vec3 pContrast = clamp(pBrightness.rgb * scale + translate, 0.0, 1.0);\n"
            + "   float sr = (1.0 - saturation) * 0.213;\n"
            + "   float sg = (1.0 - saturation) * 0.715;\n"
            + "   float sb = (1.0 - saturation) * 0.072;\n"
            + "   float psr = clamp(pContrast.r * (sr + saturation) + pContrast.g * sg + pContrast.b * sb, 0.0, 1.0);\n"
            + "   float psg = clamp(pContrast.r * sr + pContrast.g * (sg + saturation) + pContrast.b * sb, 0.0, 1.0);\n"
            + "   float psb = clamp(pContrast.r * sr + pContrast.g * sg + pContrast.b * (sb + saturation), 0.0, 1.0);\n"
            + "   gl_FragColor = vec4(psr, psg, psb, tex.a);\n"
            + "}";
    private static final String fragmentShaderCodeImage
            = "precision mediump float;\n"
            + "uniform sampler2D sTexture;\n"
            + "uniform float brightness;\n"
            + "uniform float contrast;\n"
            + "uniform float saturation;\n"
            + "varying vec2 texCoord;\n"
            + "void main() {\n"
            + "   vec4 tex = texture2D(sTexture, texCoord);\n"
            + "   vec3 pBrightness = clamp(tex.rgb + brightness, 0.0, 1.0);\n"
            + "   float scale = contrast + 1.0;\n"
            + "   float translate = -0.5 * scale + 0.5;\n"
            + "   vec3 pContrast = clamp(pBrightness.rgb * scale + translate, 0.0, 1.0);\n"
            + "   float sr = (1.0 - saturation) * 0.213;\n"
            + "   float sg = (1.0 - saturation) * 0.715;\n"
            + "   float sb = (1.0 - saturation) * 0.072;\n"
            + "   float psr = clamp(pContrast.r * (sr + saturation) + pContrast.g * sg + pContrast.b * sb, 0.0, 1.0);\n"
            + "   float psg = clamp(pContrast.r * sr + pContrast.g * (sg + saturation) + pContrast.b * sb, 0.0, 1.0);\n"
            + "   float psb = clamp(pContrast.r * sr + pContrast.g * sg + pContrast.b * (sb + saturation), 0.0, 1.0);\n"
            + "   gl_FragColor = vec4(psr, psg, psb, tex.a);\n"
            + "}";
    private static float[] mSquareCoords = {
            1.0f, 1.0f, 0.0f,
            -1.0f, 1.0f, 0.0f,
            -1.0f, -1.0f, 0.0f,
            1.0f, -1.0f, 0.0f
    };
    private static float[] mTextureCoordsR = {
            0.0f, 0.0f,
            0.0f, 1.0f,
            1.0f, 1.0f,
            1.0f, 0.0f
    };
    private static float[] mTextureCoordsF = {
            1.0f, 0.0f,
            1.0f, 1.0f,
            0.0f, 1.0f,
            0.0f, 0.0f
    };
    private static short[] mDrawOrder = {0, 1, 2, 0, 2, 3};

    private FloatBuffer mVertexBuffer;
    private FloatBuffer mTexCoordBuffer;
    private ShortBuffer mDrawListBuffer;
    private SurfaceTexture mSTexture;
    private static int mProgramPreview, mProgramImage;
    private static int[] mTextureNames;
    private boolean mUpdateST = false;

    ......

    CameraRenderer(CameraView view, CameraParams cameraParams, CameraActivity.CameraHandler handler) {
        ......

        mVertexBuffer = ByteBuffer.allocateDirect(mSquareCoords.length*4).order(ByteOrder.nativeOrder()).asFloatBuffer();
        mVertexBuffer.put(mSquareCoords).position(0);

        if (mCameraParams.mCameraMode) {
            mTexCoordBuffer = ByteBuffer.allocateDirect(mTextureCoordsR.length*4).order(ByteOrder.nativeOrder()).asFloatBuffer();
            mTexCoordBuffer.put(mTextureCoordsR).position(0);
        }
        else {
            mTexCoordBuffer = ByteBuffer.allocateDirect(mTextureCoordsF.length*4).order(ByteOrder.nativeOrder()).asFloatBuffer();
            mTexCoordBuffer.put(mTextureCoordsF).position(0);
        }

        mDrawListBuffer = ByteBuffer.allocateDirect(mDrawOrder.length*2).order(ByteOrder.nativeOrder()).asShortBuffer();
        mDrawListBuffer.put(mDrawOrder).position(0);
    }

    @Override
    public void onSurfaceCreated(GL10 gl10, EGLConfig eglConfig) {
        if (mCameraParams.mLoadSuccess == false) initCameraParams();

        initTexture();
        mSTexture = new SurfaceTexture(mTextureNames[0]);
        mSTexture.setOnFrameAvailableListener(this);

        ......

        mCamera = Camera.open(mCameraParams.mCameraID);

        ......

        mCamera.setPreviewTexture(mSTexture);

        ......

        GLES20.glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
        loadShader();

        mPreviewThread = new PreviewThread();
        mPreviewThread.start();
    }

    @Override
    public void onSurfaceChanged(GL10 gl10, int width, int height) {
        GLES20.glViewport(0, 0, width, height);
    }

    @Override
    public void onFrameAvailable(SurfaceTexture surfaceTexture) {
        mUpdateST = true;
        if (mCameraView != null) mCameraView.requestRender();
    }

    @Override
    public void onDrawFrame(GL10 gl10) {
        synchronized (this) {
            GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);

            if (mUpdateST) {
                mSTexture.updateTexImage();
                mUpdateST = false;
            }

            GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
            GLES20.glUseProgram(mProgramPreview);

            int positionHandle = GLES20.glGetAttribLocation(mProgramPreview, "vPosition");
            int texCoordHandle = GLES20.glGetAttribLocation(mProgramPreview, "vTexCoord");
            int TextureHandle = GLES20.glGetUniformLocation(mProgramPreview, "sTexture");
            GLES20.glEnableVertexAttribArray(positionHandle);
            GLES20.glEnableVertexAttribArray(texCoordHandle);
            GLES20.glVertexAttribPointer(positionHandle, 3, GLES20.GL_FLOAT, false, 0, mVertexBuffer);
            GLES20.glVertexAttribPointer(texCoordHandle, 2, GLES20.GL_FLOAT, false, 0, mTexCoordBuffer);
            GLES20.glUniform1i(TextureHandle, 0);

            int bright = GLES20.glGetUniformLocation(mProgramPreview, "brightness");
            int contr = GLES20.glGetUniformLocation(mProgramPreview, "contrast");
            int satur = GLES20.glGetUniformLocation(mProgramPreview, "saturation");
            GLES20.glUniform1f(bright, mCameraParams.mBrightness / 200.0f);
            GLES20.glUniform1f(contr, mCameraParams.mContrast / 200.0f);
            GLES20.glUniform1f(satur, (mCameraParams.mSaturation + 100) / 100.0f);    
            GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
            GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureNames[0]);
            GLES20.glDrawElements(GLES20.GL_TRIANGLES, mDrawOrder.length, GLES20.GL_UNSIGNED_SHORT, mDrawListBuffer);

            GLES20.glDisableVertexAttribArray(positionHandle);
            GLES20.glDisableVertexAttribArray(texCoordHandle);
        }
    }

    public void initTexture() {
        mTextureNames = new int[2];
        GLES20.glGenTextures(mTextureNames.length, mTextureNames, 0);
        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureNames[0]);
        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
        GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureNames[1]);
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST);
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
    }

    public void onSurfaceDestroyed() {
        ......
    }

    public static void loadShader() {
        int vShader = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);
        GLES20.glShaderSource(vShader, vertexShaderCode);
        GLES20.glCompileShader(vShader);

        int[] compiled = new int[1];
        GLES20.glGetShaderiv(vShader, GLES20.GL_COMPILE_STATUS, compiled, 0);

        if (compiled[0] == 0) {
            Log.e("CheckLog", "Could not compile vshader");
            Log.v("CheckLog", "Could not compile vshader:" + GLES20.glGetShaderInfoLog(vShader));
            GLES20.glDeleteShader(vShader);
            vShader = 0;
        }

        int fShaderPreview = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
        GLES20.glShaderSource(fShaderPreview, fragmentShaderCodePreview);
        GLES20.glCompileShader(fShaderPreview);
        GLES20.glGetShaderiv(fShaderPreview, GLES20.GL_COMPILE_STATUS, compiled, 0);
        if (compiled[0] == 0) {
            Log.e("CheckLog", "Could not compile fshader");
            Log.v("CheckLog", "Could not compile fshader:" + GLES20.glGetShaderInfoLog(fShaderPreview));
            GLES20.glDeleteShader(fShaderPreview);
            fShaderPreview = 0;
        }

        mProgramPreview = GLES20.glCreateProgram();
        GLES20.glAttachShader(mProgramPreview, vShader);
        GLES20.glAttachShader(mProgramPreview, fShaderPreview);
        GLES20.glLinkProgram(mProgramPreview);

        int fShaderImage = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
        GLES20.glShaderSource(fShaderImage, fragmentShaderCodeImage);
        GLES20.glCompileShader(fShaderImage);
        GLES20.glGetShaderiv(fShaderImage, GLES20.GL_COMPILE_STATUS, compiled, 0);
        if (compiled[0] == 0) {
            Log.e("CheckLog", "Could not compile fshader");
            Log.v("CheckLog", "Could not compile fshader:" + GLES20.glGetShaderInfoLog(fShaderImage));
            GLES20.glDeleteShader(fShaderImage);
            fShaderImage = 0;
        }

        mProgramImage = GLES20.glCreateProgram();
        GLES20.glAttachShader(mProgramImage, vShader);
        GLES20.glAttachShader(mProgramImage, fShaderImage);
        GLES20.glLinkProgram(mProgramImage);
    }

    public static void deleteTextures() {
        if (mTextureNames != null) {
            GLES20.glDeleteTextures(mTextureNames.length, mTextureNames, 0);
            mTextureNames = null;
        }
    }

    public Bitmap renderBitmap(Bitmap preBitmap) {
        synchronized (this) {
            GLES20.glEnable(GLES20.GL_TEXTURE_2D);
            GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
            GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, preBitmap, 0);
            preBitmap.recycle();

            GLES20.glUseProgram(mProgramImage);

            int positionHandle = GLES20.glGetAttribLocation(mProgramImage, "vPosition");
            int texCoordHandle = GLES20.glGetAttribLocation(mProgramImage, "vTexCoord");
            GLES20.glEnableVertexAttribArray(positionHandle);
            GLES20.glEnableVertexAttribArray(texCoordHandle);
            GLES20.glVertexAttribPointer(positionHandle, 3, GLES20.GL_FLOAT, false, 0, mVertexBuffer);
            GLES20.glVertexAttribPointer(texCoordHandle, 2, GLES20.GL_FLOAT, false, 0, mTexCoordBuffer);

            int bright = GLES20.glGetUniformLocation(mProgramImage, "brightness");
            int contr = GLES20.glGetUniformLocation(mProgramImage, "contrast");
            int satur = GLES20.glGetUniformLocation(mProgramImage, "saturation");
            GLES20.glUniform1f(bright, mCameraParams.mBrightness / 200.0f);
            GLES20.glUniform1f(contr, mCameraParams.mContrast / 200.0f);
            GLES20.glUniform1f(satur, (mCameraParams.mSaturation + 100) / 100.0f);

            GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
            GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureNames[1]);
            GLES20.glDrawElements(GLES20.GL_TRIANGLES, mDrawOrder.length, GLES20.GL_UNSIGNED_SHORT, mDrawListBuffer);

            GLES20.glDisableVertexAttribArray(positionHandle);
            GLES20.glDisableVertexAttribArray(texCoordHandle);
            GLES20.glDisable(GLES20.GL_TEXTURE_2D);

            IntBuffer intBuffer = IntBuffer.allocate(preBitmap.getWidth() * preBitmap.getHeight());
            GLES20.glReadPixels(0, 0, preBitmap.getWidth(), preBitmap.getHeight(), GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, intBuffer);

            Bitmap postBitmap = Bitmap.createBitmap(preBitmap.getWidth(), preBitmap.getHeight(), Bitmap.Config.ARGB_8888);
            postBitmap.copyPixelsFromBuffer(intBuffer);

            return postBitmap;
        }
    }

    ......
}

As you can check it in this code, I tried to make two OpenGL programs and generate two OpenGL textures for multi-renderer. However when glDrawElements method in renderBitmap method was called, the error was occurred as I told. I don't know why, because I'm not good at OpenGL programming. Please help me. :-(

By : THK


Video about How to use multi-renderers of OpenGL ES 2.0 on android