Libgdx masking with image

Question

I have a framebuffer, where some shapes are drawn using ShapeRenderer. And now I want to mask this framebuffer with mask from image. Before that I got it working with simple circle mask drawn by ShapeRenderer. But I need to use more complex mask so I have to use an image. Mask is a png with black mask and transparent background. Here's my code:

   @Override
   public void draw(Batch batch, float parentAlpha) {

      //disable RGB color, only enable ALPHA to the frame buffer
      Gdx.gl.glColorMask(false, false, false, true);

      //change the blending function for our alpha map
      batch.setBlendFunction(GL20.GL_ZERO, GL20.GL_SRC_ALPHA);

      //draw alpha mask sprite(s)
      batch.draw(maskTexture, MASK_OFFSET_X + getX(), MASK_OFFSET_Y + getY());   

      //flush the batch to the GPU
      batch.flush();

      Gdx.gl.glColorMask(true, true, true, true);
      batch.setBlendFunction(GL20.GL_DST_ALPHA, GL20.GL_ONE_MINUS_DST_ALPHA);

      //The scissor test is optional, but it depends 
      Gdx.gl.glEnable(GL20.GL_SCISSOR_TEST);
      Gdx.gl.glScissor(MASK_OFFSET_X + (int) getX(), MASK_OFFSET_Y + (int) getY(), maskTexture.getWidth(), maskTexture.getHeight());

      //draw framebuffer to be masked
      batch.draw(frm, getX(), getY(), frmSizeX, frmSizeY);

      //remember to flush before changing GL states again
      batch.flush();

      //disable scissor before continuing
      Gdx.gl.glDisable(GL20.GL_SCISSOR_TEST);

      //set default blend function
      batch.setBlendFunction(GL20.GL_SRC_ALPHA, GL20.GL_ONE_MINUS_SRC_ALPHA);
   }

My image is masked indeed, but there's a black background from mask image (it should be transparent). It's looking like this now:

And it should look for example like this (except this example is without mask so ofc paint shoudn't go outside head):

Also take a note that paint is half-transparent. (I don't know if it will change some code).

Ofc I'm using RGBA8888 format, here's initialization code:

frmBuff = new FrameBuffer(Format.RGBA8888, frmSizeX, frmSizeY, false);
frm = new TextureRegion(frmBuff.getColorBufferTexture());
frmCam = new OrthographicCamera(frmSizeX, frmSizeY);
frmCam.translate(frmSizeX / 2, frmSizeY / 2);

maskTexture = game.manager.get("my_mask.png", Texture.class);
maskTexture.setFilter(TextureFilter.Linear, TextureFilter.Linear);

I was messing around setBlendFunction and achieved very different results, but none of them was actually right.

How can I fix this?

Btw my code is based on this example: https://gist.github.com/mattdesl/6076846

I've also already read this: https://github.com/mattdesl/lwjgl-basics/wiki/LibGDX-Masking

Answer 1

I know this is an old post. But all my research on this issue only lead me here. I read the post on https://gist.github.com/mattdesl/6076846 and realized that the issue is that you are only taking into consideration the alpha of the mask, but really you want the alpha of the product of the mask and the foreground (the purple paint).

//disable RGB color, only enable ALPHA to the frame buffer
Gdx.gl.glColorMask(false, false, false, true);

//change the blending function for our alpha map
batch.setBlendFunction(GL20.GL_ZERO, GL20.GL_SRC_ALPHA);

//draw alpha mask sprite(s)
batch.draw(maskTexture, MASK_OFFSET_X + getX(), MASK_OFFSET_Y + getY());

//change the function for our source
batch.setBlendFunction(GL20.GL_ONE_MINUS_SRC_ALPHA, GL20.GL_SRC_ALPHA);

//draw the source alpha
river_sprite.draw(batch);

//flush the batch to the GPU
batch.flush();

Answer 2

I finally did it using shaders.

My code is based on this tutorial: https://github.com/mattdesl/lwjgl-basics/wiki/ShaderLesson4

private SpriteBatch spriteBatch;
private FrameBuffer frmBuff;
private TextureRegion frm;
private OrthographicCamera frmCam;
private FrameBuffer maskBuff;
private TextureRegion msk;

int frmSizeX = 500;
int frmSizeY = 700;

private ShapeRenderer renderer;

private ShaderProgram shader;

public static final int MASK_OFFSET_X = 55 - (int) Constants.BACKGROUND_OFFSET_X - 8;
public static final int MASK_OFFSET_Y = 150;

Texture tex0;
Texture mask;
SpriteBatch myBatch;

final String VERT =  
        "attribute vec4 "+ShaderProgram.POSITION_ATTRIBUTE+";\n" +
        "attribute vec4 "+ShaderProgram.COLOR_ATTRIBUTE+";\n" +
        "attribute vec2 "+ShaderProgram.TEXCOORD_ATTRIBUTE+"0;\n" +

        "uniform mat4 u_projTrans;\n" + 
        " \n" + 
        "varying vec4 vColor;\n" +
        "varying vec2 vTexCoord;\n" +

        "void main() {\n" +  
        "   vColor = "+ShaderProgram.COLOR_ATTRIBUTE+";\n" +
        "   vTexCoord = "+ShaderProgram.TEXCOORD_ATTRIBUTE+"0;\n" +
        "   gl_Position =  u_projTrans * " + ShaderProgram.POSITION_ATTRIBUTE + ";\n" +
        "}";

final String FRAG = 
        //GL ES specific stuff
          "#ifdef GL_ES\n" //
        + "#define LOWP lowp\n" //
        + "precision mediump float;\n" //
        + "#else\n" //
        + "#define LOWP \n" //
        + "#endif\n" + //
        "varying LOWP vec4 vColor;\n" +
        "varying vec2 vTexCoord;\n" + 
        "uniform sampler2D u_texture;\n" +  
        "uniform sampler2D u_texture1;\n" +
        "uniform sampler2D u_mask;\n" + 
        "void main(void) {\n" + 
        "   //sample the colour from the first texture\n" + 
        "   vec4 texColor0 = texture2D(u_texture, vTexCoord);\n" + 
        "\n" + 
        "   //sample the colour from the second texture\n" + 
        "   vec4 texColor1 = texture2D(u_texture1, vTexCoord);\n" + 
        "\n" + 
        "   //get the mask; we will only use the alpha channel\n" + 
        "   float mask = texture2D(u_mask, vTexCoord).a;\n" + 
        "\n" + 
        "   //interpolate the colours based on the mask\n" + 
        "   gl_FragColor = vColor * mix(texColor0, texColor1, mask);\n" + 
        "}";

public SprayRenderer(ShapeRenderer renderer) {
        super();
        this.renderer = renderer;

        ShaderProgram.pedantic = false;

        spriteBatch = new SpriteBatch();

        tex0 = new Texture(Gdx.files.internal("snowman_back.png"));
        mask = new Texture(Gdx.files.internal("balwan_maska.png"));

        maskBuff = new FrameBuffer(Format.RGBA8888, frmSizeX, frmSizeY, false);
        msk = new TextureRegion(maskBuff.getColorBufferTexture());

        frmBuff = new FrameBuffer(Format.RGBA8888, frmSizeX, frmSizeY, false);
        frm = new TextureRegion(frmBuff.getColorBufferTexture());

        frmCam = new OrthographicCamera(frmSizeX, frmSizeY);
        frmCam.translate(frmSizeX / 2, frmSizeY / 2);


        frmCam.update();
        renderer.setProjectionMatrix(frmCam.combined);
        frmBuff.begin();

        Gdx.gl.glClearColor(0, 0, 0, 0);
        Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT);

        frmBuff.end();



        SpriteBatch batch = new SpriteBatch();

        frmCam.update();
        batch.setProjectionMatrix(frmCam.combined);
        maskBuff.begin();

        Gdx.gl.glClearColor(0, 0, 0, 0);
        Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT);

        batch.begin();
        batch.draw(mask, 0, 0, 0, 0, 500, 700, 1, 1, 0, 0, 0, 500, 700, false, true);
        batch.end();

        maskBuff.end();

        shader = new ShaderProgram(VERT, FRAG);
        if (!shader.isCompiled()) {
            System.err.println(shader.getLog());
            System.exit(0);
        }
        if (shader.getLog().length()!=0)
            System.out.println(shader.getLog());


        shader.begin();
        shader.setUniformi("u_texture1", 1);
        shader.setUniformi("u_mask", 2);
        shader.end();

        //bind mask to glActiveTexture(GL_TEXTURE2)
        msk.getTexture().bind(2);

        //bind dirt to glActiveTexture(GL_TEXTURE1)
//      tex1.bind(1);
        frm.getTexture().bind(1);

        //now we need to reset glActiveTexture to zero!!!! since sprite batch does not do this for us
        Gdx.gl.glActiveTexture(GL20.GL_TEXTURE0);       
    }

        @Override
    public void draw(Batch batch, float parentAlpha) {      
        batch.flush();
        batch.setShader(shader);    
        batch.draw(tex0, getX() + MASK_OFFSET_X, getY() + MASK_OFFSET_Y);
        batch.flush();
        batch.setShader(null);
    }
}