AlphaMask in monogame not working properly

Question

Im trying to build a rendertarget alpha mask for a fog of war example.. I tell it that I want to clear the rendertarget with solid white (Alpha 255)

Then I draw my image with a transparent circle, onto the alpha channel, from there I would expect to have an image in the rendertarget, with holes in it. This however is not the case.. I'm very familiar with alpha masking and have done so successfully when drawing a single image to the alpha mask, but when drawing a bunch like we have here, it fails entirely to give me proper alpha values.

if (mFogOfWarRT == null)
{
    mFogOfWarRT = new RenderTarget2D(pGraphics.GraphicsDevice, MapSize, MapSize);
    pGraphics.GraphicsDevice.SetRenderTarget(mFogOfWarRT);
    //pGraphicsDevice.Clear(Color.Black);
}
else
{
    pGraphics.GraphicsDevice.SetRenderTarget(mFogOfWarRT);
}

pGraphicsDevice.Clear(Color.White);
pSpriteBatch.Begin();
BlendState lKeep = new BlendState();
lKeep.AlphaSourceBlend = Blend.One;
lKeep.AlphaDestinationBlend = Blend.Zero;
lKeep.ColorSourceBlend = Blend.Zero;
lKeep.ColorDestinationBlend = Blend.One;
lKeep.ColorBlendFunction = BlendFunction.Subtract;
lKeep.AlphaBlendFunction = BlendFunction.Add;
pGraphicsDevice.BlendState = lKeep;
pGraphicsDevice.BlendState.ColorWriteChannels = ColorWriteChannels.Alpha;

foreach (ClearArea lArea in mDrawQueue)
{
    pSpriteBatch.Draw(mAlphaMask, new Rectangle((int)lArea.X, lArea.Y, lArea.Diameter, lArea.Diameter), Color.White);
}
//pSpriteBatch.Draw(mDot, new Rectangle(0, 0, 100, 100), Color.White);
pSpriteBatch.End();

pGraphicsDevice.SetRenderTarget(null);

This is my drawing code for the alpha mask.. when draws to the screen we get a white square, nothing else.

down below I draw my "Map" to the RGB, then set the A, and draw my alphamask texture and it fails miserably

pGraphicsDevice.SetRenderTarget(null);

// Draw minimap texture
pGraphicsDevice.Clear(Color.CornflowerBlue);
pGraphicsDevice.BlendState = lKeep;
pGraphicsDevice.BlendState.ColorWriteChannels = ColorWriteChannels.Red | ColorWriteChannels.Blue | ColorWriteChannels.Green;

pSpriteBatch.Begin();
pSpriteBatch.Draw(mDot, new Rectangle(0, 0, 400, 400), Color.Blue);
pGraphicsDevice.BlendState.ColorWriteChannels = ColorWriteChannels.Alpha;
pSpriteBatch.Draw(mFogOfWarRT, Vector2.Zero, Color.White);

pSpriteBatch.End();

Any help with this would be much appreciate..

paste bin of full code

http://pastebin.com/KsEVaW0d

screen shot of outcome available if needed.

willthiswork89 willthiswork89 · Accepted Answer · 2014-02-05T05:01:25

Decided that alpha masking wasn't the best approach after doing research I found Stencils.

The code below fixes the issue by writing to the stencil buffer.

graphics.PreferredDepthStencilFormat = DepthFormat.Depth24Stencil8;
graphics.ApplyChanges();

AlphaTestEffect alphaTestEffect = new AlphaTestEffect(pGraphicsDevice);
alphaTestEffect.VertexColorEnabled = true;
alphaTestEffect.DiffuseColor = Color.White.ToVector3();
alphaTestEffect.AlphaFunction = CompareFunction.Equal;
alphaTestEffect.ReferenceAlpha = 0;
alphaTestEffect.World = Matrix.Identity;
alphaTestEffect.View = Matrix.Identity;
Matrix projection = Matrix.CreateOrthographicOffCenter(0, 400,400, 0, 0, 1);
alphaTestEffect.Projection = projection;
        // Create fog of war mask
        if (mFogOfWarRT == null)
        {
            mFogOfWarRT = new RenderTarget2D(pGraphics.GraphicsDevice, MapSize, MapSize, false, SurfaceFormat.Color, DepthFormat.Depth24Stencil8);
            pGraphics.GraphicsDevice.SetRenderTarget(mFogOfWarRT);

        }
        else
        {
            pGraphics.GraphicsDevice.SetRenderTarget(mFogOfWarRT);
        }
        //important both stencil states be created in their own object, cannot modify once set for some reason.
        DepthStencilState lState = new DepthStencilState();
        lState.StencilEnable = true;
        lState.StencilFunction = CompareFunction.Always;
        lState.ReferenceStencil = 1;
        lState.StencilPass = StencilOperation.Replace;
        lState.DepthBufferEnable = false;
        pGraphicsDevice.Clear(ClearOptions.Target | ClearOptions.Stencil,
                                  new Color(0, 0, 0, 1), 0, 0);
        pSpriteBatch.Begin(SpriteSortMode.Deferred, BlendState.AlphaBlend, null, lState, null, alphaTestEffect);
        foreach (ClearArea lArea in mDrawQueue)
        {
//draw whatever you want "visible" anything in the texture with an alpha of 0 will be allowed to draw.
            pSpriteBatch.Draw(mAlphaMask, new Rectangle((int)lArea.X, lArea.Y, lArea.Diameter, lArea.Diameter), Color.White);
        }

        pSpriteBatch.End();

        // Draw minimap texture
        DepthStencilState lState2 = new DepthStencilState();
        lState2.StencilEnable = true;
        lState2.StencilFunction = CompareFunction.Equal;
        lState2.ReferenceStencil = 0;
        lState2.StencilPass = StencilOperation.Keep;
        lState2.DepthBufferEnable = false;

        pSpriteBatch.Begin(SpriteSortMode.Deferred, BlendState.AlphaBlend, null, lState2, null);
        pSpriteBatch.Draw(mDot, new Rectangle(0, 0, 400, 400), Color.Black);

        pSpriteBatch.End();
        //done drawing to the render target
        pGraphicsDevice.SetRenderTarget(null);
        pGraphicsDevice.Clear(Color.Gray);
        pSpriteBatch.Begin();
        pSpriteBatch.Draw(mDot, new Rectangle(0, 0, 400, 400), Color.Blue);
        pSpriteBatch.Draw(mFogOfWarRT,Vector2.Zero,Color.White);
        pSpriteBatch.End();

AlphaMask in monogame not working properly

1 Answers