You must multiply each RGB value with the alpha value and afterwards divide the result with the maximum possible alpha.
Let's say you want to average just three pixels:
newBlue = (
src[-1].Blue * src[-1].Alpha +
src[0].Blue * src[0].Alpha +
src[1].Blue * src[1].Alpha ) / (255*3);
As you can see: if all three pixels are solid (alpha=255) then this calculation won't cange anything compared to ignoring the alpha channel (which is indeed what we want).
Here is a 3x3 convolution without alpha:
for (var i = nWidth - 2; i > 0; i--)
{
n = ((((pT[-sourcePixelSize]*m.TL) + (pT[0]*m.TM) + (pT[sourcePixelSize]*m.TR) +
(pM[-sourcePixelSize]*m.ML) + (pM[0]*m.MM) + (pM[sourcePixelSize]*m.MR) +
(pB[-sourcePixelSize]*m.BL) + (pB[0]*m.BM) + (pB[sourcePixelSize]*m.BR) + 5)/m.Factor) + m.Offset);
*pD = (byte) (n <= 0 ? 0 : n >= 255 ? 255 : n);
pT += sourcePixelSize;
pM += sourcePixelSize;
pB += sourcePixelSize;
pD += 4;
}
Here is the eqvivalent with alpha:
for (var i = nWidth - 2; i > 0; i--)
{
alphaSum = (pT[-4 + ao] + pT[ao] + pT[4 + ao] +
pM[-4 + ao] + pM[ao] + pM[4 + ao] +
pB[-4 + ao] + pB[ao] + pB[4 + ao] + 5)/9;
n = alphaSum != 0
? ((((pT[-4]*pT[-4 + ao]*m.TL) + (pT[0]*pT[ao]*m.TM) + (pT[4]*pT[4 + ao]*m.TR) +
(pM[-4]*pM[-4 + ao]*m.ML) + (pM[0]*pM[ao]*m.MM) + (pM[4]*pM[4 + ao]*m.MR) +
(pB[-4]*pB[-4 + ao]*m.BL) + (pB[0]*pB[ao]*m.BM) + (pB[4]*pB[4 + ao]*m.BR) + 5)/
(m.Factor*alphaSum)) + m.Offset)
: 0;
*pD = (byte) (n <= 0 ? 0 : n >= 255 ? 255 : n);
pT += 4;
pM += 4;
pB += 4;
pD += 4;
}