Shadow mapping with wrong results (HLSL, Shader model 3.0)

Question

(Sorry for my bad English.)

I'm new to Stack Overflow and writing a 3D game application with MS Visual C++ 2015 compiler, Direct3D 9 and HLSL(Shader model 3.0).

I've implemented a deferred rendering logic with 4 render target textures. I stored depth values of pixels in a render target texture and created a shadow map. Here are the results. (All meshes have black color because the meshes have small size and close to the camera. The far plane distance value is 1000.0f.)

The depth texture and the shadow map.

I rendered a full screen quad with shadow mapping shaders and outputted shadows with red color to confirm the shader is working correctly. But, It seems that the shaders output wrong results. The shadow map texture output repeats on the mesh surfaces.

https://www.youtube.com/watch?v=1URGgoCR6Zc

Here is the shadow mapping vertex shader to draw the quad.

struct VsInput {
   float4 position : POSITION0;
};

struct VsOutput {
   float4 position : POSITION0;
   float4 cameraViewRay : TEXCOORD0;
};

float4x4 matInverseCameraViewProjection;
float4 cameraWorldPosition;
float farDistance;

VsOutput vs_main(VsInput input) {
   VsOutput output = (VsOutput)0;

   output.position = input.position;

   output.cameraViewRay = mul(float4(input.position.xy, 1.0f, 1.0f) * farDistance, matInverseCameraViewProjection);
   output.cameraViewRay /= output.cameraViewRay.w;
   output.cameraViewRay.xyz -= cameraWorldPosition.xyz;

   return output;
}

And here is the shadow mapping pixel shader to draw the quad.

struct PsInput {
   float2 screenPosition : VPOS;
   float4 viewRay : TEXCOORD0;
};

struct PsOutput {
   float4 color : COLOR0;
};

texture depthMap;
texture shadowMap;

sampler depthMapSampler = sampler_state {
    Texture = (depthMap);
    AddressU = CLAMP;
    AddressV = CLAMP;
    MagFilter = POINT;
    MinFilter = POINT;
    MipFilter = POINT;
};

sampler shadowMapSampler = sampler_state {
    Texture = (shadowMap);
    AddressU = CLAMP;
    AddressV = CLAMP;
    MagFilter = POINT;
    MinFilter = POINT;
    MipFilter = POINT;
};

//float4x4 matCameraView;
float4x4 matLightView;
float4x4 matLightProjection;
float4 cameraWorldPosition;
float4 lightWorldPosition;
float2 halfPixel;
float epsilon;
float farDistance;

PsOutput ps_main(PsInput input) {
   PsOutput output = (PsOutput)0;
   output.color.a = 1.0f;

   //Reconstruct the world position using the view-space linear depth value.
   float2 textureUv = input.screenPosition * halfPixel * 2.0f - halfPixel;
   float viewDepth = tex2D(depthMapSampler, textureUv).r;
   float3 eye = input.viewRay.xyz * viewDepth;
   float4 worldPosition = float4((eye + cameraWorldPosition.xyz), 1.0f);

   //Test if the reconstructed world position has right coordinate values.
   //output.color = mul(worldPosition, matCameraView).z / farDistance;

   float4 positionInLightView = mul(worldPosition, matLightView);
   float lightDepth = positionInLightView.z / farDistance;
   float4 positionInLightProjection = mul(positionInLightView, matLightProjection);
   positionInLightProjection /= positionInLightProjection.w;

   //If-statement doesn't work???
   float condition = positionInLightProjection.x >= -1.0f;
   condition *= positionInLightProjection.x <= 1.0f;
   condition *= positionInLightProjection.y >= -1.0f;
   condition *= positionInLightProjection.y <= 1.0f;
   condition *= positionInLightProjection.z >= 0.0f;
   condition *= positionInLightProjection.z <= 1.0f;
   condition *= positionInLightProjection.w > 0.0f;

   float2 shadowMapUv = float2(
      positionInLightProjection.x * 0.5f + 0.5f,
      -positionInLightProjection.y * 0.5f + 0.5f
   );

   //If-statement doesn't work???
   float condition2 = shadowMapUv.x >= 0.0f;
   condition2 *= shadowMapUv.x <= 1.0f;
   condition2 *= shadowMapUv.y >= 0.0f;
   condition2 *= shadowMapUv.y <= 1.0f;

   float viewDepthInShadowMap = tex2D(
      shadowMapSampler,
      shadowMapUv
   ).r;
   output.color.r = lightDepth > viewDepthInShadowMap + epsilon;
   output.color.r *= condition;
   output.color.r *= condition2;

   return output;
}

It seems that the uv for the shadow map has some wrong values, but i can't figure out what's the real problem.

Many thanks for any help.

EDIT : I've updated the shader codes. I decided to use view-space linear depth and confirmed that the world position has right value. I really don't understand why the shadow map coordinate values have wrong values...

Answer 1

It really looks like you are using a wrong bias. Google up "Shadow Acne" and you should find your answer to your problem. Also the resolution of the shadowmap could be a problem.

Answer 2

I found the solution.

The first problem was that the render target texture had wrong texture format. I should have used D3DFMT_R32F. (I had used D3DFMT_A8R8G8B8.)

And i added these lines in my shadow mapping pixel shader.

//Reconstruct the world position using the view-space linear depth value.
float2 textureUv = input.screenPosition * halfPixel * 2.0f - halfPixel;
float4 viewPosition = float4(input.viewRay.xyz * tex2D(depthMapSampler, textureUv).r, 1.0f);
float4 worldPosition = mul(viewPosition, matInverseCameraView);

...

//If-statement doesn't work???
float condition = positionInLightProjection.x >= -1.0f;
condition *= positionInLightProjection.x <= 1.0f;
condition *= positionInLightProjection.y >= -1.0f;
condition *= positionInLightProjection.y <= 1.0f;
condition *= positionInLightProjection.z >= 0.0f;
condition *= positionInLightProjection.z <= 1.0f;
condition *= viewPosition.z < farDistance;

The last line was the key and solved my second problem. The 'farDistance' is the far plane distance of the camera frustum. I'm still trying to understand why that is needed.