Ray picking mess

Question

I am using ray picking to find the boundaries for a character. It's not optimal, but it's the best I can do and will have to do; I need to have (close to) pixelperfect collisions and also I have lots and lots of objects.

I do not however get the raypicking to work correctly. It collides, but not at the right places. I tried adjust the size of the ray, and more to no avail.

Excuse me for the messy code, I just threw it together.

private void renderCollision(Vector3f dir){
    //
    // Render the models
    //

    glPushMatrix();
    glPushAttrib(GL_ALL_ATTRIB_BITS);

    float size = 10.0f;
    Picker.startPicking2D(10, 10, 20, 20, -1.0f, size);
    //Picker.startPicking2D(10, 10, 20, 20, 0.1f, 20.0f);

    glClear (GL_COLOR_BUFFER_BIT);
    glDisable(GL_BLEND);
    glDisable(GL_ALPHA);
    glEnable(GL_DEPTH_TEST);
    glDepthFunc(GL_LESS);
    glDepthMask(true);
    glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
    glClearDepth(100.0f);
    glClear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
    glDisable(GL_TEXTURE_2D);

    glLoadIdentity (); 


    //box.drawMoving(camera);
    player.move(dir);
    glTranslatef(-player.position.x, -player.position.y + size / 2.0f, -player.position.z);
    glRotatef(90.0f, 0.0f, 1.0f, 0.0f);

    box.drawAll();

    glDisable(GL_DEPTH_TEST);

    boolean hit = Picker.getHit();
    if (hit) {
        player.move(new Vector3f(-dir.x, -dir.y, -dir.z));
    }

    Picker.stopPicking();

    glPopAttrib();
    glPopMatrix();
}

public class Picker {

    private static IntBuffer selBuffer;
    private static int hits;

    private static int xSelected;
    private static int ySelected;

    /**
     * Makes the game available for picking (when in 3D mode)
     *
     * @param xMouse The x coordinate of the mouse on the screen
     * @param yMouse The y coordinate of the mouse on the screen
     */
    public static void startPicking3D(int xMouse, int yMouse, int screenWidth, int screenHeight, float near, float far) {
        startPickingGeneric(xMouse, yMouse);

        GLU.gluPerspective(SCREEN_FOV, SCREEN_RAT,
                           near, far);

        GL11.glMatrixMode(GL11.GL_MODELVIEW);
        GL11.glLoadIdentity();
    }

    /**
     * Makes the game available for picking (when in 2D mode)
     *
     * @param xMouse The x coordinate of the mouse on the screen
     * @param yMouse The y coordinate of the mouse on the screen
     */
    public static void startPicking2D(int xMouse, int yMouse, int screenWidth, int screenHeight, float near, float far) {
        startPickingGeneric(xMouse, yMouse);

        GL11.glOrtho(0, screenWidth, 0, screenHeight, near, far);

        GL11.glMatrixMode(GL11.GL_MODELVIEW);
        GL11.glLoadIdentity();
    }

    /**
     * Makes the game available for picking (generic)
     *
     * @param xMouse The x coordinate of the mouse on the screen
     * @param yMouse The y coordinate of the mouse on the screen
     */
    private static void startPickingGeneric(int xMouse, int yMouse){
        // The selection buffer
        selBuffer = ByteBuffer.allocateDirect(1024).order(ByteOrder.nativeOrder()).
                    asIntBuffer();
        IntBuffer vpBuffer = ByteBuffer.allocateDirect(64).
                             order(ByteOrder.nativeOrder()).asIntBuffer();
        // Size of the viewport. [0] Is <x>, [1] Is <y>, [2] Is <width>, [3] Is <height>
        int[] viewport = new int[4];

        // Get the viewport info
        GL11.glGetInteger(GL11.GL_VIEWPORT, vpBuffer);
        vpBuffer.get(viewport);

        // Set the buffer that OpenGL uses for selection to our buffer
        GL11.glSelectBuffer(selBuffer);

        // Change to selection mode
        GL11.glRenderMode(GL11.GL_SELECT);

        // Initialize the name stack (used for identifying which object was selected)
        GL11.glInitNames();

        GL11.glMatrixMode(GL11.GL_PROJECTION);
        GL11.glPushMatrix();
        GL11.glLoadIdentity();

        // Create 5x5 pixel picking region near cursor location
        GLU.gluPickMatrix((float) xMouse, (float) yMouse,
                          5.0f, 5.0f, IntBuffer.wrap(viewport));
    }

    /**
     * Stops the picking mode
     */
    public static void stopPicking(){
        GL11.glMatrixMode(GL11.GL_PROJECTION);
    GL11.glPopMatrix();
        GL11.glMatrixMode(GL11.GL_MODELVIEW);

        hits = 0;
        hits = GL11.glRenderMode(GL11.GL_RENDER);
    }

    /**
     * Gets the tile the mouse points to
     *
     * @return TileCoords object with the coordinates of the selected tile
     */
    public static boolean getHit(){

        int[] buffer = new int[256];
        xSelected = -1000;
        ySelected = -1000;

        selBuffer.get(buffer);

        if (hits > 0) {
              // If there were more than 0 hits
              xSelected = buffer[3]; // Make our selection the first object
              ySelected = buffer[4];
              int depth = buffer[1]; // Store how far away it is
              for (int i = 1; i < hits; i++) {
                    // Loop through all the detected hits
                    // If this object is closer to us than the one we have selected
                    if (buffer[i * 4 + 1] < (int) depth) {
                        xSelected = buffer[i * 4 + 3]; // Select the closest object
                        ySelected = buffer[i * 4 + 4];
                        depth = buffer[i * 4 + 1]; // Store how far away it is
                     }
              }
              return true;
        }
        return false;
    }

}

Answer 1

I should rotate around x-axis instead of y-axis and then translate.