three.js Vector3 to 2D screen coordinate with rotated scene

Question

I am trying to position a div such that it is always at the highest point of an object offset left and up a little so it is not right on top of the vertex. You can see that here. Please see the working with no scene rotation snippet below if the jsfiddle link no longer works. I can tell you, it works fairly marvellously.

However, for a project at work, I need to rotate the scene, itself. Apparently, this messes up the conversion to 2D screen coordinates. You can view this here. You can view the not working with scene rotation snippet below as well. As you can see, the label does not update in the vertical ("y-") direction, but it does update "horizontally". This would be because the camera's position does, indeed, change around the x-z plane (phi changes), but it never changes its y-position (theta never changes)--rather the scene is rotated when the mouse is dragged up/down. As I wrote, I need to rotate the scene to achieve the desired affect.

If someone could point me in the right direction or make a quick example in any of the html/js/css snippet sites (like jsfiddle, etc, etc), {,s}he would be a life-saver!

side-note: I tried (with no success, obviously) to jump through some hoops to convert the x- and y-coordinates for each vertex to a "properly-rotated" position (ie, by multiplying by sin(sceneRotation) and cos(sceneRotation), but that just made it even worse.
side-note 2: I also just spent an hour rotating each object in the scene, but since the objects are actually stored in THREE.Groups, it has the exact same effect.

working with no scene rotation

click run snippet below

var scene = new THREE.Scene();
var w = window.innerWidth, h = window.innerHeight;
var camera = new THREE.PerspectiveCamera(60, w / h, 1, 1000);
camera.position.set(0, 0, -60);
var renderer = new THREE.WebGLRenderer({
    alpha: true
});
renderer.setSize(window.innerWidth, window.innerHeight);
renderer.domElement.style.backgroundColor = "#bbbbbb"
document.body.appendChild(renderer.domElement);

var label = document.getElementById("label");
var controls = new THREE.OrbitControls(camera, renderer.domElement);
controls.addEventListener("change", updateLabel);

var geom = new THREE.Geometry();
geom.vertices.push(new THREE.Vector3(-10, 10, -10));
geom.vertices.push(new THREE.Vector3(10, 10, -10));
geom.vertices.push(new THREE.Vector3(10, -10, -10));
geom.vertices.push(new THREE.Vector3(-10, -10, -10));
geom.vertices.push(new THREE.Vector3(-10, 10, 10));
geom.vertices.push(new THREE.Vector3(10, 10, 10));
geom.vertices.push(new THREE.Vector3(10, -10, 10));
geom.vertices.push(new THREE.Vector3(-10, -10, 10));
geom.faces.push(new THREE.Face3(0, 1, 2));
geom.faces.push(new THREE.Face3(0, 2, 3));
geom.faces.push(new THREE.Face3(7, 6, 5));
geom.faces.push(new THREE.Face3(7, 5, 4));
geom.faces.push(new THREE.Face3(4, 5, 1));
geom.faces.push(new THREE.Face3(4, 1, 0));
geom.faces.push(new THREE.Face3(3, 2, 6));
geom.faces.push(new THREE.Face3(3, 6, 7));
geom.faces.push(new THREE.Face3(4, 0, 3));
geom.faces.push(new THREE.Face3(4, 3, 7));
geom.faces.push(new THREE.Face3(1, 5, 6));
geom.faces.push(new THREE.Face3(1, 6, 2));
var mat = new THREE.MeshBasicMaterial({ color: 0x3377dd, transparent: true, opacity: .65, wireframe: false });
var cube = new THREE.Mesh(geom, mat);
scene.add(cube);
var matWire = new THREE.MeshBasicMaterial({ color: 0xffffff, wireframe: true });
var cube1 = new THREE.Mesh(geom, matWire);
scene.add(cube1);

render();

function render() {
    requestAnimationFrame(render);
    renderer.render(scene, camera);
}

function getScreenPosition(position) {
    var vector = new THREE.Vector3( position.x, position.y, position.z );

    vector.project(camera);

    vector.x = Math.round( (   vector.x + 1 ) * w / 2 );
    vector.y = Math.round( ( - vector.y + 1 ) * h / 2 );

    return vector;
}

function updateLabel() {
    var minY = null, x = null,
    	verts = cube.geometry.vertices;
    for (var i = 0, iLen = verts.length; i < iLen; i++) {
        var pos = getScreenPosition(verts[i]);
        if (minY === null || pos.y < minY) {
        	minY = pos.y;
            x = pos.x;
        }
    }
    label.style.left = (x - 3) + "px";
    label.style.top = (minY - 28) + "px";
}

body {
    overflow: hidden;
    margin: 0;
}

#label {
    position: absolute;
    left: 20px; top: 20px;
}

<div id="label">label</div>
<script src="https://threejs.org/build/three.min.js"></script>
<script src="https://threejs.org/examples/js/controls/OrbitControls.js"></script>

not working with scene rotation

click run snippet below

/* "globals" */
/* ~~~~~~~~~ */
var PI = Math.PI;
/* camera stuff */
var lastX, lastY, r = 60, phi = 0, c = new THREE.Vector3(0, 0, 0);
/* three.js stuff */
var scene = new THREE.Scene();
var w = window.innerWidth, h = window.innerHeight;
var camera = new THREE.PerspectiveCamera(60, w / h, 1, 1000);
//camera.position.set(0, 0, -60);
updateCamera();
var renderer = new THREE.WebGLRenderer({
    alpha: true
});
renderer.setSize(window.innerWidth, window.innerHeight);
renderer.domElement.style.backgroundColor = "#bbbbbb"
document.body.appendChild(renderer.domElement);

var label = document.getElementById("label");
//var controls = new THREE.OrbitControls(camera, renderer.domElement);
//controls.addEventListener("change", updateLabel);
document.body.addEventListener("mousedown", handleMouseDown);
document.body.addEventListener("touchstart", handleTouchStart);

var geom = new THREE.Geometry();
geom.vertices.push(new THREE.Vector3(-10, 10, -10));
geom.vertices.push(new THREE.Vector3(10, 10, -10));
geom.vertices.push(new THREE.Vector3(10, -10, -10));
geom.vertices.push(new THREE.Vector3(-10, -10, -10));
geom.vertices.push(new THREE.Vector3(-10, 10, 10));
geom.vertices.push(new THREE.Vector3(10, 10, 10));
geom.vertices.push(new THREE.Vector3(10, -10, 10));
geom.vertices.push(new THREE.Vector3(-10, -10, 10));
geom.faces.push(new THREE.Face3(0, 1, 2));
geom.faces.push(new THREE.Face3(0, 2, 3));
geom.faces.push(new THREE.Face3(7, 6, 5));
geom.faces.push(new THREE.Face3(7, 5, 4));
geom.faces.push(new THREE.Face3(4, 5, 1));
geom.faces.push(new THREE.Face3(4, 1, 0));
geom.faces.push(new THREE.Face3(3, 2, 6));
geom.faces.push(new THREE.Face3(3, 6, 7));
geom.faces.push(new THREE.Face3(4, 0, 3));
geom.faces.push(new THREE.Face3(4, 3, 7));
geom.faces.push(new THREE.Face3(1, 5, 6));
geom.faces.push(new THREE.Face3(1, 6, 2));
var mat = new THREE.MeshBasicMaterial({ color: 0x3377dd, transparent: true, opacity: .65, wireframe: false });
var cube = new THREE.Mesh(geom, mat);
//cube.translateX(10);
scene.add(cube);
var matWire = new THREE.MeshBasicMaterial({ color: 0xffffff, wireframe: true });
var cube1 = new THREE.Mesh(geom, matWire);
//cube1.translateX(10);
scene.add(cube1);

render();

function render() {
    requestAnimationFrame(render);
    renderer.render(scene, camera);
}

function getScreenPosition(position) {
    var vector = new THREE.Vector3( position.x, position.y, position.z );

    vector.project(camera);

    vector.x = Math.round( (   vector.x + 1 ) * w / 2 );
    vector.y = Math.round( ( - vector.y + 1 ) * h / 2 );

    return vector;
}

function updateLabel() {
    var minY = null, x = null,
    	verts = cube.geometry.vertices;
    for (var i = 0, iLen = verts.length; i < iLen; i++) {
        var pos = getScreenPosition(verts[i]);
        if (minY === null || pos.y < minY) {
        	minY = pos.y;
            x = pos.x;
        }
    }
    label.style.left = (x - 3) + "px";
    label.style.top = (minY - 28) + "px";
}

function handleMouseDown(ev) {
    ev.preventDefault();
    mouseOrTouchDown(ev.pageX, ev.pageY);
}

function handleTouchStart(ev) {
    var touches = ev.touches;
    if (touches.length !== 1) {
        return;
    }
    ev.preventDefault();
    mouseOrTouchDown(touches.item(0).pageX, touches.item(0).pageY, true);
}

function mouseOrTouchDown(downX, downY, touch) {
    if (touch === undefined) { touch = false; }
    lastX = downX;
    lastY = downY;
    if (touch) {
        document.ontouchmove = handleTouchMove;
        document.addEventListener("touchend", function(ev) {
            document.ontouchmove = null;
        });
        document.addEventListener("touchcancel", function(ev) {
            document.removeEventListener("touchmove", handleTouchMove);
        });
    } else {
        document.addEventListener("mousemove", handleMouseMove);
        document.addEventListener("mouseup", function(ev) {
            document.removeEventListener("mousemove", handleMouseMove);
        });
    }
}

function handleMouseMove(ev) {
    ev.preventDefault();
    mouseOrTouchMove(ev.pageX, ev.pageY);
}

function handleTouchMove(ev) {
    var touches = ev.touches;
    if (touches.length !== 1) {
        return;
    }
    ev.preventDefault();
    mouseOrTouchMove(touches.item(0).pageX, touches.item(0).pageY);
}

function mouseOrTouchMove(x, y) {
    var dx = lastX - x, dy = y - lastY; /* change in {x, y} */

    phi -= dx / 100;
    if (phi > 2 * PI) {
        phi -= 2 * PI;
    } else if (phi < 0) {
        phi += 2 * PI;
    }

    if (phi < PI / 2 || phi > 3 * PI / 2) {
        sign = -1;
    } else {
        sign = 1;
    }
    if (scene.rotation.z + sign * dy / 100 < -PI) {
        scene.rotation.z = -PI;
    } else if (scene.rotation.z + sign * dy / 100 > 0) {
        scene.rotation.z = 0;
    } else {
        scene.rotateZ(sign * dy / 100);
    }

    lastX = x;
    lastY = y;

    updateCamera();
    updateLabel();
}

function updateCamera() {
    var z = r * Math.sin(phi); /* new y pos (z-axis) in x-z plane */
    var x = r * Math.cos(phi); /* new x pos (x-axis) in x-z plane */
    camera.position.set(x, 1, z);
    camera.lookAt(c);
}

body {
    overflow: hidden;
    margin: 0;
}

#label {
    position: absolute;
    left: 20px; top: 20px;
}

<div id="label">label</div>
<script src="https://threejs.org/build/three.min.js"></script>
<script src="https://threejs.org/examples/js/controls/OrbitControls.js"></script>

What effect are you trying to accomplish by rotating the scene, that you can't by rotating the camera? — TheJim01
@TheJim01 I have conferred with my colleague about this, and I have thought about it. Unfortunately, I am not Euler, so I am having a hard time figuring out how I might set the cameras rotation such that I could get the desired effect. I need the camera to rotate in the x-z plane (phi, as you may know) when I drag left or right, and I need the scene to rotate (or appear to rotate) about a particular axis (let's say the x-axis) when I drag up or down. That's what I need to do. Gimbal lock or Euler Angles or whatever have me all tied up in a knot. Maybe you know something I don't! — dylnmc
essentially, if I did not do it the way I have done it (either by rotating the scene or all objects in the scene), then I would have to find the camera's position with respect to the center of the scene and rotate it accordingly ... something I am not too comfortable with, for ... well, maybe ... obvious reasons. That's some hard 3D calc right there >_< Also, thank you for the expeditious reply! ^_^ — dylnmc

Rabbid76 Rabbid76 · Accepted Answer · 2017-10-10T16:09:39

In a rendering, each mesh of the scene usually is transformed by the model matrix, the view matrix and the projection matrix.

Model matrix:
The model matrix defines the location, orientation and the relative size of an mesh in the scene. The model matrix transforms the vertex positions from of the mesh to the world space.
View matrix:
The view matrix describes the direction and position from which the scene is looked at. The view matrix transforms from the world space to the view (eye) space. In the coordinate system on the viewport, the X-axis points to the left, the Y-axis up and the Z-axis out of the view (Note in a right hand system the Z-Axis is the cross product of the X-Axis and the Y-Axis).
Projection matrix:
The projection matrix describes the mapping from 3D points of a scene, to 2D points of the viewport. The projection matrix transforms from view space to the clip space, and the coordinates in the clip space are transformed to the normalized device coordinates (NDC) in the range (-1, -1, -1) to (1, 1, 1) by dividing with the w component of the clip coordinates.

If you want know, where a point from the geometry is seen on the viewport, then you have to do all these transformation and you have to convert from normalized device coordinates (NDC) to window coordinates (pixel).

The transformation by the view matrix and the projection matrix is done by project:

vector.project(camera);

The transformation from normalized device coordinates (NDC) to window coordinates (pixel) is done like this:

vector.x = Math.round( (   vector.x + 1 ) * w / 2 );
vector.y = Math.round( ( - vector.y + 1 ) * h / 2 );

You forgot the transformation with the model matrix, which can be done like this:

var modelMat = cube.matrixWorld;
vector.applyMatrix4(modelMat);

    // world to view and view to NDC
    vector.project(camera);

    // NDC to pixel
    vector.x = Math.round( (   vector.x + 1 ) * w / 2 );
    vector.y = Math.round( ( - vector.y + 1 ) * h / 2 );
   
    return vector;
}

/* can't see top line; thanks jsfiddle */
    /* "globals" */
    /* ~~~~~~~~~ */
    var PI = Math.PI;
    /* camera stuff */
    var lastX, lastY, r = 60, phi = 0, c = new THREE.Vector3(0, 0, 0);
    /* three.js stuff */
    var scene = new THREE.Scene();
    var w = window.innerWidth, h = window.innerHeight;
    var camera = new THREE.PerspectiveCamera(60, w / h, 1, 1000);
    //camera.position.set(0, 0, -60);
    updateCamera();
    var renderer = new THREE.WebGLRenderer({
        alpha: true
    });
    renderer.setSize(window.innerWidth, window.innerHeight);
    renderer.domElement.style.backgroundColor = "#bbbbbb"
    document.body.appendChild(renderer.domElement);

    var label = document.getElementById("label");
    //var controls = new THREE.OrbitControls(camera, renderer.domElement);
    //controls.addEventListener("change", updateLabel);
    document.body.addEventListener("mousedown", handleMouseDown);
    document.body.addEventListener("touchstart", handleTouchStart);

    var geom = new THREE.Geometry();
    geom.vertices.push(new THREE.Vector3(-10, 10, -10));
    geom.vertices.push(new THREE.Vector3(10, 10, -10));
    geom.vertices.push(new THREE.Vector3(10, -10, -10));
    geom.vertices.push(new THREE.Vector3(-10, -10, -10));
    geom.vertices.push(new THREE.Vector3(-10, 10, 10));
    geom.vertices.push(new THREE.Vector3(10, 10, 10));
    geom.vertices.push(new THREE.Vector3(10, -10, 10));
    geom.vertices.push(new THREE.Vector3(-10, -10, 10));
    geom.faces.push(new THREE.Face3(0, 1, 2));
    geom.faces.push(new THREE.Face3(0, 2, 3));
    geom.faces.push(new THREE.Face3(7, 6, 5));
    geom.faces.push(new THREE.Face3(7, 5, 4));
    geom.faces.push(new THREE.Face3(4, 5, 1));
    geom.faces.push(new THREE.Face3(4, 1, 0));
    geom.faces.push(new THREE.Face3(3, 2, 6));
    geom.faces.push(new THREE.Face3(3, 6, 7));
    geom.faces.push(new THREE.Face3(4, 0, 3));
    geom.faces.push(new THREE.Face3(4, 3, 7));
    geom.faces.push(new THREE.Face3(1, 5, 6));
    geom.faces.push(new THREE.Face3(1, 6, 2));
    var mat = new THREE.MeshBasicMaterial({ color: 0x3377dd, transparent: true, opacity: .65, wireframe: false });
    var cube = new THREE.Mesh(geom, mat);
    //cube.translateX(10);
    scene.add(cube);
    var matWire = new THREE.MeshBasicMaterial({ color: 0xffffff, wireframe: true });
    var cube1 = new THREE.Mesh(geom, matWire);
    //cube1.translateX(10);
    scene.add(cube1);

    render();

    function render() {
        requestAnimationFrame(render);
        renderer.render(scene, camera);
    }

    function getScreenPosition(position, object) {
        var vector = new THREE.Vector3( position.x, position.y, position.z );

        // model to world
        if ( object != null ) {
            var modelMat = cube.matrixWorld;
            vector.applyMatrix4(modelMat);
        }

        // world to view and view to NDC
        vector.project(camera);

        // NDC to pixel
        vector.x = Math.round( (   vector.x + 1 ) * w / 2 );
        vector.y = Math.round( ( - vector.y + 1 ) * h / 2 );

        return vector;
    }

    function updateLabel() {
        var minY = null, x = null,
            verts = cube.geometry.vertices;
        for (var i = 0, iLen = verts.length; i < iLen; i++) {
            var pos = getScreenPosition(verts[i], cube);
            if (minY === null || pos.y < minY) {
                minY = pos.y;
                x = pos.x;
            }
        }
        label.style.left = (x - 3) + "px";
        label.style.top = (minY - 28) + "px";
    }

    function handleMouseDown(ev) {
        ev.preventDefault();
        mouseOrTouchDown(ev.pageX, ev.pageY);
    }

    function handleTouchStart(ev) {
        var touches = ev.touches;
        if (touches.length !== 1) {
            return;
        }
        ev.preventDefault();
        mouseOrTouchDown(touches.item(0).pageX, touches.item(0).pageY, true);
    }

    function mouseOrTouchDown(downX, downY, touch) {
        if (touch === undefined) { touch = false; }
        lastX = downX;
        lastY = downY;
        if (touch) {
            document.ontouchmove = handleTouchMove;
            document.addEventListener("touchend", function(ev) {
                document.ontouchmove = null;
            });
            document.addEventListener("touchcancel", function(ev) {
                document.removeEventListener("touchmove", handleTouchMove);
            });
        } else {
            document.addEventListener("mousemove", handleMouseMove);
            document.addEventListener("mouseup", function(ev) {
                document.removeEventListener("mousemove", handleMouseMove);
            });
        }
    }

    function handleMouseMove(ev) {
        ev.preventDefault();
        mouseOrTouchMove(ev.pageX, ev.pageY);
    }

    function handleTouchMove(ev) {
        var touches = ev.touches;
        if (touches.length !== 1) {
            return;
        }
        ev.preventDefault();
        mouseOrTouchMove(touches.item(0).pageX, touches.item(0).pageY);
    }

    function mouseOrTouchMove(x, y) {
        var dx = lastX - x, dy = y - lastY; /* change in {x, y} */

        phi -= dx / 100;
        if (phi > 2 * PI) {
            phi -= 2 * PI;
        } else if (phi < 0) {
            phi += 2 * PI;
        }

        if (phi < PI / 2 || phi > 3 * PI / 2) {
            sign = -1;
        } else {
            sign = 1;
        }
        if (scene.rotation.z + sign * dy / 100 < -PI) {
            scene.rotation.z = -PI;
        } else if (scene.rotation.z + sign * dy / 100 > 0) {
            scene.rotation.z = 0;
        } else {
            scene.rotateZ(sign * dy / 100);
        }

        lastX = x;
        lastY = y;

        updateCamera();
        updateLabel();
    }

    function updateCamera() {
        var z = r * Math.sin(phi); /* new y pos (z-axis) in x-z plane */
        var x = r * Math.cos(phi); /* new x pos (x-axis) in x-z plane */
        camera.position.set(x, 1, z);
        camera.lookAt(c);
    }

body {
    overflow: hidden;
    margin: 0;
}

#label {
    position: absolute;
    left: 20px; top: 20px;
}

<div id="label">label</div>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/build/three.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/examples/js/controls/OrbitControls.js"></script>

three.js Vector3 to 2D screen coordinate with rotated scene

working with no scene rotation

not working with scene rotation

1 Answers