Three.js: Strategies to progressively increase resolution of texture

Question

I'm working on a Three.js chart that basically represents a bunch of images in a 2D plane.

Right now the individual images are each 32px by 32px segments of larger 2048px by 2048px image atlas files. I want to increase the size of those individual images when users zoom in to particular regions of the scene. For example, if users start to zoom in on the images in the far right region of the space, I plan to update the 32px by 32px individual images in that region with 64px by 64px images with the same content (to show more detail).

My question is: what's the Three.js way to accomplish this goal?

My flat-footed plan is to load the higher-resolution assets, map them to the proper geometry coordinates, then simply delete the old mesh with 32px subimages and add the new mesh with 64px subimages. I originally thought I could just update the texture/material for an extant geometry, but I've read that one shouldn't use textures larger than 2048px by 2048px, and a geometry with n points won't allow me to continually increase the fidelity of the images in that geometry without surpassing that maximum texture size.

I would be very grateful for any insight Three.js veterans can offer on how they would approach this task!

Full code:

/**
* Globals
**/

// Identify data endpoint
var dataUrl = 'https://s3.amazonaws.com/duhaime/blog/tsne-webgl/data/';

// Create global stores for image and atlas sizes
var image, atlas;

// Create a store for image position information
var imagePositions = null;

// Create a store for the load progress. Data structure:
// {atlas0: percentLoaded, atlas1: percentLoaded}
var loadProgress = {};

// Create a store for the image atlas materials. Data structure:
// {subImageSize: {atlas0: material, atlas1: material}}
var materials = {32: {}, 64: {}};

// Create a store for meshes
var meshes = [];

/**
* Create Scene
**/

// Create the scene and a camera to view it
var scene = new THREE.Scene();

/**
* Camera
**/

// Specify the portion of the scene visiable at any time (in degrees)
var fieldOfView = 75;

// Specify the camera's aspect ratio
var aspectRatio = window.innerWidth / window.innerHeight;

/*
Specify the near and far clipping planes. Only objects
between those planes will be rendered in the scene
(these values help control the number of items rendered
at any given time)
*/
var nearPlane = 100;
var farPlane = 50000;

// Use the values specified above to create a camera
var camera = new THREE.PerspectiveCamera(
  fieldOfView, aspectRatio, nearPlane, farPlane
);

// Finally, set the camera's position
camera.position.z = 12000;
camera.position.y = -2000;

/**
* Lights
**/

// Add a point light with #fff color, .7 intensity, and 0 distance
var light = new THREE.PointLight( 0xffffff, 1, 0 );

// Specify the light's position
light.position.set( 1, 1, 100 );

// Add the light to the scene
scene.add(light)

/**
* Renderer
**/

// Create the canvas with a renderer
var renderer = new THREE.WebGLRenderer({ antialias: true });

// Add support for retina displays
renderer.setPixelRatio( window.devicePixelRatio );

// Specify the size of the canvas
renderer.setSize( window.innerWidth, window.innerHeight );

// Add the canvas to the DOM
document.body.appendChild( renderer.domElement );

/**
* Load External Data
**/

// Load the image position JSON file
var fileLoader = new THREE.FileLoader();
var url = dataUrl + 'image_tsne_projections.json';
fileLoader.load(url, function(data) {
  imagePositions = JSON.parse(data);
  conditionallyBuildGeometries(32)
})

/**
* Load Atlas Textures
**/

// List of all textures to be loaded, the size of subimages
// in each, and the total count of atlas files for each size
var textureSets = {
  32: { size: 32, count: 5 },
  64: { size: 64, count: 20 }
}

// Create a texture loader so we can load our image files
var textureLoader = new AjaxTextureLoader();

function loadTextures(size, onProgress) {
  setImageAndAtlasSize(size)
  for (var i=0; i<textureSets[size].count; i++) {
    var url = dataUrl + 'atlas_files/' + size + 'px/atlas-' + i + '.jpg';
    if (onProgress) {
      textureLoader.load(url,
        handleTexture.bind(null, size, i),
        onProgress.bind(null, size, i));
    } else {
      textureLoader.load(url, handleTexture.bind(null, size, i));
    }
  }
}

function handleProgress(size, idx, xhr) {
  loadProgress[idx] = xhr.loaded / xhr.total;
  var sum = 0;
  Object.keys(loadProgress).forEach(function(k) { sum += loadProgress[k]; })
  var progress = sum/textureSets[size].count;
  var loader = document.querySelector('#loader');
  progress < 1
    ? loader.innerHTML = parseInt(progress * 100) + '%'
    : loader.style.display = 'none';
}

// Create a material from the new texture and call
// the geometry builder if all textures have loaded 
function handleTexture(size, idx, texture) {
  var material = new THREE.MeshBasicMaterial({ map: texture });
  materials[size][idx] = material;
  conditionallyBuildGeometries(size, idx)
}

// If the textures and the mapping from image idx to positional information
// are all loaded, create the geometries
function conditionallyBuildGeometries(size, idx) {
  if (size === 32) {
    var nLoaded = Object.keys(materials[size]).length;
    var nRequired = textureSets[size].count;
    if (nLoaded === nRequired && imagePositions) {  
      // Add the low-res textures and load the high-res textures
      buildGeometry(size);
      loadTextures(64)
    }
  } else {
    // Add the new high-res texture to the scene
    updateMesh(size, idx)
  }
}

loadTextures(32, handleProgress)

/**
* Build Image Geometry
**/

// Iterate over the textures in the current texture set
// and for each, add a new mesh to the scene
function buildGeometry(size) {
  for (var i=0; i<textureSets[size].count; i++) {
    // Create one new geometry per set of 1024 images
    var geometry = new THREE.Geometry();
    geometry.faceVertexUvs[0] = [];
    for (var j=0; j<atlas.cols*atlas.rows; j++) {
      var coords = getCoords(i, j);
      geometry = updateVertices(geometry, coords);
      geometry = updateFaces(geometry);
      geometry = updateFaceVertexUvs(geometry, j);
      if ((j+1)%1024 === 0) {
        var idx = (i*textureSets[size].count) + j;
        buildMesh(geometry, materials[size][i], idx);
        var geometry = new THREE.Geometry();
      }
    }
  }
}

// Get the x, y, z coords for the subimage at index position j
// of atlas in index position i
function getCoords(i, j) {
  var idx = (i * atlas.rows * atlas.cols) + j;
  var coords = imagePositions[idx];
  coords.x *= 2200;
  coords.y *= 1200;
  coords.z = (-200 + j/10);
  return coords;
}

// Add one vertex for each corner of the image, using the 
// following order: lower left, lower right, upper right, upper left
function updateVertices(geometry, coords) {
  // Retrieve the x, y, z coords for this subimage
  geometry.vertices.push(
    new THREE.Vector3(
      coords.x,
      coords.y,
      coords.z
    ),
    new THREE.Vector3(
      coords.x + image.shownWidth,
      coords.y,
      coords.z
    ),
    new THREE.Vector3(
      coords.x + image.shownWidth,
      coords.y + image.shownHeight,
      coords.z
    ),
    new THREE.Vector3(
      coords.x,
      coords.y + image.shownHeight,
      coords.z
    )
  );
  return geometry;
}

// Create two new faces for a given subimage, then add those
// faces to the geometry
function updateFaces(geometry) {
  // Add the first face (the lower-right triangle)
  var faceOne = new THREE.Face3(
    geometry.vertices.length-4,
    geometry.vertices.length-3,
    geometry.vertices.length-2
  )
  // Add the second face (the upper-left triangle)
  var faceTwo = new THREE.Face3(
    geometry.vertices.length-4,
    geometry.vertices.length-2,
    geometry.vertices.length-1
  )
  // Add those faces to the geometry
  geometry.faces.push(faceOne, faceTwo);
  return geometry;
}

function updateFaceVertexUvs(geometry, j) {  
  // Identify the relative width and height of the subimages
  // within the image atlas
  var relativeW = image.width / atlas.width;
  var relativeH = image.height / atlas.height;

  // Identify this subimage's offset in the x dimension
  // An xOffset of 0 means the subimage starts flush with
  // the left-hand edge of the atlas
  var xOffset = (j % atlas.cols) * relativeW;
  
  // Identify this subimage's offset in the y dimension
  // A yOffset of 0 means the subimage starts flush with
  // the bottom edge of the atlas
  var yOffset = 1 - (Math.floor(j/atlas.cols) * relativeH) - relativeH;

  // Determine the faceVertexUvs index position
  var faceIdx = 2 * (j%1024);

  // Use the xOffset and yOffset (and the knowledge that
  // each row and column contains only 32 images) to specify
  // the regions of the current image. Use .set() if the given
  // faceVertex is already defined, due to a bug in updateVertexUvs:
  // https://github.com/mrdoob/three.js/issues/7179
  if (geometry.faceVertexUvs[0][faceIdx]) {
    geometry.faceVertexUvs[0][faceIdx][0].set(xOffset, yOffset)
    geometry.faceVertexUvs[0][faceIdx][1].set(xOffset + relativeW, yOffset)
    geometry.faceVertexUvs[0][faceIdx][2].set(xOffset + relativeW, yOffset + relativeH)
  } else {
    geometry.faceVertexUvs[0][faceIdx] = [
      new THREE.Vector2(xOffset, yOffset),
      new THREE.Vector2(xOffset + relativeW, yOffset),
      new THREE.Vector2(xOffset + relativeW, yOffset + relativeH)
    ]
  }
  // Map the region of the image described by the lower-left, 
  // upper-right, and upper-left vertices to `faceTwo`
  if (geometry.faceVertexUvs[0][faceIdx+1]) {
    geometry.faceVertexUvs[0][faceIdx+1][0].set(xOffset, yOffset)
    geometry.faceVertexUvs[0][faceIdx+1][1].set(xOffset + relativeW, yOffset + relativeH)
    geometry.faceVertexUvs[0][faceIdx+1][2].set(xOffset, yOffset + relativeH)
  } else {
    geometry.faceVertexUvs[0][faceIdx+1] = [
      new THREE.Vector2(xOffset, yOffset),
      new THREE.Vector2(xOffset + relativeW, yOffset + relativeH),
      new THREE.Vector2(xOffset, yOffset + relativeH)
    ]
  }
  return geometry;
}

function buildMesh(geometry, material, idx) {
  // Convert the geometry to a BuferGeometry for additional performance
  //var geometry = new THREE.BufferGeometry().fromGeometry(geometry);
  // Combine the image geometry and material into a mesh
  var mesh = new THREE.Mesh(geometry, material);
  // Store this image's index position in the mesh
  mesh.userData.idx = idx;
  // Set the position of the image mesh in the x,y,z dimensions
  mesh.position.set(0,0,0)
  // Add the image to the scene
  scene.add(mesh);
  // Save this mesh
  meshes.push(mesh);
  return mesh;
}

/**
* Update Geometries with new VertexUvs and materials
**/

function updateMesh(size, idx) {
  // Update the appropriate material
  meshes[idx].material = materials[size][idx];
  meshes[idx].material.needsUpdate = true;
  // Update the facevertexuvs
  for (var j=0; j<atlas.cols*atlas.rows; j++) {
    meshes[idx].geometry = updateFaceVertexUvs(meshes[idx].geometry, j);
  }
  meshes[idx].geometry.uvsNeedUpdate = true;
  meshes[idx].geometry.verticesNeedUpdate = true;
}

/**
* Helpers
**/

function setImageAndAtlasSize(size) {
  // Identify the subimage size in px (width/height) and the
  // size of the image as it will be displayed in the map
  image = { width: size,  height: size, shownWidth: 64, shownHeight: 64 };
  
  // Identify the total number of cols & rows in the image atlas
  atlas = { width: 2048, height: 2048, cols: 2048/size, rows: 2048/size };
}

/**
* Add Controls
**/

var controls = new THREE.TrackballControls(camera, renderer.domElement);

/**
* Add Raycaster
**/

var raycaster = new THREE.Raycaster();
var mouse = new THREE.Vector2();

function onMouseMove( event ) {
  // Calculate mouse position in normalized device coordinates
  // (-1 to +1) for both components
  mouse.x = ( event.clientX / window.innerWidth ) * 2 - 1;
  mouse.y = - ( event.clientY / window.innerHeight ) * 2 + 1;
}

function onClick( event ) {
  // Determine which image is selected (if any)
  var selected = raycaster.intersectObjects( scene.children );
  // Intersecting elements are ordered by their distance (increasing)
  if (!selected) return;
  if (selected.length) {
    selected = selected[0];
    console.log('clicked', selected.object.userData.idx)
  }
}

window.addEventListener('mousemove', onMouseMove)
window.addEventListener('click', onClick)

/**
* Handle window resizes
**/

window.addEventListener('resize', function() {
  camera.aspect = window.innerWidth / window.innerHeight;
  camera.updateProjectionMatrix();
  renderer.setSize( window.innerWidth, window.innerHeight );
  controls.handleResize();
});

/**
* Render!
**/

// The main animation function that re-renders the scene each animation frame
function animate() {
requestAnimationFrame( animate );
  raycaster.setFromCamera( mouse, camera );
  renderer.render( scene, camera );
  controls.update();
}
animate();

* {
  margin: 0;
  padding: 0;
  background: #000;
  color: #fff;
}

<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/88/three.min.js"></script>
<script src="https://s3.amazonaws.com/duhaime/blog/tsne-webgl/assets/js/texture-loader.js"></script>
<script src="https://s3.amazonaws.com/duhaime/blog/tsne-webgl/assets/js/trackball-controls.js"></script>
<div id='loader'>0%</div>

Answer 1

You can potentially use mutli-materials and geometry groups (or in your case, material indices).

This depends on your texture dimensions scaling 1::1. In other words, if your first resolution as dimensions 32x64, then double that resolution should have dimensions of 64x128. UVs are percentage-based, so moving from an image at one resolution to the same image at another resolution "just works".

At this point, you really only need to change the texture image source. But it sounds like you don't want to do that. So instead, we need to assign ALL of your textures to the same Mesh at once. Three.js makes this really easy...

var myMesh = new THREE.Mesh(myGeometry, [ material1, material2, material3 ]);

Notice that the material parameters is defined as an array. Each material has a different texture, which in your case are the different resolution images.

Now, debug into your Mesh. Under the goemetry property, you'll see an a property called faces, which is an array of Face3 objects. Each face has a property named materialIndex. This is the face's reference to the array of materials.

When you reach a point where you want to trigger a change (such as your camera being a certain distance from a mesh), you can change the material index, then trigger the mesh to change its material:

var distance = camera.position.distanceTo(myMesh.position);
if(distance < 50){
  myMesh.faces.forEach(function(face){
    face.materialIndex = 2;
  });
}
else if(distance => 50 && distance < 100){
  myMesh.faces.forEach(function(face){
    face.materialIndex = 1;
  });
}
else{
  myMesh.faces.forEach(function(face){
    face.materialIndex = 0;
  });
}
myMesh.groupsNeedUpdate = true;

The last line (myMesh.groupsNeedUpdate = true;) tells the renderer that the material indices changed, so it will need to update the materials for the render.

Answer 2

Perhaps you could use THREE.LOD. It basically allows you to define different meshes for a range of distances. The meshes would be the same Quads, but you could change their materials to use different textures... Here is the LOD example in the THREE.js web.

Hope it helps!!