path: root/public/bluebey-studio/src/face.js

import * as THREE from 'three';
import { drawEyes, drawMouth, drawHeadCoverWrap, canvasLayout } from './faceArt.js';

// The hair plate's texture is drawn at this multiple of the plate's natural UV
// size. The hairline crosses the head over only a fraction of that height, so at
// 1x its edge comes out visibly jagged; doubling it (with the finer curve
// subdivision in `drawHeadCoverWrap`) keeps the edge smooth.
const HAIR_SCALE = 2;

/**
 * Owns the two procedural textures (eyes and mouth) and keeps them in sync with
 * the face parameters.
 *
 * The artwork is carried by the model's face plates, but those plates are shells
 * of the *whole* front half of the body, so their UVs cover far more than the
 * 0..1 window the drawing lives in. `remapPlate` rewrites each plate's UVs onto
 * 0..1 and the canvas is enlarged to match, which is what removes the
 * clamp-to-edge smear at the border and what gives a moved tear, brow or mouth
 * room to move (see `canvasLayout` in src/faceArt.js).
 *
 * Both parts can independently fall back to the original hand-drawn texture
 * that ships inside the GLB.
 */
export class Face {
  constructor({ eyeMesh, mouthMesh, hairMesh = null, originals }) {
    this.eyeMesh = eyeMesh;
    this.mouthMesh = mouthMesh;
    // 頭の模様 can be carried by a plate that wraps right round the head (the
    // `hair-plate`), so hair is visible from behind as well. Older models have no
    // such plate and draw the covering on the front one instead.
    this.hairMesh = hairMesh;

    // Rewrite each plate's UVs so its own range fills 0..1, and work out the
    // canvas that keeps the artwork exactly where the 2022 textures put it.
    this.eyeLayout = remapPlate(eyeMesh);
    this.mouthLayout = remapPlate(mouthMesh);
    this.hairLayout = hairMesh ? remapPlate(hairMesh) : null;

    // The hand-drawn eye textures bake the face colour into their background,
    // which is invisible in the lit "real" style but shows up as a flat purple
    // patch in the flat and line-art styles. Key it out once, here. Both kinds
    // of original are window-sized artwork, so they belong *in* the window, not
    // stretched across the enlarged canvas.
    this.originals = {
      eyes: Object.fromEntries(
        Object.entries(originals.eyes).map(([key, texture]) => [key, keyOutBackground(texture, this.eyeLayout)]),
      ),
      mouth: placeInWindowTexture(originals.mouth, this.mouthLayout),
    };

    this.customNames = new Map();
    // Drawings supplied for a beard kind (assets/beards/<shape>.png). Loaded by the
    // app; a kind without one falls back to the built-in drawing (see drawBeard).
    this.beardImages = {};
    // Drawings for the brows (assets/brows/<kind>.png), loaded the same way.
    this.browImages = {};
    this.eyeCanvas = makeCanvas(this.eyeLayout);
    this.mouthCanvas = makeCanvas(this.mouthLayout);
    this.hairCanvas = this.hairLayout ? makeCanvas(this.hairLayout, HAIR_SCALE) : null;
    this.eyeCtx = this.eyeCanvas.getContext('2d');
    this.mouthCtx = this.mouthCanvas.getContext('2d');
    this.hairCtx = this.hairCanvas ? this.hairCanvas.getContext('2d') : null;

    this.eyeTexture = makeTexture(this.eyeCanvas);
    this.mouthTexture = makeTexture(this.mouthCanvas);
    this.hairTexture = this.hairCanvas ? makeTexture(this.hairCanvas) : null;

    this.params = null;
    this.styleMode = 'paint';
    this.eyeSource = 'parametric';
    this.mouthSource = 'parametric';
    this.dirty = true;
    this.lastDraw = -Infinity;

    // The face plates are shells of the body surface, pushed a hair outwards by
    // the loader, so they win the depth test against the body by themselves.
    // Nudging them with a polygon offset as well keeps the two from z-fighting
    // along grazing angles. Depth *testing* stays on: that is what lets the nose
    // (which pokes further out) and an arm waved in front of the face hide the
    // artwork the way they should - it only ever hid the old planes because
    // those did not reach far enough down the head.
    const orders = new Map([[eyeMesh, 2], [mouthMesh, 1]]);
    if (hairMesh) orders.set(hairMesh, 3);
    for (const mesh of [eyeMesh, mouthMesh, ...(hairMesh ? [hairMesh] : [])]) {
      const material = mesh.material;
      material.polygonOffset = true;
      material.polygonOffsetFactor = -1;
      material.polygonOffsetUnits = -2;
      material.depthTest = true;
      material.depthWrite = false;
      // The GLB may describe the plate material as opaque; the artwork is a
      // texture with alpha, so blend instead of replacing what is behind it.
      material.transparent = true;
      material.side = THREE.DoubleSide;
      material.needsUpdate = true;
      // Tears are drawn on the eye plate and the mouth on the other one, and
      // both sit on the same shell: draw the mouth first so a teardrop can fall
      // across it instead of being painted over.
      mesh.renderOrder = orders.get(mesh) ?? 0;
    }

    this.applyTextures();
  }

  /** Show or hide the mouth without touching the artwork. */
  setMouthVisible(visible) {
    if (this.mouthMesh.visible !== visible) this.mouthMesh.visible = visible;
  }

  applyTextures() {
    const eyeMap = this.eyeSource === 'parametric'
      ? this.eyeTexture
      : (this.originals.eyes[this.eyeSource] ?? this.eyeTexture);
    const mouthMap = this.mouthSource === 'parametric'
      ? this.mouthTexture
      : (this.originals.mouth ?? this.mouthTexture);

    if (this.eyeMesh.material.map !== eyeMap) {
      this.eyeMesh.material.map = eyeMap;
      this.eyeMesh.material.needsUpdate = true;
    }
    if (this.mouthMesh.material.map !== mouthMap) {
      this.mouthMesh.material.map = mouthMap;
      this.mouthMesh.material.needsUpdate = true;
    }
    if (this.hairMesh && this.hairMesh.material.map !== this.hairTexture) {
      this.hairMesh.material.map = this.hairTexture;
      this.hairMesh.material.needsUpdate = true;
    }
  }

  /**
   * Load a hand-drawn image and use it for the eyes or the mouth. Any size is
   * accepted; it is scaled to the layout the model expects (1024 x 380).
   * Returns the key the new artwork is registered under.
   */
  async loadImage(kind, file) {
    const layout = kind === 'eyes' ? this.eyeLayout : this.mouthLayout;
    const bitmap = await createImageBitmap(file);
    // Any size is accepted; it is scaled into the artwork window, which is where
    // the plate's rewritten UVs expect it.
    const canvas = placeInWindow(bitmap, layout);
    if (typeof bitmap.close === 'function') bitmap.close();
    const texture = makeTexture(canvas);

    if (kind === 'eyes') {
      const key = `custom-${this.customNames.size + 1}`;
      this.customNames.set(key, file.name);
      this.originals.eyes[key] = texture;
      this.setSource('eyes', key);
      return key;
    }
    this.originals.mouth = texture;
    this.setSource('mouth', 'original');
    return 'original';
  }

  /**
   * Set the drawings used for the beards, keyed by beard kind (`scotch`, `kaiser`,
   * `apron`, `cat`). Called by the app once the files are loaded; the next redraw
   * picks them up.
   */
  setBeardImages(map) {
    this.beardImages = map ?? {};
    this.dirty = true;
  }

  /**
   * Set the drawings used for the brows, keyed by brow kind (`gol`). Called by the
   * app once the files are loaded; the next redraw picks them up.
   */
  setBrowImages(map) {
    this.browImages = map ?? {};
    this.dirty = true;
  }

  /** `kind` is `'eyes'` or `'mouth'`; `source` is `'parametric'` or a variant key. */
  setSource(kind, source) {
    if (kind === 'eyes') {
      if (this.eyeSource === source) return;
      this.eyeSource = source;
    } else {
      if (this.mouthSource === source) return;
      this.mouthSource = source;
    }
    this.applyTextures();
    this.dirty = true;
  }

  setParams(params, styleMode = this.styleMode) {
    this.params = params;
    this.styleMode = styleMode;
    this.dirty = true;
  }

  /**
   * Redraw the procedural textures when they are stale. `minInterval` throttles
   * redraws while animating; pass 0 before an export so nothing is left pending.
   */
  flush(now = performance.now(), minInterval = 0) {
    if (!this.dirty) return false;
    if (now - this.lastDraw < minInterval) return false;
    this.redraw();
    this.lastDraw = now;
    return true;
  }

  redraw() {
    const p = this.params;
    if (!p) {
      this.dirty = false;
      return;
    }
    // Hiding the mouth has to hide the plane itself, not just stop drawing on
    // it - otherwise the last drawing stays on screen.
    this.setMouthVisible(p.mouth.visible !== false);
    const mode = this.styleMode === 'line' ? 'line' : 'paint';

    if (this.eyeSource === 'parametric') {
      const layout = this.eyeLayout;
      const ctx = this.eyeCtx;
      ctx.clearRect(0, 0, layout.width, layout.height);
      ctx.save();
      // The artwork window sits at an offset inside the bigger canvas.
      ctx.translate(layout.offsetX, layout.offsetY);
      drawEyes(ctx, {
        mode,
        eyes: { left: p.eyes.left, right: p.eyes.right },
        style: {
          white: p.eyes.white,
          iris: p.eyes.iris,
          line: p.eyes.line,
          irisScale: p.eyes.irisScale,
          lookMax: p.eyes.lookMax,
          highlight: p.eyes.highlight,
          lidWidth: p.eyes.lidWidth,
          lowerLid: p.eyes.lowerLid,
          lidShape: p.eyes.lidShape,
          lidTilt: p.eyes.lidTilt,
          lashes: p.eyes.lashes,
          brow: p.eyes.brow,
          // ほっぺ and 頭の模様 are shared by the whole face, so they travel with
          // the other shared eye fields (see `drawEyes` in src/faceArt.js).
          cheeks: p.eyes.cheeks,
          // When the model has a wrapping hair plate the covering is drawn there
          // instead (below), so it is switched off on the front plate to keep it
          // from being drawn twice.
          headMark: this.hairMesh ? { shape: 'off' } : p.eyes.headMark,
          // ひげ is shared by the whole face as well. (鼻ちょうちん is a real 3D
          // object hung off the nose - see src/main.js - so it is not drawn here.)
          beard: p.eyes.beard,
          beards: this.beardImages,
          brows: this.browImages,
          // 眼鏡 / サングラス are shared by both eyes and drawn into this same
          // texture, so they travel with the other shared eye fields.
          glasses: p.eyes.glasses,
          heartScale: p.eyes.heartScale,
          heartColor: p.eyes.heartColor,
          tearColor: p.eyes.tearColor,
          // Where the artwork's real edges are, so a brow lifted too far or a
          // tear dropped too low can stop inside the artwork (see `canvasLayout`).
          limits: layout.limits,
        },
      });
      ctx.restore();
      this.eyeTexture.needsUpdate = true;
    }

    if (this.mouthSource === 'parametric' && p.mouth.visible) {
      const layout = this.mouthLayout;
      const ctx = this.mouthCtx;
      ctx.clearRect(0, 0, layout.width, layout.height);
      ctx.save();
      ctx.translate(layout.offsetX, layout.offsetY);
      drawMouth(ctx, { mode, ...p.mouth, limits: layout.limits });
      ctx.restore();
      this.mouthTexture.needsUpdate = true;
    }

    if (this.hairMesh) {
      const hairShape = p.eyes.headMark?.shape ?? 'off';
      // Shapes that draw nothing (the retired front-plate ones, or なし). A fully
      // transparent texture still tinted the head on iOS, so the plate is hidden
      // outright when there is no hair rather than left showing an empty texture.
      const hairShown = !(hairShape === 'off' || hairShape === 'none'
        || hairShape === 'split' || hairShape === 'wave' || hairShape === 'side');
      this.hairMesh.visible = hairShown;
      if (hairShown) {
        // `overEyes` decides the draw order: the hair plate sits in front of the
        // eyes (bangs over the face) or behind them.
        this.hairMesh.renderOrder = p.eyes.headMark?.overEyes === false ? 0 : 3;
        // The colour lives on the material (the texture is a white mask), so it
        // follows the same colour management as the body on every platform.
        this.hairMesh.material.color.set(p.eyes.headMark?.color ?? '#8a4fe0');
        const layout = this.hairLayout;
        const ctx = this.hairCtx;
        ctx.setTransform(1, 0, 0, 1, 0, 0);
        ctx.clearRect(0, 0, layout.width * HAIR_SCALE, layout.height * HAIR_SCALE);
        ctx.scale(HAIR_SCALE, HAIR_SCALE);
        // `drawHeadCoverWrap` works in canvas coordinates (its x axis is the UV
        // angle already normalised), so it is not shifted by the window offset.
        drawHeadCoverWrap(ctx, p.eyes.headMark ?? {}, layout);
        ctx.setTransform(1, 0, 0, 1, 0, 0);
        this.hairTexture.needsUpdate = true;
      }
    }

    this.dirty = false;
  }
}

function makeCanvas({ width, height }, scale = 1) {
  const canvas = document.createElement('canvas');
  canvas.width = Math.round(width * scale);
  canvas.height = Math.round(height * scale);
  return canvas;
}

/**
 * Rewrite a plate's UVs so its own range maps onto 0..1, and return the canvas
 * that keeps the artwork at its original pixel size (`canvasLayout`).
 *
 * The plate covers the whole front half of the body, so its UVs used to run well
 * outside the artwork (v -1.02..1.93 on the eye plate). Everything past the edge
 * clamped to the canvas border, which is what stretched a tear or a brow that
 * reached it. After this the whole plate samples real canvas, so nothing clamps.
 */
function remapPlate(mesh) {
  const uv = mesh?.geometry?.attributes?.uv;
  if (!uv) return canvasLayout(null);
  let u0 = Infinity;
  let u1 = -Infinity;
  let v0 = Infinity;
  let v1 = -Infinity;
  for (let i = 0; i < uv.count; i += 1) {
    const u = uv.getX(i);
    const v = uv.getY(i);
    if (u < u0) u0 = u;
    if (u > u1) u1 = u;
    if (v < v0) v0 = v;
    if (v > v1) v1 = v;
  }
  const du = Math.max(1e-6, u1 - u0);
  const dv = Math.max(1e-6, v1 - v0);
  for (let i = 0; i < uv.count; i += 1) {
    uv.setXY(i, (uv.getX(i) - u0) / du, (uv.getY(i) - v0) / dv);
  }
  uv.needsUpdate = true;
  return canvasLayout({ u0, u1, v0, v1 });
}

/**
 * Draw a hand-drawn 1024 x 380 image where the artwork window now sits, on a
 * canvas as big as the plate's own UV range. The plates cover far more than the
 * window (see `canvasLayout`) and the UVs were rewritten to match, so the window
 * is exactly where that image belongs.
 */
function placeInWindow(source, layout) {
  const canvas = document.createElement('canvas');
  canvas.width = layout.width;
  canvas.height = layout.height;
  if (source) {
    canvas.getContext('2d').drawImage(
      source, layout.offsetX, layout.offsetY, layout.window.width, layout.window.height,
    );
  }
  return canvas;
}

/** `placeInWindow` for an existing texture (keeps `null` as `null`). */
function placeInWindowTexture(texture, layout) {
  const image = texture?.image;
  if (!image || !image.width || !image.height) return texture ?? null;
  return makeTexture(placeInWindow(image, layout));
}

function makeTexture(canvas) {
  const texture = new THREE.CanvasTexture(canvas);
  // glTF puts v = 0 at the top of the image and the loader uploads the model's
  // own textures with flipY = false; matching that keeps the artwork aligned
  // with the mesh UVs.
  texture.flipY = false;
  texture.colorSpace = THREE.SRGBColorSpace;
  texture.premultiplyAlpha = false;
  texture.needsUpdate = true;
  return clampToEdge(texture);
}

/**
 * The face plates are shells of the whole body, so their UVs run well past the
 * 0..1 of the artwork (a plate corner can sit at v = -1.26). glTF's default
 * sampler repeats, which would stamp the drawing back onto the model several
 * times - a frown flicked up onto the forehead, a smile's end onto its side.
 * Clamping sends everything outside the artwork to the blank edge of the canvas
 * instead, where nothing is drawn.
 */
function clampToEdge(texture) {
  if (!texture) return texture;
  texture.wrapS = THREE.ClampToEdgeWrapping;
  texture.wrapT = THREE.ClampToEdgeWrapping;
  texture.needsUpdate = true;
  return texture;
}

/**
 * Make the flat colour that fills the background of a hand-drawn texture
 * transparent. The edge is feathered rather than hard, so the anti-aliased
 * pixels along the artwork do not leave a pale fringe.
 */
function keyOutBackground(texture, layout) {
  const image = texture?.image;
  if (!image || !image.width || !image.height) return texture;

  const canvas = document.createElement('canvas');
  canvas.width = image.width;
  canvas.height = image.height;
  const ctx = canvas.getContext('2d', { willReadFrequently: true });
  ctx.drawImage(image, 0, 0);
  const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
  const pixels = imageData.data;

  // The background is uniform, so the top-left pixel is a reliable sample.
  const baseR = pixels[0];
  const baseG = pixels[1];
  const baseB = pixels[2];
  const fullyClear = 30;  // at or below this distance: transparent
  const fullySolid = 96;  // at or above this distance: untouched

  for (let i = 0; i < pixels.length; i += 4) {
    const distance = Math.abs(pixels[i] - baseR)
      + Math.abs(pixels[i + 1] - baseG)
      + Math.abs(pixels[i + 2] - baseB);
    if (distance <= fullyClear) {
      pixels[i + 3] = 0;
    } else if (distance < fullySolid) {
      const ratio = (distance - fullyClear) / (fullySolid - fullyClear);
      pixels[i + 3] = Math.round(pixels[i + 3] * ratio);
    }
  }
  ctx.putImageData(imageData, 0, 0);

  return makeTexture(placeInWindow(canvas, layout));
}