diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index fc1b4a8820..bb21dc7e19 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -24,10 +24,10 @@ import { cropWithRotation, fetchImageBitmap, getLocalFileImageBitmap, - getPixelBilinear, getThumbnailImageBitmap, imageBitmapToBlob, normalizePixelBetween0And1, + pixelRGBBilinear, warpAffineFloat32List, } from "./image"; import { transformFaceDetections } from "./transform-box"; @@ -204,7 +204,7 @@ const convertToYOLOInputFloat32ChannelsFirst = (imageBitmap: ImageBitmap) => { const { r, g, b } = w >= scaledWidth || h >= scaledHeight ? { r: 114, g: 114, b: 114 } - : getPixelBilinear( + : pixelRGBBilinear( w / scale, h / scale, pixelData, diff --git a/web/apps/photos/src/services/face/image.ts b/web/apps/photos/src/services/face/image.ts index c52b16975f..957b938b0a 100644 --- a/web/apps/photos/src/services/face/image.ts +++ b/web/apps/photos/src/services/face/image.ts @@ -65,33 +65,18 @@ export function unnormalizePixelFromBetweenMinus1And1(pixelValue: number) { return clamp(Math.round((pixelValue + 1.0) * 127.5), 0, 255); } -export function readPixelColor( - imageData: Uint8ClampedArray, - width: number, - height: number, - x: number, - y: number, -) { - if (x < 0 || x >= width || y < 0 || y >= height) { - return { r: 0, g: 0, b: 0, a: 0 }; - } - const index = (y * width + x) * 4; - return { - r: imageData[index], - g: imageData[index + 1], - b: imageData[index + 2], - a: imageData[index + 3], - }; -} - -export function getPixelBicubic( +/** + * Returns the pixel value (RGB) at the given coordinates ({@link fx}, + * {@link fy}) using bicubic interpolation. + */ +export function pixelRGBBicubic( fx: number, fy: number, imageData: Uint8ClampedArray, imageWidth: number, imageHeight: number, ) { - // Clamp to image boundaries + // Clamp to image boundaries. fx = clamp(fx, 0, imageWidth - 1); fy = clamp(fy, 0, imageHeight - 1); @@ -106,40 +91,35 @@ export function getPixelBicubic( const dx = fx - x; const dy = fy - y; - function cubic( + const cubic = ( dx: number, ipp: number, icp: number, inp: number, iap: number, - ) { - return ( - icp + - 0.5 * - (dx * (-ipp + inp) + - dx * dx * (2 * ipp - 5 * icp + 4 * inp - iap) + - dx * dx * dx * (-ipp + 3 * icp - 3 * inp + iap)) - ); - } + ) => + icp + + 0.5 * + (dx * (-ipp + inp) + + dx * dx * (2 * ipp - 5 * icp + 4 * inp - iap) + + dx * dx * dx * (-ipp + 3 * icp - 3 * inp + iap)); - const icc = readPixelColor(imageData, imageWidth, imageHeight, x, y); + const icc = pixelRGBA(imageData, imageWidth, imageHeight, x, y); const ipp = px < 0 || py < 0 ? icc - : readPixelColor(imageData, imageWidth, imageHeight, px, py); + : pixelRGBA(imageData, imageWidth, imageHeight, px, py); const icp = - px < 0 - ? icc - : readPixelColor(imageData, imageWidth, imageHeight, x, py); + px < 0 ? icc : pixelRGBA(imageData, imageWidth, imageHeight, x, py); const inp = py < 0 || nx >= imageWidth ? icc - : readPixelColor(imageData, imageWidth, imageHeight, nx, py); + : pixelRGBA(imageData, imageWidth, imageHeight, nx, py); const iap = ax >= imageWidth || py < 0 ? icc - : readPixelColor(imageData, imageWidth, imageHeight, ax, py); + : pixelRGBA(imageData, imageWidth, imageHeight, ax, py); const ip0 = cubic(dx, ipp.r, icp.r, inp.r, iap.r); const ip1 = cubic(dx, ipp.g, icp.g, inp.g, iap.g); @@ -147,17 +127,15 @@ export function getPixelBicubic( // const ip3 = cubic(dx, ipp.a, icp.a, inp.a, iap.a); const ipc = - px < 0 - ? icc - : readPixelColor(imageData, imageWidth, imageHeight, px, y); + px < 0 ? icc : pixelRGBA(imageData, imageWidth, imageHeight, px, y); const inc = nx >= imageWidth ? icc - : readPixelColor(imageData, imageWidth, imageHeight, nx, y); + : pixelRGBA(imageData, imageWidth, imageHeight, nx, y); const iac = ax >= imageWidth ? icc - : readPixelColor(imageData, imageWidth, imageHeight, ax, y); + : pixelRGBA(imageData, imageWidth, imageHeight, ax, y); const ic0 = cubic(dx, ipc.r, icc.r, inc.r, iac.r); const ic1 = cubic(dx, ipc.g, icc.g, inc.g, iac.g); @@ -167,19 +145,19 @@ export function getPixelBicubic( const ipn = px < 0 || ny >= imageHeight ? icc - : readPixelColor(imageData, imageWidth, imageHeight, px, ny); + : pixelRGBA(imageData, imageWidth, imageHeight, px, ny); const icn = ny >= imageHeight ? icc - : readPixelColor(imageData, imageWidth, imageHeight, x, ny); + : pixelRGBA(imageData, imageWidth, imageHeight, x, ny); const inn = nx >= imageWidth || ny >= imageHeight ? icc - : readPixelColor(imageData, imageWidth, imageHeight, nx, ny); + : pixelRGBA(imageData, imageWidth, imageHeight, nx, ny); const ian = ax >= imageWidth || ny >= imageHeight ? icc - : readPixelColor(imageData, imageWidth, imageHeight, ax, ny); + : pixelRGBA(imageData, imageWidth, imageHeight, ax, ny); const in0 = cubic(dx, ipn.r, icn.r, inn.r, ian.r); const in1 = cubic(dx, ipn.g, icn.g, inn.g, ian.g); @@ -189,19 +167,19 @@ export function getPixelBicubic( const ipa = px < 0 || ay >= imageHeight ? icc - : readPixelColor(imageData, imageWidth, imageHeight, px, ay); + : pixelRGBA(imageData, imageWidth, imageHeight, px, ay); const ica = ay >= imageHeight ? icc - : readPixelColor(imageData, imageWidth, imageHeight, x, ay); + : pixelRGBA(imageData, imageWidth, imageHeight, x, ay); const ina = nx >= imageWidth || ay >= imageHeight ? icc - : readPixelColor(imageData, imageWidth, imageHeight, nx, ay); + : pixelRGBA(imageData, imageWidth, imageHeight, nx, ay); const iaa = ax >= imageWidth || ay >= imageHeight ? icc - : readPixelColor(imageData, imageWidth, imageHeight, ax, ay); + : pixelRGBA(imageData, imageWidth, imageHeight, ax, ay); const ia0 = cubic(dx, ipa.r, ica.r, ina.r, iaa.r); const ia1 = cubic(dx, ipa.g, ica.g, ina.g, iaa.g); @@ -216,19 +194,41 @@ export function getPixelBicubic( return { r: c0, g: c1, b: c2 }; } -/// Returns the pixel value (RGB) at the given coordinates using bilinear interpolation. -export function getPixelBilinear( +const pixelRGBA = ( + imageData: Uint8ClampedArray, + width: number, + height: number, + x: number, + y: number, +) => { + if (x < 0 || x >= width || y < 0 || y >= height) { + return { r: 0, g: 0, b: 0, a: 0 }; + } + const index = (y * width + x) * 4; + return { + r: imageData[index], + g: imageData[index + 1], + b: imageData[index + 2], + a: imageData[index + 3], + }; +}; + +/** + * Returns the pixel value (RGB) at the given coordinates ({@link fx}, + * {@link fy}) using bilinear interpolation. + */ +export function pixelRGBBilinear( fx: number, fy: number, imageData: Uint8ClampedArray, imageWidth: number, imageHeight: number, ) { - // Clamp to image boundaries + // Clamp to image boundaries. fx = clamp(fx, 0, imageWidth - 1); fy = clamp(fy, 0, imageHeight - 1); - // Get the surrounding coordinates and their weights + // Get the surrounding coordinates and their weights. const x0 = Math.floor(fx); const x1 = Math.ceil(fx); const y0 = Math.floor(fy); @@ -239,10 +239,10 @@ export function getPixelBilinear( const dy1 = 1.0 - dy; // Get the original pixels - const pixel1 = readPixelColor(imageData, imageWidth, imageHeight, x0, y0); - const pixel2 = readPixelColor(imageData, imageWidth, imageHeight, x1, y0); - const pixel3 = readPixelColor(imageData, imageWidth, imageHeight, x0, y1); - const pixel4 = readPixelColor(imageData, imageWidth, imageHeight, x1, y1); + const pixel1 = pixelRGBA(imageData, imageWidth, imageHeight, x0, y0); + const pixel2 = pixelRGBA(imageData, imageWidth, imageHeight, x1, y0); + const pixel3 = pixelRGBA(imageData, imageWidth, imageHeight, x0, y1); + const pixel4 = pixelRGBA(imageData, imageWidth, imageHeight, x1, y1); function bilinear(val1: number, val2: number, val3: number, val4: number) { return Math.round( @@ -268,7 +268,7 @@ export function warpAffineFloat32List( inputData: Float32Array, inputStartIndex: number, ): void { - // Get the pixel data + // Get the pixel data. const offscreenCanvas = new OffscreenCanvas( imageBitmap.width, imageBitmap.height, @@ -308,8 +308,8 @@ export function warpAffineFloat32List( const yOrigin = a10Prime * (xTrans - b00) + a11Prime * (yTrans - b10); - // Get the pixel from interpolation - const pixel = getPixelBicubic( + // Get the pixel RGB using bicubic interpolation. + const { r, g, b } = pixelRGBBicubic( xOrigin, yOrigin, pixelData, @@ -320,11 +320,11 @@ export function warpAffineFloat32List( // Set the pixel in the input data const index = (yTrans * faceSize + xTrans) * 3; inputData[inputStartIndex + index] = - normalizePixelBetweenMinus1And1(pixel.r); + normalizePixelBetweenMinus1And1(r); inputData[inputStartIndex + index + 1] = - normalizePixelBetweenMinus1And1(pixel.g); + normalizePixelBetweenMinus1And1(g); inputData[inputStartIndex + index + 2] = - normalizePixelBetweenMinus1And1(pixel.b); + normalizePixelBetweenMinus1And1(b); } } }