From 515c084639e54ac5e2590b950e3d3e4efbad82de Mon Sep 17 00:00:00 2001
From: Ike Saunders <ikesau@protonmail.com>
Date: Fri, 6 Dec 2024 18:14:31 -0500
Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20add=20hash=20table=20to=20images?=
 =?UTF-8?q?=20table?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 adminSiteServer/apiRouter.ts                  | 39 ++++++++++++++++---
 adminSiteServer/imagesHelpers.ts              |  4 ++
 .../1731360326761-CloudflareImages.ts         |  5 ++-
 .../types/src/dbTypes/Images.ts               |  3 +-
 4 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts
index c1f890ae41d..fa10ea2789f 100644
--- a/adminSiteServer/apiRouter.ts
+++ b/adminSiteServer/apiRouter.ts
@@ -3124,7 +3124,21 @@ postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => {
         }
     }
 
-    const { asBlob, dimensions } = await processImageContent(content, type)
+    const { asBlob, dimensions, hash } = await processImageContent(
+        content,
+        type
+    )
+
+    const collision = await trx<DbEnrichedImage>("images")
+        .where("hash", "=", hash)
+        .first()
+
+    if (collision) {
+        return {
+            success: false,
+            error: `An image with this content already exists (filename: ${collision.filename})`,
+        }
+    }
 
     const cloudflareId = await uploadToCloudflare(filename, asBlob)
 
@@ -3140,10 +3154,9 @@ postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => {
         originalWidth: dimensions.width,
         originalHeight: dimensions.height,
         cloudflareId,
-        // TODO: make defaultAlt nullable
-        defaultAlt: "Default alt text",
         updatedAt: new Date().getTime(),
         userId: res.locals.user.id,
+        hash,
     })
 
     const image = await db.getCloudflareImage(trx, filename)
@@ -3180,10 +3193,23 @@ putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => {
         )
     }
 
-    await deleteFromCloudflare(originalCloudflareId)
-
     const { type, content } = validateImagePayload(req.body)
-    const { asBlob, dimensions } = await processImageContent(content, type)
+    const { asBlob, dimensions, hash } = await processImageContent(
+        content,
+        type
+    )
+    const collision = await trx<DbEnrichedImage>("images")
+        .where("hash", "=", hash)
+        .first()
+
+    if (collision) {
+        return {
+            success: false,
+            error: `An image with this content already exists (filename: ${collision.filename})`,
+        }
+    }
+
+    await deleteFromCloudflare(originalCloudflareId)
     const newCloudflareId = await uploadToCloudflare(originalFilename, asBlob)
 
     if (!newCloudflareId) {
@@ -3194,6 +3220,7 @@ putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => {
         originalWidth: dimensions.width,
         originalHeight: dimensions.height,
         updatedAt: new Date().getTime(),
+        hash,
     })
 
     const updated = await db.getCloudflareImage(trx, originalFilename)
diff --git a/adminSiteServer/imagesHelpers.ts b/adminSiteServer/imagesHelpers.ts
index d6ab707121c..d2e90bea7e6 100644
--- a/adminSiteServer/imagesHelpers.ts
+++ b/adminSiteServer/imagesHelpers.ts
@@ -1,3 +1,4 @@
+import crypto from "crypto"
 import { JsonError } from "@ourworldindata/types"
 import sharp from "sharp"
 import {
@@ -32,9 +33,11 @@ export async function processImageContent(
 ): Promise<{
     asBlob: Blob
     dimensions: { width: number; height: number }
+    hash: string
 }> {
     const stripped = content.slice(content.indexOf(",") + 1)
     const asBuffer = Buffer.from(stripped, "base64")
+    const hash = crypto.createHash("sha256").update(asBuffer).digest("hex")
     const asBlob = new Blob([asBuffer], { type })
     const { width, height } = await sharp(asBuffer)
         .metadata()
@@ -50,6 +53,7 @@ export async function processImageContent(
             width,
             height,
         },
+        hash,
     }
 }
 
diff --git a/db/migration/1731360326761-CloudflareImages.ts b/db/migration/1731360326761-CloudflareImages.ts
index 1a1adb47530..fc93ecd61e5 100644
--- a/db/migration/1731360326761-CloudflareImages.ts
+++ b/db/migration/1731360326761-CloudflareImages.ts
@@ -5,6 +5,8 @@ export class CloudflareImages1731360326761 implements MigrationInterface {
         await queryRunner.query(`-- sql
             ALTER TABLE images
             ADD COLUMN cloudflareId VARCHAR(255) NULL,
+            ADD CONSTRAINT images_cloudflareId_unique UNIQUE (cloudflareId),
+            ADD COLUMN hash VARCHAR(255) NULL,
             MODIFY COLUMN googleId VARCHAR(255) NULL,
             MODIFY COLUMN defaultAlt VARCHAR(1600) NULL;`)
 
@@ -19,7 +21,8 @@ export class CloudflareImages1731360326761 implements MigrationInterface {
     public async down(queryRunner: QueryRunner): Promise<void> {
         await queryRunner.query(`-- sql
             ALTER TABLE images
-            DROP COLUMN cloudflareId
+            DROP COLUMN cloudflareId,
+            DROP COLUMN hash
         `)
 
         await queryRunner.query(`-- sql
diff --git a/packages/@ourworldindata/types/src/dbTypes/Images.ts b/packages/@ourworldindata/types/src/dbTypes/Images.ts
index e60157f9983..a17431a5a44 100644
--- a/packages/@ourworldindata/types/src/dbTypes/Images.ts
+++ b/packages/@ourworldindata/types/src/dbTypes/Images.ts
@@ -2,7 +2,7 @@ import { DbPlainUser } from "./Users.js"
 
 export const ImagesTableName = "images"
 export interface DbInsertImage {
-    googleId: string
+    googleId: string | null
     defaultAlt: string
     filename: string
     id?: number
@@ -10,6 +10,7 @@ export interface DbInsertImage {
     originalHeight?: number | null
     updatedAt?: string | null // MySQL Date objects round to the nearest second, whereas Google includes milliseconds so we store as an epoch of type bigint to avoid any conversion issues
     cloudflareId?: string | null
+    hash?: string | null
     userId?: number | null
 }
 export type DbRawImage = Required<DbInsertImage>