feat(Db): Use SnowflakeId for previews

Allow to get an id for the storing the preview on disk before inserting
the preview on the DB.

Signed-off-by: Carl Schwan <[email protected]>

Author: CarlSchwan

core/BackgroundJobs/MovePreviewJob.php

@@ -26,6 +26,7 @@
 use OCP\IAppConfig;
 use OCP\IConfig;
 use OCP\IDBConnection;
+use OCP\Snowflake\IGenerator;
 use Override;
 use Psr\Log\LoggerInterface;
 
@@ -44,6 +45,7 @@ public function __construct(
 		private readonly IMimeTypeDetector $mimeTypeDetector,
 		private readonly IMimeTypeLoader $mimeTypeLoader,
 		private readonly LoggerInterface $logger,
+		private readonly IGenerator $generator,
 		IAppDataFactory $appDataFactory,
 	) {
 		parent::__construct($time);
@@ -136,6 +138,7 @@ private function processPreviews(int $fileId, bool $flatPath): void {
 			$path = $fileId . '/' . $previewFile->getName();
 			/** @var SimpleFile $previewFile */
 			$preview = Preview::fromPath($path, $this->mimeTypeDetector);
+			$preview->setId($this->generator->nextId());
 			if (!$preview) {
 				$this->logger->error('Unable to import old preview at path.');
 				continue;

core/Migrations/Version33000Date20251023110529.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+namespace OC\Core\Migrations;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\Migration\Attributes\ModifyColumn;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * Migrate away from auto-increment
+ */
+#[ModifyColumn(table: 'preview_locations', name: 'id', description: 'Remove auto-increment')]
+#[ModifyColumn(table: 'previews', name: 'id', description: 'Remove auto-increment')]
+#[ModifyColumn(table: 'preview_versions', name: 'id', description: 'Remove auto-increment')]
+class Version33000Date20251023110529 extends SimpleMigrationStep {
+	/**
+	 * @param Closure(): ISchemaWrapper $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+	 */
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		$schema = $schemaClosure();
+
+		if ($schema->hasTable('preview_locations')) {
+			$schema->dropAutoincrementColumn('preview_locations', 'id');
+		}
+
+		if ($schema->hasTable('preview_versions')) {
+			$schema->dropAutoincrementColumn('preview_versions', 'id');
+		}
+
+		if ($schema->hasTable('previews')) {
+			$schema->dropAutoincrementColumn('previews', 'id');
+		}
+
+		return $schema;
+	}
+}

core/Migrations/Version33000Date20251023120529.php

@@ -0,0 +1,86 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+namespace OC\Core\Migrations;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\IDBConnection;
+use OCP\Migration\Attributes\AddIndex;
+use OCP\Migration\Attributes\IndexType;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * Use unique index for preview_locations
+ */
+#[AddIndex(table: 'preview_locations', type: IndexType::UNIQUE)]
+class Version33000Date20251023120529 extends SimpleMigrationStep {
+	public function __construct(
+		private readonly IDBConnection $connection,
+	) {
+	}
+
+	/**
+	 * @param Closure(): ISchemaWrapper $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+	 */
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		if ($schema->hasTable('preview_locations')) {
+			$table = $schema->getTable('preview_locations');
+			$table->addUniqueIndex(['bucket_name', 'object_store_name'], 'unique_bucket_store');
+		}
+
+		return $schema;
+	}
+
+	public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options) {
+		// This shouldn't run on a production instance, only daily
+		$qb = $this->connection->getQueryBuilder();
+		$qb->select('*')
+			->from('preview_locations');
+		$result = $qb->executeQuery();
+
+		$set = [];
+
+		while ($row = $result->fetch()) {
+			// Iterate over all the rows with duplicated rows
+			$id = $row['id'];
+
+			if (isset($set[$row['bucket_name'] . '_' . $row['object_store_name']])) {
+				// duplicate
+				$authoritativeId = $set[$row['bucket_name'] . '_' . $row['object_store_name']];
+				$qb = $this->connection->getQueryBuilder();
+				$qb->select('id')
+					->from('preview_locations')
+					->where($qb->expr()->eq('bucket_name', $qb->createNamedParameter($row['bucket_name'])))
+					->andWhere($qb->expr()->eq('object_store_name', $qb->createNamedParameter($row['object_store_name'])))
+					->andWhere($qb->expr()->neq('id', $qb->createNamedParameter($authoritativeId)));
+
+				$result = $qb->executeQuery();
+				while ($row = $result->fetch()) {
+					// Update previews entries to the now de-duplicated id
+					$qb = $this->connection->getQueryBuilder();
+					$qb->update('previews')
+						->set('location_id', $qb->createNamedParameter($id))
+						->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])));
+					$qb->executeStatement();
+
+					$qb = $this->connection->getQueryBuilder();
+					$qb->delete('preview_locations')
+						->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])));
+					$qb->executeStatement();
+				}
+				break;
+			}
+			$set[$row['bucket_name'] . '_' . $row['object_store_name']] = $row['id'];
+		}
+	}
+}

lib/composer/composer/autoload_classmap.php

@@ -1530,6 +1530,8 @@
     'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php',
     'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php',
     'OC\\Core\\Migrations\\Version33000Date20250819110529' => $baseDir . '/core/Migrations/Version33000Date20250819110529.php',
+    'OC\\Core\\Migrations\\Version33000Date20251023110529' => $baseDir . '/core/Migrations/Version33000Date20251023110529.php',
+    'OC\\Core\\Migrations\\Version33000Date20251023120529' => $baseDir . '/core/Migrations/Version33000Date20251023120529.php',
     'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php',
     'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php',
     'OC\\Core\\Service\\CronService' => $baseDir . '/core/Service/CronService.php',

lib/composer/composer/autoload_static.php

@@ -1571,6 +1571,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php',
         'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php',
         'OC\\Core\\Migrations\\Version33000Date20250819110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110529.php',
+        'OC\\Core\\Migrations\\Version33000Date20251023110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20251023110529.php',
+        'OC\\Core\\Migrations\\Version33000Date20251023120529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20251023120529.php',
         'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php',
         'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php',
         'OC\\Core\\Service\\CronService' => __DIR__ . '/../../..' . '/core/Service/CronService.php',

lib/private/AppFramework/Http/Dispatcher.php

@@ -204,7 +204,7 @@ private function executeController(Controller $controller, string $methodName):
 		try {
 			$response = \call_user_func_array([$controller, $methodName], $arguments);
 		} catch (\TypeError $e) {
-			// Only intercept TypeErrors occuring on the first line, meaning that the invocation of the controller method failed.
+			// Only intercept TypeErrors occurring on the first line, meaning that the invocation of the controller method failed.
 			// Any other TypeError happens inside the controller method logic and should be logged as normal.
 			if ($e->getFile() === $this->reflector->getFile() && $e->getLine() === $this->reflector->getStartLine()) {
 				$this->logger->debug('Failed to call controller method: ' . $e->getMessage(), ['exception' => $e]);

lib/private/DB/SchemaWrapper.php

@@ -8,8 +8,11 @@
 
 use Doctrine\DBAL\Exception;
 use Doctrine\DBAL\Platforms\AbstractPlatform;
+use Doctrine\DBAL\Platforms\OraclePlatform;
 use Doctrine\DBAL\Schema\Schema;
 use OCP\DB\ISchemaWrapper;
+use OCP\Server;
+use Psr\Log\LoggerInterface;
 
 class SchemaWrapper implements ISchemaWrapper {
 	/** @var Connection */
@@ -131,4 +134,18 @@ public function getTables() {
 	public function getDatabasePlatform() {
 		return $this->connection->getDatabasePlatform();
 	}
+
+	public function dropAutoincrementColumn(string $table, string $column): void {
+		$tableObj = $this->schema->getTable($this->connection->getPrefix() . $table);
+		$tableObj->modifyColumn('id', ['autoincrement' => false]);
+		$platform = $this->getDatabasePlatform();
+		if ($platform instanceof OraclePlatform) {
+			try {
+				$this->connection->executeStatement('DROP TRIGGER "' . $this->connection->getPrefix() . $table . '_AI_PK"');
+				$this->connection->executeStatement('DROP SEQUENCE "' . $this->connection->getPrefix() . $table . '_SEQ"');
+			} catch (Exception $e) {
+				Server::get(LoggerInterface::class)->error($e->getMessage(), ['exception' => $e]);
+			}
+		}
+	}
 }

lib/private/Preview/Db/Preview.php

@@ -17,14 +17,16 @@
 /**
  * Preview entity mapped to the oc_previews and oc_preview_locations table.
  *
+ * @method string getId()
+ * @method void setId(string $id)
  * @method int getFileId() Get the file id of the original file.
  * @method void setFileId(int $fileId)
  * @method int getStorageId() Get the storage id of the original file.
  * @method void setStorageId(int $fileId)
  * @method int getOldFileId() Get the old location in the file-cache table, for legacy compatibility.
  * @method void setOldFileId(int $oldFileId)
- * @method int getLocationId() Get the location id in the preview_locations table. Only set when using an object store as primary storage.
- * @method void setLocationId(int $locationId)
+ * @method string getLocationId() Get the location id in the preview_locations table. Only set when using an object store as primary storage.
+ * @method void setLocationId(string $locationId)
  * @method string|null getBucketName() Get the bucket name where the preview is stored. This is stored in the preview_locations table.
  * @method string|null getObjectStoreName() Get the object store name where the preview is stored. This is stored in the preview_locations table.
  * @method int getWidth() Get the width of the preview.
@@ -46,7 +48,7 @@
  * @method string getEtag() Get the etag of the preview.
  * @method void setEtag(string $etag)
  * @method string|null getVersion() Get the version for files_versions_s3
- * @method void setVersionId(int $versionId)
+ * @method void setVersionId(string $versionId)
  * @method bool|null getIs() Get the version for files_versions_s3
  * @method bool isEncrypted() Get whether the preview is encrypted. At the moment every preview is unencrypted.
  * @method void setEncrypted(bool $encrypted)
@@ -57,7 +59,7 @@ class Preview extends Entity {
 	protected ?int $fileId = null;
 	protected ?int $oldFileId = null;
 	protected ?int $storageId = null;
-	protected ?int $locationId = null;
+	protected ?string $locationId = null;
 	protected ?string $bucketName = null;
 	protected ?string $objectStoreName = null;
 	protected ?int $width = null;
@@ -72,14 +74,15 @@ class Preview extends Entity {
 	protected ?bool $cropped = null;
 	protected ?string $etag = null;
 	protected ?string $version = null;
-	protected ?int $versionId = null;
+	protected ?string $versionId = null;
 	protected ?bool $encrypted = null;
 
 	public function __construct() {
+		$this->addType('id', Types::STRING);
 		$this->addType('fileId', Types::BIGINT);
 		$this->addType('storageId', Types::BIGINT);
 		$this->addType('oldFileId', Types::BIGINT);
-		$this->addType('locationId', Types::BIGINT);
+		$this->addType('locationId', Types::STRING);
 		$this->addType('width', Types::INTEGER);
 		$this->addType('height', Types::INTEGER);
 		$this->addType('mimetypeId', Types::INTEGER);

lib/private/Preview/Db/PreviewMapper.php

@@ -15,6 +15,7 @@
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\IMimeTypeLoader;
 use OCP\IDBConnection;
+use OCP\Snowflake\IGenerator;
 use Override;
 
 /**
@@ -29,6 +30,7 @@ class PreviewMapper extends QBMapper {
 	public function __construct(
 		IDBConnection $db,
 		private readonly IMimeTypeLoader $mimeTypeLoader,
+		private readonly IGenerator $snowflake,
 	) {
 		parent::__construct($db, self::TABLE_NAME, Preview::class);
 	}
@@ -50,13 +52,15 @@ public function insert(Entity $entity): Entity {
 
 		if ($preview->getVersion() !== null && $preview->getVersion() !== '') {
 			$qb = $this->db->getQueryBuilder();
+			$id = $this->snowflake->nextId();
 			$qb->insert(self::VERSION_TABLE_NAME)
 				->values([
+					'id' => $id,
 					'version' => $preview->getVersion(),
 					'file_id' => $preview->getFileId(),
 				])
 				->executeStatement();
-			$entity->setVersionId($qb->getLastInsertId());
+			$entity->setVersionId($id);
 		}
 		return parent::insert($preview);
 	}
@@ -148,7 +152,13 @@ protected function joinLocation(IQueryBuilder $qb): IQueryBuilder {
 			));
 	}
 
-	public function getLocationId(string $bucket, string $objectStore): int {
+	/**
+	 * Get the location id corresponding to the $bucket and $objectStore. Create one
+	 * if not existing yet.
+	 *
+	 * @throws Exception
+	 */
+	public function getLocationId(string $bucket, string $objectStore): string {
 		$qb = $this->db->getQueryBuilder();
 		$result = $qb->select('id')
 			->from(self::LOCATION_TABLE_NAME)
@@ -157,14 +167,33 @@ public function getLocationId(string $bucket, string $objectStore): int {
 			->executeQuery();
 		$data = $result->fetchOne();
 		if ($data) {
-			return $data;
+			return (string)$data;
 		} else {
-			$qb->insert(self::LOCATION_TABLE_NAME)
-				->values([
-					'bucket_name' => $qb->createNamedParameter($bucket),
-					'object_store_name' => $qb->createNamedParameter($objectStore),
-				])->executeStatement();
-			return $qb->getLastInsertId();
+			try {
+				$id = $this->snowflake->nextId();
+				$qb->insert(self::LOCATION_TABLE_NAME)
+					->values([
+						'id' => $qb->createNamedParameter($id),
+						'bucket_name' => $qb->createNamedParameter($bucket),
+						'object_store_name' => $qb->createNamedParameter($objectStore),
+					])->executeStatement();
+				return $id;
+			} catch (Exception $e) {
+				if ($e->getReason() === Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
+					// Fetch again as there seems to be another entry added meanwhile
+					$result = $qb->select('id')
+						->from(self::LOCATION_TABLE_NAME)
+						->where($qb->expr()->eq('bucket_name', $qb->createNamedParameter($bucket)))
+						->andWhere($qb->expr()->eq('object_store_name', $qb->createNamedParameter($objectStore)))
+						->executeQuery();
+					$data = $result->fetchOne();
+					if ($data) {
+						return (string)$data;
+					}
+				}
+
+				throw $e;
+			}
 		}
 	}