Merge pull request #44899 from nextcloud/backport/43574/stable29

[stable29] enh(metadata): migrate metadata to lazy appconfig

lib/private/Files/Cache/Cache.php

@@ -171,7 +171,7 @@ public function get($file) {
 		} elseif (!$data) {
 			return $data;
 		} else {
-			$data['metadata'] = $metadataQuery?->extractMetadata($data)->asArray() ?? [];
+			$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
 			return self::cacheEntryFromData($data, $this->mimetypeLoader);
 		}
 	}
@@ -243,7 +243,7 @@ public function getFolderContentsById($fileId) {
 			$result->closeCursor();
 
 			return array_map(function (array $data) use ($metadataQuery) {
-				$data['metadata'] = $metadataQuery?->extractMetadata($data)->asArray() ?? [];
+				$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
 				return self::cacheEntryFromData($data, $this->mimetypeLoader);
 			}, $files);
 		}

lib/private/Files/Cache/CacheQueryBuilder.php

@@ -138,11 +138,11 @@ public function whereParentInParameter(string $parameter) {
 	/**
 	 * join metadata to current query builder and returns an helper
 	 *
-	 * @return IMetadataQuery|null NULL if no metadata have never been generated
+	 * @return IMetadataQuery
 	 */
-	public function selectMetadata(): ?IMetadataQuery {
+	public function selectMetadata(): IMetadataQuery {
 		$metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid');
-		$metadataQuery?->retrieveMetadata();
+		$metadataQuery->retrieveMetadata();
 		return $metadataQuery;
 	}
 }

lib/private/Files/Cache/QuerySearchHelper.php

@@ -195,12 +195,7 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array
 		$files = $result->fetchAll();
 
 		$rawEntries = array_map(function (array $data) use ($metadataQuery) {
-			// migrate to null safe ...
-			if ($metadataQuery === null) {
-				$data['metadata'] = [];
-			} else {
-				$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
-			}
+			$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
 			return Cache::cacheEntryFromData($data, $this->mimetypeLoader);
 		}, $files);
 

lib/private/FilesMetadata/FilesMetadataManager.php

@@ -51,8 +51,7 @@
 use OCP\FilesMetadata\IMetadataQuery;
 use OCP\FilesMetadata\Model\IFilesMetadata;
 use OCP\FilesMetadata\Model\IMetadataValueWrapper;
-use OCP\IConfig;
-use OCP\IDBConnection;
+use OCP\IAppConfig;
 use Psr\Log\LoggerInterface;
 
 /**
@@ -69,7 +68,7 @@ class FilesMetadataManager implements IFilesMetadataManager {
 	public function __construct(
 		private IEventDispatcher $eventDispatcher,
 		private IJobList $jobList,
-		private IConfig $config,
+		private IAppConfig $appConfig,
 		private LoggerInterface $logger,
 		private MetadataRequestService $metadataRequestService,
 		private IndexRequestService $indexRequestService,
@@ -206,7 +205,7 @@ public function saveMetadata(IFilesMetadata $filesMetadata): void {
 		// update metadata types list
 		$current = $this->getKnownMetadata();
 		$current->import($filesMetadata->jsonSerialize(true));
-		$this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current));
+		$this->appConfig->setValueArray('core', self::CONFIG_KEY, $current->jsonSerialize(), lazy: true);
 	}
 
 	/**
@@ -235,20 +234,16 @@ public function deleteMetadata(int $fileId): void {
 	 * @param string $fileIdField alias of the field that contains file ids
 	 *
 	 * @inheritDoc
-	 * @return IMetadataQuery|null
+	 * @return IMetadataQuery
 	 * @see IMetadataQuery
 	 * @since 28.0.0
 	 */
 	public function getMetadataQuery(
 		IQueryBuilder $qb,
 		string $fileTableAlias,
 		string $fileIdField
-	): ?IMetadataQuery {
-		if (!$this->metadataInitiated()) {
-			return null;
-		}
-
-		return new MetadataQuery($qb, $this->getKnownMetadata(), $fileTableAlias, $fileIdField);
+	): IMetadataQuery {
+		return new MetadataQuery($qb, $this, $fileTableAlias, $fileIdField);
 	}
 
 	/**
@@ -263,8 +258,7 @@ public function getKnownMetadata(): IFilesMetadata {
 		$this->all = new FilesMetadata();
 
 		try {
-			$data = json_decode($this->config->getAppValue('core', self::CONFIG_KEY, '[]'), true, 127, JSON_THROW_ON_ERROR);
-			$this->all->import($data);
+			$this->all->import($this->appConfig->getValueArray('core', self::CONFIG_KEY, lazy: true));
 		} catch (JsonException) {
 			$this->logger->warning('issue while reading stored list of metadata. Advised to run ./occ files:scan --all --generate-metadata');
 		}
@@ -310,7 +304,7 @@ public function initMetadata(
 		}
 
 		$current->import([$key => ['type' => $type, 'indexed' => $indexed, 'editPermission' => $editPermission]]);
-		$this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current));
+		$this->appConfig->setValueArray('core', self::CONFIG_KEY, $current->jsonSerialize(), lazy: true);
 		$this->all = $current;
 	}
 
@@ -323,26 +317,4 @@ public static function loadListeners(IEventDispatcher $eventDispatcher): void {
 		$eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class);
 		$eventDispatcher->addServiceListener(CacheEntryRemovedEvent::class, MetadataDelete::class);
 	}
-
-	/**
-	 * Will confirm that tables were created and store an app value to cache the result.
-	 * Can be removed in 29 as this is to avoid strange situation when Nextcloud files were
-	 * replaced but the upgrade was not triggered yet.
-	 *
-	 * @return bool
-	 */
-	private function metadataInitiated(): bool {
-		if ($this->config->getAppValue('core', self::MIGRATION_DONE, '0') === '1') {
-			return true;
-		}
-
-		$dbConnection = \OCP\Server::get(IDBConnection::class);
-		if ($dbConnection->tableExists(MetadataRequestService::TABLE_METADATA)) {
-			$this->config->setAppValue('core', self::MIGRATION_DONE, '1');
-
-			return true;
-		}
-
-		return false;
-	}
 }

lib/private/FilesMetadata/MetadataQuery.php

@@ -31,9 +31,11 @@
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException;
 use OCP\FilesMetadata\Exceptions\FilesMetadataTypeException;
+use OCP\FilesMetadata\IFilesMetadataManager;
 use OCP\FilesMetadata\IMetadataQuery;
 use OCP\FilesMetadata\Model\IFilesMetadata;
 use OCP\FilesMetadata\Model\IMetadataValueWrapper;
+use Psr\Log\LoggerInterface;
 
 /**
  * @inheritDoc
@@ -43,12 +45,23 @@ class MetadataQuery implements IMetadataQuery {
 	private array $knownJoinedIndex = [];
 	public function __construct(
 		private IQueryBuilder $queryBuilder,
-		private IFilesMetadata $knownMetadata,
+		private IFilesMetadata|IFilesMetadataManager $manager,
 		private string $fileTableAlias = 'fc',
 		private string $fileIdField = 'fileid',
 		private string $alias = 'meta',
 		private string $aliasIndexPrefix = 'meta_index'
 	) {
+		if ($manager instanceof IFilesMetadata) {
+			/**
+			 * Since 29, because knownMetadata is stored in lazy appconfig, it seems smarter
+			 * to not call getKnownMetadata() at the load of this class as it is only needed
+			 * in {@see getMetadataValueField}.
+			 *
+			 * FIXME: remove support for IFilesMetadata
+			 */
+			$logger = \OCP\Server::get(LoggerInterface::class);
+			$logger->debug('It is deprecated to use IFilesMetadata as second parameter when calling MetadataQuery::__construct()');
+		}
 	}
 
 	/**
@@ -158,7 +171,20 @@ public function getMetadataKeyField(string $metadataKey): string {
 	 * @since 28.0.0
 	 */
 	public function getMetadataValueField(string $metadataKey): string {
-		return match ($this->knownMetadata->getType($metadataKey)) {
+		if ($this->manager instanceof IFilesMetadataManager) {
+			/**
+			 * Since 29, because knownMetadata is stored in lazy appconfig, it seems smarter
+			 * to not call getKnownMetadata() at the load of this class as it is only needed
+			 * in this method.
+			 *
+			 * FIXME: keep only this line and remove support for previous IFilesMetadata in constructor
+			 */
+			$knownMetadata = $this->manager->getKnownMetadata();
+		} else {
+			$knownMetadata = $this->manager;
+		}
+
+		return match ($knownMetadata->getType($metadataKey)) {
 			IMetadataValueWrapper::TYPE_STRING => $this->joinedTableAlias($metadataKey) . '.meta_value_string',
 			IMetadataValueWrapper::TYPE_INT, IMetadataValueWrapper::TYPE_BOOL => $this->joinedTableAlias($metadataKey) . '.meta_value_int',
 			default => throw new FilesMetadataTypeException('metadata is not set as indexed'),

lib/public/FilesMetadata/IFilesMetadataManager.php

@@ -122,30 +122,35 @@ public function deleteMetadata(int $fileId): void;
 	 * @param string $fileTableAlias alias of the table that contains data about files
 	 * @param string $fileIdField alias of the field that contains file ids
 	 *
-	 * @return IMetadataQuery|null NULL if table are not set yet or never used
+	 * @return IMetadataQuery
 	 * @see IMetadataQuery
 	 * @since 28.0.0
 	 */
 	public function getMetadataQuery(
 		IQueryBuilder $qb,
 		string $fileTableAlias,
 		string $fileIdField
-	): ?IMetadataQuery;
+	): IMetadataQuery;
 
 	/**
 	 * returns all type of metadata currently available.
 	 * The list is stored in a IFilesMetadata with null values but correct type.
 	 *
+	 * Note: this method loads lazy appconfig values.
+	 *
 	 * @return IFilesMetadata
 	 * @since 28.0.0
 	 */
 	public function getKnownMetadata(): IFilesMetadata;
 
 	/**
-	 * initiate a metadata key with its type.
+	 * Initiate a metadata key with its type.
+	 *
 	 * The call is mandatory before using the metadata property in a webdav request.
-	 * It is not needed to only use this method when the app is enabled: the method can be
-	 * called each time during the app loading as the metadata will only be initiated if not known
+	 * The call should be part of a migration/repair step and not be called on app's boot
+	 * process as it is using lazy-appconfig value
+	 *
+	 * Note: this method loads lazy appconfig values.
 	 *
 	 * @param string $key metadata key
 	 * @param string $type metadata type
@@ -164,6 +169,7 @@ public function getKnownMetadata(): IFilesMetadata;
 	 * @see IMetadataValueWrapper::EDIT_REQ_WRITE_PERMISSION
 	 * @see IMetadataValueWrapper::EDIT_REQ_READ_PERMISSION
 	 * @since 28.0.0
+	 * @since 29.0.0 uses lazy config value - do not use this method out of repair steps
 	 */
 	public function initMetadata(string $key, string $type, bool $indexed, int $editPermission): void;
 }

lib/public/FilesMetadata/IMetadataQuery.php

@@ -83,6 +83,8 @@ public function getMetadataKeyField(string $metadataKey): string;
 	/**
 	 * returns the name of the field for metadata string value to be used in query expressions
 	 *
+	 * Note: this method loads lazy appconfig values.
+	 *
 	 * @param string $metadataKey metadata key
 	 *
 	 * @return string table field