fix(server): make library exclusion matching consistently case-insensitive (#31973)

fix(server): make library exclusion matching consistently case-insensitive (#31970)

(cherry picked from commit de6dcf669c)

Co-authored-by: Jonathan Jogenfors <[email protected]>

docs/docs/features/libraries.md

@@ -43,7 +43,7 @@ To validate that Immich can reach your external library, start a shell inside th
 
 ### Exclusion Patterns
 
-By default, all files in the import paths will be added to the library. If there are files that should not be added, exclusion patterns can be used to exclude them. Exclusion patterns are glob patterns are matched against the full file path. If a file matches an exclusion pattern, it will not be added to the library. Exclusion patterns can be added in the Scan Settings page for each library.
+By default, all files in the import paths will be added to the library. If there are files that should not be added, exclusion patterns can be used to exclude them. Exclusion patterns are glob patterns matched against the full file path. If a file matches an exclusion pattern, it will not be added to the library. If it has already been imported, it will be offlined on the next library scan. Exclusion patterns can be added in the Scan Settings page for each library.
 
 Some basic examples:
 
@@ -51,15 +51,18 @@ Some basic examples:
 - `**/hidden.jpg` will exclude all files named `hidden.jpg`
 - `**/Raw/**` will exclude all files in any directory named `Raw`
 - `**/*.{tif,jpg}` will exclude all files with the extension `.tif` or `.jpg`
+- `**/*.{arw,dng}` will exclude all files with the extension `.arw` or `.dng`
 
-Note that `*` is a wildcard matching zero or more characters (i.e., withinin a filename or single directory name). `**` matches zero or more subdirectories, recursively. It also includes any/all files within a subdirectory, i.e., when used at the end of a pattern. For example, `**/exclude_me/**` will exclude all files in any directory named `exclude_me`, as well as all files in any subdirectories of `exclude_me`, recursively.
+Exclusion patterns are case-insensitive for both new files and already imported assets. For example, `**/*.{arw,dng}` also excludes files with uppercase or mixed-case extensions such as `.ARW` and `.DnG`.
+
+Note that `*` is a wildcard matching zero or more characters (i.e., within a filename or single directory name). `**` matches zero or more subdirectories, recursively. It also includes any/all files within a subdirectory, i.e., when used at the end of a pattern. For example, `**/exclude_me/**` will exclude all files in any directory named `exclude_me`, as well as all files in any subdirectories of `exclude_me`, recursively.
 
 Special characters such as @ should be escaped, for instance:
 
 - `**/\@eaDir/**` will exclude all files in any directory named `@eaDir`
 
 :::info
-Internally, Immich uses the [glob](https://www.npmjs.com/package/glob) package to process exclusion patterns, and sometimes those patterns are translated into [Postgres LIKE patterns](https://www.postgresql.org/docs/current/functions-matching.html). The intention is to support basic folder exclusions but we recommend against advanced usage since those can't reliably be translated to the Postgres syntax. Please refer to the [glob documentation](https://github.com/isaacs/node-glob#glob-primer) for a basic overview on glob patterns.
+For more information on the glob syntax supported by exclusion patterns, refer to the [Picomatch documentation](https://github.com/micromatch/picomatch#globbing-features).
 :::
 
 ### Automatic watching (EXPERIMENTAL)

server/src/queries/asset.repository.sql

@@ -532,7 +532,7 @@ where
   and "libraryId" = $5::uuid
   and (
     not "originalPath" like $6
-    or "originalPath" ~ $7
+    or "originalPath" ~* $7
   )
 
 -- AssetRepository.filterNewExternalAssetPaths

server/src/repositories/asset.repository.ts

@@ -1102,7 +1102,7 @@ export class AssetRepository {
       .where((eb) =>
         eb.or([
           eb.not(eb.or(paths.map((path) => eb('originalPath', 'like', path)))),
-          eb.or(exclusions.map((pattern) => eb('originalPath', '~', pattern))),
+          eb.or(exclusions.map((pattern) => eb('originalPath', '~*', pattern))),
         ]),
       )
       .executeTakeFirstOrThrow();

server/src/services/library.service.ts

@@ -521,7 +521,9 @@ export class LibraryService extends BaseService {
             break;
           }
 
-          const isExcluded = job.exclusionPatterns.some((pattern) => picomatch.isMatch(asset.originalPath, pattern));
+          const isExcluded = job.exclusionPatterns.some((pattern) =>
+            picomatch.isMatch(asset.originalPath, pattern, { nocase: true }),
+          );
 
           if (!isExcluded) {
             this.logger.debug(`Offline asset ${asset.originalPath} is now online in library ${job.libraryId}`);

server/test/medium/specs/services/library.service.spec.ts

@@ -476,6 +476,50 @@ describe(LibraryService.name, () => {
       await expect(ctx.getAssetPaths(library.id)).resolves.toEqual([]);
     });
 
+    // https://github.com/immich-app/immich/issues/23619
+    it.each([
+      { pattern: '**/*.{arw,ARW,dng,DNG}', extensions: ['arw', 'ARW', 'dng', 'DNG'] },
+      { pattern: '**/*.{arw,dng}', extensions: ['arw', 'ARW', 'ArW', 'dng', 'DNG', 'DnG'] },
+      { pattern: '**/*.ARW', extensions: ['arw', 'ARW', 'ArW'] },
+      { pattern: '**/*.{tif,jpg}', extensions: ['tif', 'TIF', 'TiF', 'jpg', 'JPG', 'JpG'] },
+    ])(
+      'should offline existing assets covered by $pattern regardless of extension case',
+      async ({ pattern, extensions }) => {
+        const { sut, ctx } = setup();
+
+        const excludedAssets = await Promise.all(
+          extensions.map((extension, index) => createFile(join(importRoot, 'nested', `asset${index}.${extension}`))),
+        );
+        const includedAsset = await createFile(join(importRoot, 'included.png'));
+        const library = await ctx.createLibrary({ importPaths: [importRoot] });
+
+        await ctx.scan(library.id);
+        await expect(ctx.getAssetPaths(library.id)).resolves.toEqual([...excludedAssets, includedAsset].sort());
+
+        await sut.update(library.id, { exclusionPatterns: [pattern] });
+        await ctx.scan(library.id);
+
+        await expect(ctx.getAssetPaths(library.id)).resolves.toEqual([includedAsset]);
+        const excluded = await ctx.database
+          .selectFrom('asset')
+          .select(['originalPath', 'isOffline', 'deletedAt'])
+          .where('libraryId', '=', library.id)
+          .where('originalPath', 'in', excludedAssets)
+          .orderBy('originalPath')
+          .execute();
+        expect(excluded).toEqual(
+          excludedAssets.sort().map((originalPath) => ({
+            originalPath,
+            isOffline: true,
+            deletedAt: expect.any(Date),
+          })),
+        );
+
+        await ctx.scan(library.id);
+        await expect(ctx.getAssetPaths(library.id)).resolves.toEqual([includedAsset]);
+      },
+    );
+
     // https://github.com/immich-app/immich/issues/17121
     it('should respect exclusion patterns when using multiple import paths', async () => {
       const { sut, ctx } = setup();