fix(server): "view all" for cities only showing 12 cities (#8035)

* view all cities * increase limit * rename endpoint * optimize query * remove pagination * update sql * linting * revert sort by count in explore page for now * fix query * fix * update sql * move to search, add partner support * update sql * pr feedback * euphemism * parameters as separate variable * move comment * update sql * linting
2025-11-14 17:36:12 +00:00 · 2024-03-19 23:23:57 -04:00 · 2024-03-19 23:23:57 -04:00 · f392fe7702
commit f392fe7702
parent 2daed747cd
14 changed files with 358 additions and 32 deletions
--- a/server/src/domain/repositories/search.repository.ts
+++ b/server/src/domain/repositories/search.repository.ts
@ -187,5 +187,6 @@ export interface ISearchRepository {
  searchFaces(search: FaceEmbeddingSearch): Promise<FaceSearchResult[]>;
  upsert(smartInfo: Partial<SmartInfoEntity>, embedding?: Embedding): Promise<void>;
  searchPlaces(placeName: string): Promise<GeodataPlacesEntity[]>;
+  getAssetsByCity(userIds: string[]): Promise<AssetEntity[]>;
  deleteAllSearchEmbeddings(): Promise<void>;
 }
--- a/server/src/domain/search/search.service.ts
+++ b/server/src/domain/search/search.service.ts
@ -115,6 +115,32 @@ export class SearchService {
    return this.mapResponse(items, hasNextPage ? (page + 1).toString() : null);
  }

+  async getAssetsByCity(auth: AuthDto): Promise<AssetResponseDto[]> {
+    const userIds = await this.getUserIdsToSearch(auth);
+    const assets = await this.searchRepository.getAssetsByCity(userIds);
+    return assets.map((asset) => mapAsset(asset));
+  }
+
+  getSearchSuggestions(auth: AuthDto, dto: SearchSuggestionRequestDto): Promise<string[]> {
+    switch (dto.type) {
+      case SearchSuggestionType.COUNTRY: {
+        return this.metadataRepository.getCountries(auth.user.id);
+      }
+      case SearchSuggestionType.STATE: {
+        return this.metadataRepository.getStates(auth.user.id, dto.country);
+      }
+      case SearchSuggestionType.CITY: {
+        return this.metadataRepository.getCities(auth.user.id, dto.country, dto.state);
+      }
+      case SearchSuggestionType.CAMERA_MAKE: {
+        return this.metadataRepository.getCameraMakes(auth.user.id, dto.model);
+      }
+      case SearchSuggestionType.CAMERA_MODEL: {
+        return this.metadataRepository.getCameraModels(auth.user.id, dto.make);
+      }
+    }
+  }
+
  // TODO: remove after implementing new search filters
  /** @deprecated */
  async search(auth: AuthDto, dto: SearchDto): Promise<SearchResponseDto> {
@ -191,24 +217,4 @@ export class SearchService {
      },
    };
  }
-
-  async getSearchSuggestions(auth: AuthDto, dto: SearchSuggestionRequestDto): Promise<string[]> {
-    switch (dto.type) {
-      case SearchSuggestionType.COUNTRY: {
-        return this.metadataRepository.getCountries(auth.user.id);
-      }
-      case SearchSuggestionType.STATE: {
-        return this.metadataRepository.getStates(auth.user.id, dto.country);
-      }
-      case SearchSuggestionType.CITY: {
-        return this.metadataRepository.getCities(auth.user.id, dto.country, dto.state);
-      }
-      case SearchSuggestionType.CAMERA_MAKE: {
-        return this.metadataRepository.getCameraMakes(auth.user.id, dto.model);
-      }
-      case SearchSuggestionType.CAMERA_MODEL: {
-        return this.metadataRepository.getCameraModels(auth.user.id, dto.make);
-      }
-    }
-  }
 }
--- a/server/src/immich/controllers/search.controller.ts
+++ b/server/src/immich/controllers/search.controller.ts
@ -1,4 +1,5 @@
 import {
+  AssetResponseDto,
  AuthDto,
  MetadataSearchDto,
  PersonResponseDto,
@ -55,6 +56,11 @@ export class SearchController {
    return this.service.searchPlaces(dto);
  }

+  @Get('cities')
+  getAssetsByCity(@Auth() auth: AuthDto): Promise<AssetResponseDto[]> {
+    return this.service.getAssetsByCity(auth);
+  }
+
  @Get('suggestions')
  getSearchSuggestions(@Auth() auth: AuthDto, @Query() dto: SearchSuggestionRequestDto): Promise<string[]> {
    return this.service.getSearchSuggestions(auth, dto);
--- a/server/src/infra/repositories/search.repository.ts
+++ b/server/src/infra/repositories/search.repository.ts
@ -15,6 +15,7 @@ import { getCLIPModelInfo } from '@app/domain/smart-info/smart-info.constant';
 import {
  AssetEntity,
  AssetFaceEntity,
+  AssetType,
  GeodataPlacesEntity,
  SmartInfoEntity,
  SmartSearchEntity,
@ -33,6 +34,7 @@ import { Instrumentation } from '../instrumentation';
 export class SearchRepository implements ISearchRepository {
  private logger = new ImmichLogger(SearchRepository.name);
  private faceColumns: string[];
+  private assetsByCityQuery: string;

  constructor(
    @InjectRepository(SmartInfoEntity) private repository: Repository<SmartInfoEntity>,
@ -45,6 +47,14 @@ export class SearchRepository implements ISearchRepository {
      .getMetadata(AssetFaceEntity)
      .ownColumns.map((column) => column.propertyName)
      .filter((propertyName) => propertyName !== 'embedding');
+    this.assetsByCityQuery =
+      assetsByCityCte +
+      this.assetRepository
+        .createQueryBuilder('asset')
+        .innerJoinAndSelect('asset.exifInfo', 'exif')
+        .withDeleted()
+        .getQuery() +
+      ' INNER JOIN cte ON asset.id = cte."assetId"';
  }

  async init(modelName: string): Promise<void> {
@ -220,6 +230,27 @@ export class SearchRepository implements ISearchRepository {
      .getMany();
  }

+  @GenerateSql({ params: [[DummyValue.UUID]] })
+  async getAssetsByCity(userIds: string[]): Promise<AssetEntity[]> {
+    const parameters = [userIds.join(', '), true, false, AssetType.IMAGE];
+    const rawRes = await this.repository.query(this.assetsByCityQuery, parameters);
+
+    const items: AssetEntity[] = [];
+    for (const res of rawRes) {
+      const item = { exifInfo: {} as Record<string, any> } as Record<string, any>;
+      for (const [key, value] of Object.entries(res)) {
+        if (key.startsWith('exif_')) {
+          item.exifInfo[key.replace('exif_', '')] = value;
+        } else {
+          item[key.replace('asset_', '')] = value;
+        }
+      }
+      items.push(item as AssetEntity);
+    }
+
+    return items;
+  }
+
  async upsert(smartInfo: Partial<SmartInfoEntity>, embedding?: Embedding): Promise<void> {
    await this.repository.upsert(smartInfo, { conflictPaths: ['assetId'] });
    if (!smartInfo.assetId || !embedding) {
@ -290,3 +321,30 @@ export class SearchRepository implements ISearchRepository {
    return runtimeConfig;
  }
 }
+
+// the performance difference between this and the normal way is too huge to ignore, e.g. 3s vs 4ms
+const assetsByCityCte = `
+WITH RECURSIVE cte AS (
+  (
+    SELECT city, "assetId"
+    FROM exif
+    INNER JOIN assets ON exif."assetId" = assets.id
+    WHERE "ownerId" IN ($1) AND "isVisible" = $2 AND "isArchived" = $3 AND type = $4
+    ORDER BY city
+    LIMIT 1
+  )
+
+  UNION ALL
+  
+  SELECT l.city, l."assetId"
+  FROM cte c
+    , LATERAL (
+    SELECT city, "assetId"
+    FROM exif
+    INNER JOIN assets ON exif."assetId" = assets.id
+    WHERE city > c.city AND "ownerId" IN ($1) AND "isVisible" = $2 AND "isArchived" = $3 AND type = $4
+    ORDER BY city
+    LIMIT 1
+    ) l
+)
+`;
--- a/server/src/infra/sql/search.repository.sql
+++ b/server/src/infra/sql/search.repository.sql
@ -266,3 +266,111 @@ ORDER BY
  ) ASC
 LIMIT
  20
+
+-- SearchRepository.getAssetsByCity
+WITH RECURSIVE
+  cte AS (
+    (
+      SELECT
+        city,
+        "assetId"
+      FROM
+        exif
+        INNER JOIN assets ON exif."assetId" = assets.id
+      WHERE
+        "ownerId" IN ($1)
+        AND "isVisible" = $2
+        AND "isArchived" = $3
+        AND type = $4
+      ORDER BY
+        city
+      LIMIT
+        1
+    )
+    UNION ALL
+    SELECT
+      l.city,
+      l."assetId"
+    FROM
+      cte c,
+      LATERAL (
+        SELECT
+          city,
+          "assetId"
+        FROM
+          exif
+          INNER JOIN assets ON exif."assetId" = assets.id
+        WHERE
+          city > c.city
+          AND "ownerId" IN ($1)
+          AND "isVisible" = $2
+          AND "isArchived" = $3
+          AND type = $4
+        ORDER BY
+          city
+        LIMIT
+          1
+      ) l
+  )
+SELECT
+  "asset"."id" AS "asset_id",
+  "asset"."deviceAssetId" AS "asset_deviceAssetId",
+  "asset"."ownerId" AS "asset_ownerId",
+  "asset"."libraryId" AS "asset_libraryId",
+  "asset"."deviceId" AS "asset_deviceId",
+  "asset"."type" AS "asset_type",
+  "asset"."originalPath" AS "asset_originalPath",
+  "asset"."resizePath" AS "asset_resizePath",
+  "asset"."webpPath" AS "asset_webpPath",
+  "asset"."thumbhash" AS "asset_thumbhash",
+  "asset"."encodedVideoPath" AS "asset_encodedVideoPath",
+  "asset"."createdAt" AS "asset_createdAt",
+  "asset"."updatedAt" AS "asset_updatedAt",
+  "asset"."deletedAt" AS "asset_deletedAt",
+  "asset"."fileCreatedAt" AS "asset_fileCreatedAt",
+  "asset"."localDateTime" AS "asset_localDateTime",
+  "asset"."fileModifiedAt" AS "asset_fileModifiedAt",
+  "asset"."isFavorite" AS "asset_isFavorite",
+  "asset"."isArchived" AS "asset_isArchived",
+  "asset"."isExternal" AS "asset_isExternal",
+  "asset"."isReadOnly" AS "asset_isReadOnly",
+  "asset"."isOffline" AS "asset_isOffline",
+  "asset"."checksum" AS "asset_checksum",
+  "asset"."duration" AS "asset_duration",
+  "asset"."isVisible" AS "asset_isVisible",
+  "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
+  "asset"."originalFileName" AS "asset_originalFileName",
+  "asset"."sidecarPath" AS "asset_sidecarPath",
+  "asset"."stackId" AS "asset_stackId",
+  "exif"."assetId" AS "exif_assetId",
+  "exif"."description" AS "exif_description",
+  "exif"."exifImageWidth" AS "exif_exifImageWidth",
+  "exif"."exifImageHeight" AS "exif_exifImageHeight",
+  "exif"."fileSizeInByte" AS "exif_fileSizeInByte",
+  "exif"."orientation" AS "exif_orientation",
+  "exif"."dateTimeOriginal" AS "exif_dateTimeOriginal",
+  "exif"."modifyDate" AS "exif_modifyDate",
+  "exif"."timeZone" AS "exif_timeZone",
+  "exif"."latitude" AS "exif_latitude",
+  "exif"."longitude" AS "exif_longitude",
+  "exif"."projectionType" AS "exif_projectionType",
+  "exif"."city" AS "exif_city",
+  "exif"."livePhotoCID" AS "exif_livePhotoCID",
+  "exif"."autoStackId" AS "exif_autoStackId",
+  "exif"."state" AS "exif_state",
+  "exif"."country" AS "exif_country",
+  "exif"."make" AS "exif_make",
+  "exif"."model" AS "exif_model",
+  "exif"."lensModel" AS "exif_lensModel",
+  "exif"."fNumber" AS "exif_fNumber",
+  "exif"."focalLength" AS "exif_focalLength",
+  "exif"."iso" AS "exif_iso",
+  "exif"."exposureTime" AS "exif_exposureTime",
+  "exif"."profileDescription" AS "exif_profileDescription",
+  "exif"."colorspace" AS "exif_colorspace",
+  "exif"."bitsPerSample" AS "exif_bitsPerSample",
+  "exif"."fps" AS "exif_fps"
+FROM
+  "assets" "asset"
+  INNER JOIN "exif" "exif" ON "exif"."assetId" = "asset"."id"
+  INNER JOIN cte ON asset.id = cte."assetId"
--- a/server/test/repositories/search.repository.mock.ts
+++ b/server/test/repositories/search.repository.mock.ts
@ -8,6 +8,7 @@ export const newSearchRepositoryMock = (): jest.Mocked<ISearchRepository> => {
    searchFaces: jest.fn(),
    upsert: jest.fn(),
    searchPlaces: jest.fn(),
+    getAssetsByCity: jest.fn(),
    deleteAllSearchEmbeddings: jest.fn(),
  };
 };