fix: partner asset and exif sync backfill (#19224)

* fix: partner asset sync backfill

* fix: add partner asset exif backfill

* ci: output content of files that have changed

server/test/medium.factory.ts

@@ -33,7 +33,7 @@ import { BaseService } from 'src/services/base.service';
 import { SyncService } from 'src/services/sync.service';
 import { RepositoryInterface } from 'src/types';
 import { factory, newDate, newEmbedding, newUuid } from 'test/small.factory';
-import { automock, ServiceOverrides } from 'test/utils';
+import { automock, ServiceOverrides, wait } from 'test/utils';
 import { Mocked } from 'vitest';
 
 const sha256 = (value: string) => createHash('sha256').update(value).digest('base64');
@@ -120,7 +120,7 @@ export const newSyncTest = (options: SyncTestOptions) => {
   const testSync = async (auth: AuthDto, types: SyncRequestType[]) => {
     const stream = mediumFactory.syncStream();
     // Wait for 2ms to ensure all updates are available and account for setTimeout inaccuracy
-    await new Promise((resolve) => setTimeout(resolve, 2));
+    await wait(2);
     await sut.stream(auth, stream, { types });
 
     return stream.getResponse();

server/test/medium/specs/sync/sync-partner-asset-exif.spec.ts

@@ -3,7 +3,7 @@ import { DB } from 'src/db';
 import { SyncEntityType, SyncRequestType } from 'src/enum';
 import { mediumFactory, newSyncAuthUser, newSyncTest } from 'test/medium.factory';
 import { factory } from 'test/small.factory';
-import { getKyselyDB } from 'test/utils';
+import { getKyselyDB, wait } from 'test/utils';
 
 let defaultDatabase: Kysely<DB>;
 
@@ -126,4 +126,154 @@ describe.concurrent(SyncRequestType.PartnerAssetExifsV1, () => {
     await expect(testSync(authUser3, [SyncRequestType.AssetExifsV1])).resolves.toHaveLength(1);
     await expect(testSync(auth, [SyncRequestType.PartnerAssetExifsV1])).resolves.toHaveLength(0);
   });
+
+  it('should backfill partner asset exif when a partner shared their library with you', async () => {
+    const { auth, sut, getRepository, testSync } = await setup();
+
+    const userRepo = getRepository('user');
+    const user2 = mediumFactory.userInsert();
+    const user3 = mediumFactory.userInsert();
+    await userRepo.create(user2);
+    await userRepo.create(user3);
+
+    const assetRepo = getRepository('asset');
+    const assetUser3 = mediumFactory.assetInsert({ ownerId: user3.id });
+    const assetUser2 = mediumFactory.assetInsert({ ownerId: user2.id });
+    await assetRepo.create(assetUser3);
+    await assetRepo.upsertExif({ assetId: assetUser3.id, make: 'Canon' });
+    await wait(2);
+    await assetRepo.create(assetUser2);
+    await assetRepo.upsertExif({ assetId: assetUser2.id, make: 'Canon' });
+
+    const partnerRepo = getRepository('partner');
+    await partnerRepo.create({ sharedById: user2.id, sharedWithId: auth.user.id });
+
+    const response = await testSync(auth, [SyncRequestType.PartnerAssetExifsV1]);
+
+    expect(response).toHaveLength(1);
+    expect(response).toEqual(
+      expect.arrayContaining([
+        {
+          ack: expect.any(String),
+          data: expect.objectContaining({
+            assetId: assetUser2.id,
+          }),
+          type: SyncEntityType.PartnerAssetExifV1,
+        },
+      ]),
+    );
+
+    const acks = response.map(({ ack }) => ack);
+    await sut.setAcks(auth, { acks });
+
+    await partnerRepo.create({ sharedById: user3.id, sharedWithId: auth.user.id });
+    const backfillResponse = await testSync(auth, [SyncRequestType.PartnerAssetExifsV1]);
+
+    expect(backfillResponse).toHaveLength(2);
+    expect(backfillResponse).toEqual(
+      expect.arrayContaining([
+        {
+          ack: expect.any(String),
+          data: expect.objectContaining({
+            assetId: assetUser3.id,
+          }),
+          type: SyncEntityType.PartnerAssetExifBackfillV1,
+        },
+        {
+          ack: expect.any(String),
+          data: {},
+          type: SyncEntityType.SyncAckV1,
+        },
+      ]),
+    );
+
+    const backfillAck = backfillResponse[1].ack;
+    await sut.setAcks(auth, { acks: [backfillAck] });
+
+    const finalResponse = await testSync(auth, [SyncRequestType.PartnerAssetExifsV1]);
+
+    const finalAcks = finalResponse.map(({ ack }) => ack);
+    expect(finalAcks).toEqual([]);
+  });
+
+  it('should only backfill partner assets created prior to the current partner asset checkpoint', async () => {
+    const { auth, sut, getRepository, testSync } = await setup();
+
+    const userRepo = getRepository('user');
+    const user2 = mediumFactory.userInsert();
+    const user3 = mediumFactory.userInsert();
+    await userRepo.create(user2);
+    await userRepo.create(user3);
+
+    const assetRepo = getRepository('asset');
+    const assetUser3 = mediumFactory.assetInsert({ ownerId: user3.id });
+    const assetUser2 = mediumFactory.assetInsert({ ownerId: user2.id });
+    const asset2User3 = mediumFactory.assetInsert({ ownerId: user3.id });
+    await assetRepo.create(assetUser3);
+    await assetRepo.upsertExif({ assetId: assetUser3.id, make: 'Canon' });
+    await wait(2);
+    await assetRepo.create(assetUser2);
+    await assetRepo.upsertExif({ assetId: assetUser2.id, make: 'Canon' });
+    await wait(2);
+    await assetRepo.create(asset2User3);
+    await assetRepo.upsertExif({ assetId: asset2User3.id, make: 'Canon' });
+
+    const partnerRepo = getRepository('partner');
+    await partnerRepo.create({ sharedById: user2.id, sharedWithId: auth.user.id });
+
+    const response = await testSync(auth, [SyncRequestType.PartnerAssetExifsV1]);
+
+    expect(response).toHaveLength(1);
+    expect(response).toEqual(
+      expect.arrayContaining([
+        {
+          ack: expect.any(String),
+          data: expect.objectContaining({
+            assetId: assetUser2.id,
+          }),
+          type: SyncEntityType.PartnerAssetExifV1,
+        },
+      ]),
+    );
+
+    const acks = response.map(({ ack }) => ack);
+    await sut.setAcks(auth, { acks });
+
+    await partnerRepo.create({ sharedById: user3.id, sharedWithId: auth.user.id });
+    const backfillResponse = await testSync(auth, [SyncRequestType.PartnerAssetExifsV1]);
+
+    expect(backfillResponse).toHaveLength(3);
+    expect(backfillResponse).toEqual(
+      expect.arrayContaining([
+        {
+          ack: expect.any(String),
+          data: expect.objectContaining({
+            assetId: assetUser3.id,
+          }),
+          type: SyncEntityType.PartnerAssetExifBackfillV1,
+        },
+        {
+          ack: expect.stringContaining(SyncEntityType.PartnerAssetExifBackfillV1),
+          data: {},
+          type: SyncEntityType.SyncAckV1,
+        },
+        {
+          ack: expect.any(String),
+          data: expect.objectContaining({
+            assetId: asset2User3.id,
+          }),
+          type: SyncEntityType.PartnerAssetExifV1,
+        },
+      ]),
+    );
+
+    const backfillAck = backfillResponse[1].ack;
+    const partnerAssetAck = backfillResponse[2].ack;
+    await sut.setAcks(auth, { acks: [backfillAck, partnerAssetAck] });
+
+    const finalResponse = await testSync(auth, [SyncRequestType.PartnerAssetExifsV1]);
+
+    const finalAcks = finalResponse.map(({ ack }) => ack);
+    expect(finalAcks).toEqual([]);
+  });
 });

server/test/medium/specs/sync/sync-partner-asset.spec.ts

@@ -3,7 +3,7 @@ import { DB } from 'src/db';
 import { SyncEntityType, SyncRequestType } from 'src/enum';
 import { mediumFactory, newSyncAuthUser, newSyncTest } from 'test/medium.factory';
 import { factory } from 'test/small.factory';
-import { getKyselyDB } from 'test/utils';
+import { getKyselyDB, wait } from 'test/utils';
 
 let defaultDatabase: Kysely<DB>;
 
@@ -19,7 +19,7 @@ beforeAll(async () => {
   defaultDatabase = await getKyselyDB();
 });
 
-describe.concurrent(SyncRequestType.PartnerAssetsV1, () => {
+describe(SyncRequestType.PartnerAssetsV1, () => {
   it('should detect and sync the first partner asset', async () => {
     const { auth, sut, getRepository, testSync } = await setup();
 
@@ -210,4 +210,149 @@ describe.concurrent(SyncRequestType.PartnerAssetsV1, () => {
     await expect(testSync(auth2, [SyncRequestType.AssetsV1])).resolves.toHaveLength(1);
     await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(0);
   });
+
+  it('should backfill partner assets when a partner shared their library with you', async () => {
+    const { auth, sut, getRepository, testSync } = await setup();
+
+    const userRepo = getRepository('user');
+    const user2 = mediumFactory.userInsert();
+    const user3 = mediumFactory.userInsert();
+    await userRepo.create(user2);
+    await userRepo.create(user3);
+
+    const assetRepo = getRepository('asset');
+    const assetUser3 = mediumFactory.assetInsert({ ownerId: user3.id });
+    const assetUser2 = mediumFactory.assetInsert({ ownerId: user2.id });
+    await assetRepo.create(assetUser3);
+    await wait(2);
+    await assetRepo.create(assetUser2);
+
+    const partnerRepo = getRepository('partner');
+    await partnerRepo.create({ sharedById: user2.id, sharedWithId: auth.user.id });
+
+    const response = await testSync(auth, [SyncRequestType.PartnerAssetsV1]);
+
+    expect(response).toHaveLength(1);
+    expect(response).toEqual(
+      expect.arrayContaining([
+        {
+          ack: expect.any(String),
+          data: expect.objectContaining({
+            id: assetUser2.id,
+          }),
+          type: SyncEntityType.PartnerAssetV1,
+        },
+      ]),
+    );
+
+    const acks = response.map(({ ack }) => ack);
+    await sut.setAcks(auth, { acks });
+
+    await partnerRepo.create({ sharedById: user3.id, sharedWithId: auth.user.id });
+    const backfillResponse = await testSync(auth, [SyncRequestType.PartnerAssetsV1]);
+
+    expect(backfillResponse).toHaveLength(2);
+    expect(backfillResponse).toEqual(
+      expect.arrayContaining([
+        {
+          ack: expect.any(String),
+          data: expect.objectContaining({
+            id: assetUser3.id,
+          }),
+          type: SyncEntityType.PartnerAssetBackfillV1,
+        },
+        {
+          ack: expect.stringContaining(SyncEntityType.PartnerAssetBackfillV1),
+          data: {},
+          type: SyncEntityType.SyncAckV1,
+        },
+      ]),
+    );
+
+    const backfillAck = backfillResponse[1].ack;
+    await sut.setAcks(auth, { acks: [backfillAck] });
+
+    const finalResponse = await testSync(auth, [SyncRequestType.PartnerAssetsV1]);
+
+    const finalAcks = finalResponse.map(({ ack }) => ack);
+    expect(finalAcks).toEqual([]);
+  });
+
+  it('should only backfill partner assets created prior to the current partner asset checkpoint', async () => {
+    const { auth, sut, getRepository, testSync } = await setup();
+
+    const userRepo = getRepository('user');
+    const user2 = mediumFactory.userInsert();
+    const user3 = mediumFactory.userInsert();
+    await userRepo.create(user2);
+    await userRepo.create(user3);
+
+    const assetRepo = getRepository('asset');
+    const assetUser3 = mediumFactory.assetInsert({ ownerId: user3.id });
+    const assetUser2 = mediumFactory.assetInsert({ ownerId: user2.id });
+    const asset2User3 = mediumFactory.assetInsert({ ownerId: user3.id });
+    await assetRepo.create(assetUser3);
+    await wait(2);
+    await assetRepo.create(assetUser2);
+    await wait(2);
+    await assetRepo.create(asset2User3);
+
+    const partnerRepo = getRepository('partner');
+    await partnerRepo.create({ sharedById: user2.id, sharedWithId: auth.user.id });
+
+    const response = await testSync(auth, [SyncRequestType.PartnerAssetsV1]);
+
+    expect(response).toHaveLength(1);
+    expect(response).toEqual(
+      expect.arrayContaining([
+        {
+          ack: expect.any(String),
+          data: expect.objectContaining({
+            id: assetUser2.id,
+          }),
+          type: SyncEntityType.PartnerAssetV1,
+        },
+      ]),
+    );
+
+    const acks = response.map(({ ack }) => ack);
+    await sut.setAcks(auth, { acks });
+
+    await partnerRepo.create({ sharedById: user3.id, sharedWithId: auth.user.id });
+    const backfillResponse = await testSync(auth, [SyncRequestType.PartnerAssetsV1]);
+
+    expect(backfillResponse).toHaveLength(3);
+    expect(backfillResponse).toEqual(
+      expect.arrayContaining([
+        {
+          ack: expect.any(String),
+          data: expect.objectContaining({
+            id: assetUser3.id,
+          }),
+          type: SyncEntityType.PartnerAssetBackfillV1,
+        },
+        {
+          ack: expect.stringContaining(SyncEntityType.PartnerAssetBackfillV1),
+          data: {},
+          type: SyncEntityType.SyncAckV1,
+        },
+        {
+          ack: expect.any(String),
+          data: expect.objectContaining({
+            id: asset2User3.id,
+          }),
+          type: SyncEntityType.PartnerAssetV1,
+        },
+      ]),
+    );
+
+    const backfillAck = backfillResponse[1].ack;
+    const partnerAssetAck = backfillResponse[2].ack;
+    await sut.setAcks(auth, { acks: [backfillAck, partnerAssetAck] });
+
+    const finalResponse = await testSync(auth, [SyncRequestType.PartnerAssetsV1]);
+
+    const finalAcks = finalResponse.map(({ ack }) => ack);
+    expect(finalAcks).toEqual([]);
+  });
 });

server/test/small.factory.ts

@@ -24,7 +24,7 @@ export const newUuids = () =>
     .fill(0)
     .map(() => newUuid());
 export const newDate = () => new Date();
-export const newUpdateId = () => 'uuid-v7';
+export const newUuidV7 = () => 'uuid-v7';
 export const newSha1 = () => Buffer.from('this is a fake hash');
 export const newEmbedding = () => {
   const embedding = Array.from({ length: 512 })
@@ -110,9 +110,10 @@ const partnerFactory = (partner: Partial<Partner> = {}) => {
     sharedBy,
     sharedWithId: sharedWith.id,
     sharedWith,
+    createId: newUuidV7(),
     createdAt: newDate(),
     updatedAt: newDate(),
-    updateId: newUpdateId(),
+    updateId: newUuidV7(),
     inTimeline: true,
     ...partner,
   };
@@ -122,7 +123,7 @@ const sessionFactory = (session: Partial<Session> = {}) => ({
   id: newUuid(),
   createdAt: newDate(),
   updatedAt: newDate(),
-  updateId: newUpdateId(),
+  updateId: newUuidV7(),
   deviceOS: 'android',
   deviceType: 'mobile',
   token: 'abc123',
@@ -201,7 +202,7 @@ const assetFactory = (asset: Partial<MapAsset> = {}) => ({
   createdAt: newDate(),
   updatedAt: newDate(),
   deletedAt: null,
-  updateId: newUpdateId(),
+  updateId: newUuidV7(),
   status: AssetStatus.ACTIVE,
   checksum: newSha1(),
   deviceAssetId: '',
@@ -240,7 +241,7 @@ const activityFactory = (activity: Partial<Activity> = {}) => {
     albumId: newUuid(),
     createdAt: newDate(),
     updatedAt: newDate(),
-    updateId: newUpdateId(),
+    updateId: newUuidV7(),
     ...activity,
   };
 };
@@ -250,7 +251,7 @@ const apiKeyFactory = (apiKey: Partial<ApiKey> = {}) => ({
   userId: newUuid(),
   createdAt: newDate(),
   updatedAt: newDate(),
-  updateId: newUpdateId(),
+  updateId: newUuidV7(),
   name: 'Api Key',
   permissions: [Permission.ALL],
   ...apiKey,
@@ -260,7 +261,7 @@ const libraryFactory = (library: Partial<Library> = {}) => ({
   id: newUuid(),
   createdAt: newDate(),
   updatedAt: newDate(),
-  updateId: newUpdateId(),
+  updateId: newUuidV7(),
   deletedAt: null,
   refreshedAt: null,
   name: 'Library',
@@ -275,7 +276,7 @@ const memoryFactory = (memory: Partial<Memory> = {}) => ({
   id: newUuid(),
   createdAt: newDate(),
   updatedAt: newDate(),
-  updateId: newUpdateId(),
+  updateId: newUuidV7(),
   deletedAt: null,
   ownerId: newUuid(),
   type: MemoryType.ON_THIS_DAY,

server/test/utils.ts

@@ -438,3 +438,7 @@ export async function* makeStream<T>(items: T[] = []): AsyncIterableIterator<T>
     yield item;
   }
 }
+
+export const wait = (ms: number) => {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+};