feat(server): asset entity audit (#3824)

* feat(server): audit log

* feedback

* Insert to database

* migration

* test

* controller/repository/service

* test

* module

* feat(server): implement audit endpoint

* directly return changed assets

* add daily cleanup of audit table

* fix tests

* review feedback

* ci

* refactor(server): audit implementation

* chore: open api

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
Co-authored-by: Fynn Petersen-Frey <zoodyy@users.noreply.github.com>
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
This commit is contained in:
Fynn Petersen-Frey 2023-08-24 21:28:50 +02:00 committed by GitHub
parent d6887117ac
commit cf9e04c8ec
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
57 changed files with 1381 additions and 36 deletions

View file

@ -13,7 +13,7 @@ import {
import { when } from 'jest-when';
import { Readable } from 'stream';
import { ICryptoRepository } from '../crypto';
import { IJobRepository, JobName } from '../index';
import { IJobRepository, JobName } from '../job';
import { IStorageRepository } from '../storage';
import { AssetStats, IAssetRepository } from './asset.repository';
import { AssetService, UploadFieldName } from './asset.service';

View file

@ -16,18 +16,23 @@ import {
AssetIdsDto,
AssetJobName,
AssetJobsDto,
AssetStatsDto,
DownloadArchiveInfo,
DownloadInfoDto,
DownloadResponseDto,
MapMarkerDto,
mapStats,
MemoryLaneDto,
TimeBucketAssetDto,
TimeBucketDto,
} from './dto';
import { AssetStatsDto, mapStats } from './dto/asset-statistics.dto';
import { MapMarkerDto } from './dto/map-marker.dto';
import { AssetResponseDto, mapAsset, MapMarkerResponseDto } from './response-dto';
import { MemoryLaneResponseDto } from './response-dto/memory-lane-response.dto';
import { TimeBucketResponseDto } from './response-dto/time-bucket-response.dto';
import {
AssetResponseDto,
mapAsset,
MapMarkerResponseDto,
MemoryLaneResponseDto,
TimeBucketResponseDto,
} from './response-dto';
export enum UploadFieldName {
ASSET_DATA = 'assetData',

View file

@ -84,3 +84,8 @@ export function mapAssetWithoutExif(entity: AssetEntity): AssetResponseDto {
checksum: entity.checksum.toString('base64'),
};
}
export class MemoryLaneResponseDto {
title!: string;
assets!: AssetResponseDto[];
}

View file

@ -1,6 +0,0 @@
import { AssetResponseDto } from './asset-response.dto';
export class MemoryLaneResponseDto {
title!: string;
assets!: AssetResponseDto[];
}

View file

@ -0,0 +1,61 @@
import { DatabaseAction, EntityType } from '@app/infra/entities';
import { auditStub, authStub, IAccessRepositoryMock, newAccessRepositoryMock, newAuditRepositoryMock } from '@test';
import { IAuditRepository } from './audit.repository';
import { AuditService } from './audit.service';
describe(AuditService.name, () => {
let sut: AuditService;
let accessMock: IAccessRepositoryMock;
let auditMock: jest.Mocked<IAuditRepository>;
beforeEach(async () => {
accessMock = newAccessRepositoryMock();
auditMock = newAuditRepositoryMock();
sut = new AuditService(accessMock, auditMock);
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('handleCleanup', () => {
it('should delete old audit entries', async () => {
await expect(sut.handleCleanup()).resolves.toBe(true);
expect(auditMock.removeBefore).toBeCalledWith(expect.any(Date));
});
});
describe('getDeletes', () => {
it('should require full sync if the request is older than 100 days', async () => {
auditMock.getAfter.mockResolvedValue([]);
const date = new Date(2022, 0, 1);
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
needsFullSync: true,
ids: [],
});
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
action: DatabaseAction.DELETE,
ownerId: authStub.admin.id,
entityType: EntityType.ASSET,
});
});
it('should get any new or updated assets and deleted ids', async () => {
auditMock.getAfter.mockResolvedValue([auditStub.delete]);
const date = new Date();
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
needsFullSync: false,
ids: ['asset-deleted'],
});
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
action: DatabaseAction.DELETE,
ownerId: authStub.admin.id,
entityType: EntityType.ASSET,
});
});
});
});

View file

@ -0,0 +1,24 @@
import { EntityType } from '@app/infra/entities';
import { ApiProperty } from '@nestjs/swagger';
import { Type } from 'class-transformer';
import { IsDate, IsEnum, IsOptional, IsUUID } from 'class-validator';
export class AuditDeletesDto {
@IsDate()
@Type(() => Date)
after!: Date;
@ApiProperty({ enum: EntityType, enumName: 'EntityType' })
@IsEnum(EntityType)
entityType!: EntityType;
@IsOptional()
@IsUUID('4')
@ApiProperty({ format: 'uuid' })
userId?: string;
}
export class AuditDeletesResponseDto {
needsFullSync!: boolean;
ids!: string[];
}

View file

@ -0,0 +1,14 @@
import { AuditEntity, DatabaseAction, EntityType } from '@app/infra/entities';
export const IAuditRepository = 'IAuditRepository';
export interface AuditSearch {
action?: DatabaseAction;
entityType?: EntityType;
ownerId?: string;
}
export interface IAuditRepository {
getAfter(since: Date, options: AuditSearch): Promise<AuditEntity[]>;
removeBefore(before: Date): Promise<void>;
}

View file

@ -0,0 +1,43 @@
import { DatabaseAction } from '@app/infra/entities';
import { Inject, Injectable } from '@nestjs/common';
import { DateTime } from 'luxon';
import { AccessCore, IAccessRepository, Permission } from '../access';
import { AuthUserDto } from '../auth';
import { AUDIT_LOG_MAX_DURATION } from '../domain.constant';
import { AuditDeletesDto, AuditDeletesResponseDto } from './audit.dto';
import { IAuditRepository } from './audit.repository';
@Injectable()
export class AuditService {
private access: AccessCore;
constructor(
@Inject(IAccessRepository) accessRepository: IAccessRepository,
@Inject(IAuditRepository) private repository: IAuditRepository,
) {
this.access = new AccessCore(accessRepository);
}
async handleCleanup(): Promise<boolean> {
await this.repository.removeBefore(DateTime.now().minus(AUDIT_LOG_MAX_DURATION).toJSDate());
return true;
}
async getDeletes(authUser: AuthUserDto, dto: AuditDeletesDto): Promise<AuditDeletesResponseDto> {
const userId = dto.userId || authUser.id;
await this.access.requirePermission(authUser, Permission.LIBRARY_READ, userId);
const audits = await this.repository.getAfter(dto.after, {
ownerId: userId,
entityType: dto.entityType,
action: DatabaseAction.DELETE,
});
const duration = DateTime.now().diff(DateTime.fromJSDate(dto.after));
return {
needsFullSync: duration > AUDIT_LOG_MAX_DURATION,
ids: audits.map(({ entityId }) => entityId),
};
}
}

View file

@ -0,0 +1,3 @@
export * from './audit.dto';
export * from './audit.repository';
export * from './audit.service';

View file

@ -1,8 +1,11 @@
import { AssetType } from '@app/infra/entities';
import { BadRequestException } from '@nestjs/common';
import { Duration } from 'luxon';
import { extname } from 'node:path';
import pkg from 'src/../../package.json';
export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 });
const [major, minor, patch] = pkg.version.split('.');
export interface IServerVersion {

View file

@ -2,6 +2,7 @@ import { DynamicModule, Global, Module, ModuleMetadata, OnApplicationShutdown, P
import { AlbumService } from './album';
import { APIKeyService } from './api-key';
import { AssetService } from './asset';
import { AuditService } from './audit';
import { AuthService } from './auth';
import { FacialRecognitionService } from './facial-recognition';
import { JobService } from './job';
@ -23,6 +24,7 @@ const providers: Provider[] = [
AlbumService,
APIKeyService,
AssetService,
AuditService,
AuthService,
FacialRecognitionService,
JobService,

View file

@ -2,6 +2,7 @@ export * from './access';
export * from './album';
export * from './api-key';
export * from './asset';
export * from './audit';
export * from './auth';
export * from './communication';
export * from './crypto';

View file

@ -55,6 +55,7 @@ export enum JobName {
// cleanup
DELETE_FILES = 'delete-files',
CLEAN_OLD_AUDIT_LOGS = 'clean-old-audit-logs',
// search
SEARCH_INDEX_ASSETS = 'search-index-assets',
@ -84,6 +85,7 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.USER_DELETE_CHECK]: QueueName.BACKGROUND_TASK,
[JobName.USER_DELETION]: QueueName.BACKGROUND_TASK,
[JobName.DELETE_FILES]: QueueName.BACKGROUND_TASK,
[JobName.CLEAN_OLD_AUDIT_LOGS]: QueueName.BACKGROUND_TASK,
[JobName.PERSON_CLEANUP]: QueueName.BACKGROUND_TASK,
// conversion

View file

@ -68,6 +68,9 @@ export type JobItem =
// Filesystem
| { name: JobName.DELETE_FILES; data: IDeleteFilesJob }
// Audit log cleanup
| { name: JobName.CLEAN_OLD_AUDIT_LOGS; data?: IBaseJob }
// Asset Deletion
| { name: JobName.PERSON_CLEANUP; data?: IBaseJob }

View file

@ -51,6 +51,7 @@ describe(JobService.name, () => {
[{ name: JobName.USER_DELETE_CHECK }],
[{ name: JobName.PERSON_CLEANUP }],
[{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }],
[{ name: JobName.CLEAN_OLD_AUDIT_LOGS }],
]);
});
});

View file

@ -136,6 +136,7 @@ export class JobService {
await this.jobRepository.queue({ name: JobName.USER_DELETE_CHECK });
await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
await this.jobRepository.queue({ name: JobName.CLEAN_OLD_AUDIT_LOGS });
}
/**

View file

@ -1,7 +1,7 @@
import { AssetEntity, ExifEntity } from '@app/infra/entities';
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { IsNull, Not } from 'typeorm';
import { IsNull, MoreThan, Not } from 'typeorm';
import { In } from 'typeorm/find-options/operator/In';
import { Repository } from 'typeorm/repository/Repository';
import { AssetSearchDto } from './dto/asset-search.dto';
@ -131,6 +131,7 @@ export class AssetRepository implements IAssetRepository {
isVisible: true,
isFavorite: dto.isFavorite,
isArchived: dto.isArchived,
updatedAt: dto.updatedAfter ? MoreThan(dto.updatedAfter) : undefined,
},
relations: {
exifInfo: true,

View file

@ -1,7 +1,7 @@
import { toBoolean } from '@app/domain';
import { ApiProperty } from '@nestjs/swagger';
import { Transform } from 'class-transformer';
import { IsBoolean, IsNotEmpty, IsNumber, IsOptional, IsUUID } from 'class-validator';
import { Transform, Type } from 'class-transformer';
import { IsBoolean, IsDate, IsNotEmpty, IsNumber, IsOptional, IsUUID } from 'class-validator';
export class AssetSearchDto {
@IsOptional()
@ -32,4 +32,9 @@ export class AssetSearchDto {
@IsUUID('4')
@ApiProperty({ format: 'uuid' })
userId?: string;
@IsOptional()
@IsDate()
@Type(() => Date)
updatedAfter?: Date;
}

View file

@ -16,6 +16,7 @@ import {
APIKeyController,
AppController,
AssetController,
AuditController,
AuthController,
JobController,
OAuthController,
@ -42,6 +43,7 @@ import {
AppController,
AlbumController,
APIKeyController,
AuditController,
AuthController,
JobController,
OAuthController,

View file

@ -9,14 +9,14 @@ import {
AuthUserDto,
DownloadInfoDto,
DownloadResponseDto,
MapMarkerDto,
MapMarkerResponseDto,
MemoryLaneDto,
MemoryLaneResponseDto,
TimeBucketAssetDto,
TimeBucketDto,
TimeBucketResponseDto,
} from '@app/domain';
import { MapMarkerDto } from '@app/domain/asset/dto/map-marker.dto';
import { MemoryLaneResponseDto } from '@app/domain/asset/response-dto/memory-lane-response.dto';
import { Body, Controller, Get, HttpCode, HttpStatus, Param, Post, Put, Query, StreamableFile } from '@nestjs/common';
import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
import { Authenticated, AuthUser, SharedLinkRoute } from '../app.guard';

View file

@ -0,0 +1,18 @@
import { AuditDeletesDto, AuditDeletesResponseDto, AuditService, AuthUserDto } from '@app/domain';
import { Controller, Get, Query } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { Authenticated, AuthUser } from '../app.guard';
import { UseValidation } from '../app.utils';
@ApiTags('Audit')
@Controller('audit')
@Authenticated()
@UseValidation()
export class AuditController {
constructor(private service: AuditService) {}
@Get('deletes')
getAuditDeletes(@AuthUser() authUser: AuthUserDto, @Query() dto: AuditDeletesDto): Promise<AuditDeletesResponseDto> {
return this.service.getDeletes(authUser, dto);
}
}

View file

@ -2,6 +2,7 @@ export * from './album.controller';
export * from './api-key.controller';
export * from './app.controller';
export * from './asset.controller';
export * from './audit.controller';
export * from './auth.controller';
export * from './job.controller';
export * from './oauth.controller';

View file

@ -17,6 +17,7 @@ export const databaseConfig: PostgresConnectionOptions = {
entities: [__dirname + '/entities/*.entity.{js,ts}'],
synchronize: false,
migrations: [__dirname + '/migrations/*.{js,ts}'],
subscribers: [__dirname + '/subscribers/*.{js,ts}'],
migrationsRun: true,
connectTimeoutMS: 10000, // 10 seconds
...urlOrParts,

View file

@ -0,0 +1,34 @@
import { Column, CreateDateColumn, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';
export enum DatabaseAction {
CREATE = 'CREATE',
UPDATE = 'UPDATE',
DELETE = 'DELETE',
}
export enum EntityType {
ASSET = 'ASSET',
ALBUM = 'ALBUM',
}
@Entity('audit')
@Index('IDX_ownerId_createdAt', ['ownerId', 'createdAt'])
export class AuditEntity {
@PrimaryGeneratedColumn('increment')
id!: number;
@Column()
entityType!: EntityType;
@Column({ type: 'uuid' })
entityId!: string;
@Column()
action!: DatabaseAction;
@Column({ type: 'uuid' })
ownerId!: string;
@CreateDateColumn({ type: 'timestamptz' })
createdAt!: Date;
}

View file

@ -2,6 +2,7 @@ import { AlbumEntity } from './album.entity';
import { APIKeyEntity } from './api-key.entity';
import { AssetFaceEntity } from './asset-face.entity';
import { AssetEntity } from './asset.entity';
import { AuditEntity } from './audit.entity';
import { PartnerEntity } from './partner.entity';
import { PersonEntity } from './person.entity';
import { SharedLinkEntity } from './shared-link.entity';
@ -15,6 +16,7 @@ export * from './album.entity';
export * from './api-key.entity';
export * from './asset-face.entity';
export * from './asset.entity';
export * from './audit.entity';
export * from './exif.entity';
export * from './partner.entity';
export * from './person.entity';
@ -30,6 +32,7 @@ export const databaseEntities = [
APIKeyEntity,
AssetEntity,
AssetFaceEntity,
AuditEntity,
PartnerEntity,
PersonEntity,
SharedLinkEntity,

View file

@ -2,6 +2,7 @@ import {
IAccessRepository,
IAlbumRepository,
IAssetRepository,
IAuditRepository,
ICommunicationRepository,
ICryptoRepository,
IFaceRepository,
@ -35,6 +36,7 @@ import {
AlbumRepository,
APIKeyRepository,
AssetRepository,
AuditRepository,
CommunicationRepository,
CryptoRepository,
FaceRepository,
@ -58,6 +60,7 @@ const providers: Provider[] = [
{ provide: IAccessRepository, useClass: AccessRepository },
{ provide: IAlbumRepository, useClass: AlbumRepository },
{ provide: IAssetRepository, useClass: AssetRepository },
{ provide: IAuditRepository, useClass: AuditRepository },
{ provide: ICommunicationRepository, useClass: CommunicationRepository },
{ provide: ICryptoRepository, useClass: CryptoRepository },
{ provide: IFaceRepository, useClass: FaceRepository },

View file

@ -0,0 +1,16 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class AddAuditTable1692804658140 implements MigrationInterface {
name = 'AddAuditTable1692804658140'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`CREATE TABLE "audit" ("id" SERIAL NOT NULL, "entityType" character varying NOT NULL, "entityId" uuid NOT NULL, "action" character varying NOT NULL, "ownerId" uuid NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "PK_1d3d120ddaf7bc9b1ed68ed463a" PRIMARY KEY ("id"))`);
await queryRunner.query(`CREATE INDEX "IDX_ownerId_createdAt" ON "audit" ("ownerId", "createdAt") `);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX "public"."IDX_ownerId_createdAt"`);
await queryRunner.query(`DROP TABLE "audit"`);
}
}

View file

@ -0,0 +1,26 @@
import { AuditSearch, IAuditRepository } from '@app/domain';
import { InjectRepository } from '@nestjs/typeorm';
import { LessThan, MoreThan, Repository } from 'typeorm';
import { AuditEntity } from '../entities';
export class AuditRepository implements IAuditRepository {
constructor(@InjectRepository(AuditEntity) private repository: Repository<AuditEntity>) {}
getAfter(since: Date, options: AuditSearch): Promise<AuditEntity[]> {
return this.repository
.createQueryBuilder('audit')
.where({
createdAt: MoreThan(since),
action: options.action,
entityType: options.entityType,
ownerId: options.ownerId,
})
.distinctOn(['audit.entityId', 'audit.entityType'])
.orderBy('audit.entityId, audit.entityType, audit.createdAt', 'DESC')
.getMany();
}
async removeBefore(before: Date): Promise<void> {
await this.repository.delete({ createdAt: LessThan(before) });
}
}

View file

@ -2,6 +2,7 @@ export * from './access.repository';
export * from './album.repository';
export * from './api-key.repository';
export * from './asset.repository';
export * from './audit.repository';
export * from './communication.repository';
export * from './crypto.repository';
export * from './face.repository';

View file

@ -0,0 +1,38 @@
import { EntitySubscriberInterface, EventSubscriber, RemoveEvent } from 'typeorm';
import { AlbumEntity, AssetEntity, AuditEntity, DatabaseAction, EntityType } from '../entities';
@EventSubscriber()
export class AuditSubscriber implements EntitySubscriberInterface<AssetEntity | AlbumEntity> {
async afterRemove(event: RemoveEvent<AssetEntity>): Promise<void> {
await this.onEvent(DatabaseAction.DELETE, event);
}
private async onEvent<T>(action: DatabaseAction, event: RemoveEvent<T>): Promise<any> {
const audit = this.getAudit(event.metadata.name, { ...event.entity, id: event.entityId });
if (audit && audit.entityId && audit.ownerId) {
await event.manager.getRepository(AuditEntity).save({ ...audit, action });
}
}
private getAudit(entityName: string, entity: any): Partial<AuditEntity> | null {
switch (entityName) {
case AssetEntity.name:
const asset = entity as AssetEntity;
return {
entityType: EntityType.ASSET,
entityId: asset.id,
ownerId: asset.ownerId,
};
case AlbumEntity.name:
const album = entity as AlbumEntity;
return {
entityType: EntityType.ALBUM,
entityId: album.id,
ownerId: album.ownerId,
};
}
return null;
}
}

View file

@ -1,4 +1,5 @@
import {
AuditService,
FacialRecognitionService,
IDeleteFilesJob,
JobName,
@ -35,11 +36,13 @@ export class AppService {
private storageService: StorageService,
private systemConfigService: SystemConfigService,
private userService: UserService,
private auditService: AuditService,
) {}
async init() {
await this.jobService.registerHandlers({
[JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data),
[JobName.CLEAN_OLD_AUDIT_LOGS]: () => this.auditService.handleCleanup(),
[JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(),
[JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data),
[JobName.QUEUE_OBJECT_TAGGING]: (data) => this.smartInfoService.handleQueueObjectTagging(data),

View file

@ -408,7 +408,11 @@ export class MetadataExtractionProcessor {
}
await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] });
await this.assetRepository.save({ id: asset.id, fileCreatedAt: fileCreatedAt || undefined });
await this.assetRepository.save({
id: asset.id,
fileCreatedAt: fileCreatedAt || undefined,
updatedAt: new Date(),
});
return true;
}