feat(cli): refactor and add album support (#4434)

* Allow building and installing cli

* feat: add format fix

* docs: remove cli folder

* feat: use immich scoped package

* feat: rewrite cli readme

* docs: add info on running without building

* cleanup

* chore: remove import functionality from cli

* feat: add logout to cli

* docs: add todo for file format from server

* docs: add compilation step to cli

* fix: success message spacing

* feat: can create albums

* fix: add check step to cli

* fix: typos

* feat: pull file formats from server

* chore: use crawl service from server

* chore: fix lint

* docs: add cli documentation

* chore: rename ignore pattern

* chore: add version number to cli

* feat: use sdk

* fix: cleanup

* feat: album name on windows

* chore: remove skipped asset field

* feat: add more info to server-info command

* chore: cleanup

* chore: remove unneeded packages

* chore: fix docs links

* feat: add cli v2 milestone

* fix: set correct cli date

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>
This commit is contained in:
Jonathan Jogenfors 2023-11-19 23:16:24 +01:00 committed by GitHub
parent 88b5f5b500
commit 7e38e7c949
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 637 additions and 800 deletions

View file

@ -1,3 +0,0 @@
// ./__mocks__/axios.js
import mockAxios from 'jest-mock-axios';
export default mockAxios;

View file

@ -9,7 +9,6 @@ import { ServerVersionResponseDto, UserResponseDto } from 'src/api/open-api';
export abstract class BaseCommand {
protected sessionService!: SessionService;
protected immichApi!: ImmichApi;
protected deviceId!: string;
protected user!: UserResponseDto;
protected serverVersion!: ServerVersionResponseDto;

View file

@ -1,15 +1,19 @@
import { BaseCommand } from '../cli/base-command';
export default class ServerInfo extends BaseCommand {
static description = 'Display server information';
static enableJsonFlag = true;
public async run() {
console.log('Getting server information');
await this.connect();
const { data: versionInfo } = await this.immichApi.serverInfoApi.getServerVersion();
console.log(versionInfo);
console.log(`Server is running version ${versionInfo.major}.${versionInfo.minor}.${versionInfo.patch}`);
const { data: supportedmedia } = await this.immichApi.serverInfoApi.getSupportedMediaTypes();
console.log(`Supported image types: ${supportedmedia.image.map((extension) => extension.replace('.', ''))}`);
console.log(`Supported video types: ${supportedmedia.video.map((extension) => extension.replace('.', ''))}`);
const { data: statistics } = await this.immichApi.assetApi.getAssetStatistics();
console.log(`Images: ${statistics.images}, Videos: ${statistics.videos}, Total: ${statistics.total}`);
}
}

View file

@ -1,43 +1,36 @@
import { BaseCommand } from '../cli/base-command';
import { CrawledAsset } from '../cores/models/crawled-asset';
import { CrawlService, UploadService } from '../services';
import * as si from 'systeminformation';
import FormData from 'form-data';
import { Asset } from '../cores/models/asset';
import { CrawlService } from '../services';
import { UploadOptionsDto } from '../cores/dto/upload-options-dto';
import { CrawlOptionsDto } from '../cores/dto/crawl-options-dto';
import cliProgress from 'cli-progress';
import byteSize from 'byte-size';
import { BaseCommand } from '../cli/base-command';
export default class Upload extends BaseCommand {
private crawlService = new CrawlService();
private uploadService!: UploadService;
deviceId!: string;
uploadLength!: number;
dryRun = false;
public async run(paths: string[], options: UploadOptionsDto): Promise<void> {
await this.connect();
const uuid = await si.uuid();
this.deviceId = uuid.os || 'CLI';
this.uploadService = new UploadService(this.immichApi.apiConfiguration);
const deviceId = 'CLI';
this.dryRun = options.dryRun;
const formatResponse = await this.immichApi.serverInfoApi.getSupportedMediaTypes();
const crawlService = new CrawlService(formatResponse.data.image, formatResponse.data.video);
const crawlOptions = new CrawlOptionsDto();
crawlOptions.pathsToCrawl = paths;
crawlOptions.recursive = options.recursive;
crawlOptions.excludePatterns = options.excludePatterns;
crawlOptions.exclusionPatterns = options.exclusionPatterns;
const crawledFiles: string[] = await this.crawlService.crawl(crawlOptions);
const crawledFiles: string[] = await crawlService.crawl(crawlOptions);
if (crawledFiles.length === 0) {
console.log('No assets found, exiting');
return;
}
const assetsToUpload = crawledFiles.map((path) => new CrawledAsset(path));
const assetsToUpload = crawledFiles.map((path) => new Asset(path, deviceId));
const uploadProgress = new cliProgress.SingleBar(
{
@ -58,117 +51,86 @@ export default class Upload extends BaseCommand {
totalSize += asset.fileSize;
}
const existingAlbums = (await this.immichApi.albumApi.getAllAlbums()).data;
uploadProgress.start(totalSize, 0);
uploadProgress.update({ value_formatted: 0, total_formatted: byteSize(totalSize) });
for (const asset of assetsToUpload) {
uploadProgress.update({
filename: asset.path,
});
try {
for (const asset of assetsToUpload) {
uploadProgress.update({
filename: asset.path,
});
try {
if (options.import) {
const importData = {
assetPath: asset.path,
sidecarPath: asset.sidecarPath,
deviceAssetId: asset.deviceAssetId,
deviceId: this.deviceId,
fileCreatedAt: asset.fileCreatedAt,
fileModifiedAt: asset.fileModifiedAt,
isFavorite: false,
isReadOnly: options.readOnly,
};
let skipUpload = false;
if (!options.skipHash) {
const assetBulkUploadCheckDto = { assets: [{ id: asset.path, checksum: await asset.hash() }] };
if (!this.dryRun) {
await this.uploadService.import(importData);
}
} else {
await this.uploadAsset(asset, options.skipHash);
const checkResponse = await this.immichApi.assetApi.checkBulkUpload({
assetBulkUploadCheckDto,
});
skipUpload = checkResponse.data.results[0].action === 'reject';
}
} catch (error) {
uploadProgress.stop();
throw error;
}
sizeSoFar += asset.fileSize;
if (!asset.skipped) {
totalSizeUploaded += asset.fileSize;
uploadCounter++;
}
if (!skipUpload) {
if (!options.dryRun) {
const res = await this.immichApi.assetApi.uploadFile(asset.getUploadFileRequest());
uploadProgress.update(sizeSoFar, { value_formatted: byteSize(sizeSoFar) });
if (options.album && asset.albumName) {
let album = existingAlbums.find((album) => album.albumName === asset.albumName);
if (!album) {
const res = await this.immichApi.albumApi.createAlbum({
createAlbumDto: { albumName: asset.albumName },
});
album = res.data;
existingAlbums.push(album);
}
await this.immichApi.albumApi.addAssetsToAlbum({ id: album.id, bulkIdsDto: { ids: [res.data.id] } });
}
}
totalSizeUploaded += asset.fileSize;
uploadCounter++;
}
sizeSoFar += asset.fileSize;
uploadProgress.update(sizeSoFar, { value_formatted: byteSize(sizeSoFar) });
}
} finally {
uploadProgress.stop();
}
uploadProgress.stop();
let messageStart;
if (this.dryRun) {
messageStart = 'Would have ';
if (options.dryRun) {
messageStart = 'Would have';
} else {
messageStart = 'Successfully ';
messageStart = 'Successfully';
}
if (options.import) {
console.log(`${messageStart} imported ${uploadCounter} assets (${byteSize(totalSizeUploaded)})`);
if (uploadCounter === 0) {
console.log('All assets were already uploaded, nothing to do.');
} else {
if (uploadCounter === 0) {
console.log('All assets were already uploaded, nothing to do.');
console.log(`${messageStart} uploaded ${uploadCounter} assets (${byteSize(totalSizeUploaded)})`);
}
if (options.delete) {
if (options.dryRun) {
console.log(`Would now have deleted assets, but skipped due to dry run`);
} else {
console.log(`${messageStart} uploaded ${uploadCounter} assets (${byteSize(totalSizeUploaded)})`);
}
if (options.delete) {
if (this.dryRun) {
console.log(`Would now have deleted assets, but skipped due to dry run`);
} else {
console.log('Deleting assets that have been uploaded...');
const deletionProgress = new cliProgress.SingleBar(cliProgress.Presets.shades_classic);
deletionProgress.start(crawledFiles.length, 0);
console.log('Deleting assets that have been uploaded...');
const deletionProgress = new cliProgress.SingleBar(cliProgress.Presets.shades_classic);
deletionProgress.start(crawledFiles.length, 0);
for (const asset of assetsToUpload) {
if (!this.dryRun) {
await asset.delete();
}
deletionProgress.increment();
for (const asset of assetsToUpload) {
if (!options.dryRun) {
await asset.delete();
}
deletionProgress.stop();
console.log('Deletion complete');
deletionProgress.increment();
}
}
}
}
private async uploadAsset(asset: CrawledAsset, skipHash = false) {
await asset.readData();
let skipUpload = false;
if (!skipHash) {
const checksum = await asset.hash();
const checkResponse = await this.uploadService.checkIfAssetAlreadyExists(asset.path, checksum);
skipUpload = checkResponse.data.results[0].action === 'reject';
}
if (skipUpload) {
asset.skipped = true;
} else {
const uploadFormData = new FormData();
uploadFormData.append('deviceAssetId', asset.deviceAssetId);
uploadFormData.append('deviceId', this.deviceId);
uploadFormData.append('fileCreatedAt', asset.fileCreatedAt);
uploadFormData.append('fileModifiedAt', asset.fileModifiedAt);
uploadFormData.append('isFavorite', String(false));
uploadFormData.append('assetData', asset.assetData, { filename: asset.path });
if (asset.sidecarData) {
uploadFormData.append('sidecarData', asset.sidecarData, {
filename: asset.sidecarPath,
contentType: 'application/xml',
});
}
if (!this.dryRun) {
await this.uploadService.upload(uploadFormData);
deletionProgress.stop();
console.log('Deletion complete');
}
}
}

View file

@ -1,59 +0,0 @@
// Check asset-upload.config.spec.ts for complete list
// TODO: we should get this list from the server via API in the future
// Videos
const videos = ['mp4', 'webm', 'mov', '3gp', 'avi', 'm2ts', 'mts', 'mpg', 'flv', 'mkv', 'wmv'];
// Images
const heic = ['heic', 'heif'];
const jpeg = ['jpg', 'jpeg'];
const png = ['png'];
const gif = ['gif'];
const tiff = ['tif', 'tiff'];
const webp = ['webp'];
const dng = ['dng'];
const other = [
'3fr',
'ari',
'arw',
'avif',
'cap',
'cin',
'cr2',
'cr3',
'crw',
'dcr',
'nef',
'erf',
'fff',
'iiq',
'jxl',
'k25',
'kdc',
'mrw',
'orf',
'ori',
'pef',
'psd',
'raf',
'raw',
'rwl',
'sr2',
'srf',
'srw',
'orf',
'ori',
'x3f',
];
export const ACCEPTED_FILE_EXTENSIONS = [
...videos,
...jpeg,
...png,
...heic,
...gif,
...tiff,
...webp,
...dng,
...other,
];

View file

@ -1,6 +1,6 @@
export class CrawlOptionsDto {
pathsToCrawl!: string[];
recursive = false;
includeHidden = false;
excludePatterns!: string[];
recursive? = false;
includeHidden? = false;
exclusionPatterns?: string[];
}

View file

@ -1,9 +1,9 @@
export class UploadOptionsDto {
recursive = false;
excludePatterns!: string[];
exclusionPatterns!: string[];
dryRun = false;
skipHash = false;
delete = false;
import = false;
readOnly = true;
album = false;
}

View file

@ -1,2 +1 @@
export * from './constants';
export * from './models';

View file

@ -0,0 +1,91 @@
import * as fs from 'fs';
import { basename } from 'node:path';
import crypto from 'crypto';
import { AssetApiUploadFileRequest } from 'src/api/open-api';
import Os from 'os';
export class Asset {
readonly path: string;
readonly deviceId!: string;
assetData?: File;
deviceAssetId?: string;
fileCreatedAt?: string;
fileModifiedAt?: string;
sidecarData?: File;
sidecarPath?: string;
fileSize!: number;
albumName?: string;
constructor(path: string, deviceId: string) {
this.path = path;
this.deviceId = deviceId;
}
async process() {
const stats = await fs.promises.stat(this.path);
this.deviceAssetId = `${basename(this.path)}-${stats.size}`.replace(/\s+/g, '');
this.fileCreatedAt = stats.mtime.toISOString();
this.fileModifiedAt = stats.mtime.toISOString();
this.fileSize = stats.size;
this.albumName = this.extractAlbumName();
this.assetData = await this.getFileObject(this.path);
// TODO: doesn't xmp replace the file extension? Will need investigation
const sideCarPath = `${this.path}.xmp`;
try {
fs.accessSync(sideCarPath, fs.constants.R_OK);
this.sidecarData = await this.getFileObject(sideCarPath);
} catch (error) {}
}
getUploadFileRequest(): AssetApiUploadFileRequest {
if (!this.assetData) throw new Error('Asset data not set');
if (!this.deviceAssetId) throw new Error('Device asset id not set');
if (!this.fileCreatedAt) throw new Error('File created at not set');
if (!this.fileModifiedAt) throw new Error('File modified at not set');
if (!this.deviceId) throw new Error('Device id not set');
return {
assetData: this.assetData,
deviceAssetId: this.deviceAssetId,
deviceId: this.deviceId,
fileCreatedAt: this.fileCreatedAt,
fileModifiedAt: this.fileModifiedAt,
isFavorite: false,
sidecarData: this.sidecarData,
};
}
private async getFileObject(path: string): Promise<File> {
const buffer = await fs.promises.readFile(path);
return new File([buffer], basename(path));
}
async delete(): Promise<void> {
return fs.promises.unlink(this.path);
}
public async hash(): Promise<string> {
const sha1 = (filePath: string) => {
const hash = crypto.createHash('sha1');
return new Promise<string>((resolve, reject) => {
const rs = fs.createReadStream(filePath);
rs.on('error', reject);
rs.on('data', (chunk) => hash.update(chunk));
rs.on('end', () => resolve(hash.digest('hex')));
});
};
return await sha1(this.path);
}
private extractAlbumName(): string {
if (Os.platform() === 'win32') {
return this.path.split('\\').slice(-2)[0];
} else {
return this.path.split('/').slice(-2)[0];
}
}
}

View file

@ -1,58 +0,0 @@
import * as fs from 'fs';
import { basename } from 'node:path';
import crypto from 'crypto';
export class CrawledAsset {
public path: string;
public assetData?: fs.ReadStream;
public deviceAssetId?: string;
public fileCreatedAt?: string;
public fileModifiedAt?: string;
public sidecarData?: Buffer;
public sidecarPath?: string;
public fileSize!: number;
public skipped = false;
constructor(path: string) {
this.path = path;
}
async readData() {
this.assetData = fs.createReadStream(this.path);
}
async process() {
const stats = await fs.promises.stat(this.path);
this.deviceAssetId = `${basename(this.path)}-${stats.size}`.replace(/\s+/g, '');
this.fileCreatedAt = stats.mtime.toISOString();
this.fileModifiedAt = stats.mtime.toISOString();
this.fileSize = stats.size;
// TODO: doesn't xmp replace the file extension? Will need investigation
const sideCarPath = `${this.path}.xmp`;
try {
fs.accessSync(sideCarPath, fs.constants.R_OK);
this.sidecarData = await fs.promises.readFile(sideCarPath);
this.sidecarPath = sideCarPath;
} catch (error) {}
}
async delete(): Promise<void> {
return fs.promises.unlink(this.path);
}
public async hash(): Promise<string> {
const sha1 = (filePath: string) => {
const hash = crypto.createHash('sha1');
return new Promise<string>((resolve, reject) => {
const rs = fs.createReadStream(filePath);
rs.on('error', reject);
rs.on('data', (chunk) => hash.update(chunk));
rs.on('end', () => resolve(hash.digest('hex')));
});
};
return await sha1(this.path);
}
}

View file

@ -1 +1 @@
export * from './crawled-asset';
export * from './asset';

View file

@ -1,7 +1,10 @@
#! /usr/bin/env node
import { program, Option } from 'commander';
import Upload from './commands/upload';
import ServerInfo from './commands/server-info';
import LoginKey from './commands/login/key';
import Logout from './commands/logout';
program.name('immich').description('Immich command line interface');
@ -12,6 +15,11 @@ program
.addOption(new Option('-r, --recursive', 'Recursive').env('IMMICH_RECURSIVE').default(false))
.addOption(new Option('-i, --ignore [paths...]', 'Paths to ignore').env('IMMICH_IGNORE_PATHS'))
.addOption(new Option('-h, --skip-hash', "Don't hash files before upload").env('IMMICH_SKIP_HASH').default(false))
.addOption(
new Option('-a, --album', 'Automatically create albums based on folder name')
.env('IMMICH_AUTO_CREATE_ALBUM')
.default(false),
)
.addOption(
new Option('-n, --dry-run', "Don't perform any actions, just show what will be done")
.env('IMMICH_DRY_RUN')
@ -20,33 +28,13 @@ program
.addOption(new Option('--delete', 'Delete local assets after upload').env('IMMICH_DELETE_ASSETS'))
.argument('[paths...]', 'One or more paths to assets to be uploaded')
.action(async (paths, options) => {
options.excludePatterns = options.ignore;
await new Upload().run(paths, options);
});
program
.command('import')
.description('Import existing assets')
.usage('[options] [paths...]')
.addOption(new Option('-r, --recursive', 'Recursive').env('IMMICH_RECURSIVE').default(false))
.addOption(
new Option('-n, --dry-run', "Don't perform any actions, just show what will be done")
.env('IMMICH_DRY_RUN')
.default(false),
)
.addOption(new Option('-i, --ignore [paths...]', 'Paths to ignore').env('IMMICH_IGNORE_PATHS').default(false))
.addOption(new Option('--no-read-only', 'Import files without read-only protection, allowing Immich to manage them'))
.argument('[paths...]', 'One or more paths to assets to be imported')
.action(async (paths, options) => {
options.import = true;
options.excludePatterns = options.ignore;
options.exclusionPatterns = options.ignore;
await new Upload().run(paths, options);
});
program
.command('server-info')
.description('Display server information')
.action(async () => {
await new ServerInfo().run();
});
@ -60,4 +48,11 @@ program
await new LoginKey().run(paths, options);
});
program
.command('logout')
.description('Remove stored credentials')
.action(async () => {
await new Logout().run();
});
program.parse(process.argv);

View file

@ -1,235 +1,206 @@
/* eslint-disable @typescript-eslint/no-var-requires */
/* eslint-disable @typescript-eslint/no-unused-vars */
import { CrawlService } from './crawl.service';
import mockfs from 'mock-fs';
import { toIncludeSameMembers } from 'jest-extended';
import { CrawlOptionsDto } from '../cores/dto/crawl-options-dto';
import { CrawlOptionsDto } from 'src/cores/dto/crawl-options-dto';
import { CrawlService } from '.';
const matchers = require('jest-extended');
expect.extend(matchers);
interface Test {
test: string;
options: CrawlOptionsDto;
files: Record<string, boolean>;
}
const crawlService = new CrawlService();
const cwd = process.cwd();
describe('CrawlService', () => {
beforeAll(() => {
// Write a dummy output before mock-fs to prevent some annoying errors
console.log();
});
const tests: Test[] = [
{
test: 'should return empty when crawling an empty path list',
options: {
pathsToCrawl: [],
},
files: {},
},
{
test: 'should crawl a single path',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
},
},
{
test: 'should exclude by file extension',
options: {
pathsToCrawl: ['/photos/'],
exclusionPatterns: ['**/*.tif'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.tif': false,
},
},
{
test: 'should exclude by file extension without case sensitivity',
options: {
pathsToCrawl: ['/photos/'],
exclusionPatterns: ['**/*.TIF'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.tif': false,
},
},
{
test: 'should exclude by folder',
options: {
pathsToCrawl: ['/photos/'],
exclusionPatterns: ['**/raw/**'],
},
files: {
'/photos/image.jpg': true,
'/photos/raw/image.jpg': false,
'/photos/raw2/image.jpg': true,
'/photos/folder/raw/image.jpg': false,
'/photos/crawl/image.jpg': true,
},
},
{
test: 'should crawl multiple paths',
options: {
pathsToCrawl: ['/photos/', '/images/', '/albums/'],
},
files: {
'/photos/image1.jpg': true,
'/images/image2.jpg': true,
'/albums/image3.jpg': true,
},
},
{
test: 'should support globbing paths',
options: {
pathsToCrawl: ['/photos*'],
},
files: {
'/photos1/image1.jpg': true,
'/photos2/image2.jpg': true,
'/images/image3.jpg': false,
},
},
{
test: 'should crawl a single path without trailing slash',
options: {
pathsToCrawl: ['/photos'],
},
files: {
'/photos/image.jpg': true,
},
},
{
test: 'should crawl a single path',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/subfolder/image1.jpg': true,
'/photos/subfolder/image2.jpg': true,
'/image1.jpg': false,
},
},
{
test: 'should filter file extensions',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.txt': false,
'/photos/1': false,
},
},
{
test: 'should include photo and video extensions',
options: {
pathsToCrawl: ['/photos/', '/videos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.jpeg': true,
'/photos/image.heic': true,
'/photos/image.heif': true,
'/photos/image.png': true,
'/photos/image.gif': true,
'/photos/image.tif': true,
'/photos/image.tiff': true,
'/photos/image.webp': true,
'/photos/image.dng': true,
'/photos/image.nef': true,
'/videos/video.mp4': true,
'/videos/video.mov': true,
'/videos/video.webm': true,
},
},
{
test: 'should check file extensions without case sensitivity',
options: {
pathsToCrawl: ['/photos/'],
},
files: {
'/photos/image.jpg': true,
'/photos/image.Jpg': true,
'/photos/image.jpG': true,
'/photos/image.JPG': true,
'/photos/image.jpEg': true,
'/photos/image.TIFF': true,
'/photos/image.tif': true,
'/photos/image.dng': true,
'/photos/image.NEF': true,
},
},
{
test: 'should normalize the path',
options: {
pathsToCrawl: ['/photos/1/../2'],
},
files: {
'/photos/1/image.jpg': false,
'/photos/2/image.jpg': true,
},
},
{
test: 'should return absolute paths',
options: {
pathsToCrawl: ['photos'],
},
files: {
[`${cwd}/photos/1.jpg`]: true,
[`${cwd}/photos/2.jpg`]: true,
[`/photos/3.jpg`]: false,
},
},
];
it('should crawl a single directory', async () => {
mockfs({
'/photos/image.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should crawl a single file', async () => {
mockfs({
'/photos/image.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/image.jpg'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should crawl a file and a directory', async () => {
mockfs({
'/photos/image.jpg': '',
'/images/photo.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/image.jpg', '/images/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg', '/images/photo.jpg']);
});
it('should exclude by file extension', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/image.tif': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
options.excludePatterns = ['**/*.tif'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should exclude by file extension without case sensitivity', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/image.tif': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
options.excludePatterns = ['**/*.TIF'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should exclude by folder', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/raw/image.jpg': '',
'/photos/raw2/image.jpg': '',
'/photos/folder/raw/image.jpg': '',
'/photos/crawl/image.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
options.excludePatterns = ['**/raw/**'];
options.recursive = true;
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg', '/photos/raw2/image.jpg', '/photos/crawl/image.jpg']);
});
it('should crawl multiple paths', async () => {
mockfs({
'/photos/image1.jpg': '',
'/images/image2.jpg': '',
'/albums/image3.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/', '/images/', '/albums/'];
options.recursive = false;
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image1.jpg', '/images/image2.jpg', '/albums/image3.jpg']);
});
it('should crawl a single path without trailing slash', async () => {
mockfs({
'/photos/image.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should crawl a single path without recursion', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/subfolder/image1.jpg': '',
'/photos/subfolder/image2.jpg': '',
'/image1.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should crawl a single path with recursion', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/subfolder/image1.jpg': '',
'/photos/subfolder/image2.jpg': '',
'/image1.jpg': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
options.recursive = true;
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers([
'/photos/image.jpg',
'/photos/subfolder/image1.jpg',
'/photos/subfolder/image2.jpg',
]);
});
it('should filter file extensions', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/image.txt': '',
'/photos/1': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers(['/photos/image.jpg']);
});
it('should include photo and video extensions', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/image.jpeg': '',
'/photos/image.heic': '',
'/photos/image.heif': '',
'/photos/image.png': '',
'/photos/image.gif': '',
'/photos/image.tif': '',
'/photos/image.tiff': '',
'/photos/image.webp': '',
'/photos/image.dng': '',
'/photos/image.nef': '',
'/videos/video.mp4': '',
'/videos/video.mov': '',
'/videos/video.webm': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/', '/videos/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers([
'/photos/image.jpg',
'/photos/image.jpeg',
'/photos/image.heic',
'/photos/image.heif',
'/photos/image.png',
'/photos/image.gif',
'/photos/image.tif',
'/photos/image.tiff',
'/photos/image.webp',
'/photos/image.dng',
'/photos/image.nef',
'/videos/video.mp4',
'/videos/video.mov',
'/videos/video.webm',
]);
});
it('should check file extensions without case sensitivity', async () => {
mockfs({
'/photos/image.jpg': '',
'/photos/image.Jpg': '',
'/photos/image.jpG': '',
'/photos/image.JPG': '',
'/photos/image.jpEg': '',
'/photos/image.TIFF': '',
'/photos/image.tif': '',
'/photos/image.dng': '',
'/photos/image.NEF': '',
});
const options = new CrawlOptionsDto();
options.pathsToCrawl = ['/photos/'];
const paths: string[] = await crawlService.crawl(options);
expect(paths).toIncludeSameMembers([
'/photos/image.jpg',
'/photos/image.Jpg',
'/photos/image.jpG',
'/photos/image.JPG',
'/photos/image.jpEg',
'/photos/image.TIFF',
'/photos/image.tif',
'/photos/image.dng',
'/photos/image.NEF',
]);
});
describe(CrawlService.name, () => {
const sut = new CrawlService(
['.jpg', '.jpeg', '.png', '.heif', '.heic', '.tif', '.nef', '.webp', '.tiff', '.dng', '.gif'],
['.mov', '.mp4', '.webm'],
);
afterEach(() => {
mockfs.restore();
});
describe('crawl', () => {
for (const { test, options, files } of tests) {
it(test, async () => {
mockfs(Object.fromEntries(Object.keys(files).map((file) => [file, ''])));
const actual = await sut.crawl(options);
const expected = Object.entries(files)
.filter((entry) => entry[1])
.map(([file]) => file);
expect(actual.sort()).toEqual(expected.sort());
});
}
});
});

View file

@ -1,47 +1,28 @@
import { CrawlOptionsDto } from 'src/cores/dto/crawl-options-dto';
import { ACCEPTED_FILE_EXTENSIONS } from '../cores';
import { glob } from 'glob';
import * as fs from 'fs';
export class CrawlService {
public async crawl(crawlOptions: CrawlOptionsDto): Promise<string[]> {
const pathsToCrawl: string[] = crawlOptions.pathsToCrawl;
private readonly extensions!: string[];
const directories: string[] = [];
const crawledFiles: string[] = [];
constructor(image: string[], video: string[]) {
this.extensions = image.concat(video).map((extension) => extension.replace('.', ''));
}
for await (const currentPath of pathsToCrawl) {
const stats = await fs.promises.stat(currentPath);
if (stats.isFile() || stats.isSymbolicLink()) {
crawledFiles.push(currentPath);
} else {
directories.push(currentPath);
}
crawl(crawlOptions: CrawlOptionsDto): Promise<string[]> {
const { pathsToCrawl, exclusionPatterns, includeHidden } = crawlOptions;
if (!pathsToCrawl) {
return Promise.resolve([]);
}
let searchPattern: string;
if (directories.length === 1) {
searchPattern = directories[0];
} else if (directories.length === 0) {
return crawledFiles;
} else {
searchPattern = '{' + directories.join(',') + '}';
}
const base = pathsToCrawl.length === 1 ? pathsToCrawl[0] : `{${pathsToCrawl.join(',')}}`;
const extensions = `*{${this.extensions}}`;
if (crawlOptions.recursive) {
searchPattern = searchPattern + '/**/';
}
searchPattern = `${searchPattern}/*.{${ACCEPTED_FILE_EXTENSIONS.join(',')}}`;
const globbedFiles = await glob(searchPattern, {
return glob(`${base}/**/${extensions}`, {
absolute: true,
nocase: true,
nodir: true,
ignore: crawlOptions.excludePatterns,
dot: includeHidden,
ignore: exclusionPatterns,
});
const returnedFiles = crawledFiles.concat(globbedFiles);
returnedFiles.sort();
return returnedFiles;
}
}

View file

@ -1,2 +1 @@
export * from './upload.service';
export * from './crawl.service';

View file

@ -46,7 +46,7 @@ export class SessionService {
// Check if server and api key are valid
const { data: userInfo } = await this.api.userApi.getMyUserInfo().catch((error) => {
throw new LoginError(`Failed to connect to the server: ${error.message}`);
throw new LoginError(`Failed to connect to server ${instanceUrl}: ${error.message}`);
});
console.log(`Logged in as ${userInfo.email}`);
@ -78,7 +78,7 @@ export class SessionService {
private async ping(): Promise<void> {
const { data: pingResponse } = await this.api.serverInfoApi.pingServer().catch((error) => {
throw new Error(`Failed to connect to the server: ${error.message}`);
throw new Error(`Failed to connect to server ${this.api.apiConfiguration.instanceUrl}: ${error.message}`);
});
if (pingResponse.res !== 'pong') {

View file

@ -1,24 +0,0 @@
import { UploadService } from './upload.service';
import axios from 'axios';
import FormData from 'form-data';
import { ApiConfiguration } from '../cores/api-configuration';
jest.mock('axios', () => jest.fn());
describe('UploadService', () => {
let uploadService: UploadService;
beforeEach(() => {
const apiConfiguration = new ApiConfiguration('https://example.com/api', 'key');
uploadService = new UploadService(apiConfiguration);
});
it('should call axios', async () => {
const data = new FormData();
await uploadService.upload(data);
expect(axios).toHaveBeenCalled();
});
});

View file

@ -1,65 +0,0 @@
import axios, { AxiosRequestConfig } from 'axios';
import FormData from 'form-data';
import { ApiConfiguration } from '../cores/api-configuration';
export class UploadService {
private readonly uploadConfig: AxiosRequestConfig<any>;
private readonly checkAssetExistenceConfig: AxiosRequestConfig<any>;
private readonly importConfig: AxiosRequestConfig<any>;
constructor(apiConfiguration: ApiConfiguration) {
this.uploadConfig = {
method: 'post',
maxRedirects: 0,
url: `${apiConfiguration.instanceUrl}/asset/upload`,
headers: {
'x-api-key': apiConfiguration.apiKey,
},
maxContentLength: Number.POSITIVE_INFINITY,
maxBodyLength: Number.POSITIVE_INFINITY,
};
this.importConfig = {
method: 'post',
maxRedirects: 0,
url: `${apiConfiguration.instanceUrl}/asset/import`,
headers: {
'x-api-key': apiConfiguration.apiKey,
'Content-Type': 'application/json',
},
maxContentLength: Number.POSITIVE_INFINITY,
maxBodyLength: Number.POSITIVE_INFINITY,
};
this.checkAssetExistenceConfig = {
method: 'post',
maxRedirects: 0,
url: `${apiConfiguration.instanceUrl}/asset/bulk-upload-check`,
headers: {
'x-api-key': apiConfiguration.apiKey,
'Content-Type': 'application/json',
},
};
}
public checkIfAssetAlreadyExists(path: string, checksum: string) {
this.checkAssetExistenceConfig.data = JSON.stringify({ assets: [{ id: path, checksum: checksum }] });
// TODO: retry on 500 errors?
return axios(this.checkAssetExistenceConfig);
}
public upload(data: FormData) {
this.uploadConfig.data = data;
// TODO: retry on 500 errors?
return axios(this.uploadConfig);
}
public import(data: any) {
this.importConfig.data = data;
// TODO: retry on 500 errors?
return axios(this.importConfig);
}
}