Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ module.exports = {
},
],
rules: {
'no-await-in-loop': 'warn',
'no-await-in-loop': 'off',
'@typescript-eslint/no-use-before-define': ['warn', { functions: false, classes: true, variables: true }],
'array-callback-return': 'warn',
'max-len': [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,15 @@ import { TemporalFileByFolderFinder } from '../../../../context/storage/Temporal
import { TemporalFileByPathFinder } from '../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder';
import { TemporalFilePathsByFolderFinder } from '../../../../context/storage/TemporalFiles/application/find/TemporalFilePathsByFolderFinder';
import { TemporalFileTruncater } from '../../../../context/storage/TemporalFiles/application/truncate/TemporalFileTruncater';
import { createTemporalFileUploadQueueService } from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/create-temporal-file-upload-queue-service';
import { TemporalFileUploadQueue } from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types';
import { TemporalFileUploader } from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploader';
import { TemporalFileWriter } from '../../../../context/storage/TemporalFiles/application/write/TemporalFileWriter';
import { TemporalFileRepository } from '../../../../context/storage/TemporalFiles/domain/TemporalFileRepository';
import { TemporalFileUploaderFactory } from '../../../../context/storage/TemporalFiles/domain/upload/TemporalFileUploaderFactory';
import { NodeTemporalFileRepository } from '../../../../context/storage/TemporalFiles/infrastructure/NodeTemporalFileRepository';
import { EnvironmentTemporalFileUploaderFactory } from '../../../../context/storage/TemporalFiles/infrastructure/upload/EnvironmentTemporalFileUploaderFactory';
import { FirstsFileSearcher } from '../../../../context/virtual-drive/files/application/search/FirstsFileSearcher';
import { DependencyInjectionUserProvider } from '../../../shared/dependency-injection/DependencyInjectionUserProvider';
import { PATHS } from '../../../../core/electron/paths';

Expand Down Expand Up @@ -52,6 +55,17 @@ export async function registerTemporalFilesServices(builder: ContainerBuilder) {
builder.registerAndUse(TemporalFileTruncater);
builder.registerAndUse(TemporalFileByteByByteComparator);
builder.registerAndUse(TemporalFileByFolderFinder);
builder
.register(TemporalFileUploadQueue)
.useFactory((container) =>
createTemporalFileUploadQueueService({
repository: container.get(TemporalFileRepository),
uploader: container.get(TemporalFileUploader),
deleter: container.get(TemporalFileDeleter),
fileSearcher: container.get(FirstsFileSearcher),
}),
)
.asSingleton();

// Event handlers
builder.registerAndUse(DeleteTemporalFileOnFileCreated).addTag('event-handler');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import { Environment } from '@internxt/inxt-js';
export async function registerFilesServices(builder: ContainerBuilder): Promise<void> {
// Infra

builder.register(FileRepository).use(InMemoryFileRepository).asSingleton().private();
builder.register(FileRepository).use(InMemoryFileRepository).asSingleton();

const user = DependencyInjectionUserProvider.get();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ export async function registerFolderServices(builder: ContainerBuilder): Promise
builder.register(SyncFolderMessenger).use(MainProcessSyncFolderMessenger);
// TODO: can be private?

builder.register(FolderRepository).use(InMemoryFolderRepository).asSingleton().private();
builder.register(FolderRepository).use(InMemoryFolderRepository).asSingleton();

builder.register(RemoteFileSystem).use(HttpRemoteFileSystem).private();

Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,35 @@
import { ContainerBuilder } from 'diod';
import { AbsolutePathToRelativeConverter } from '../../../../context/virtual-drive/shared/application/AbsolutePathToRelativeConverter';
import { RelativePathToAbsoluteConverter } from '../../../../context/virtual-drive/shared/application/RelativePathToAbsoluteConverter';
import { FileRepository } from '../../../../context/virtual-drive/files/domain/FileRepository';
import { FolderRepository } from '../../../../context/virtual-drive/folders/domain/FolderRepository';
import { PATHS } from '../../../../core/electron/paths';
import {
createDirectoryStateSqliteRepository,
DirectoryStateSqliteRepository,
} from '../../../../backend/features/virtual-drive/services/lazy/DirectoryStateSqliteRepository';
import {
createLazyVirtualDriveHydratorService,
LazyVirtualDriveHydrator,
} from '../../../../backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service';

export async function registerVirtualDriveSharedServices(builder: ContainerBuilder): Promise<void> {
const downloaded = PATHS.DOWNLOADED;

builder.register(RelativePathToAbsoluteConverter).useFactory(() => new RelativePathToAbsoluteConverter(downloaded));
builder.register(AbsolutePathToRelativeConverter).useFactory(() => new AbsolutePathToRelativeConverter(downloaded));
builder
.register(DirectoryStateSqliteRepository)
.useFactory(() => createDirectoryStateSqliteRepository())
.asSingleton();
builder
.register(LazyVirtualDriveHydrator)
.useFactory((container) =>
createLazyVirtualDriveHydratorService({
folderRepository: container.get(FolderRepository),
fileRepository: container.get(FileRepository),
directoryStateRepository: container.get(DirectoryStateSqliteRepository),
}),
)
.asSingleton();
}
36 changes: 36 additions & 0 deletions src/apps/main/database/data-source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,28 @@ import { TypeOrmStorageFile } from '../../../context/storage/StorageFiles/infras
import { logger } from '@internxt/drive-desktop-core/build/backend';
import { PATHS } from '../../../core/electron/paths';

const SQLITE_BOOTSTRAP_STATEMENTS = [
'PRAGMA journal_mode=WAL;',
'PRAGMA synchronous=NORMAL;',
'PRAGMA temp_store=MEMORY;',
'PRAGMA foreign_keys=ON;',
'PRAGMA busy_timeout=5000;',
'PRAGMA wal_autocheckpoint=1000;',
'PRAGMA mmap_size=268435456;',
`CREATE TABLE IF NOT EXISTS drive_directory_state (
folder_id INTEGER NOT NULL,
status_scope TEXT NOT NULL,
children_loaded_at TEXT NULL,
children_last_error_at TEXT NULL,
fetch_state TEXT NOT NULL DEFAULT 'idle',
PRIMARY KEY (folder_id, status_scope)
);`,
'CREATE INDEX IF NOT EXISTS idx_drive_file_folder_status_updated ON drive_file(folderId, status, updatedAt);',
'CREATE INDEX IF NOT EXISTS idx_drive_file_status_updated ON drive_file(status, updatedAt);',
'CREATE INDEX IF NOT EXISTS idx_drive_folder_parent_status_updated ON drive_folder(parentId, status, updatedAt);',
'CREATE INDEX IF NOT EXISTS idx_drive_folder_status_updated ON drive_folder(status, updatedAt);',
];

export const AppDataSource = new DataSource({
type: 'better-sqlite3',
database: PATHS.DATABASE,
Expand All @@ -16,6 +38,20 @@ export const AppDataSource = new DataSource({

logger.debug({ msg: `Using database file at ${PATHS.DATABASE}` });

export async function initializeVirtualDriveSqlite() {
if (!AppDataSource.isInitialized) {
return;
}

for (const statement of SQLITE_BOOTSTRAP_STATEMENTS) {
try {
await AppDataSource.query(statement);
} catch (error) {
logger.error({ msg: 'Error running SQLite bootstrap statement', statement, error });
}
}
}
Comment thread
egalvis27 marked this conversation as resolved.

export async function resetAppDataSourceOnLogout() {
if (!AppDataSource.isInitialized) {
return;
Expand Down
6 changes: 4 additions & 2 deletions src/apps/main/remote-sync/RemoteSyncManager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ describe('RemoteSyncManager', () => {
await sut.startRemoteSync();

expect(mockedGet).toBeCalledTimes(2);
expect(mockedGet.mock.calls[0]?.[1]).toMatchObject({ query: { status: 'EXISTS', limit: 2 } });
expect(sut.getSyncStatus()).toBe('SYNCED');
});
it('Should sync all the folders', async () => {
Expand Down Expand Up @@ -207,6 +208,7 @@ describe('RemoteSyncManager', () => {
await sut.startRemoteSync();

expect(mockedGet).toBeCalledTimes(3);
expect(mockedGet.mock.calls[0]?.[1]).toMatchObject({ query: { status: 'EXISTS', limit: 2 } });
expect(sut.getSyncStatus()).toBe('SYNCED');
});

Expand Down Expand Up @@ -252,7 +254,7 @@ describe('RemoteSyncManager', () => {

expect(mockedGet).toBeCalledTimes(6);
expect(sut.getSyncStatus()).toBe('SYNC_FAILED');
});
}, 15000);

it('Should fail the sync if some files or folders cannot be retrieved', async () => {
mockedGet.mockResolvedValueOnce({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') });
Expand All @@ -261,7 +263,7 @@ describe('RemoteSyncManager', () => {

expect(mockedGet).toBeCalledTimes(6);
expect(sut.getSyncStatus()).toBe('SYNC_FAILED');
});
}, 15000);

it('should handle the error while syncing files by calling the error handler properly', async () => {
const sut = new RemoteSyncManager(
Expand Down
33 changes: 16 additions & 17 deletions src/apps/main/remote-sync/RemoteSyncManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,20 +93,19 @@ export class RemoteSyncManager {
logger.debug({ tag: 'SYNC-ENGINE', msg: 'Starting' });
this.changeStatus('SYNCING');
try {
await Promise.all([
this.config.syncFiles
? this.syncRemoteFiles({
retry: 1,
maxRetries: 3,
})
: Promise.resolve(),
this.config.syncFolders
? this.syncRemoteFolders({
retry: 1,
maxRetries: 3,
})
: Promise.resolve(),
]);
if (this.config.syncFolders) {
await this.syncRemoteFolders({
retry: 1,
maxRetries: 3,
});
}

if (this.config.syncFiles) {
await this.syncRemoteFiles({
retry: 1,
maxRetries: 3,
});
}
} catch (error) {
this.changeStatus('SYNC_FAILED');
logger.error({
Expand Down Expand Up @@ -333,7 +332,7 @@ export class RemoteSyncManager {
const { data, error } = await fetchFiles({
limit: this.config.fetchFilesLimitPerRequest,
offset: 0,
status: 'ALL',
status: 'EXISTS',
updatedAt: updatedAtCheckpoint?.toISOString(),
});

Expand All @@ -357,9 +356,9 @@ export class RemoteSyncManager {
result: RemoteSyncedFolder[];
}> {
const { data, error } = await fetchFolders({
limit: this.config.fetchFilesLimitPerRequest,
limit: this.config.fetchFoldersLimitPerRequest,
offset: 0,
status: 'ALL',
status: 'EXISTS',
updatedAt: updatedAtCheckpoint?.toISOString(),
});

Expand Down
4 changes: 2 additions & 2 deletions src/apps/main/remote-sync/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ export const remoteSyncManager = new RemoteSyncManager(
folders: driveFoldersCollection,
},
{
fetchFilesLimitPerRequest: 1000,
fetchFoldersLimitPerRequest: 1000,
fetchFilesLimitPerRequest: 250,
fetchFoldersLimitPerRequest: 500,
syncFiles: true,
syncFolders: true,
},
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
import { MODAL_DEBOUNCE_MS } from './constants';
import { showMaxFileSizeRejectionModal } from './show-max-file-size-rejection-modal';
import { type UploadFileSizeValidation } from './validate-upload-file-size';

const uploadSizeLimitBlockedPaths = new Set<string>();
import {
clearUploadSizeLimitBlockedPath,
isUploadSizeLimitBlockedPath,
markUploadSizeLimitBlockedPath,
} from './upload-size-limit-blocked-paths';
export { clearUploadSizeLimitBlockedPath, isUploadSizeLimitBlockedPath, markUploadSizeLimitBlockedPath };

Check warning on line 9 in src/backend/features/user/file-size-limit/add-max-file-size-rejection.ts

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Use `export…from` to re-export `clearUploadSizeLimitBlockedPath`.

See more on https://sonarcloud.io/project/issues?id=internxt_drive-desktop-linux&issues=AZ8_rdg-3vq2AwgGmHdd&open=AZ8_rdg-3vq2AwgGmHdd&pullRequest=405

Check warning on line 9 in src/backend/features/user/file-size-limit/add-max-file-size-rejection.ts

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Use `export…from` to re-export `isUploadSizeLimitBlockedPath`.

See more on https://sonarcloud.io/project/issues?id=internxt_drive-desktop-linux&issues=AZ8_rdg-3vq2AwgGmHde&open=AZ8_rdg-3vq2AwgGmHde&pullRequest=405

type MaxFileSizeRejectionModalState = {
rejectedFilesCount: number;
Expand All @@ -23,18 +27,6 @@

let state: MaxFileSizeRejectionModalState | undefined;

export function markUploadSizeLimitBlockedPath(path: string): void {
uploadSizeLimitBlockedPaths.add(path);
}

export function isUploadSizeLimitBlockedPath(path: string): boolean {
return uploadSizeLimitBlockedPaths.has(path);
}

export function clearUploadSizeLimitBlockedPath(path: string): void {
uploadSizeLimitBlockedPaths.delete(path);
}

export function addMaxFileSizeRejection(rejection: MaxFileSizeRejection): void {
if (rejection.blockUploadPath ?? true) {
markUploadSizeLimitBlockedPath(rejection.path);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
const uploadSizeLimitBlockedPaths = new Set<string>();

export function markUploadSizeLimitBlockedPath(path: string): void {
uploadSizeLimitBlockedPaths.add(path);
}

export function isUploadSizeLimitBlockedPath(path: string): boolean {
return uploadSizeLimitBlockedPaths.has(path);
}

export function clearUploadSizeLimitBlockedPath(path: string): void {
uploadSizeLimitBlockedPaths.delete(path);
}
Comment on lines +1 to +13

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📐 Maintainability & Code Quality | 🔴 Critical | ⚡ Quick win

Fix Prettier formatting failures before merge.

The pipeline's format check (npm run format) reported code style issues in 2 files. As a new file in this PR, this file may be one of them. Run npm run format to auto-fix.

#!/bin/bash
# Identify which files have Prettier formatting issues
npx prettier src --check 2>&1 | head -20
🧰 Tools
🪛 GitHub Actions: Lint / 👨‍🎨 Format Check

[error] 1-1: Command failed: npm run format (prettier src --check). Prettier reported code style issues in 2 files, exiting with code 1.

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/backend/features/user/file-size-limit/upload-size-limit-blocked-paths.ts`
around lines 1 - 13, This file likely has Prettier formatting issues flagged by
the format check; run the repository formatting command to auto-fix the style in
the upload-size-limit-blocked-paths module. Verify the exported helpers
markUploadSizeLimitBlockedPath, isUploadSizeLimitBlockedPath, and
clearUploadSizeLimitBlockedPath remain unchanged functionally, then commit the
formatted output.

Source: Pipeline failures

Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ import * as releaseServiceModule from '../../services/operations/release.service
import { partialSpyOn } from '../../../../../../tests/vitest/utils.helper';
import { FuseError } from '../../../../../apps/drive/fuse/callbacks/FuseErrors';
import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes';
import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder';
import { TemporalFileDeleter } from '../../../../../context/storage/TemporalFiles/application/deletion/TemporalFileDeleter';
import { TemporalFileUploadQueue } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types';

vi.mock(import('@internxt/drive-desktop-core/build/backend'));

Expand All @@ -19,6 +22,11 @@ describe('releaseController', () => {
req = mockDeep<Request>();
res = mockDeep<Response>();
container = mockDeep<Container>();
container.get.calledWith(TemporalFileByPathFinder).mockReturnValue({ run: vi.fn() } as TemporalFileByPathFinder);
container.get.calledWith(TemporalFileDeleter).mockReturnValue({ run: vi.fn() } as TemporalFileDeleter);
container.get.calledWith(TemporalFileUploadQueue).mockReturnValue({
enqueue: vi.fn(),
} as unknown as { enqueue: (props: unknown) => Promise<void> });
});

it('should return errno 0 when release succeeds', async () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,28 @@ import { Container } from 'diod';
import { logger } from '@internxt/drive-desktop-core/build/backend';
import { release } from '../../services/operations/release.service';
import { ensureLeadingSlash } from '../ensure-leading-slash';
import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder';
import { TemporalFileDeleter } from '../../../../../context/storage/TemporalFiles/application/deletion/TemporalFileDeleter';
import { TemporalFileUploadQueue } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types';

export async function releaseController(req: Request, res: Response, container: Container) {
const rawPath: string = req.body.path ?? '';
const processName: string = req.body.processName ?? '';
logger.debug({
msg: `[FUSE DAEMON] Release signal received for path: ${rawPath} by process: ${processName}`,
});
const normalizedPath = ensureLeadingSlash(rawPath);
const finder = container.get(TemporalFileByPathFinder);
const deleter = container.get(TemporalFileDeleter);
const uploadQueue = container.get(TemporalFileUploadQueue);

const result = await release({ path: normalizedPath, processName, container });
const result = await release({
path: normalizedPath,
processName,
findTemporalFileByPath: finder.run.bind(finder),
deleteTemporalFile: deleter.run.bind(deleter),
enqueueTemporalFile: uploadQueue.enqueue.bind(uploadQueue),
});

if (result.error) {
res.json({ errno: result.error.code });
Expand Down
8 changes: 6 additions & 2 deletions src/backend/features/virtual-drive/ipc/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,15 @@ import { updateVirtualDriveContainer } from '../services/update-virtual-drive-co
import { DependencyInjectionUserProvider } from '../../../../apps/shared/dependency-injection/DependencyInjectionUserProvider';
import { logger } from '@internxt/drive-desktop-core/build/backend';
import { getVirtualDriveState } from '../services/daemon.service';
import { DirectoryStateSqliteRepository } from '../services/lazy/DirectoryStateSqliteRepository';

function remoteChangesSyncedHandler() {
const container = getVirtualDriveContainer();
if (container) {
updateVirtualDriveContainer({ container, user: DependencyInjectionUserProvider.get() });
void Promise.all([
updateVirtualDriveContainer({ container, user: DependencyInjectionUserProvider.get() }),
container.get(DirectoryStateSqliteRepository).clear(),
]);
} else {
logger.warn({ msg: '[FUSE] updateVirtualDriveContainer called before container was initialized' });
}
Expand All @@ -24,7 +28,7 @@ function syncRootChangedHandler({ oldPath, newPath }: { oldPath: string; newPath
}

export function registerVirtualDriveHandlers() {
eventBus.on('INITIAL_SYNC_READY', startVirtualDrive);
eventBus.on('APP_DATA_SOURCE_INITIALIZED', startVirtualDrive);
eventBus.on('REMOTE_CHANGES_SYNCHED', remoteChangesSyncedHandler);
eventBus.on('SYNC_ROOT_CHANGED', syncRootChangedHandler);
ipcMain.handle('get-virtual-drive-status', getVirtualDriveState);
Expand Down
Loading
Loading