From e09de644d573ea179b21694a3d9259b92c2c05af Mon Sep 17 00:00:00 2001 From: ido Date: Tue, 14 Jan 2025 22:25:49 +0200 Subject: [PATCH 01/37] fix(downloadProgram): different default chunk size --- .../download-file/download-engine-file.ts | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/download/download-engine/download-file/download-engine-file.ts b/src/download/download-engine/download-file/download-engine-file.ts index f7b3b5c..7ca3816 100644 --- a/src/download/download-engine/download-file/download-engine-file.ts +++ b/src/download/download-engine/download-file/download-engine-file.ts @@ -42,9 +42,12 @@ export type DownloadEngineFileEvents = { [key: string]: any }; +const DEFAULT_CHUNKS_SIZE_FOR_CHUNKS_PROGRAM = 1024 * 1024 * 5; // 5MB +const DEFAULT_CHUNKS_SIZE_FOR_STREAM_PROGRAM = 1024 * 1024; // 1MB + const DEFAULT_OPTIONS: Omit = { - chunkSize: 1024 * 1024 * 5, - parallelStreams: 1 + chunkSize: 0, + parallelStreams: 3 }; export default class DownloadEngineFile extends EventEmitter { @@ -70,7 +73,23 @@ export default class DownloadEngineFile extends EventEmitter Date: Tue, 21 Jan 2025 00:43:13 +0200 Subject: [PATCH 02/37] feat: debounce write - faster chunks write --- .../download-file/download-engine-file.ts | 5 +- .../engine/base-download-engine.ts | 13 ++- .../engine/download-engine-nodejs.ts | 8 ++ .../engine/utils/concurrency.ts | 42 +++++---- .../download-engine-write-stream-nodejs.ts | 82 ++++++++++++++--- .../utils/BytesWriteDebounce.ts | 92 +++++++++++++++++++ test/daynamic-content-length.test.ts | 6 +- 7 files changed, 207 insertions(+), 41 deletions(-) create mode 100644 src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts diff --git a/src/download/download-engine/download-file/download-engine-file.ts b/src/download/download-engine/download-file/download-engine-file.ts index 7ca3816..04bf6c6 100644 --- a/src/download/download-engine/download-file/download-engine-file.ts +++ b/src/download/download-engine/download-file/download-engine-file.ts @@ -19,6 +19,7 @@ export type DownloadEngineFileOptions = { onFinishAsync?: () => Promise onStartedAsync?: () => Promise onCloseAsync?: () => Promise + onPausedAsync?: () => Promise onSaveProgressAsync?: (progress: SaveProgressInfo) => Promise programType?: AvailablePrograms @@ -322,14 +323,14 @@ export default class DownloadEngineFile extends EventEmitter void paused: () => void resumed: () => void - progress: (progress: ProgressStatusWithIndex) => void + progress: (progress: FormattedStatus) => void save: (progress: SaveProgressInfo) => void finished: () => void closed: () => void @@ -36,7 +37,11 @@ export default class BaseDownloadEngine extends EventEmitter; protected _latestStatus?: ProgressStatusWithIndex; protected constructor(engine: DownloadEngineFile, options: DownloadEngineFileOptions) { @@ -106,8 +111,8 @@ export default class BaseDownloadEngine extends EventEmitter { + await this.options.writeStream.ensureBytesSynced(); + }; + // Try to clone the file if it's a single part download this._engine.options.onStartedAsync = async () => { if (this.options.skipExisting || this.options.fetchStrategy !== "localFile" || this.options.partURLs.length !== 1) return; diff --git a/src/download/download-engine/engine/utils/concurrency.ts b/src/download/download-engine/engine/utils/concurrency.ts index 7763584..408318b 100644 --- a/src/download/download-engine/engine/utils/concurrency.ts +++ b/src/download/download-engine/engine/utils/concurrency.ts @@ -1,24 +1,28 @@ -export function concurrency(array: Value[], concurrencyCount: number, callback: (value: Value) => Promise): Promise { - return new Promise((resolve, reject) => { - let index = 0; - let activeCount = 0; +export function concurrency(array: Value[], concurrencyCount: number, callback: (value: Value) => Promise) { + const {resolve, reject, promise} = Promise.withResolvers(); + let index = 0; + let activeCount = 0; - function next() { - if (index === array.length && activeCount === 0) { - resolve(); - return; - } + function reload() { + if (index === array.length && activeCount === 0) { + resolve(); + return; + } - while (activeCount < concurrencyCount && index < array.length) { - activeCount++; - callback(array[index++]) - .then(() => { - activeCount--; - next(); - }, reject); - } + while (activeCount < concurrencyCount && index < array.length) { + activeCount++; + callback(array[index++]) + .then(() => { + activeCount--; + reload(); + }, reject); } + } + + reload(); - next(); - }); + return { + promise, + reload + }; } diff --git a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts index ebf6032..ac3e7ed 100644 --- a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts +++ b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts @@ -4,27 +4,70 @@ import retry from "async-retry"; import {withLock} from "lifecycle-utils"; import BaseDownloadEngineWriteStream from "./base-download-engine-write-stream.js"; import WriterIsClosedError from "./errors/writer-is-closed-error.js"; +import {BytesWriteDebounce} from "./utils/BytesWriteDebounce.js"; export type DownloadEngineWriteStreamOptionsNodeJS = { retry?: retry.Options mode: string; + debounceWrite?: { + maxTime?: number + maxSize?: number + } }; -const DEFAULT_OPTIONS: DownloadEngineWriteStreamOptionsNodeJS = { - mode: "r+" -}; +const DEFAULT_OPTIONS = { + mode: "r+", + debounceWrite: { + maxTime: 1000 * 5, // 5 seconds + maxSize: 1024 * 1024 * 2 // 2 MB + } +} satisfies DownloadEngineWriteStreamOptionsNodeJS; +const MAX_AUTO_DEBOUNCE_SIZE = 1024 * 1024 * 100; // 100 MB +const AUTO_DEBOUNCE_SIZE_PERCENT = 0.05; const NOT_ENOUGH_SPACE_ERROR_CODE = "ENOSPC"; export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineWriteStream { private _fd: FileHandle | null = null; private _fileWriteFinished = false; + private _writeDebounce: BytesWriteDebounce; + private _fileSize = 0; + public readonly options: DownloadEngineWriteStreamOptionsNodeJS; - public fileSize = 0; + public autoDebounceMaxSize = false; constructor(public path: string, public finalPath: string, options: Partial = {}) { super(); - this.options = {...DEFAULT_OPTIONS, ...options}; + + this.autoDebounceMaxSize = !options.debounceWrite?.maxSize; + const optionsWithDefaults = this.options = { + ...DEFAULT_OPTIONS, + ...options, + debounceWrite: { + ...DEFAULT_OPTIONS.debounceWrite, + ...options.debounceWrite + } + }; + + this._writeDebounce = new BytesWriteDebounce({ + ...optionsWithDefaults.debounceWrite, + writev: (cursor, buffer) => this._writeWithoutDebounce(cursor, buffer) + }); + } + + public get fileSize() { + return this._fileSize; + } + + public set fileSize(value) { + this._fileSize = value; + + if (this.autoDebounceMaxSize) { + this.options.debounceWrite!.maxSize = Math.max( + Math.min(value * AUTO_DEBOUNCE_SIZE_PERCENT, MAX_AUTO_DEBOUNCE_SIZE), + DEFAULT_OPTIONS.debounceWrite.maxSize + ); + } } private async _ensureFileOpen() { @@ -41,11 +84,15 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW } async write(cursor: number, buffer: Uint8Array) { + await this._writeDebounce.addChunk(cursor, buffer); + } + + async _writeWithoutDebounce(cursor: number, buffers: Uint8Array[]) { let throwError: Error | false = false; await retry(async () => { try { - return await this._writeWithoutRetry(cursor, buffer); + return await this._writeWithoutRetry(cursor, buffers); } catch (error: any) { if (error?.code === NOT_ENOUGH_SPACE_ERROR_CODE) { throwError = error; @@ -60,7 +107,12 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW } } - async ftruncate(size = this.fileSize) { + async ensureBytesSynced() { + await this._writeDebounce.writeAll(); + } + + async ftruncate(size = this._fileSize) { + await this.ensureBytesSynced(); this._fileWriteFinished = true; await retry(async () => { const fd = await this._ensureFileOpen(); @@ -78,7 +130,7 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW const encoder = new TextEncoder(); const uint8Array = encoder.encode(jsonString); - await this.write(this.fileSize, uint8Array); + await this.write(this._fileSize, uint8Array); } async loadMetadataAfterFileWithoutRetry() { @@ -89,13 +141,13 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW const fd = await this._ensureFileOpen(); try { const state = await fd.stat(); - const metadataSize = state.size - this.fileSize; + const metadataSize = state.size - this._fileSize; if (metadataSize <= 0) { return; } const metadataBuffer = Buffer.alloc(metadataSize); - await fd.read(metadataBuffer, 0, metadataSize, this.fileSize); + await fd.read(metadataBuffer, 0, metadataSize, this._fileSize); const decoder = new TextDecoder(); const metadataString = decoder.decode(metadataBuffer); @@ -108,10 +160,12 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW } } - private async _writeWithoutRetry(cursor: number, buffer: Uint8Array) { - const fd = await this._ensureFileOpen(); - const {bytesWritten} = await fd.write(buffer, 0, buffer.length, cursor); - return bytesWritten; + private async _writeWithoutRetry(cursor: number, buffers: Uint8Array[]) { + return await withLock(this, "lockWriteOperation", async () => { + const fd = await this._ensureFileOpen(); + const {bytesWritten} = await fd.writev(buffers, cursor); + return bytesWritten; + }); } override async close() { diff --git a/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts b/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts new file mode 100644 index 0000000..f68d47e --- /dev/null +++ b/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts @@ -0,0 +1,92 @@ +import sleep from "sleep-promise"; + +export type BytesWriteDebounceOptions = { + maxTime: number; + maxSize: number; + writev: (index: number, buffers: Uint8Array[]) => Promise; +} + +export class BytesWriteDebounce { + private _writeChunks: { + index: number; + buffer: Uint8Array; + }[] = []; + private _lastWriteTime = Date.now(); + private _totalSizeOfChunks = 0; + private _checkWriteInterval = false; + + constructor(private _options: BytesWriteDebounceOptions) { + + } + + async addChunk(index: number, buffer: Uint8Array) { + this._writeChunks.push({index, buffer}); + this._totalSizeOfChunks += buffer.byteLength; + + await this._writeIfNeeded(); + this.checkIfWriteNeededInterval(); + } + + private async _writeIfNeeded() { + if (this._totalSizeOfChunks >= this._options.maxSize || Date.now() - this._lastWriteTime >= this._options.maxTime) { + await this.writeAll(); + } + } + + private async checkIfWriteNeededInterval() { + if (this._checkWriteInterval) { + return; + } + this._checkWriteInterval = true; + + while (this._writeChunks.length > 0) { + await this._writeIfNeeded(); + const timeUntilMaxLimitAfterWrite = this._options.maxTime - (Date.now() - this._lastWriteTime); + await sleep(Math.max(timeUntilMaxLimitAfterWrite, 0)); + } + + this._checkWriteInterval = false; + } + + writeAll() { + if (this._writeChunks.length === 0) { + return; + } + + this._writeChunks = this._writeChunks.sort((a, b) => a.index - b.index); + const firstWrite = this._writeChunks[0]; + + let writeIndex = firstWrite.index; + let buffer: Uint8Array[] = [firstWrite.buffer]; + let bufferTotalLength = firstWrite.buffer.length; + + const writePromises: Promise[] = []; + + for (let i = 0; i < this._writeChunks.length; i++) { + const nextWriteLocation = writeIndex + bufferTotalLength; + const currentWrite = this._writeChunks[i]; + + if (currentWrite.index < nextWriteLocation) { //overlapping, prefer the last buffer (newer data) + const lastBuffer = buffer.pop()!; + buffer.push(currentWrite.buffer); + bufferTotalLength += currentWrite.buffer.length - lastBuffer.length; + } else if (nextWriteLocation === currentWrite.index) { + buffer.push(currentWrite.buffer); + bufferTotalLength += currentWrite.buffer.length; + } else { + writePromises.push(this._options.writev(writeIndex, buffer)); + + writeIndex = currentWrite.index; + buffer = [currentWrite.buffer]; + bufferTotalLength = currentWrite.buffer.length; + } + } + + writePromises.push(this._options.writev(writeIndex, buffer)); + + this._writeChunks = []; + this._totalSizeOfChunks = 0; + this._lastWriteTime = Date.now(); + return Promise.all(writePromises); + } +} diff --git a/test/daynamic-content-length.test.ts b/test/daynamic-content-length.test.ts index 3625c69..ef1c232 100644 --- a/test/daynamic-content-length.test.ts +++ b/test/daynamic-content-length.test.ts @@ -15,10 +15,12 @@ describe("Dynamic content download", async () => { beforeAll(async () => { regularDownload = await ensureLocalFile(DYNAMIC_DOWNLOAD_FILE, ORIGINAL_FILE); originalFileHash = await fileHash(regularDownload); - }); + }, 1000 * 30); afterAll(async () => { - await fs.remove(regularDownload); + if (regularDownload) { + await fs.remove(regularDownload); + } }); From 739ab73b418a053eb8511650cae139c0b147b6f3 Mon Sep 17 00:00:00 2001 From: ido Date: Tue, 21 Jan 2025 00:43:32 +0200 Subject: [PATCH 03/37] feat: scoped multi download engine --- .../engine/download-engine-multi-download.ts | 97 +++++++++++++------ 1 file changed, 68 insertions(+), 29 deletions(-) diff --git a/src/download/download-engine/engine/download-engine-multi-download.ts b/src/download/download-engine/engine/download-engine-multi-download.ts index ebdbadf..528ff0e 100644 --- a/src/download/download-engine/engine/download-engine-multi-download.ts +++ b/src/download/download-engine/engine/download-engine-multi-download.ts @@ -1,15 +1,13 @@ import {EventEmitter} from "eventemitter3"; import {FormattedStatus} from "../../transfer-visualize/format-transfer-status.js"; -import ProgressStatisticsBuilder, {ProgressStatusWithIndex} from "../../transfer-visualize/progress-statistics-builder.js"; +import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js"; import BaseDownloadEngine, {BaseDownloadEngineEvents} from "./base-download-engine.js"; import DownloadAlreadyStartedError from "./error/download-already-started-error.js"; import {concurrency} from "./utils/concurrency.js"; import {DownloadFlags, DownloadStatus} from "../download-file/progress-status-file.js"; import {NoDownloadEngineProvidedError} from "./error/no-download-engine-provided-error.js"; -const DEFAULT_PARALLEL_DOWNLOADS = 1; - -type DownloadEngineMultiAllowedEngines = BaseDownloadEngine; +type DownloadEngineMultiAllowedEngines = BaseDownloadEngine | DownloadEngineMultiDownload; type DownloadEngineMultiDownloadEvents = BaseDownloadEngineEvents & { childDownloadStarted: (engine: Engine) => void @@ -18,28 +16,58 @@ type DownloadEngineMultiDownloadEvents extends EventEmitter { public readonly downloads: Engine[]; - public readonly options: DownloadEngineMultiDownloadOptions; + protected _options: DownloadEngineMultiDownloadOptions; protected _aborted = false; protected _activeEngines = new Set(); protected _progressStatisticsBuilder = new ProgressStatisticsBuilder(); - protected _downloadStatues: (ProgressStatusWithIndex | FormattedStatus)[] = []; + protected _downloadStatues: FormattedStatus[] | FormattedStatus[][] = []; protected _closeFiles: (() => Promise)[] = []; - protected _lastStatus?: ProgressStatusWithIndex; + protected _lastStatus?: FormattedStatus; protected _loadingDownloads = 0; + protected _reloadDownloadParallelisms?: () => void; + /** + * @internal + */ + _downloadStarted?: Promise; protected constructor(engines: (DownloadEngineMultiAllowedEngines | DownloadEngineMultiDownload)[], options: DownloadEngineMultiDownloadOptions) { super(); this.downloads = DownloadEngineMultiDownload._extractEngines(engines); - this.options = options; + this._options = {...DEFAULT_OPTIONS, ...options}; this._init(); } + public get parallelDownloads() { + return this._options.parallelDownloads; + } + + public set parallelDownloads(value) { + if (this._options.parallelDownloads === value) return; + this._options.parallelDownloads = value; + this._reloadDownloadParallelisms?.(); + } + public get downloadStatues() { - return this._downloadStatues; + return this._downloadStatues.flat(); } public get status() { @@ -74,15 +102,21 @@ export default class DownloadEngineMultiDownload + this._options.unpackInnerMultiDownloadsStatues && engine instanceof DownloadEngineMultiDownload ? engine.downloadStatues : defaultProgress; + + this._downloadStatues[index] = getStatus(); engine.on("progress", (progress) => { - this._downloadStatues[index] = progress; + this._downloadStatues[index] = getStatus(progress); }); - this._changeEngineFinishDownload(engine); + if (this._options.finalizeDownloadAfterAllSettled) { + this._changeEngineFinishDownload(engine); + } + this._reloadDownloadParallelisms?.(); } - public async addDownload(engine: Engine | DownloadEngineMultiDownload | Promise>) { + public async addDownload(engine: Engine | Promise) { const index = this.downloads.length + this._loadingDownloads; this._downloadStatues[index] = ProgressStatisticsBuilder.loadingStatusEmptyStatistics(); @@ -92,18 +126,9 @@ export default class DownloadEngineMultiDownload { @@ -114,19 +139,25 @@ export default class DownloadEngineMultiDownload { + const concurrencyCount = this._options.parallelDownloads ?? DEFAULT_OPTIONS.parallelDownloads; + const {reload, promise} = concurrency(this.downloads, concurrencyCount, async (engine) => { if (this._aborted) return; this._activeEngines.add(engine); this.emit("childDownloadStarted", engine); - await engine.download(); + if (engine._downloadStarted) { + await engine._downloadStarted; + } else { + await engine.download(); + } this.emit("childDownloadClosed", engine); this._activeEngines.delete(engine); }); + this._reloadDownloadParallelisms = reload; + this._downloadStarted = promise; + await promise; this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Finished; this.emit("finished"); await this._finishEnginesDownload(); @@ -134,6 +165,14 @@ export default class DownloadEngineMultiDownload { + }; + this._closeFiles.push(_finishEnginesDownload); + return; + } + const options = engine._fileEngineOptions; const onFinishAsync = options.onFinishAsync; const onCloseAsync = options.onCloseAsync; From 216ff94d34ccfad3630e819b2d5afb43817459cb Mon Sep 17 00:00:00 2001 From: ido Date: Tue, 21 Jan 2025 01:10:22 +0200 Subject: [PATCH 04/37] feat: add id to download --- package-lock.json | 61 +++++++++++++------ package.json | 3 +- .../download-file/download-engine-file.ts | 4 ++ .../download-file/progress-status-file.ts | 3 + src/download/download-engine/types.ts | 1 + .../progress-statistics-builder.ts | 1 + 6 files changed, 53 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index 80da83d..d151e19 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,7 +27,8 @@ "sleep-promise": "^9.1.0", "slice-ansi": "^7.1.0", "stdout-update": "^4.0.1", - "strip-ansi": "^7.1.0" + "strip-ansi": "^7.1.0", + "uid": "^2.0.2" }, "bin": { "ipull": "dist/cli/cli.js" @@ -1247,6 +1248,15 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@lukeed/csprng": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@lukeed/csprng/-/csprng-1.1.0.tgz", + "integrity": "sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/@material/material-color-utilities": { "version": "0.2.7", "resolved": "https://registry.npmjs.org/@material/material-color-utilities/-/material-color-utilities-0.2.7.tgz", @@ -7307,24 +7317,6 @@ "thenify-all": "^1.0.0" } }, - "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -10530,6 +10522,25 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/postcss/node_modules/nanoid": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -12407,6 +12418,18 @@ "node": ">=0.8.0" } }, + "node_modules/uid": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/uid/-/uid-2.0.2.tgz", + "integrity": "sha512-u3xV3X7uzvi5b1MncmZo3i2Aw222Zk1keqLA1YkHldREkAhAqi65wuPfe7lHx8H/Wzy+8CE7S7uS3jekIM5s8g==", + "license": "MIT", + "dependencies": { + "@lukeed/csprng": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", diff --git a/package.json b/package.json index 4616151..5655c5e 100644 --- a/package.json +++ b/package.json @@ -147,6 +147,7 @@ "sleep-promise": "^9.1.0", "slice-ansi": "^7.1.0", "stdout-update": "^4.0.1", - "strip-ansi": "^7.1.0" + "strip-ansi": "^7.1.0", + "uid": "^2.0.2" } } diff --git a/src/download/download-engine/download-file/download-engine-file.ts b/src/download/download-engine/download-file/download-engine-file.ts index 04bf6c6..d9eea3b 100644 --- a/src/download/download-engine/download-file/download-engine-file.ts +++ b/src/download/download-engine/download-file/download-engine-file.ts @@ -8,6 +8,7 @@ import {withLock} from "lifecycle-utils"; import switchProgram, {AvailablePrograms} from "./download-programs/switch-program.js"; import BaseDownloadProgram from "./download-programs/base-download-program.js"; import {pushComment} from "./utils/push-comment.js"; +import {uid} from "uid"; export type DownloadEngineFileOptions = { chunkSize?: number; @@ -56,6 +57,7 @@ export default class DownloadEngineFile extends EventEmitter Date: Fri, 14 Feb 2025 17:50:36 +0200 Subject: [PATCH 05/37] feat: Global CLI management --- src/browser.ts | 10 +- src/download/browser-download.ts | 21 +- .../engine/base-download-engine.ts | 23 ++- .../engine/download-engine-multi-download.ts | 186 ++++++++++++------ .../engine/download-engine-nodejs.ts | 12 +- .../no-download-engine-provided-error.ts | 7 - .../download-engine-write-stream-nodejs.ts | 5 +- .../utils/BytesWriteDebounce.ts | 4 +- src/download/node-download.ts | 33 +--- .../progress-statistics-builder.ts | 98 ++++++--- .../transfer-cli/GlobalCLI.ts | 157 +++++++++++++++ .../transfer-cli/cli-animation-wrapper.ts | 89 --------- .../base-loading-animation.ts | 74 ------- .../cli-spinners-loading-animation.ts | 23 --- .../multiProgressBars/BaseMultiProgressBar.ts | 17 +- .../SummaryMultiProgressBar.ts | 8 +- .../transfer-cli/transfer-cli.ts | 40 +--- src/index.ts | 4 +- 18 files changed, 423 insertions(+), 388 deletions(-) delete mode 100644 src/download/download-engine/engine/error/no-download-engine-provided-error.ts create mode 100644 src/download/transfer-visualize/transfer-cli/GlobalCLI.ts delete mode 100644 src/download/transfer-visualize/transfer-cli/cli-animation-wrapper.ts delete mode 100644 src/download/transfer-visualize/transfer-cli/loading-animation/base-loading-animation.ts delete mode 100644 src/download/transfer-visualize/transfer-cli/loading-animation/cli-spinners-loading-animation.ts diff --git a/src/browser.ts b/src/browser.ts index d81a030..f6b4b16 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -1,4 +1,4 @@ -import {downloadFileBrowser, DownloadFileBrowserOptions, downloadSequenceBrowser} from "./download/browser-download.js"; +import {downloadFileBrowser, DownloadFileBrowserOptions, downloadSequenceBrowser, DownloadSequenceBrowserOptions} from "./download/browser-download.js"; import DownloadEngineBrowser from "./download/download-engine/engine/download-engine-browser.js"; import EmptyResponseError from "./download/download-engine/streams/download-engine-fetch-stream/errors/empty-response-error.js"; import StatusCodeError from "./download/download-engine/streams/download-engine-fetch-stream/errors/status-code-error.js"; @@ -14,7 +14,6 @@ import HttpError from "./download/download-engine/streams/download-engine-fetch- import BaseDownloadEngine from "./download/download-engine/engine/base-download-engine.js"; import {InvalidOptionError} from "./download/download-engine/engine/error/InvalidOptionError.js"; import {DownloadFlags, DownloadStatus} from "./download/download-engine/download-file/progress-status-file.js"; -import {NoDownloadEngineProvidedError} from "./download/download-engine/engine/error/no-download-engine-provided-error.js"; export { DownloadFlags, @@ -29,15 +28,14 @@ export { FetchStreamError, IpullError, EngineError, - InvalidOptionError, - NoDownloadEngineProvidedError + InvalidOptionError }; - export type { BaseDownloadEngine, DownloadFileBrowserOptions, DownloadEngineBrowser, DownloadEngineMultiDownload, FormattedStatus, - SaveProgressInfo + SaveProgressInfo, + DownloadSequenceBrowserOptions }; diff --git a/src/download/browser-download.ts b/src/download/browser-download.ts index b4f4ebf..0dc4765 100644 --- a/src/download/browser-download.ts +++ b/src/download/browser-download.ts @@ -1,6 +1,7 @@ import DownloadEngineBrowser, {DownloadEngineOptionsBrowser} from "./download-engine/engine/download-engine-browser.js"; -import DownloadEngineMultiDownload from "./download-engine/engine/download-engine-multi-download.js"; -import {NoDownloadEngineProvidedError} from "./download-engine/engine/error/no-download-engine-provided-error.js"; +import DownloadEngineMultiDownload, {DownloadEngineMultiDownloadOptions} from "./download-engine/engine/download-engine-multi-download.js"; +import BaseDownloadEngine from "./download-engine/engine/base-download-engine.js"; +import {DownloadSequenceOptions} from "./node-download.js"; const DEFAULT_PARALLEL_STREAMS_FOR_BROWSER = 3; @@ -9,6 +10,8 @@ export type DownloadFileBrowserOptions = DownloadEngineOptionsBrowser & { partsURL?: string[]; }; +export type DownloadSequenceBrowserOptions = DownloadEngineMultiDownloadOptions; + /** * Download one file in the browser environment. */ @@ -25,10 +28,16 @@ export async function downloadFileBrowser(options: DownloadFileBrowserOptions) { /** * Download multiple files in the browser environment. */ -export async function downloadSequenceBrowser(...downloads: (DownloadEngineBrowser | Promise)[]) { - if (downloads.length === 0) { - throw new NoDownloadEngineProvidedError(); +export function downloadSequenceBrowser(options?: DownloadSequenceBrowserOptions | DownloadEngineBrowser | Promise, ...downloads: (DownloadEngineBrowser | Promise)[]) { + let downloadOptions: DownloadSequenceOptions = {}; + if (options instanceof BaseDownloadEngine || options instanceof Promise) { + downloads.unshift(options); + } else if (options) { + downloadOptions = options; } - return await DownloadEngineMultiDownload.fromEngines(downloads); + const downloader = new DownloadEngineMultiDownload(downloadOptions); + downloader.addDownload(...downloads); + + return downloader; } diff --git a/src/download/download-engine/engine/base-download-engine.ts b/src/download/download-engine/engine/base-download-engine.ts index cb13bb7..22fc104 100644 --- a/src/download/download-engine/engine/base-download-engine.ts +++ b/src/download/download-engine/engine/base-download-engine.ts @@ -3,8 +3,7 @@ import DownloadEngineFile, {DownloadEngineFileOptions} from "../download-file/do import BaseDownloadEngineFetchStream, {BaseDownloadEngineFetchStreamOptions} from "../streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js"; import UrlInputError from "./error/url-input-error.js"; import {EventEmitter} from "eventemitter3"; -import ProgressStatisticsBuilder, {ProgressStatusWithIndex} from "../../transfer-visualize/progress-statistics-builder.js"; -import DownloadAlreadyStartedError from "./error/download-already-started-error.js"; +import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js"; import retry from "async-retry"; import {AvailablePrograms} from "../download-file/download-programs/switch-program.js"; import StatusCodeError from "../streams/download-engine-fetch-stream/errors/status-code-error.js"; @@ -41,8 +40,12 @@ export default class BaseDownloadEngine extends EventEmitter; - protected _latestStatus?: ProgressStatusWithIndex; + _downloadEndPromise = Promise.withResolvers(); + /** + * @internal + */ + _downloadStarted = false; + protected _latestStatus?: FormattedStatus; protected constructor(engine: DownloadEngineFile, options: DownloadEngineFileOptions) { super(); @@ -101,18 +104,22 @@ export default class BaseDownloadEngine extends EventEmitter { this._latestStatus = status; - return this.emit("progress", status); + this.emit("progress", status); }); } async download() { if (this._downloadStarted) { - throw new DownloadAlreadyStartedError(); + return this._downloadEndPromise.promise; } try { - this._downloadStarted = this._engine.download(); - await this._downloadStarted; + this._downloadStarted = true; + const promise = this._engine.download(); + promise + .then(this._downloadEndPromise.resolve) + .catch(this._downloadEndPromise.reject); + await promise; } finally { await this.close(); } diff --git a/src/download/download-engine/engine/download-engine-multi-download.ts b/src/download/download-engine/engine/download-engine-multi-download.ts index 528ff0e..8652ba8 100644 --- a/src/download/download-engine/engine/download-engine-multi-download.ts +++ b/src/download/download-engine/engine/download-engine-multi-download.ts @@ -2,16 +2,15 @@ import {EventEmitter} from "eventemitter3"; import {FormattedStatus} from "../../transfer-visualize/format-transfer-status.js"; import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js"; import BaseDownloadEngine, {BaseDownloadEngineEvents} from "./base-download-engine.js"; -import DownloadAlreadyStartedError from "./error/download-already-started-error.js"; import {concurrency} from "./utils/concurrency.js"; import {DownloadFlags, DownloadStatus} from "../download-file/progress-status-file.js"; -import {NoDownloadEngineProvidedError} from "./error/no-download-engine-provided-error.js"; -type DownloadEngineMultiAllowedEngines = BaseDownloadEngine | DownloadEngineMultiDownload; +export type DownloadEngineMultiAllowedEngines = BaseDownloadEngine | DownloadEngineMultiDownload; type DownloadEngineMultiDownloadEvents = BaseDownloadEngineEvents & { childDownloadStarted: (engine: Engine) => void childDownloadClosed: (engine: Engine) => void + downloadAdded: (engine: Engine) => void }; export type DownloadEngineMultiDownloadOptions = { @@ -25,6 +24,15 @@ export type DownloadEngineMultiDownloadOptions = { * Finalize download (change .ipull file to original extension) after all downloads are settled */ finalizeDownloadAfterAllSettled?: boolean + + /** + * Do not start download automatically + * @internal + */ + naturalDownloadStart?: boolean + + downloadName?: string + downloadComment?: string }; const DEFAULT_OPTIONS = { @@ -34,32 +42,71 @@ const DEFAULT_OPTIONS = { } satisfies DownloadEngineMultiDownloadOptions; export default class DownloadEngineMultiDownload extends EventEmitter { - public readonly downloads: Engine[]; + public readonly downloads: Engine[] = []; protected _options: DownloadEngineMultiDownloadOptions; protected _aborted = false; protected _activeEngines = new Set(); protected _progressStatisticsBuilder = new ProgressStatisticsBuilder(); protected _downloadStatues: FormattedStatus[] | FormattedStatus[][] = []; protected _closeFiles: (() => Promise)[] = []; - protected _lastStatus?: FormattedStatus; + protected _lastStatus: FormattedStatus = null!; protected _loadingDownloads = 0; protected _reloadDownloadParallelisms?: () => void; + protected _engineWaitPromises = new Set>(); /** * @internal */ - _downloadStarted?: Promise; + _downloadEndPromise = Promise.withResolvers(); + /** + * @internal + */ + _downloadStarted = false; - protected constructor(engines: (DownloadEngineMultiAllowedEngines | DownloadEngineMultiDownload)[], options: DownloadEngineMultiDownloadOptions) { + /** + * @internal + */ + constructor(options: DownloadEngineMultiDownloadOptions = {}) { super(); - this.downloads = DownloadEngineMultiDownload._extractEngines(engines); this._options = {...DEFAULT_OPTIONS, ...options}; this._init(); } + public get activeDownloads() { + return Array.from(this._activeEngines); + } + public get parallelDownloads() { return this._options.parallelDownloads; } + public get loadingDownloads() { + if (!this._options.unpackInnerMultiDownloadsStatues) { + return this._loadingDownloads; + } + + let totalLoading = this._loadingDownloads; + for (const download of this.downloads) { + if (download instanceof DownloadEngineMultiDownload) { + totalLoading += download.loadingDownloads; + } + } + + return totalLoading; + } + + /** + * @internal + */ + public get _flatEngines(): Engine[] { + return this.downloads.map(engine => { + if (engine instanceof DownloadEngineMultiDownload) { + return engine._flatEngines; + } + return engine; + }) + .flat(); + } + public set parallelDownloads(value) { if (this._options.parallelDownloads === value) return; this._options.parallelDownloads = value; @@ -67,13 +114,11 @@ export default class DownloadEngineMultiDownload statues.findIndex(x => x.downloadId === status.downloadId) === index)); } public get status() { - if (!this._lastStatus) { - throw new NoDownloadEngineProvidedError(); - } return this._lastStatus; } @@ -86,24 +131,25 @@ export default class DownloadEngineMultiDownload { progress = { ...progress, + fileName: this._options.downloadName ?? progress.fileName, + comment: this._options.downloadComment ?? progress.comment, downloadFlags: progress.downloadFlags.concat([DownloadFlags.DownloadSequence]) }; this._lastStatus = progress; this.emit("progress", progress); }); - let index = 0; - for (const engine of this.downloads) { - this._addEngine(engine, index++); - } - - // Prevent multiple progress events on adding engines - this._progressStatisticsBuilder.add(...this.downloads); + const originalProgress = this._progressStatisticsBuilder.status; + this._lastStatus = { + ...originalProgress, + downloadFlags: originalProgress.downloadFlags.concat([DownloadFlags.DownloadSequence]) + }; } private _addEngine(engine: Engine, index: number) { + this.emit("downloadAdded", engine); const getStatus = (defaultProgress = engine.status) => - this._options.unpackInnerMultiDownloadsStatues && engine instanceof DownloadEngineMultiDownload ? engine.downloadStatues : defaultProgress; + (this._options.unpackInnerMultiDownloadsStatues && engine instanceof DownloadEngineMultiDownload ? engine.downloadStatues : defaultProgress); this._downloadStatues[index] = getStatus(); engine.on("progress", (progress) => { @@ -116,52 +162,80 @@ export default class DownloadEngineMultiDownload) { + public async _addDownloadNoStatisticUpdate(engine: Engine | Promise) { const index = this.downloads.length + this._loadingDownloads; this._downloadStatues[index] = ProgressStatisticsBuilder.loadingStatusEmptyStatistics(); this._loadingDownloads++; this._progressStatisticsBuilder._totalDownloadParts++; - const awaitEngine = engine instanceof Promise ? await engine : engine; + this._progressStatisticsBuilder._sendLatestProgress(); + + const isPromise = engine instanceof Promise; + if (isPromise) { + this._engineWaitPromises.add(engine); + } + const awaitEngine = isPromise ? await engine : engine; + if (isPromise) { + this._engineWaitPromises.delete(engine); + } this._progressStatisticsBuilder._totalDownloadParts--; this._loadingDownloads--; this._addEngine(awaitEngine, index); this.downloads.push(awaitEngine); - this._progressStatisticsBuilder.add(awaitEngine); + this._progressStatisticsBuilder.add(awaitEngine, true); + + return awaitEngine; + } + + public async addDownload(...engines: (Engine | Promise)[]) { + await Promise.all(engines.map(this._addDownloadNoStatisticUpdate.bind(this))); } public async download(): Promise { - if (this._activeEngines.size) { - throw new DownloadAlreadyStartedError(); + if (this._downloadStarted) { + return this._downloadEndPromise.promise; } - this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Active; - this.emit("start"); - - const concurrencyCount = this._options.parallelDownloads ?? DEFAULT_OPTIONS.parallelDownloads; - const {reload, promise} = concurrency(this.downloads, concurrencyCount, async (engine) => { - if (this._aborted) return; - this._activeEngines.add(engine); - - this.emit("childDownloadStarted", engine); - if (engine._downloadStarted) { - await engine._downloadStarted; - } else { - await engine.download(); + try { + this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Active; + this._downloadStarted = true; + this.emit("start"); + + const concurrencyCount = this._options.parallelDownloads || DEFAULT_OPTIONS.parallelDownloads; + let continueIteration = true; + while (this._loadingDownloads > 0 || continueIteration) { + continueIteration = false; + const {reload, promise} = concurrency(this.downloads, concurrencyCount, async (engine) => { + if (this._aborted) return; + this._activeEngines.add(engine); + + this.emit("childDownloadStarted", engine); + if (engine._downloadStarted || this._options.naturalDownloadStart) { + await engine._downloadEndPromise.promise; + } else { + await engine.download(); + } + this.emit("childDownloadClosed", engine); + + this._activeEngines.delete(engine); + }); + this._reloadDownloadParallelisms = reload; + await promise; + continueIteration = this._engineWaitPromises.size > 0; + await Promise.race(this._engineWaitPromises); } - this.emit("childDownloadClosed", engine); - - this._activeEngines.delete(engine); - }); - this._reloadDownloadParallelisms = reload; - this._downloadStarted = promise; - await promise; - this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Finished; - this.emit("finished"); - await this._finishEnginesDownload(); - await this.close(); + this._downloadEndPromise = Promise.withResolvers(); + this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Finished; + this.emit("finished"); + await this._finishEnginesDownload(); + await this.close(); + this._downloadEndPromise.resolve(); + } catch (error) { + this._downloadEndPromise.reject(error); + throw error; + } } private _changeEngineFinishDownload(engine: Engine) { @@ -214,18 +288,4 @@ export default class DownloadEngineMultiDownload(engines: Engine[]) { - return engines.map(engine => { - if (engine instanceof DownloadEngineMultiDownload) { - return engine.downloads; - } - return engine; - }) - .flat(); - } - - public static async fromEngines(engines: (Engine | Promise)[], options: DownloadEngineMultiDownloadOptions = {}) { - return new DownloadEngineMultiDownload(await Promise.all(engines), options); - } } diff --git a/src/download/download-engine/engine/download-engine-nodejs.ts b/src/download/download-engine/engine/download-engine-nodejs.ts index 9709a64..a27f1e3 100644 --- a/src/download/download-engine/engine/download-engine-nodejs.ts +++ b/src/download/download-engine/engine/download-engine-nodejs.ts @@ -16,7 +16,7 @@ export const PROGRESS_FILE_EXTENSION = ".ipull"; type PathOptions = { directory: string } | { savePath: string }; export type DownloadEngineOptionsNodejs = PathOptions & BaseDownloadEngineOptions & { fileName?: string; - fetchStrategy?: "localFile" | "fetch"; + fetchStrategy?: "local" | "remote"; skipExisting?: boolean; debounceWrite?: { maxTime: number @@ -51,7 +51,7 @@ export default class DownloadEngineNodejs { if (this.options.skipExisting) return; - await this.options.writeStream.saveMedataAfterFile(progress); + await this.options.writeStream.saveMetadataAfterFile(progress); }; this._engine.options.onPausedAsync = async () => { @@ -60,7 +60,7 @@ export default class DownloadEngineNodejs { - if (this.options.skipExisting || this.options.fetchStrategy !== "localFile" || this.options.partURLs.length !== 1) return; + if (this.options.skipExisting || this.options.fetchStrategy !== "local" || this.options.partURLs.length !== 1) return; try { const {reflinkFile} = await import("@reflink/reflink"); @@ -138,7 +138,7 @@ export default class DownloadEngineNodejs= MAX_META_SIZE) { return; } diff --git a/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts b/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts index f68d47e..68810db 100644 --- a/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts +++ b/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts @@ -4,7 +4,7 @@ export type BytesWriteDebounceOptions = { maxTime: number; maxSize: number; writev: (index: number, buffers: Uint8Array[]) => Promise; -} +}; export class BytesWriteDebounce { private _writeChunks: { @@ -66,7 +66,7 @@ export class BytesWriteDebounce { const nextWriteLocation = writeIndex + bufferTotalLength; const currentWrite = this._writeChunks[i]; - if (currentWrite.index < nextWriteLocation) { //overlapping, prefer the last buffer (newer data) + if (currentWrite.index < nextWriteLocation) { // overlapping, prefer the last buffer (newer data) const lastBuffer = buffer.pop()!; buffer.push(currentWrite.buffer); bufferTotalLength += currentWrite.buffer.length - lastBuffer.length; diff --git a/src/download/node-download.ts b/src/download/node-download.ts index 69647fe..be05706 100644 --- a/src/download/node-download.ts +++ b/src/download/node-download.ts @@ -1,32 +1,20 @@ import DownloadEngineNodejs, {DownloadEngineOptionsNodejs} from "./download-engine/engine/download-engine-nodejs.js"; import BaseDownloadEngine from "./download-engine/engine/base-download-engine.js"; import DownloadEngineMultiDownload, {DownloadEngineMultiDownloadOptions} from "./download-engine/engine/download-engine-multi-download.js"; -import CliAnimationWrapper, {CliProgressDownloadEngineOptions} from "./transfer-visualize/transfer-cli/cli-animation-wrapper.js"; -import {CLI_LEVEL} from "./transfer-visualize/transfer-cli/transfer-cli.js"; -import {NoDownloadEngineProvidedError} from "./download-engine/engine/error/no-download-engine-provided-error.js"; +import {CliProgressDownloadEngineOptions, globalCLI} from "./transfer-visualize/transfer-cli/GlobalCLI.js"; const DEFAULT_PARALLEL_STREAMS_FOR_NODEJS = 3; -export type DownloadFileOptions = DownloadEngineOptionsNodejs & CliProgressDownloadEngineOptions & { - /** @deprecated use partURLs instead */ - partsURL?: string[]; -}; +export type DownloadFileOptions = DownloadEngineOptionsNodejs & CliProgressDownloadEngineOptions; /** * Download one file with CLI progress */ export async function downloadFile(options: DownloadFileOptions) { - // TODO: Remove in the next major version - if (!("url" in options) && options.partsURL) { - options.partURLs ??= options.partsURL; - } - - options.parallelStreams ??= DEFAULT_PARALLEL_STREAMS_FOR_NODEJS; const downloader = DownloadEngineNodejs.createFromOptions(options); - const wrapper = new CliAnimationWrapper(downloader, options); + globalCLI.addDownload(downloader, options); - await wrapper.attachAnimation(); return await downloader; } @@ -38,7 +26,7 @@ export type DownloadSequenceOptions = CliProgressDownloadEngineOptions & Downloa /** * Download multiple files with CLI progress */ -export async function downloadSequence(options?: DownloadSequenceOptions | DownloadEngineNodejs | Promise, ...downloads: (DownloadEngineNodejs | Promise)[]) { +export function downloadSequence(options?: DownloadSequenceOptions | DownloadEngineNodejs | Promise, ...downloads: (DownloadEngineNodejs | Promise)[]) { let downloadOptions: DownloadSequenceOptions = {}; if (options instanceof BaseDownloadEngine || options instanceof Promise) { downloads.unshift(options); @@ -46,14 +34,9 @@ export async function downloadSequence(options?: DownloadSequenceOptions | Downl downloadOptions = options; } - if (downloads.length === 0) { - throw new NoDownloadEngineProvidedError(); - } - - downloadOptions.cliLevel = CLI_LEVEL.HIGH; - const downloader = DownloadEngineMultiDownload.fromEngines(downloads, downloadOptions); - const wrapper = new CliAnimationWrapper(downloader, downloadOptions); + const downloader = new DownloadEngineMultiDownload(downloadOptions); + downloader.addDownload(...downloads); + globalCLI.addDownload(downloader, downloadOptions); - await wrapper.attachAnimation(); - return await downloader; + return downloader; } diff --git a/src/download/transfer-visualize/progress-statistics-builder.ts b/src/download/transfer-visualize/progress-statistics-builder.ts index cc79022..aec3682 100644 --- a/src/download/transfer-visualize/progress-statistics-builder.ts +++ b/src/download/transfer-visualize/progress-statistics-builder.ts @@ -16,10 +16,11 @@ interface CliProgressBuilderEvents { export type AnyEngine = DownloadEngineFile | BaseDownloadEngine | DownloadEngineMultiDownload; export default class ProgressStatisticsBuilder extends EventEmitter { - private _engines: AnyEngine[] = []; + private _engines = new Set(); private _activeTransfers: { [index: number]: number } = {}; private _totalBytes = 0; private _transferredBytes = 0; + private _latestEngine: AnyEngine | null = null; /** * @internal */ @@ -27,8 +28,35 @@ export default class ProgressStatisticsBuilder extends EventEmitter { this._sendProgress(data, index, downloadPartStart); @@ -71,10 +97,20 @@ export default class ProgressStatisticsBuilder extends EventEmitter string) + cliName?: string; + loadingAnimation?: cliSpinners.SpinnerName; +}; + +class GlobalCLI { + private _multiDownloadEngine = this._createMultiDownloadEngine(); + private _transferCLI = GlobalCLI._createOptions({}); + private _cliActive = false; + private _downloadOptions = new WeakMap(); + + constructor() { + this._registerCLIEvents(); + } + + async addDownload(engine: AllowedDownloadEngine | Promise, cliOptions: CliProgressDownloadEngineOptions = {}) { + if (!this._cliActive && cliOptions.cliProgress) { + this._transferCLI = GlobalCLI._createOptions(cliOptions); + } + + if (engine instanceof Promise) { + engine.then((engine) => this._downloadOptions.set(engine, cliOptions)); + } else { + this._downloadOptions.set(engine, cliOptions); + } + + await this._multiDownloadEngine.addDownload(engine); + if (this._multiDownloadEngine.activeDownloads.length === 0) { + await this._multiDownloadEngine.download(); + } + } + + private _createMultiDownloadEngine() { + return new DownloadEngineMultiDownload({ + unpackInnerMultiDownloadsStatues: true, + finalizeDownloadAfterAllSettled: false, + naturalDownloadStart: true, + parallelDownloads: Number.MAX_VALUE + }); + } + + private _registerCLIEvents() { + const isDownloadActive = (parentEngine: DownloadEngineMultiDownload = this._multiDownloadEngine) => { + if (parentEngine.loadingDownloads > 0) { + return true; + } + + for (const engine of parentEngine.activeDownloads) { + if (engine instanceof DownloadEngineMultiDownload) { + if (isDownloadActive(engine)) { + return true; + } + } + + if (engine.status.downloadStatus === DownloadStatus.Active || parentEngine.status.downloadStatus === DownloadStatus.Active && [DownloadStatus.Loading, DownloadStatus.NotStarted].includes(engine.status.downloadStatus)) { + return true; + } + } + + return false; + }; + + const checkPauseCLI = () => { + if (!isDownloadActive()) { + this._transferCLI.stop(); + this._cliActive = false; + } + }; + + const checkResumeCLI = (engine: DownloadEngineMultiAllowedEngines) => { + if (engine.status.downloadStatus === DownloadStatus.Active) { + this._transferCLI.start(); + this._cliActive = true; + } + }; + + this._multiDownloadEngine.on("start", () => { + this._transferCLI.start(); + this._cliActive = true; + }); + + this._multiDownloadEngine.on("finished", () => { + this._transferCLI.stop(); + this._cliActive = false; + this._multiDownloadEngine = this._createMultiDownloadEngine(); + }); + + this._multiDownloadEngine.on("childDownloadStarted", function registerEngineStatus(engine) { + engine.on("closed", checkPauseCLI); + engine.on("paused", checkPauseCLI); + engine.on("resumed", () => checkResumeCLI(engine)); + + if (engine instanceof DownloadEngineMultiDownload) { + engine.on("childDownloadStarted", registerEngineStatus); + } + }); + + const getCLIEngines = (multiEngine: DownloadEngineMultiDownload) => { + const enginesToShow: AllowedDownloadEngine[] = []; + for (const engine of multiEngine.activeDownloads) { + const isShowEngine = this._downloadOptions.get(engine)?.cliProgress; + if (engine instanceof DownloadEngineMultiDownload) { + if (isShowEngine) { + enginesToShow.push(...engine._flatEngines); + continue; + } + enginesToShow.push(...getCLIEngines(engine)); + } else if (isShowEngine) { + enginesToShow.push(engine); + } + } + + return enginesToShow.filter((engine, index, self) => self.indexOf(engine) === index); + }; + + this._multiDownloadEngine.on("progress", (progress) => { + if (!this._cliActive) return; + const statues = getCLIEngines(this._multiDownloadEngine) + .map(x => x.status); + this._transferCLI.updateStatues(statues, progress, this._multiDownloadEngine.loadingDownloads); + }); + } + + private static _createOptions(options: CliProgressDownloadEngineOptions) { + const cliOptions: Partial = {...options}; + cliOptions.createProgressBar ??= typeof options.cliStyle === "function" ? + { + createStatusLine: options.cliStyle, + multiProgressBar: options.createMultiProgressBar ?? BaseMultiProgressBar + } : + switchCliProgressStyle(options.cliStyle ?? DEFAULT_CLI_STYLE, { + truncateName: options.truncateName, + loadingSpinner: options.loadingAnimation + }); + return new TransferCli(cliOptions); + } +} + +export const globalCLI = new GlobalCLI(); diff --git a/src/download/transfer-visualize/transfer-cli/cli-animation-wrapper.ts b/src/download/transfer-visualize/transfer-cli/cli-animation-wrapper.ts deleted file mode 100644 index d1a2c48..0000000 --- a/src/download/transfer-visualize/transfer-cli/cli-animation-wrapper.ts +++ /dev/null @@ -1,89 +0,0 @@ -import DownloadEngineNodejs from "../../download-engine/engine/download-engine-nodejs.js"; -import DownloadEngineMultiDownload from "../../download-engine/engine/download-engine-multi-download.js"; -import switchCliProgressStyle, {AvailableCLIProgressStyle} from "./progress-bars/switch-cli-progress-style.js"; -import {CliFormattedStatus} from "./progress-bars/base-transfer-cli-progress-bar.js"; -import TransferCli, {CLI_LEVEL, TransferCliOptions} from "./transfer-cli.js"; -import {BaseMultiProgressBar} from "./multiProgressBars/BaseMultiProgressBar.js"; -import cliSpinners from "cli-spinners"; - -const DEFAULT_CLI_STYLE: AvailableCLIProgressStyle = "auto"; -type AllowedDownloadEngines = DownloadEngineNodejs | DownloadEngineMultiDownload; - -export type CliProgressDownloadEngineOptions = { - truncateName?: boolean | number; - cliProgress?: boolean; - maxViewDownloads?: number; - createMultiProgressBar?: typeof BaseMultiProgressBar, - cliStyle?: AvailableCLIProgressStyle | ((status: CliFormattedStatus) => string) - cliName?: string; - cliAction?: string; - fetchStrategy?: "localFile" | "fetch"; - loadingAnimation?: cliSpinners.SpinnerName; - /** @internal */ - cliLevel?: CLI_LEVEL; -}; - -export default class CliAnimationWrapper { - private readonly _downloadEngine: Promise; - private readonly _options: CliProgressDownloadEngineOptions; - private _activeCLI?: TransferCli; - - public constructor(downloadEngine: Promise, _options: CliProgressDownloadEngineOptions) { - this._options = _options; - this._downloadEngine = downloadEngine; - this._init(); - } - - private _init() { - if (!this._options.cliProgress) { - return; - } - this._options.cliAction ??= this._options.fetchStrategy === "localFile" ? "Copying" : "Downloading"; - - const cliOptions: Partial = {...this._options}; - if (this._options.cliAction) { - cliOptions.action = this._options.cliAction; - } - if (this._options.cliName) { - cliOptions.name = this._options.cliName; - } - - cliOptions.createProgressBar = typeof this._options.cliStyle === "function" ? - { - createStatusLine: this._options.cliStyle, - multiProgressBar: this._options.createMultiProgressBar ?? BaseMultiProgressBar - } : - switchCliProgressStyle(this._options.cliStyle ?? DEFAULT_CLI_STYLE, { - truncateName: this._options.truncateName, - loadingSpinner: this._options.loadingAnimation - }); - - this._activeCLI = new TransferCli(cliOptions, this._options.cliLevel); - } - - public async attachAnimation() { - if (!this._activeCLI) { - return; - } - this._activeCLI.loadingAnimation.start(); - - try { - const engine = await this._downloadEngine; - this._activeCLI.loadingAnimation.stop(); - - engine.once("start", () => { - this._activeCLI?.start(); - - engine.on("progress", (progress) => { - this._activeCLI?.updateStatues(engine.downloadStatues, progress); - }); - - engine.on("closed", () => { - this._activeCLI?.stop(); - }); - }); - } finally { - this._activeCLI.loadingAnimation.stop(); - } - } -} diff --git a/src/download/transfer-visualize/transfer-cli/loading-animation/base-loading-animation.ts b/src/download/transfer-visualize/transfer-cli/loading-animation/base-loading-animation.ts deleted file mode 100644 index 588bee8..0000000 --- a/src/download/transfer-visualize/transfer-cli/loading-animation/base-loading-animation.ts +++ /dev/null @@ -1,74 +0,0 @@ -import UpdateManager from "stdout-update"; -import sleep from "sleep-promise"; -import {CLIProgressPrintType} from "../multiProgressBars/BaseMultiProgressBar.js"; - -export type BaseLoadingAnimationOptions = { - updateIntervalMs?: number | null; - loadingText?: string; - logType: CLIProgressPrintType -}; - -export const DEFAULT_LOADING_ANIMATION_OPTIONS: BaseLoadingAnimationOptions = { - loadingText: "Gathering information", - logType: "update" -}; - -const DEFAULT_UPDATE_INTERVAL_MS = 300; - -export default abstract class BaseLoadingAnimation { - protected options: BaseLoadingAnimationOptions; - protected stdoutManager = UpdateManager.getInstance(); - protected _animationActive = false; - - - protected constructor(options: BaseLoadingAnimationOptions = DEFAULT_LOADING_ANIMATION_OPTIONS) { - this.options = options; - this._processExit = this._processExit.bind(this); - } - - protected _render(): void { - const frame = this.createFrame(); - - if (this.options.logType === "update") { - this.stdoutManager.update([frame]); - } else { - console.log(frame); - } - } - - protected abstract createFrame(): string; - - async start() { - process.on("SIGINT", this._processExit); - - if (this.options.logType === "update") { - this.stdoutManager.hook(); - } - - this._animationActive = true; - while (this._animationActive) { - this._render(); - await sleep(this.options.updateIntervalMs || DEFAULT_UPDATE_INTERVAL_MS); - } - } - - stop(): void { - if (!this._animationActive) { - return; - } - - this._animationActive = false; - - if (this.options.logType === "update") { - this.stdoutManager.erase(); - this.stdoutManager.unhook(false); - } - - process.off("SIGINT", this._processExit); - } - - private _processExit() { - this.stop(); - process.exit(0); - } -} diff --git a/src/download/transfer-visualize/transfer-cli/loading-animation/cli-spinners-loading-animation.ts b/src/download/transfer-visualize/transfer-cli/loading-animation/cli-spinners-loading-animation.ts deleted file mode 100644 index 54d1fda..0000000 --- a/src/download/transfer-visualize/transfer-cli/loading-animation/cli-spinners-loading-animation.ts +++ /dev/null @@ -1,23 +0,0 @@ -import BaseLoadingAnimation, {BaseLoadingAnimationOptions, DEFAULT_LOADING_ANIMATION_OPTIONS} from "./base-loading-animation.js"; -import {Spinner} from "cli-spinners"; - -export default class CliSpinnersLoadingAnimation extends BaseLoadingAnimation { - private _spinner: Spinner; - private _frameIndex = 0; - - public constructor(spinner: Spinner, options: BaseLoadingAnimationOptions) { - options = {...DEFAULT_LOADING_ANIMATION_OPTIONS, ...options}; - options.updateIntervalMs ??= spinner.interval; - super(options); - this._spinner = spinner; - } - - protected createFrame(): string { - const frame = this._spinner.frames[this._frameIndex]; - this._frameIndex++; - if (this._frameIndex >= this._spinner.frames.length) { - this._frameIndex = 0; - } - return `${frame} ${this.options.loadingText}`; - } -} diff --git a/src/download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.ts b/src/download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.ts index d393167..d4c5086 100644 --- a/src/download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.ts +++ b/src/download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.ts @@ -3,11 +3,12 @@ import {FormattedStatus} from "../../format-transfer-status.js"; import {DownloadStatus} from "../../../download-engine/download-file/progress-status-file.js"; import chalk from "chalk"; import prettyBytes from "pretty-bytes"; +import cliSpinners from "cli-spinners"; export type MultiProgressBarOptions = { maxViewDownloads: number; createProgressBar: TransferCliProgressBar - action?: string; + loadingAnimation: cliSpinners.SpinnerName, }; export type CLIProgressPrintType = "update" | "log"; @@ -16,14 +17,12 @@ export class BaseMultiProgressBar { public readonly updateIntervalMs: null | number = null; public readonly printType: CLIProgressPrintType = "update"; + public constructor(protected options: MultiProgressBarOptions) { } protected createProgresses(statuses: FormattedStatus[]): string { - return statuses.map((status) => { - status.transferAction = this.options.action ?? status.transferAction; - return this.options.createProgressBar.createStatusLine(status); - }) + return statuses.map((status) => this.options.createProgressBar.createStatusLine(status)) .join("\n"); } @@ -49,8 +48,8 @@ export class BaseMultiProgressBar { } // eslint-disable-next-line @typescript-eslint/no-unused-vars - createMultiProgressBar(statuses: FormattedStatus[], oneStatus: FormattedStatus) { - if (statuses.length < this.options.maxViewDownloads) { + createMultiProgressBar(statuses: FormattedStatus[], oneStatus: FormattedStatus, loadingDownloads = 0) { + if (statuses.length < this.options.maxViewDownloads - Math.min(loadingDownloads, 1)) { return this.createProgresses(statuses); } @@ -58,10 +57,10 @@ export class BaseMultiProgressBar { const tasksLogs = this.createProgresses(allStatusesSorted.slice(0, this.options.maxViewDownloads)); if (notFinished) { - return tasksLogs + `\nand ${chalk.gray(remaining)} more out of ${chalk.blueBright(statuses.length)} downloads.`; + return tasksLogs + `\nand ${chalk.gray((remaining + loadingDownloads).toLocaleString())} more out of ${chalk.blueBright(statuses.length.toLocaleString())} downloads.`; } const totalSize = allStatusesSorted.reduce((acc, status) => acc + status.totalBytes, 0); - return tasksLogs + `\n${chalk.green(`All ${statuses.length} downloads (${prettyBytes(totalSize)}) finished.`)}`; + return tasksLogs + `\n${chalk.green(`All ${statuses.length.toLocaleString()} downloads (${prettyBytes(totalSize)}) finished.`)}`; } } diff --git a/src/download/transfer-visualize/transfer-cli/multiProgressBars/SummaryMultiProgressBar.ts b/src/download/transfer-visualize/transfer-cli/multiProgressBars/SummaryMultiProgressBar.ts index 2bf59c0..b257edc 100644 --- a/src/download/transfer-visualize/transfer-cli/multiProgressBars/SummaryMultiProgressBar.ts +++ b/src/download/transfer-visualize/transfer-cli/multiProgressBars/SummaryMultiProgressBar.ts @@ -8,15 +8,15 @@ export class SummaryMultiProgressBar extends BaseMultiProgressBar { private _parallelDownloads = 0; private _lastStatuses: FormattedStatus[] = []; - override createMultiProgressBar(statuses: FormattedStatus[], oneStatus: FormattedStatus) { + override createMultiProgressBar(statuses: FormattedStatus[], oneStatus: FormattedStatus, loadingDownloads = 0) { oneStatus = structuredClone(oneStatus); oneStatus.downloadFlags.push(DownloadFlags.DownloadSequence); const linesToPrint: FormattedStatus[] = []; - let index = 0; for (const status of statuses) { - const isStatusChanged = this._lastStatuses[index++]?.downloadStatus !== status.downloadStatus; + const lastStatus = this._lastStatuses.find(x => x.downloadId === status.downloadId); + const isStatusChanged = lastStatus?.downloadStatus !== status.downloadStatus; const copyStatus = structuredClone(status); if (isStatusChanged) { @@ -38,7 +38,7 @@ export class SummaryMultiProgressBar extends BaseMultiProgressBar { const activeDownloads = statuses.filter((status) => status.downloadStatus === DownloadStatus.Active).length; this._parallelDownloads ||= activeDownloads; const finishedDownloads = statuses.filter((status) => status.downloadStatus === DownloadStatus.Finished).length; - oneStatus.comment = `${finishedDownloads}/${statuses.length} files done${this._parallelDownloads > 1 ? ` (${activeDownloads} active)` : ""}`; + oneStatus.comment = `${finishedDownloads.toLocaleString()}/${(statuses.length + loadingDownloads).toLocaleString()} files done${this._parallelDownloads > 1 ? ` (${activeDownloads.toLocaleString()} active)` : ""}`; return this.createProgresses(filterStatusesSliced); } diff --git a/src/download/transfer-visualize/transfer-cli/transfer-cli.ts b/src/download/transfer-visualize/transfer-cli/transfer-cli.ts index 9e64af8..ccb36e1 100644 --- a/src/download/transfer-visualize/transfer-cli/transfer-cli.ts +++ b/src/download/transfer-visualize/transfer-cli/transfer-cli.ts @@ -2,13 +2,11 @@ import UpdateManager from "stdout-update"; import debounce from "lodash.debounce"; import {TransferCliProgressBar} from "./progress-bars/base-transfer-cli-progress-bar.js"; import cliSpinners from "cli-spinners"; -import CliSpinnersLoadingAnimation from "./loading-animation/cli-spinners-loading-animation.js"; import {FormattedStatus} from "../format-transfer-status.js"; import switchCliProgressStyle from "./progress-bars/switch-cli-progress-style.js"; import {BaseMultiProgressBar} from "./multiProgressBars/BaseMultiProgressBar.js"; export type TransferCliOptions = { - action?: string, name?: string, maxViewDownloads: number; truncateName: boolean | number; @@ -17,7 +15,6 @@ export type TransferCliOptions = { createProgressBar: TransferCliProgressBar; createMultiProgressBar: typeof BaseMultiProgressBar, loadingAnimation: cliSpinners.SpinnerName, - loadingText?: string; }; export const DEFAULT_TRANSFER_CLI_OPTIONS: TransferCliOptions = { @@ -27,30 +24,19 @@ export const DEFAULT_TRANSFER_CLI_OPTIONS: TransferCliOptions = { maxDebounceWait: process.platform === "win32" ? 500 : 100, createProgressBar: switchCliProgressStyle("auto", {truncateName: true}), loadingAnimation: "dots", - loadingText: "Gathering information", createMultiProgressBar: BaseMultiProgressBar }; -export enum CLI_LEVEL { - LOW = 0, - HIGH = 2 -} - - export default class TransferCli { - public static activeCLILevel = CLI_LEVEL.LOW; - public readonly loadingAnimation: CliSpinnersLoadingAnimation; protected options: TransferCliOptions; protected stdoutManager = UpdateManager.getInstance(); - protected myCLILevel: number; - protected latestProgress: [FormattedStatus[], FormattedStatus] = null!; + protected latestProgress: [FormattedStatus[], FormattedStatus, number] = null!; private _cliStopped = true; private readonly _updateStatuesDebounce: () => void; private _multiProgressBar: BaseMultiProgressBar; private _isFirstPrint = true; - public constructor(options: Partial, myCLILevel = CLI_LEVEL.LOW) { - TransferCli.activeCLILevel = this.myCLILevel = myCLILevel; + public constructor(options: Partial) { this.options = {...DEFAULT_TRANSFER_CLI_OPTIONS, ...options}; this._multiProgressBar = new this.options.createProgressBar.multiProgressBar(this.options); @@ -59,18 +45,11 @@ export default class TransferCli { maxWait: maxDebounceWait }); - this.loadingAnimation = new CliSpinnersLoadingAnimation(cliSpinners[this.options.loadingAnimation], { - loadingText: this.options.loadingText, - updateIntervalMs: this._multiProgressBar.updateIntervalMs, - logType: this._multiProgressBar.printType - }); this._processExit = this._processExit.bind(this); - - } start() { - if (this.myCLILevel !== TransferCli.activeCLILevel) return; + if (!this._cliStopped) return; this._cliStopped = false; if (this._multiProgressBar.printType === "update") { this.stdoutManager.hook(); @@ -79,9 +58,9 @@ export default class TransferCli { } stop() { - if (this._cliStopped || this.myCLILevel !== TransferCli.activeCLILevel) return; - this._updateStatues(); + if (this._cliStopped) return; this._cliStopped = true; + this._updateStatues(); if (this._multiProgressBar.printType === "update") { this.stdoutManager.unhook(false); } @@ -93,8 +72,8 @@ export default class TransferCli { process.exit(0); } - updateStatues(statues: FormattedStatus[], oneStatus: FormattedStatus) { - this.latestProgress = [statues, oneStatus]; + updateStatues(statues: FormattedStatus[], oneStatus: FormattedStatus, loadingDownloads = 0) { + this.latestProgress = [statues, oneStatus, loadingDownloads]; if (this._isFirstPrint) { this._isFirstPrint = false; @@ -105,10 +84,7 @@ export default class TransferCli { } private _updateStatues() { - if (this._cliStopped || this.myCLILevel !== TransferCli.activeCLILevel) { - return; // Do not update if there is a higher level CLI, meaning that this CLI is sub-CLI - } - + if (!this.latestProgress) return; const printLog = this._multiProgressBar.createMultiProgressBar(...this.latestProgress); if (printLog) { this._logUpdate(printLog); diff --git a/src/index.ts b/src/index.ts index dbc4183..60f6da1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,7 +16,6 @@ import BaseDownloadEngine from "./download/download-engine/engine/base-download- import {InvalidOptionError} from "./download/download-engine/engine/error/InvalidOptionError.js"; import {BaseMultiProgressBar, MultiProgressBarOptions} from "./download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.js"; import {DownloadFlags, DownloadStatus} from "./download/download-engine/download-file/progress-status-file.js"; -import {NoDownloadEngineProvidedError} from "./download/download-engine/engine/error/no-download-engine-provided-error.js"; export { DownloadFlags, @@ -33,8 +32,7 @@ export { FetchStreamError, IpullError, EngineError, - InvalidOptionError, - NoDownloadEngineProvidedError + InvalidOptionError }; From b3949864f54cc62ea1969e07b29e1b3ec9e20def Mon Sep 17 00:00:00 2001 From: ido Date: Fri, 14 Feb 2025 18:28:33 +0200 Subject: [PATCH 06/37] feat: Remote CLI progress --- src/browser.ts | 5 +- src/download/browser-download.ts | 8 +++ .../engine/DownloadEngineRemote.ts | 72 +++++++++++++++++++ .../engine/download-engine-multi-download.ts | 22 ++++-- src/download/node-download.ts | 12 ++++ .../progress-statistics-builder.ts | 3 +- .../transfer-cli/GlobalCLI.ts | 3 +- src/index.ts | 9 ++- 8 files changed, 125 insertions(+), 9 deletions(-) create mode 100644 src/download/download-engine/engine/DownloadEngineRemote.ts diff --git a/src/browser.ts b/src/browser.ts index f6b4b16..46572e0 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -1,4 +1,4 @@ -import {downloadFileBrowser, DownloadFileBrowserOptions, downloadSequenceBrowser, DownloadSequenceBrowserOptions} from "./download/browser-download.js"; +import {downloadFileBrowser, DownloadFileBrowserOptions, downloadFileRemoteBrowser, downloadSequenceBrowser, DownloadSequenceBrowserOptions} from "./download/browser-download.js"; import DownloadEngineBrowser from "./download/download-engine/engine/download-engine-browser.js"; import EmptyResponseError from "./download/download-engine/streams/download-engine-fetch-stream/errors/empty-response-error.js"; import StatusCodeError from "./download/download-engine/streams/download-engine-fetch-stream/errors/status-code-error.js"; @@ -14,11 +14,13 @@ import HttpError from "./download/download-engine/streams/download-engine-fetch- import BaseDownloadEngine from "./download/download-engine/engine/base-download-engine.js"; import {InvalidOptionError} from "./download/download-engine/engine/error/InvalidOptionError.js"; import {DownloadFlags, DownloadStatus} from "./download/download-engine/download-file/progress-status-file.js"; +import {DownloadEngineRemote} from "./download/download-engine/engine/DownloadEngineRemote.js"; export { DownloadFlags, DownloadStatus, downloadFileBrowser, + downloadFileRemoteBrowser, downloadSequenceBrowser, EmptyResponseError, HttpError, @@ -31,6 +33,7 @@ export { InvalidOptionError }; export type { + DownloadEngineRemote, BaseDownloadEngine, DownloadFileBrowserOptions, DownloadEngineBrowser, diff --git a/src/download/browser-download.ts b/src/download/browser-download.ts index 0dc4765..0a177bc 100644 --- a/src/download/browser-download.ts +++ b/src/download/browser-download.ts @@ -2,6 +2,7 @@ import DownloadEngineBrowser, {DownloadEngineOptionsBrowser} from "./download-en import DownloadEngineMultiDownload, {DownloadEngineMultiDownloadOptions} from "./download-engine/engine/download-engine-multi-download.js"; import BaseDownloadEngine from "./download-engine/engine/base-download-engine.js"; import {DownloadSequenceOptions} from "./node-download.js"; +import {DownloadEngineRemote} from "./download-engine/engine/DownloadEngineRemote.js"; const DEFAULT_PARALLEL_STREAMS_FOR_BROWSER = 3; @@ -25,6 +26,13 @@ export async function downloadFileBrowser(options: DownloadFileBrowserOptions) { return await DownloadEngineBrowser.createFromOptions(options); } +/** + * Stream events for a download from remote session, doing so by calling `emitRemoteProgress` with the progress info. + */ +export function downloadFileRemoteBrowser() { + return new DownloadEngineRemote(); +} + /** * Download multiple files in the browser environment. */ diff --git a/src/download/download-engine/engine/DownloadEngineRemote.ts b/src/download/download-engine/engine/DownloadEngineRemote.ts new file mode 100644 index 0000000..a153b3e --- /dev/null +++ b/src/download/download-engine/engine/DownloadEngineRemote.ts @@ -0,0 +1,72 @@ +import {BaseDownloadEngineEvents} from "./base-download-engine.js"; +import {EventEmitter} from "eventemitter3"; +import {FormattedStatus} from "../../transfer-visualize/format-transfer-status.js"; +import {DownloadStatus} from "../download-file/progress-status-file.js"; +import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js"; + +export class DownloadEngineRemote extends EventEmitter { + /** + * @internal + */ + _downloadEndPromise = Promise.withResolvers(); + /** + * @internal + */ + _downloadStarted = false; + + private _latestStatus: FormattedStatus = ProgressStatisticsBuilder.loadingStatusEmptyStatistics(); + + public get status() { + return this._latestStatus!; + } + + public get downloadStatues() { + return [this.status]; + } + + public get downloadSize() { + return this._latestStatus?.totalBytes ?? 0; + } + + public get fileName() { + return this._latestStatus?.fileName ?? ""; + } + + public download() { + return this._downloadEndPromise.promise; + } + + public emitRemoteProgress(progress: FormattedStatus) { + this._latestStatus = progress; + this.emit("progress", progress); + const isStatusChanged = this._latestStatus?.downloadStatus !== progress.downloadStatus; + + if (!isStatusChanged) { + return; + } + + switch (progress.downloadStatus) { + case DownloadStatus.Active: + if (this._latestStatus?.downloadStatus === DownloadStatus.Paused) { + this.emit("resumed"); + } else { + this.emit("start"); + this._downloadStarted = true; + } + break; + case DownloadStatus.Finished: + case DownloadStatus.Cancelled: + this.emit("finished"); + this.emit("closed"); + this._downloadEndPromise.resolve(); + break; + case DownloadStatus.Paused: + this.emit("paused"); + break; + } + + if (progress.downloadStatus === DownloadStatus.Active) { + this.emit("start"); + } + } +} diff --git a/src/download/download-engine/engine/download-engine-multi-download.ts b/src/download/download-engine/engine/download-engine-multi-download.ts index 8652ba8..11a2b50 100644 --- a/src/download/download-engine/engine/download-engine-multi-download.ts +++ b/src/download/download-engine/engine/download-engine-multi-download.ts @@ -4,8 +4,9 @@ import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statist import BaseDownloadEngine, {BaseDownloadEngineEvents} from "./base-download-engine.js"; import {concurrency} from "./utils/concurrency.js"; import {DownloadFlags, DownloadStatus} from "../download-file/progress-status-file.js"; +import {DownloadEngineRemote} from "./DownloadEngineRemote.js"; -export type DownloadEngineMultiAllowedEngines = BaseDownloadEngine | DownloadEngineMultiDownload; +export type DownloadEngineMultiAllowedEngines = BaseDownloadEngine | DownloadEngineRemote | DownloadEngineMultiDownload; type DownloadEngineMultiDownloadEvents = BaseDownloadEngineEvents & { childDownloadStarted: (engine: Engine) => void @@ -247,6 +248,10 @@ export default class DownloadEngineMultiDownload engine.pause()); + this._activeEngines.forEach(engine => { + if ("pause" in engine) engine.pause(); + }); } public resume(): void { this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Active; - this._activeEngines.forEach(engine => engine.resume()); + this._activeEngines.forEach(engine => { + if ("resume" in engine) engine.resume(); + }); } public async close() { @@ -283,7 +292,12 @@ export default class DownloadEngineMultiDownload engine.close()); + .map(engine => { + if ("close" in engine) { + return engine.close(); + } + return Promise.resolve(); + }); await Promise.all(closePromises); this.emit("closed"); diff --git a/src/download/node-download.ts b/src/download/node-download.ts index be05706..5915773 100644 --- a/src/download/node-download.ts +++ b/src/download/node-download.ts @@ -2,6 +2,7 @@ import DownloadEngineNodejs, {DownloadEngineOptionsNodejs} from "./download-engi import BaseDownloadEngine from "./download-engine/engine/base-download-engine.js"; import DownloadEngineMultiDownload, {DownloadEngineMultiDownloadOptions} from "./download-engine/engine/download-engine-multi-download.js"; import {CliProgressDownloadEngineOptions, globalCLI} from "./transfer-visualize/transfer-cli/GlobalCLI.js"; +import {DownloadEngineRemote} from "./download-engine/engine/DownloadEngineRemote.js"; const DEFAULT_PARALLEL_STREAMS_FOR_NODEJS = 3; export type DownloadFileOptions = DownloadEngineOptionsNodejs & CliProgressDownloadEngineOptions; @@ -18,6 +19,17 @@ export async function downloadFile(options: DownloadFileOptions) { return await downloader; } +/** + * Stream events for a download from remote session, doing so by calling `emitRemoteProgress` with the progress info. + * - Supports CLI progress. + */ +export function downloadFileRemote(options?: CliProgressDownloadEngineOptions) { + const downloader = new DownloadEngineRemote(); + globalCLI.addDownload(downloader, options); + + return downloader; +} + export type DownloadSequenceOptions = CliProgressDownloadEngineOptions & DownloadEngineMultiDownloadOptions & { fetchStrategy?: "localFile" | "fetch"; }; diff --git a/src/download/transfer-visualize/progress-statistics-builder.ts b/src/download/transfer-visualize/progress-statistics-builder.ts index aec3682..bc21745 100644 --- a/src/download/transfer-visualize/progress-statistics-builder.ts +++ b/src/download/transfer-visualize/progress-statistics-builder.ts @@ -5,6 +5,7 @@ import {createFormattedStatus, FormattedStatus} from "./format-transfer-status.j import DownloadEngineFile from "../download-engine/download-file/download-engine-file.js"; import ProgressStatusFile, {DownloadStatus, ProgressStatus} from "../download-engine/download-file/progress-status-file.js"; import DownloadEngineMultiDownload from "../download-engine/engine/download-engine-multi-download.js"; +import {DownloadEngineRemote} from "../download-engine/engine/DownloadEngineRemote.js"; export type ProgressStatusWithIndex = FormattedStatus & { index: number, @@ -14,7 +15,7 @@ interface CliProgressBuilderEvents { progress: (progress: ProgressStatusWithIndex) => void; } -export type AnyEngine = DownloadEngineFile | BaseDownloadEngine | DownloadEngineMultiDownload; +export type AnyEngine = DownloadEngineFile | BaseDownloadEngine | DownloadEngineMultiDownload | DownloadEngineRemote; export default class ProgressStatisticsBuilder extends EventEmitter { private _engines = new Set(); private _activeTransfers: { [index: number]: number } = {}; diff --git a/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts b/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts index 74e4df6..40b4fdd 100644 --- a/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts +++ b/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts @@ -6,8 +6,9 @@ import {CliFormattedStatus} from "./progress-bars/base-transfer-cli-progress-bar import cliSpinners from "cli-spinners"; import {DownloadStatus} from "../../download-engine/download-file/progress-status-file.js"; import BaseDownloadEngine from "../../download-engine/engine/base-download-engine.js"; +import {DownloadEngineRemote} from "../../download-engine/engine/DownloadEngineRemote.js"; -type AllowedDownloadEngine = DownloadEngineMultiDownload | BaseDownloadEngine; +type AllowedDownloadEngine = DownloadEngineMultiDownload | BaseDownloadEngine | DownloadEngineRemote; const DEFAULT_CLI_STYLE: AvailableCLIProgressStyle = "auto"; diff --git a/src/index.ts b/src/index.ts index 60f6da1..caed3df 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,5 @@ import DownloadEngineNodejs from "./download/download-engine/engine/download-engine-nodejs.js"; -import {downloadFile, DownloadFileOptions, downloadSequence, DownloadSequenceOptions} from "./download/node-download.js"; +import {downloadFile, DownloadFileOptions, downloadFileRemote, downloadSequence, DownloadSequenceOptions} from "./download/node-download.js"; import {SaveProgressInfo} from "./download/download-engine/types.js"; import PathNotAFileError from "./download/download-engine/streams/download-engine-fetch-stream/errors/path-not-a-file-error.js"; import EmptyResponseError from "./download/download-engine/streams/download-engine-fetch-stream/errors/empty-response-error.js"; @@ -16,10 +16,13 @@ import BaseDownloadEngine from "./download/download-engine/engine/base-download- import {InvalidOptionError} from "./download/download-engine/engine/error/InvalidOptionError.js"; import {BaseMultiProgressBar, MultiProgressBarOptions} from "./download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.js"; import {DownloadFlags, DownloadStatus} from "./download/download-engine/download-file/progress-status-file.js"; +import {DownloadEngineRemote} from "./download/download-engine/engine/DownloadEngineRemote.js"; +import {CliProgressDownloadEngineOptions} from "./download/transfer-visualize/transfer-cli/GlobalCLI.js"; export { DownloadFlags, DownloadStatus, + downloadFileRemote, downloadFile, downloadSequence, BaseMultiProgressBar, @@ -38,13 +41,15 @@ export { export type { BaseDownloadEngine, + DownloadEngineRemote, DownloadFileOptions, DownloadSequenceOptions, DownloadEngineNodejs, DownloadEngineMultiDownload, SaveProgressInfo, FormattedStatus, - MultiProgressBarOptions + MultiProgressBarOptions, + CliProgressDownloadEngineOptions }; From 332eff03bad11a10e00219fe766c0810e0302d75 Mon Sep 17 00:00:00 2001 From: ido Date: Fri, 14 Feb 2025 18:40:18 +0200 Subject: [PATCH 07/37] fix: load download file metadata --- .../download-engine-write-stream-nodejs.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts index ececd2e..b57174e 100644 --- a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts +++ b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts @@ -144,6 +144,9 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW const state = await fd.stat(); const metadataSize = state.size - this._fileSize; if (metadataSize <= 0 || metadataSize >= MAX_META_SIZE) { + if (this._fileSize > 0 && state.size > this._fileSize) { + await this.ftruncate(); + } return; } From d71ba6f42aed250a2846f1399f5f5b1c8259a09e Mon Sep 17 00:00:00 2001 From: ido Date: Fri, 14 Feb 2025 18:43:50 +0200 Subject: [PATCH 08/37] chores: beta release --- .releaserc.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.releaserc.json b/.releaserc.json index a395035..ef8eaf5 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -1,6 +1,10 @@ { "branches": [ - "main" + "main", + { + "name": "beta", + "prerelease": true + } ], "ci": true, "plugins": [ From 0d961c332e4ed6edbe20069f608164278819da63 Mon Sep 17 00:00:00 2001 From: ido Date: Fri, 14 Feb 2025 20:55:53 +0200 Subject: [PATCH 09/37] feat: stream timeout --- .../download-file/download-engine-file.ts | 45 ++++++++-- .../download-file/progress-status-file.ts | 12 ++- .../base-download-engine-fetch-stream.ts | 42 ++++++--- .../download-engine-fetch-stream-fetch.ts | 27 +++++- .../download-engine-fetch-stream-xhr.ts | 23 +++++ .../errors/EmptyStreamTimeoutError.ts | 3 + .../progress-statistics-builder.ts | 19 +++- .../base-transfer-cli-progress-bar.ts | 11 +++ .../fancy-transfer-cli-progress-bar.ts | 89 ++++++++++--------- 9 files changed, 208 insertions(+), 63 deletions(-) create mode 100644 src/download/download-engine/streams/download-engine-fetch-stream/errors/EmptyStreamTimeoutError.ts diff --git a/src/download/download-engine/download-file/download-engine-file.ts b/src/download/download-engine/download-file/download-engine-file.ts index d9eea3b..b8e58b5 100644 --- a/src/download/download-engine/download-file/download-engine-file.ts +++ b/src/download/download-engine/download-file/download-engine-file.ts @@ -66,7 +66,14 @@ export default class DownloadEngineFile extends EventEmitter c.isRetrying); + thisStatus.retryingTotalAttempts = Math.max(...streamContexts.map(x => x.retryingAttempts)); + + return thisStatus; } protected get _activePart() { @@ -135,8 +149,8 @@ export default class DownloadEngineFile extends EventEmitter acc + bytes, 0); + const streamingBytes = Object.values(this._activeStreamContext) + .reduce((acc, cur) => acc + cur.streamBytes, 0); const streamBytes = this._activeDownloadedChunkSize + streamingBytes; const streamBytesMin = Math.min(streamBytes, this._activePart.size || streamBytes); @@ -210,7 +224,7 @@ export default class DownloadEngineFile extends EventEmitter this._activeStreamContext[startChunk] ??= {streamBytes: 0, retryingAttempts: 0}; + const fetchState = this.options.fetchStream.withSubState({ chunkSize: this._progress.chunkSize, startChunk, @@ -247,11 +263,22 @@ export default class DownloadEngineFile extends EventEmitter { - this._activeStreamBytes[startChunk] = length; + getContext().streamBytes = length; this._sendProgressDownloadPart(); } }); + fetchState.addListener("retryingOn", () => { + const context = getContext(); + context.isRetrying = true; + context.retryingAttempts++; + this._sendProgressDownloadPart(); + }); + + fetchState.addListener("retryingOff", () => { + getContext().isRetrying = false; + }); + const downloadedPartsSize = this._downloadedPartsSize; this._progress.chunks[startChunk] = ChunkStatus.IN_PROGRESS; @@ -282,7 +309,7 @@ export default class DownloadEngineFile extends EventEmitter last + current.length, 0); - delete this._activeStreamBytes[startChunk]; + getContext().streamBytes = 0; void this._saveProgress(); const nextChunk = this._progress.chunks[index + 1]; @@ -302,7 +329,7 @@ export default class DownloadEngineFile extends EventEmitter c === ChunkStatus.COMPLETE); } - delete this._activeStreamBytes[startChunk]; + delete this._activeStreamContext[startChunk]; await Promise.all(allWrites); })(); } diff --git a/src/download/download-engine/download-file/progress-status-file.ts b/src/download/download-engine/download-file/progress-status-file.ts index a3a1508..e473a1e 100644 --- a/src/download/download-engine/download-file/progress-status-file.ts +++ b/src/download/download-engine/download-file/progress-status-file.ts @@ -11,6 +11,8 @@ export type ProgressStatus = { transferAction: string downloadStatus: DownloadStatus downloadFlags: DownloadFlags[] + retrying: boolean + retryingTotalAttempts: number }; export enum DownloadStatus { @@ -41,6 +43,8 @@ export default class ProgressStatusFile { public startTime: number = 0; public endTime: number = 0; public downloadId: string = ""; + public retrying = false; + public retryingTotalAttempts = 0; public constructor( totalDownloadParts: number, @@ -70,7 +74,13 @@ export default class ProgressStatusFile { this.endTime = Date.now(); } - public createStatus(downloadPart: number, transferredBytes: number, totalBytes = this.totalBytes, downloadStatus = DownloadStatus.Active, comment = this.comment): ProgressStatusFile { + public createStatus( + downloadPart: number, + transferredBytes: number, + totalBytes = this.totalBytes, + downloadStatus = DownloadStatus.Active, + comment = this.comment + ): ProgressStatusFile { const newStatus = new ProgressStatusFile( this.totalDownloadParts, this.fileName, diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts index f0d290a..f3a4551 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts @@ -10,6 +10,11 @@ export const MIN_LENGTH_FOR_MORE_INFO_REQUEST = 1024 * 1024 * 3; // 3MB export type BaseDownloadEngineFetchStreamOptions = { retry?: retry.Options + retryFetchDownloadInfo?: retry.Options + /** + * Max wait for next data stream + */ + maxStreamWait?: number /** * If true, the engine will retry the request if the server returns a status code between 500 and 599 */ @@ -57,14 +62,23 @@ export type BaseDownloadEngineFetchStreamEvents = { resumed: () => void aborted: () => void errorCountIncreased: (errorCount: number, error: Error) => void + retryingOn: (error: Error, attempt: number) => void + retryingOff: () => void }; export type WriteCallback = (data: Uint8Array[], position: number, index: number) => void; const DEFAULT_OPTIONS: BaseDownloadEngineFetchStreamOptions = { retryOnServerError: true, + maxStreamWait: 1000 * 3, retry: { - retries: 150, + retries: 50, + factor: 1.5, + minTimeout: 200, + maxTimeout: 5_000 + }, + retryFetchDownloadInfo: { + retries: 5, factor: 1.5, minTimeout: 200, maxTimeout: 5_000 @@ -136,11 +150,17 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter const fetchDownloadInfoCallback = async (): Promise => { try { - return await this.fetchDownloadInfoWithoutRetry(url); + const response = await this.fetchDownloadInfoWithoutRetry(url); + this.emit("retryingOff"); + return response; } catch (error: any) { + this.errorCount.value++; + this.emit("errorCountIncreased", this.errorCount.value, error); + if (error instanceof HttpError && !this.retryOnServerError(error)) { if ("tryHeaders" in this.options && tryHeaders.length) { this.options.headers = tryHeaders.shift(); + this.emit("retryingOn", error, this.errorCount.value); await sleep(this.options.tryHeadersDelay ?? 0); return await fetchDownloadInfoCallback(); } @@ -149,10 +169,8 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter return null; } - this.errorCount.value++; - this.emit("errorCountIncreased", this.errorCount.value, error); - if (error instanceof StatusCodeError && error.retryAfter) { + this.emit("retryingOn", error, this.errorCount.value); await sleep(error.retryAfter * 1000); return await fetchDownloadInfoCallback(); } @@ -161,8 +179,7 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter } }; - - const response = ("defaultFetchDownloadInfo" in this.options && this.options.defaultFetchDownloadInfo) || await retry(fetchDownloadInfoCallback, this.options.retry); + const response = ("defaultFetchDownloadInfo" in this.options && this.options.defaultFetchDownloadInfo) || await retry(fetchDownloadInfoCallback, this.options.retryFetchDownloadInfo); if (throwErr) { throw throwErr; } @@ -179,16 +196,19 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter // eslint-disable-next-line no-constant-condition while (true) { try { - return await this.fetchWithoutRetryChunks(callback); + const response = await this.fetchWithoutRetryChunks(callback); + this.emit("retryingOff"); + return response; } catch (error: any) { if (error?.name === "AbortError") return; + this.errorCount.value++; + this.emit("errorCountIncreased", this.errorCount.value, error); + if (error instanceof HttpError && !this.retryOnServerError(error)) { throw error; } - this.errorCount.value++; - this.emit("errorCountIncreased", this.errorCount.value, error); - + this.emit("retryingOn", error, this.errorCount.value); if (error instanceof StatusCodeError && error.retryAfter) { await sleep(error.retryAfter * 1000); continue; diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts index 7d0b10f..ee72f05 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts @@ -5,6 +5,8 @@ import {parseContentDisposition} from "./utils/content-disposition.js"; import StatusCodeError from "./errors/status-code-error.js"; import {parseHttpContentRange} from "./utils/httpRange.js"; import {browserCheck} from "./utils/browserCheck.js"; +import {EmptyStreamTimeoutError} from "./errors/EmptyStreamTimeoutError.js"; +import prettyMilliseconds from "pretty-ms"; type GetNextChunk = () => Promise> | ReadableStreamReadResult; export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFetchStream { @@ -114,7 +116,7 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe // eslint-disable-next-line no-constant-condition while (true) { - const {done, value} = await getNextChunk(); + const {done, value} = await DownloadEngineFetchStreamFetch._wrapperStreamTimeout(getNextChunk()); await this.paused; if (done || this.aborted) break; @@ -132,4 +134,27 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe }); return headerObj; } + + protected static _wrapperStreamTimeout(promise: Promise | T, maxStreamWait = 2_000): Promise | T { + if (!(promise instanceof Promise)) { + return promise; + } + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new EmptyStreamTimeoutError(`Stream timeout after ${prettyMilliseconds(maxStreamWait)}`)); + }, maxStreamWait); + + promise.then( + (result) => { + clearTimeout(timeout); + resolve(result); + }, + (error) => { + clearTimeout(timeout); + reject(error); + } + ); + }); + } } diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts index 370ea7c..57c653f 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts @@ -7,6 +7,8 @@ import retry from "async-retry"; import {AvailablePrograms} from "../../download-file/download-programs/switch-program.js"; import {parseContentDisposition} from "./utils/content-disposition.js"; import {parseHttpContentRange} from "./utils/httpRange.js"; +import {EmptyStreamTimeoutError} from "./errors/EmptyStreamTimeoutError.js"; +import prettyMilliseconds from "pretty-ms"; export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetchStream { @@ -43,7 +45,24 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc xhr.setRequestHeader(key, value); } + let lastTimeoutIndex: any; + const clearStreamTimeout = () => { + if (lastTimeoutIndex) { + clearTimeout(lastTimeoutIndex); + } + }; + + const createStreamTimeout = () => { + clearStreamTimeout(); + lastTimeoutIndex = setTimeout(() => { + reject(new EmptyStreamTimeoutError(`Stream timeout after ${prettyMilliseconds(this.options.maxStreamWait!)}`)); + xhr.abort(); + }); + }; + + xhr.onload = () => { + clearStreamTimeout(); const contentLength = parseInt(xhr.getResponseHeader("content-length")!); if (this.state.rangeSupport && contentLength !== end - start) { @@ -63,18 +82,22 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc }; xhr.onerror = () => { + clearStreamTimeout(); reject(new XhrError(`Failed to fetch ${url}`)); }; xhr.onprogress = (event) => { + createStreamTimeout(); if (event.lengthComputable) { onProgress?.(event.loaded); } }; xhr.send(); + createStreamTimeout(); this.on("aborted", () => { + clearStreamTimeout(); xhr.abort(); }); }); diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/errors/EmptyStreamTimeoutError.ts b/src/download/download-engine/streams/download-engine-fetch-stream/errors/EmptyStreamTimeoutError.ts new file mode 100644 index 0000000..0d53a14 --- /dev/null +++ b/src/download/download-engine/streams/download-engine-fetch-stream/errors/EmptyStreamTimeoutError.ts @@ -0,0 +1,3 @@ +import FetchStreamError from "./fetch-stream-error.js"; + +export class EmptyStreamTimeoutError extends FetchStreamError {} diff --git a/src/download/transfer-visualize/progress-statistics-builder.ts b/src/download/transfer-visualize/progress-statistics-builder.ts index bc21745..704838e 100644 --- a/src/download/transfer-visualize/progress-statistics-builder.ts +++ b/src/download/transfer-visualize/progress-statistics-builder.ts @@ -34,6 +34,8 @@ export default class ProgressStatisticsBuilder extends EventEmitter { + if (isRetrying) { + this._retrying--; + this._retryingTotalAttempts -= retryingTotalAttempts; + } + + isRetrying = data.retrying; + retryingTotalAttempts = data.retryingTotalAttempts; + + this._retrying += Number(isRetrying); + this._retryingTotalAttempts += retryingTotalAttempts; + this._sendProgress(data, index, downloadPartStart); }); @@ -141,7 +156,9 @@ export default class ProgressStatisticsBuilder extends EventEmitter 0, + retryingTotalAttempts: this._retryingTotalAttempts }), index }; diff --git a/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts b/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts index c8f356f..159a6cd 100644 --- a/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts +++ b/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts @@ -64,6 +64,10 @@ export default class BaseTransferCliProgressBar implements TransferCliProgressBa return this.status.startTime < Date.now() - SKIP_ETA_START_TIME; } + protected get retryingText() { + return this.status.retrying ? `(retrying #${this.status.retryingTotalAttempts})` : ""; + } + protected getNameSize(fileName = this.status.fileName) { return this.options.truncateName === false ? fileName.length @@ -170,6 +174,7 @@ export default class BaseTransferCliProgressBar implements TransferCliProgressBa const {formattedPercentage, formattedSpeed, formatTransferredOfTotal, formatTotal} = this.status; const status = this.switchTransferToShortText(); + const retryingText = this.retryingText; return renderDataLine([ { type: "status", @@ -177,6 +182,12 @@ export default class BaseTransferCliProgressBar implements TransferCliProgressBa size: status.length, formatter: (text) => chalk.cyan(text) }, + { + type: "status", + fullText: retryingText, + size: retryingText.length, + formatter: (text) => chalk.ansi256(196)(text) + }, { type: "spacer", fullText: " ", diff --git a/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts b/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts index a9d0429..c4f2685 100644 --- a/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts +++ b/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts @@ -19,47 +19,56 @@ export default class FancyTransferCliProgressBar extends BaseTransferCliProgress const progressBarText = ` ${formattedPercentageWithPadding} (${formatTransferred}/${formatTotal}) `; const dimEta: DataLine = this.getETA(" | ", text => chalk.dim(text)); + const retryingText = this.retryingText; - return renderDataLine([{ - type: "status", - fullText: "", - size: 1, - formatter: () => STATUS_ICONS.activeDownload - }, { - type: "spacer", - fullText: " ", - size: " ".length - }, ...this.getNameAndCommentDataParts(), { - type: "spacer", - fullText: " ", - size: " ".length - }, { - type: "progressBar", - fullText: progressBarText, - size: Math.max(progressBarText.length, `100.0% (1024.00MB/${formatTotal})`.length), - flex: 4, - addEndPadding: 4, - maxSize: 40, - formatter(_, size) { - const leftPad = " ".repeat(Math.floor((size - progressBarText.length) / 2)); - return renderProgressBar({ - barText: leftPad + ` ${chalk.black.bgWhiteBright(formattedPercentageWithPadding)} ${chalk.gray(`(${formatTransferred}/${formatTotal})`)} `, - backgroundText: leftPad + ` ${chalk.yellow.bgGray(formattedPercentageWithPadding)} ${chalk.white(`(${formatTransferred}/${formatTotal})`)} `, - length: size, - loadedPercentage: percentage / 100, - barStyle: chalk.black.bgWhiteBright, - backgroundStyle: chalk.bgGray - }); - } - }, { - type: "spacer", - fullText: " ", - size: " ".length - }, { - type: "speed", - fullText: formattedSpeed, - size: Math.max("00.00kB/s".length, formattedSpeed.length) - }, ...dimEta]); + return renderDataLine([ + { + type: "status", + fullText: "", + size: 1, + formatter: () => STATUS_ICONS.activeDownload + }, + { + type: "status", + fullText: retryingText, + size: retryingText.length, + formatter: (text) => chalk.ansi256(196)(text) + }, + { + type: "spacer", + fullText: " ", + size: " ".length + }, ...this.getNameAndCommentDataParts(), { + type: "spacer", + fullText: " ", + size: " ".length + }, { + type: "progressBar", + fullText: progressBarText, + size: Math.max(progressBarText.length, `100.0% (1024.00MB/${formatTotal})`.length), + flex: 4, + addEndPadding: 4, + maxSize: 40, + formatter(_, size) { + const leftPad = " ".repeat(Math.floor((size - progressBarText.length) / 2)); + return renderProgressBar({ + barText: leftPad + ` ${chalk.black.bgWhiteBright(formattedPercentageWithPadding)} ${chalk.gray(`(${formatTransferred}/${formatTotal})`)} `, + backgroundText: leftPad + ` ${chalk.yellow.bgGray(formattedPercentageWithPadding)} ${chalk.white(`(${formatTransferred}/${formatTotal})`)} `, + length: size, + loadedPercentage: percentage / 100, + barStyle: chalk.black.bgWhiteBright, + backgroundStyle: chalk.bgGray + }); + } + }, { + type: "spacer", + fullText: " ", + size: " ".length + }, { + type: "speed", + fullText: formattedSpeed, + size: Math.max("00.00kB/s".length, formattedSpeed.length) + }, ...dimEta]); } protected override renderFinishedLine() { From 326916179be831849644c67859a8e2a3bdb828bf Mon Sep 17 00:00:00 2001 From: ido Date: Fri, 14 Feb 2025 20:59:06 +0200 Subject: [PATCH 10/37] fix: reduce log update --- src/download/transfer-visualize/transfer-cli/transfer-cli.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/download/transfer-visualize/transfer-cli/transfer-cli.ts b/src/download/transfer-visualize/transfer-cli/transfer-cli.ts index ccb36e1..a4e8052 100644 --- a/src/download/transfer-visualize/transfer-cli/transfer-cli.ts +++ b/src/download/transfer-visualize/transfer-cli/transfer-cli.ts @@ -35,6 +35,7 @@ export default class TransferCli { private readonly _updateStatuesDebounce: () => void; private _multiProgressBar: BaseMultiProgressBar; private _isFirstPrint = true; + private _lastProgressLong = ""; public constructor(options: Partial) { this.options = {...DEFAULT_TRANSFER_CLI_OPTIONS, ...options}; @@ -86,7 +87,8 @@ export default class TransferCli { private _updateStatues() { if (!this.latestProgress) return; const printLog = this._multiProgressBar.createMultiProgressBar(...this.latestProgress); - if (printLog) { + if (printLog && this._lastProgressLong != printLog) { + this._lastProgressLong = printLog; this._logUpdate(printLog); } } From 1f74705d3b8a9b67e090d3a608757cc4efbdaadb Mon Sep 17 00:00:00 2001 From: ido Date: Fri, 14 Feb 2025 21:01:06 +0200 Subject: [PATCH 11/37] fix: formatting --- .../progress-bars/fancy-transfer-cli-progress-bar.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts b/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts index c4f2685..235353c 100644 --- a/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts +++ b/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts @@ -68,7 +68,9 @@ export default class FancyTransferCliProgressBar extends BaseTransferCliProgress type: "speed", fullText: formattedSpeed, size: Math.max("00.00kB/s".length, formattedSpeed.length) - }, ...dimEta]); + }, + ...dimEta + ]); } protected override renderFinishedLine() { From e9dbe2f9eb5a9c8b92f0bec3f3adbba1ed836d26 Mon Sep 17 00:00:00 2001 From: ido Date: Sat, 15 Feb 2025 15:29:06 +0200 Subject: [PATCH 12/37] fix: xhr stream timeout --- .../download-engine-fetch-stream-xhr.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts index 57c653f..ccad81b 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts @@ -57,7 +57,7 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc lastTimeoutIndex = setTimeout(() => { reject(new EmptyStreamTimeoutError(`Stream timeout after ${prettyMilliseconds(this.options.maxStreamWait!)}`)); xhr.abort(); - }); + }, this.options.maxStreamWait); }; From 8eb5ea43ba7de0bc81466ca495fd4ab01813f880 Mon Sep 17 00:00:00 2001 From: ido Date: Sat, 15 Feb 2025 15:29:23 +0200 Subject: [PATCH 13/37] fix: local file tests --- test/copy-file.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/copy-file.test.ts b/test/copy-file.test.ts index 0115dda..b867404 100644 --- a/test/copy-file.test.ts +++ b/test/copy-file.test.ts @@ -17,7 +17,7 @@ describe("File Copy", async () => { fileName: copyFileToName, chunkSize: 4, parallelStreams: 1, - fetchStrategy: "localFile", + fetchStrategy: "local", cliProgress: false }); await engine.download(); @@ -37,7 +37,7 @@ describe("File Copy", async () => { url: fileToCopy, directory: ".", fileName: copyFileToName, - fetchStrategy: "localFile", + fetchStrategy: "local", cliProgress: false, parallelStreams: 1 }); From db22edaf1fac5b7fb94837fefac68938848c0df3 Mon Sep 17 00:00:00 2001 From: ido Date: Sat, 15 Feb 2025 15:43:31 +0200 Subject: [PATCH 14/37] fix: downloadSequence signature --- src/download/browser-download.ts | 4 ++-- src/download/node-download.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/download/browser-download.ts b/src/download/browser-download.ts index 0a177bc..ac7f2c8 100644 --- a/src/download/browser-download.ts +++ b/src/download/browser-download.ts @@ -36,7 +36,7 @@ export function downloadFileRemoteBrowser() { /** * Download multiple files in the browser environment. */ -export function downloadSequenceBrowser(options?: DownloadSequenceBrowserOptions | DownloadEngineBrowser | Promise, ...downloads: (DownloadEngineBrowser | Promise)[]) { +export async function downloadSequenceBrowser(options?: DownloadSequenceBrowserOptions | DownloadEngineBrowser | Promise, ...downloads: (DownloadEngineBrowser | Promise)[]) { let downloadOptions: DownloadSequenceOptions = {}; if (options instanceof BaseDownloadEngine || options instanceof Promise) { downloads.unshift(options); @@ -45,7 +45,7 @@ export function downloadSequenceBrowser(options?: DownloadSequenceBrowserOptions } const downloader = new DownloadEngineMultiDownload(downloadOptions); - downloader.addDownload(...downloads); + await downloader.addDownload(...downloads); return downloader; } diff --git a/src/download/node-download.ts b/src/download/node-download.ts index 5915773..5b5a1f7 100644 --- a/src/download/node-download.ts +++ b/src/download/node-download.ts @@ -38,7 +38,7 @@ export type DownloadSequenceOptions = CliProgressDownloadEngineOptions & Downloa /** * Download multiple files with CLI progress */ -export function downloadSequence(options?: DownloadSequenceOptions | DownloadEngineNodejs | Promise, ...downloads: (DownloadEngineNodejs | Promise)[]) { +export async function downloadSequence(options?: DownloadSequenceOptions | DownloadEngineNodejs | Promise, ...downloads: (DownloadEngineNodejs | Promise)[]) { let downloadOptions: DownloadSequenceOptions = {}; if (options instanceof BaseDownloadEngine || options instanceof Promise) { downloads.unshift(options); @@ -47,8 +47,8 @@ export function downloadSequence(options?: DownloadSequenceOptions | DownloadEng } const downloader = new DownloadEngineMultiDownload(downloadOptions); - downloader.addDownload(...downloads); globalCLI.addDownload(downloader, downloadOptions); + await downloader.addDownload(...downloads); return downloader; } From 0d6184cb5ae9a20fc7e43ca48faeef6aca5b7681 Mon Sep 17 00:00:00 2001 From: ido Date: Sat, 15 Feb 2025 15:43:49 +0200 Subject: [PATCH 15/37] docs: update readme --- README.md | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cc532b2..0496045 100644 --- a/README.md +++ b/README.md @@ -234,12 +234,12 @@ If the maximum reties was reached the download will fail and an error will be th ```ts import {downloadFile} from 'ipull'; -const downloader = await downloadFile({ - url: 'https://example.com/file.large', - directory: './this/path' -}); - try { + const downloader = await downloadFile({ + url: 'https://example.com/file.large', + directory: './this/path' + }); + await downloader.download(); } catch (error) { console.error(`Download failed: ${error.message}`); @@ -250,6 +250,8 @@ try { In some edge cases, the re-try mechanism may give the illusion that the download is stuck. +(You can see this in the progress object that "retrying" is true) + To debug this, disable the re-try mechanism: ```js @@ -258,6 +260,9 @@ const downloader = await downloadFile({ directory: './this/path', retry: { retries: 0 + }, + retryFetchDownloadInfo: { + retries: 0 } }); ``` @@ -286,6 +291,27 @@ downloader.on("progress", (progress) => { }); ``` +### Remote Download Listing + +If you want to show in the CLI the progress of a file downloading in remote. + +```ts +const originaldownloader = await downloadFile({ + url: 'https://example.com/file.large', + directory: './this/path' +}); + +const remoteDownloader = downloadFileRemote({ + cliProgress: true +}); + +originaldownloader.on("progress", (progress) => { + remoteDownloader.emitRemoteProgress(progress); +}); + +await originaldownloader.download(); +``` + ### Download multiple files If you want to download multiple files, you can use the `downloadSequence` function. From bbec167dff027456fc077b85716c047cbef70829 Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 26 Feb 2025 13:25:48 +0200 Subject: [PATCH 16/37] feat: auto increase parallel stream --- .../download-file/download-engine-file.ts | 104 +++++++++-------- .../base-download-program.ts | 44 ++++++-- .../download-file/downloaderProgramManager.ts | 106 ++++++++++++++++++ .../engine/download-engine-multi-download.ts | 2 +- .../base-download-engine-fetch-stream.ts | 4 +- .../download-engine-fetch-stream-fetch.ts | 1 + .../download-engine-fetch-stream-xhr.ts | 4 +- .../{engine => }/utils/concurrency.ts | 0 test/download.test.ts | 1 + 9 files changed, 211 insertions(+), 55 deletions(-) create mode 100644 src/download/download-engine/download-file/downloaderProgramManager.ts rename src/download/download-engine/{engine => }/utils/concurrency.ts (100%) diff --git a/src/download/download-engine/download-file/download-engine-file.ts b/src/download/download-engine/download-file/download-engine-file.ts index b8e58b5..8c486c1 100644 --- a/src/download/download-engine/download-file/download-engine-file.ts +++ b/src/download/download-engine/download-file/download-engine-file.ts @@ -9,6 +9,7 @@ import switchProgram, {AvailablePrograms} from "./download-programs/switch-progr import BaseDownloadProgram from "./download-programs/base-download-program.js"; import {pushComment} from "./utils/push-comment.js"; import {uid} from "uid"; +import {DownloaderProgramManager} from "./downloaderProgramManager.js"; export type DownloadEngineFileOptions = { chunkSize?: number; @@ -23,6 +24,7 @@ export type DownloadEngineFileOptions = { onPausedAsync?: () => Promise onSaveProgressAsync?: (progress: SaveProgressInfo) => Promise programType?: AvailablePrograms + autoIncreaseParallelStreams?: boolean /** @internal */ skipExisting?: boolean; @@ -49,7 +51,8 @@ const DEFAULT_CHUNKS_SIZE_FOR_STREAM_PROGRAM = 1024 * 1024; // 1MB const DEFAULT_OPTIONS: Omit = { chunkSize: 0, - parallelStreams: 3 + parallelStreams: 3, + autoIncreaseParallelStreams: true }; export default class DownloadEngineFile extends EventEmitter { @@ -160,7 +163,10 @@ export default class DownloadEngineFile extends EventEmitter 0 ? this._progress.chunks.length : Infinity; await this._downloadSlice(0, chunksToRead); @@ -252,7 +269,7 @@ export default class DownloadEngineFile extends EventEmitter this._activeStreamContext[startChunk] ??= {streamBytes: 0, retryingAttempts: 0}; const fetchState = this.options.fetchStream.withSubState({ @@ -279,59 +296,56 @@ export default class DownloadEngineFile extends EventEmitter { - const allWrites = new Set>(); + const allWrites = new Set>(); - let lastChunkSize = 0; - await fetchState.fetchChunks((chunks, writePosition, index) => { - if (this._closed || this._progress.chunks[index] != ChunkStatus.IN_PROGRESS) { - return; - } + let lastChunkSize = 0; + await fetchState.fetchChunks((chunks, writePosition, index) => { + if (this._closed || this._progress.chunks[index] != ChunkStatus.IN_PROGRESS) { + return; + } - for (const chunk of chunks) { - const writePromise = this.options.writeStream.write(downloadedPartsSize + writePosition, chunk); - writePosition += chunk.length; - if (writePromise) { - allWrites.add(writePromise); - writePromise.then(() => { - allWrites.delete(writePromise); - }); - } + for (const chunk of chunks) { + const writePromise = this.options.writeStream.write(downloadedPartsSize + writePosition, chunk); + writePosition += chunk.length; + if (writePromise) { + allWrites.add(writePromise); + writePromise.then(() => { + allWrites.delete(writePromise); + }); } + } - // if content length is 0, we do not know how many chunks we should have - if (this._activePart.size === 0) { - this._progress.chunks.push(ChunkStatus.NOT_STARTED); - } + // if content length is 0, we do not know how many chunks we should have + if (this._activePart.size === 0) { + this._progress.chunks.push(ChunkStatus.NOT_STARTED); + } - this._progress.chunks[index] = ChunkStatus.COMPLETE; - lastChunkSize = chunks.reduce((last, current) => last + current.length, 0); - getContext().streamBytes = 0; - void this._saveProgress(); + this._progress.chunks[index] = ChunkStatus.COMPLETE; + lastChunkSize = chunks.reduce((last, current) => last + current.length, 0); + getContext().streamBytes = 0; + void this._saveProgress(); - const nextChunk = this._progress.chunks[index + 1]; - const shouldReadNext = endChunk - index > 1; // grater than 1, meaning there is a next chunk + const nextChunk = this._progress.chunks[index + 1]; + const shouldReadNext = fetchState.state.endChunk - index > 1; // grater than 1, meaning there is a next chunk - if (shouldReadNext) { - if (nextChunk == null || nextChunk != ChunkStatus.NOT_STARTED) { - return fetchState.close(); - } - this._progress.chunks[index + 1] = ChunkStatus.IN_PROGRESS; + if (shouldReadNext) { + if (nextChunk == null || nextChunk != ChunkStatus.NOT_STARTED) { + return fetchState.close(); } - }); - - // On dynamic content length, we need to adjust the last chunk size - if (this._activePart.size === 0) { - this._activePart.size = this._activeDownloadedChunkSize - this.options.chunkSize + lastChunkSize; - this._progress.chunks = this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE); + this._progress.chunks[index + 1] = ChunkStatus.IN_PROGRESS; } + }); + + // On dynamic content length, we need to adjust the last chunk size + if (this._activePart.size === 0) { + this._activePart.size = this._activeDownloadedChunkSize - this.options.chunkSize + lastChunkSize; + this._progress.chunks = this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE); + } - delete this._activeStreamContext[startChunk]; - await Promise.all(allWrites); - })(); + delete this._activeStreamContext[startChunk]; + await Promise.all(allWrites); } protected _saveProgress() { diff --git a/src/download/download-engine/download-file/download-programs/base-download-program.ts b/src/download/download-engine/download-file/download-programs/base-download-program.ts index 149a4bf..0138dbb 100644 --- a/src/download/download-engine/download-file/download-programs/base-download-program.ts +++ b/src/download/download-engine/download-file/download-programs/base-download-program.ts @@ -11,24 +11,54 @@ export default abstract class BaseDownloadProgram { protected savedProgress: SaveProgressInfo; protected readonly _downloadSlice: DownloadSlice; protected _aborted = false; + protected _parallelStreams: number; + protected _reload?: () => void; + protected _activeDownloads: Promise[] = []; protected constructor(_savedProgress: SaveProgressInfo, _downloadSlice: DownloadSlice) { this._downloadSlice = _downloadSlice; this.savedProgress = _savedProgress; + this._parallelStreams = this.savedProgress.parallelStreams; + } + + get parallelStreams() { + return this._parallelStreams; + } + + set parallelStreams(value: number) { + const needReload = value > this._parallelStreams; + this._parallelStreams = value; + if (needReload) { + this._reload?.(); + } + } + + incParallelStreams() { + this.parallelStreams = this._activeDownloads.length + 1; + } + + decParallelStreams() { + this.parallelStreams = this._activeDownloads.length - 1; + } + + waitForStreamToEnd() { + return Promise.race(this._activeDownloads); } public async download(): Promise { - if (this.savedProgress.parallelStreams === 1) { + if (this._parallelStreams === 1) { return await this._downloadSlice(0, this.savedProgress.chunks.length); } - const activeDownloads: Promise[] = []; // eslint-disable-next-line no-constant-condition while (true) { - while (activeDownloads.length >= this.savedProgress.parallelStreams) { + while (this._activeDownloads.length >= this._parallelStreams) { if (this._aborted) return; - await Promise.race(activeDownloads); + const promiseResolvers = Promise.withResolvers(); + this._reload = promiseResolvers.resolve; + + await Promise.race(this._activeDownloads.concat([promiseResolvers.promise])); } const slice = this._createOneSlice(); @@ -36,13 +66,13 @@ export default abstract class BaseDownloadProgram { if (this._aborted) return; const promise = this._downloadSlice(slice.start, slice.end); - activeDownloads.push(promise); + this._activeDownloads.push(promise); promise.then(() => { - activeDownloads.splice(activeDownloads.indexOf(promise), 1); + this._activeDownloads.splice(this._activeDownloads.indexOf(promise), 1); }); } - await Promise.all(activeDownloads); + await Promise.all(this._activeDownloads); } protected abstract _createOneSlice(): ProgramSlice | null; diff --git a/src/download/download-engine/download-file/downloaderProgramManager.ts b/src/download/download-engine/download-file/downloaderProgramManager.ts new file mode 100644 index 0000000..6f15b47 --- /dev/null +++ b/src/download/download-engine/download-file/downloaderProgramManager.ts @@ -0,0 +1,106 @@ +import BaseDownloadProgram from "./download-programs/base-download-program.js"; +import DownloadEngineFile from "./download-engine-file.js"; +import {DownloadStatus, ProgressStatus} from "./progress-status-file.js"; +import sleep from "sleep-promise"; + +const BASE_AVERAGE_SPEED_TIME = 1000; +const AVERAGE_SPEED_TIME = 1000 * 8; +const ALLOW_SPEED_DECREASE_PERCENTAGE = 10; +const ADD_MORE_PARALLEL_IF_SPEED_INCREASE_PERCENTAGE = 10; + +export class DownloaderProgramManager { + // date, speed + private _speedHistory: [number, number][] = []; + private _lastResumeDate = 0; + private _lastAverageSpeed = 0; + private _increasePaused = false; + protected _removeEvent?: () => void; + + constructor(protected _program: BaseDownloadProgram, protected _download: DownloadEngineFile) { + this._initEvents(); + } + + private _initEvents() { + let lastTransferredBytes = 0; + let lastTransferredBytesDate = 0; + + let watNotActive = true; + const progressEvent = (event: ProgressStatus) => { + const now = Date.now(); + + if (event.retrying || event.downloadStatus != DownloadStatus.Active) { + watNotActive = true; + } else { + if (watNotActive) { + this._lastResumeDate = now; + watNotActive = false; + } + + const isTimeToCalculate = lastTransferredBytesDate + BASE_AVERAGE_SPEED_TIME < now; + if (lastTransferredBytesDate === 0 || isTimeToCalculate) { + if (isTimeToCalculate) { + const speedPerSec = event.transferredBytes - lastTransferredBytes; + this._speedHistory.push([now, speedPerSec]); + } + + lastTransferredBytesDate = now; + lastTransferredBytes = event.transferredBytes; + } + + } + + const deleteAllBefore = now - AVERAGE_SPEED_TIME; + this._speedHistory = this._speedHistory.filter(([date]) => date > deleteAllBefore); + + + if (!watNotActive && now - this._lastResumeDate > AVERAGE_SPEED_TIME) { + this._checkAction(); + } + }; + + this._download.on("progress", progressEvent); + this._removeEvent = () => this._download.off("progress", progressEvent); + } + + private _calculateAverageSpeed() { + const totalSpeed = this._speedHistory.reduce((acc, [, speed]) => acc + speed, 0); + return totalSpeed / (this._speedHistory.length || 1); + } + + private async _checkAction() { + const lastAverageSpeed = this._lastAverageSpeed; + const newAverageSpeed = this._calculateAverageSpeed(); + const speedDecreasedOK = (lastAverageSpeed - newAverageSpeed) / newAverageSpeed * 100 > ALLOW_SPEED_DECREASE_PERCENTAGE; + + if (!speedDecreasedOK) { + this._lastAverageSpeed = newAverageSpeed; + } + + if (this._increasePaused || newAverageSpeed > lastAverageSpeed || speedDecreasedOK) { + return; + } + + this._increasePaused = true; + this._program.incParallelStreams(); + let sleepTime = AVERAGE_SPEED_TIME; + + while (sleepTime <= AVERAGE_SPEED_TIME) { + await sleep(sleepTime); + sleepTime = Date.now() - this._lastResumeDate; + } + + const newSpeed = this._calculateAverageSpeed(); + const bestLastSpeed = Math.max(newAverageSpeed, lastAverageSpeed); + const speedIncreasedOK = newSpeed > bestLastSpeed && (newSpeed - bestLastSpeed) / bestLastSpeed * 100 > ADD_MORE_PARALLEL_IF_SPEED_INCREASE_PERCENTAGE; + + if (speedIncreasedOK) { + this._increasePaused = false; + } else { + this._program.decParallelStreams(); + } + } + + close() { + this._removeEvent?.(); + } +} diff --git a/src/download/download-engine/engine/download-engine-multi-download.ts b/src/download/download-engine/engine/download-engine-multi-download.ts index 11a2b50..dca5b40 100644 --- a/src/download/download-engine/engine/download-engine-multi-download.ts +++ b/src/download/download-engine/engine/download-engine-multi-download.ts @@ -2,7 +2,7 @@ import {EventEmitter} from "eventemitter3"; import {FormattedStatus} from "../../transfer-visualize/format-transfer-status.js"; import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js"; import BaseDownloadEngine, {BaseDownloadEngineEvents} from "./base-download-engine.js"; -import {concurrency} from "./utils/concurrency.js"; +import {concurrency} from "../utils/concurrency.js"; import {DownloadFlags, DownloadStatus} from "../download-file/progress-status-file.js"; import {DownloadEngineRemote} from "./DownloadEngineRemote.js"; diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts index f3a4551..e446dcd 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts @@ -87,8 +87,10 @@ const DEFAULT_OPTIONS: BaseDownloadEngineFetchStreamOptions = { }; export default abstract class BaseDownloadEngineFetchStream extends EventEmitter { - public readonly programType?: AvailablePrograms; + public readonly defaultProgramType?: AvailablePrograms; + public readonly availablePrograms: AvailablePrograms[] = ["chunks", "stream"]; public readonly abstract transferAction: string; + public readonly supportDynamicStreamLength: boolean = false; public readonly options: Partial = {}; public state: FetchSubState = null!; public paused?: Promise; diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts index ee72f05..9ff0d47 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts @@ -12,6 +12,7 @@ type GetNextChunk = () => Promise> | Readab export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFetchStream { private _fetchDownloadInfoWithHEAD = false; public override transferAction = "Downloading"; + public override readonly supportDynamicStreamLength = true; withSubState(state: FetchSubState): this { const fetchStream = new DownloadEngineFetchStreamFetch(this.options); diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts index ccad81b..ffd4a0b 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts @@ -13,7 +13,9 @@ import prettyMilliseconds from "pretty-ms"; export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetchStream { private _fetchDownloadInfoWithHEAD = true; - public override readonly programType: AvailablePrograms = "chunks"; + public override readonly defaultProgramType: AvailablePrograms = "chunks"; + public override readonly availablePrograms: AvailablePrograms[] = ["chunks"]; + public override transferAction = "Downloading"; withSubState(state: FetchSubState): this { diff --git a/src/download/download-engine/engine/utils/concurrency.ts b/src/download/download-engine/utils/concurrency.ts similarity index 100% rename from src/download/download-engine/engine/utils/concurrency.ts rename to src/download/download-engine/utils/concurrency.ts diff --git a/test/download.test.ts b/test/download.test.ts index 2bd3b86..1f196f8 100644 --- a/test/download.test.ts +++ b/test/download.test.ts @@ -23,6 +23,7 @@ describe("File Download", () => { const downloader = new DownloadEngineFile(file, { parallelStreams: randomNumber, chunkSize: 1024 ** 2, + autoIncreaseParallelStreams: false, fetchStream, writeStream }); From 23b28e78ac62e1171e2e52553e613d0519a97b4b Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 26 Feb 2025 13:29:53 +0200 Subject: [PATCH 17/37] chores: ci prerelease --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 45cf7a1..92a9807 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: release: name: Release - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/beta' runs-on: ubuntu-latest concurrency: release-${{ github.ref }} environment: @@ -82,7 +82,7 @@ jobs: name: pages-docs path: docs - name: Deploy docs to GitHub Pages - if: steps.set-npm-url.outputs.npm-url != '' + if: steps.set-npm-url.outputs.npm-url != '' && github.ref == 'refs/heads/main' uses: actions/deploy-pages@v2 with: artifact_name: pages-docs From e89e40eb564a060cc3f6c0a95f9c1870a108c043 Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 26 Feb 2025 13:31:56 +0200 Subject: [PATCH 18/37] chores: fix ci --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 92a9807..fc461c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,12 +22,12 @@ jobs: - name: Generate docs run: npm run generate-docs - name: Upload build artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "build" path: "dist" - name: Upload build artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "docs" path: "docs" From 42e3d10b2eef4f005d9212a0d14365aecdf444af Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 26 Feb 2025 13:34:22 +0200 Subject: [PATCH 19/37] chores: fix ci --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc461c9..2f9af30 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: "20" + node-version: "22" - name: Install modules run: npm ci --ignore-scripts - name: Build From 054e4bf3a2049fed6a4758e2f9e9ce157e5eb61b Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 26 Feb 2025 13:36:24 +0200 Subject: [PATCH 20/37] chores: fix ci --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2f9af30..0cfbfc6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: "20" + node-version: "22" - name: Install modules run: npm ci --ignore-scripts - uses: actions/download-artifact@v3 @@ -73,7 +73,7 @@ jobs: id: set-npm-url run: | if [ -f .semanticRelease.npmPackage.deployedVersion.txt ]; then - echo "npm-url=https://www.npmjs.com/package/node-llama-cpp/v/$(cat .semanticRelease.npmPackage.deployedVersion.txt)" >> $GITHUB_OUTPUT + echo "npm-url=https://www.npmjs.com/package/ipull/v/$(cat .semanticRelease.npmPackage.deployedVersion.txt)" >> $GITHUB_OUTPUT fi - name: Upload docs to GitHub Pages if: steps.set-npm-url.outputs.npm-url != '' From e036715c7e841184b78686353ba11f78c8aff70b Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 26 Feb 2025 13:41:09 +0200 Subject: [PATCH 21/37] chores: node 23 ci --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0cfbfc6..695aba9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: "22" + node-version: "23" - name: Install modules run: npm ci --ignore-scripts - name: Build @@ -52,7 +52,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: "22" + node-version: "23" - name: Install modules run: npm ci --ignore-scripts - uses: actions/download-artifact@v3 From e1e7ffafd5f3465be7b9c35af2dadecc38748883 Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 26 Feb 2025 13:54:22 +0200 Subject: [PATCH 22/37] chores: node 22 ci --- .github/workflows/build.yml | 4 ++-- .github/workflows/test.yml | 2 +- package-lock.json | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 695aba9..0cfbfc6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: "23" + node-version: "22" - name: Install modules run: npm ci --ignore-scripts - name: Build @@ -52,7 +52,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: "23" + node-version: "22" - name: Install modules run: npm ci --ignore-scripts - uses: actions/download-artifact@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 652f105..88b2acd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: "20" + node-version: "22" - name: Install modules run: npm ci --ignore-scripts - name: ESLint diff --git a/package-lock.json b/package-lock.json index d151e19..4d76fa1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12381,10 +12381,11 @@ } }, "node_modules/typescript": { - "version": "5.5.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", - "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", + "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" From 378c7a60a2db22988bf021cf582ab1bb6fea01cc Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 26 Feb 2025 18:16:47 +0200 Subject: [PATCH 23/37] feat: long stream timeout --- .../download-file/download-engine-file.ts | 14 +++- .../download-file/progress-status-file.ts | 2 + .../engine/download-engine-multi-download.ts | 3 +- .../base-download-engine-fetch-stream.ts | 26 ++++-- .../download-engine-fetch-stream-fetch.ts | 81 ++++++++++++------- .../download-engine-fetch-stream-xhr.ts | 35 ++++++-- .../progress-statistics-builder.ts | 24 +++--- .../transfer-cli/GlobalCLI.ts | 4 +- .../base-transfer-cli-progress-bar.ts | 16 ++-- .../fancy-transfer-cli-progress-bar.ts | 6 +- 10 files changed, 148 insertions(+), 63 deletions(-) diff --git a/src/download/download-engine/download-file/download-engine-file.ts b/src/download/download-engine/download-file/download-engine-file.ts index 8c486c1..c3b900e 100644 --- a/src/download/download-engine/download-file/download-engine-file.ts +++ b/src/download/download-engine/download-file/download-engine-file.ts @@ -74,6 +74,7 @@ export default class DownloadEngineFile extends EventEmitter c.isRetrying); + thisStatus.retrying = streamContexts.some(c => c.isRetrying); thisStatus.retryingTotalAttempts = Math.max(...streamContexts.map(x => x.retryingAttempts)); + thisStatus.streamsNotResponding = streamContexts.reduce((acc, cur) => acc + (cur.isStreamNotResponding ? 1 : 0), 0); return thisStatus; } @@ -296,6 +297,15 @@ export default class DownloadEngineFile extends EventEmitter { + getContext().isStreamNotResponding = true; + this._sendProgressDownloadPart(); + }); + + fetchState.addListener("streamNotRespondingOff", () => { + getContext().isStreamNotResponding = false; + }); + const downloadedPartsSize = this._downloadedPartsSize; this._progress.chunks[startChunk] = ChunkStatus.IN_PROGRESS; const allWrites = new Set>(); diff --git a/src/download/download-engine/download-file/progress-status-file.ts b/src/download/download-engine/download-file/progress-status-file.ts index e473a1e..d07359d 100644 --- a/src/download/download-engine/download-file/progress-status-file.ts +++ b/src/download/download-engine/download-file/progress-status-file.ts @@ -13,6 +13,7 @@ export type ProgressStatus = { downloadFlags: DownloadFlags[] retrying: boolean retryingTotalAttempts: number + streamsNotResponding: number }; export enum DownloadStatus { @@ -45,6 +46,7 @@ export default class ProgressStatusFile { public downloadId: string = ""; public retrying = false; public retryingTotalAttempts = 0; + public streamsNotResponding = 0; public constructor( totalDownloadParts: number, diff --git a/src/download/download-engine/engine/download-engine-multi-download.ts b/src/download/download-engine/engine/download-engine-multi-download.ts index dca5b40..077d58f 100644 --- a/src/download/download-engine/engine/download-engine-multi-download.ts +++ b/src/download/download-engine/engine/download-engine-multi-download.ts @@ -160,6 +160,8 @@ export default class DownloadEngineMultiDownload void retryingOn: (error: Error, attempt: number) => void retryingOff: () => void + streamNotRespondingOn: () => void + streamNotRespondingOff: () => void }; export type WriteCallback = (data: Uint8Array[], position: number, index: number) => void; const DEFAULT_OPTIONS: BaseDownloadEngineFetchStreamOptions = { retryOnServerError: true, - maxStreamWait: 1000 * 3, + maxStreamWait: 1000 * 15, retry: { retries: 50, factor: 1.5, @@ -149,11 +153,15 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter let throwErr: Error | null = null; const tryHeaders = "tryHeaders" in this.options && this.options.tryHeaders ? this.options.tryHeaders.slice() : []; + let retryingOn = false; const fetchDownloadInfoCallback = async (): Promise => { try { const response = await this.fetchDownloadInfoWithoutRetry(url); - this.emit("retryingOff"); + if (retryingOn) { + retryingOn = false; + this.emit("retryingOff"); + } return response; } catch (error: any) { this.errorCount.value++; @@ -162,6 +170,7 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter if (error instanceof HttpError && !this.retryOnServerError(error)) { if ("tryHeaders" in this.options && tryHeaders.length) { this.options.headers = tryHeaders.shift(); + retryingOn = true; this.emit("retryingOn", error, this.errorCount.value); await sleep(this.options.tryHeadersDelay ?? 0); return await fetchDownloadInfoCallback(); @@ -172,6 +181,7 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter } if (error instanceof StatusCodeError && error.retryAfter) { + retryingOn = true; this.emit("retryingOn", error, this.errorCount.value); await sleep(error.retryAfter * 1000); return await fetchDownloadInfoCallback(); @@ -194,13 +204,18 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter public async fetchChunks(callback: WriteCallback) { let lastStartLocation = this.state.startChunk; let retryResolvers = retryAsyncStatementSimple(this.options.retry); + let retryingOn = false; // eslint-disable-next-line no-constant-condition while (true) { try { - const response = await this.fetchWithoutRetryChunks(callback); - this.emit("retryingOff"); - return response; + return await this.fetchWithoutRetryChunks((...args) => { + if (retryingOn) { + retryingOn = false; + this.emit("retryingOff"); + } + callback(...args); + }); } catch (error: any) { if (error?.name === "AbortError") return; this.errorCount.value++; @@ -210,6 +225,7 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter throw error; } + retryingOn = true; this.emit("retryingOn", error, this.errorCount.value); if (error instanceof StatusCodeError && error.retryAfter) { await sleep(error.retryAfter * 1000); diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts index 9ff0d47..723bd27 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts @@ -1,4 +1,10 @@ -import BaseDownloadEngineFetchStream, {DownloadInfoResponse, FetchSubState, MIN_LENGTH_FOR_MORE_INFO_REQUEST, WriteCallback} from "./base-download-engine-fetch-stream.js"; +import BaseDownloadEngineFetchStream, { + DownloadInfoResponse, + FetchSubState, + MIN_LENGTH_FOR_MORE_INFO_REQUEST, + STREAM_NOT_RESPONDING_TIMEOUT, + WriteCallback +} from "./base-download-engine-fetch-stream.js"; import InvalidContentLengthError from "./errors/invalid-content-length-error.js"; import SmartChunkSplit from "./utils/smart-chunk-split.js"; import {parseContentDisposition} from "./utils/content-disposition.js"; @@ -11,6 +17,7 @@ import prettyMilliseconds from "pretty-ms"; type GetNextChunk = () => Promise> | ReadableStreamReadResult; export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFetchStream { private _fetchDownloadInfoWithHEAD = false; + private _activeController?: AbortController; public override transferAction = "Downloading"; public override readonly supportDynamicStreamLength = true; @@ -29,10 +36,14 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe headers.range = `bytes=${this._startSize}-${this._endSize - 1}`; } - const controller = new AbortController(); + if (!this._activeController?.signal.aborted) { + this._activeController?.abort(); + } + + this._activeController = new AbortController(); const response = await fetch(this.appendToURL(this.state.url), { headers, - signal: controller.signal + signal: this._activeController.signal }); if (response.status < 200 || response.status >= 300) { @@ -46,7 +57,7 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe } this.on("aborted", () => { - controller.abort(); + this._activeController?.abort(); }); const reader = response.body!.getReader(); @@ -117,7 +128,7 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe // eslint-disable-next-line no-constant-condition while (true) { - const {done, value} = await DownloadEngineFetchStreamFetch._wrapperStreamTimeout(getNextChunk()); + const {done, value} = await this._wrapperStreamNotResponding(getNextChunk()); await this.paused; if (done || this.aborted) break; @@ -128,6 +139,44 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe smartSplit.sendLeftovers(); } + protected _wrapperStreamNotResponding(promise: Promise | T): Promise | T { + if (!(promise instanceof Promise)) { + return promise; + } + + return new Promise((resolve, reject) => { + let streamNotRespondedInTime = false; + let timeoutMaxStreamWaitThrows = false; + const timeoutNotResponding = setTimeout(() => { + streamNotRespondedInTime = true; + this.emit("streamNotRespondingOn"); + }, STREAM_NOT_RESPONDING_TIMEOUT); + + const timeoutMaxStreamWait = setTimeout(() => { + timeoutMaxStreamWaitThrows = true; + reject(new EmptyStreamTimeoutError(`Stream timeout after ${prettyMilliseconds(this.options.maxStreamWait!)}`)); + this._activeController?.abort(); + }, this.options.maxStreamWait); + + promise + .then(resolve) + .catch(error => { + if (timeoutMaxStreamWaitThrows) { + return; + } + throw error; + }) + .finally(() => { + clearTimeout(timeoutNotResponding); + clearTimeout(timeoutMaxStreamWait); + if (streamNotRespondedInTime) { + this.emit("streamNotRespondingOff"); + } + }); + }); + } + + protected static convertHeadersToRecord(headers: Headers): { [key: string]: string } { const headerObj: { [key: string]: string } = {}; headers.forEach((value, key) => { @@ -136,26 +185,4 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe return headerObj; } - protected static _wrapperStreamTimeout(promise: Promise | T, maxStreamWait = 2_000): Promise | T { - if (!(promise instanceof Promise)) { - return promise; - } - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - reject(new EmptyStreamTimeoutError(`Stream timeout after ${prettyMilliseconds(maxStreamWait)}`)); - }, maxStreamWait); - - promise.then( - (result) => { - clearTimeout(timeout); - resolve(result); - }, - (error) => { - clearTimeout(timeout); - reject(error); - } - ); - }); - } } diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts index ffd4a0b..d161535 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts @@ -1,4 +1,10 @@ -import BaseDownloadEngineFetchStream, {DownloadInfoResponse, FetchSubState, MIN_LENGTH_FOR_MORE_INFO_REQUEST, WriteCallback} from "./base-download-engine-fetch-stream.js"; +import BaseDownloadEngineFetchStream, { + DownloadInfoResponse, + FetchSubState, + MIN_LENGTH_FOR_MORE_INFO_REQUEST, + STREAM_NOT_RESPONDING_TIMEOUT, + WriteCallback +} from "./base-download-engine-fetch-stream.js"; import EmptyResponseError from "./errors/empty-response-error.js"; import StatusCodeError from "./errors/status-code-error.js"; import XhrError from "./errors/xhr-error.js"; @@ -7,8 +13,8 @@ import retry from "async-retry"; import {AvailablePrograms} from "../../download-file/download-programs/switch-program.js"; import {parseContentDisposition} from "./utils/content-disposition.js"; import {parseHttpContentRange} from "./utils/httpRange.js"; -import {EmptyStreamTimeoutError} from "./errors/EmptyStreamTimeoutError.js"; import prettyMilliseconds from "pretty-ms"; +import {EmptyStreamTimeoutError} from "./errors/EmptyStreamTimeoutError.js"; export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetchStream { @@ -47,16 +53,33 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc xhr.setRequestHeader(key, value); } - let lastTimeoutIndex: any; + let lastNotRespondingTimeoutIndex: any; + let lastMaxStreamWaitTimeoutIndex: any; + let streamNotResponding = false; const clearStreamTimeout = () => { - if (lastTimeoutIndex) { - clearTimeout(lastTimeoutIndex); + if (streamNotResponding) { + this.emit("streamNotRespondingOff"); + streamNotResponding = false; + } + + if (lastNotRespondingTimeoutIndex) { + clearTimeout(lastNotRespondingTimeoutIndex); + } + + if (lastMaxStreamWaitTimeoutIndex) { + clearTimeout(lastMaxStreamWaitTimeoutIndex); } }; const createStreamTimeout = () => { clearStreamTimeout(); - lastTimeoutIndex = setTimeout(() => { + + lastNotRespondingTimeoutIndex = setTimeout(() => { + streamNotResponding = true; + this.emit("streamNotRespondingOn"); + }, STREAM_NOT_RESPONDING_TIMEOUT); + + lastMaxStreamWaitTimeoutIndex = setTimeout(() => { reject(new EmptyStreamTimeoutError(`Stream timeout after ${prettyMilliseconds(this.options.maxStreamWait!)}`)); xhr.abort(); }, this.options.maxStreamWait); diff --git a/src/download/transfer-visualize/progress-statistics-builder.ts b/src/download/transfer-visualize/progress-statistics-builder.ts index 704838e..978fb2b 100644 --- a/src/download/transfer-visualize/progress-statistics-builder.ts +++ b/src/download/transfer-visualize/progress-statistics-builder.ts @@ -36,6 +36,7 @@ export default class ProgressStatisticsBuilder extends EventEmitter { - if (isRetrying) { - this._retrying--; - this._retryingTotalAttempts -= retryingTotalAttempts; - } + const retrying = Number(data.retrying); + this._retrying += retrying - lastRetrying; + lastRetrying = retrying; - isRetrying = data.retrying; - retryingTotalAttempts = data.retryingTotalAttempts; + this._retryingTotalAttempts += data.retryingTotalAttempts - lastRetryingTotalAttempts; + lastRetryingTotalAttempts = data.retryingTotalAttempts; - this._retrying += Number(isRetrying); - this._retryingTotalAttempts += retryingTotalAttempts; + this._streamsNotResponding += data.streamsNotResponding - lastStreamsNotResponding; + lastStreamsNotResponding = data.streamsNotResponding; this._sendProgress(data, index, downloadPartStart); }); @@ -158,7 +159,8 @@ export default class ProgressStatisticsBuilder extends EventEmitter 0, - retryingTotalAttempts: this._retryingTotalAttempts + retryingTotalAttempts: this._retryingTotalAttempts, + streamsNotResponding: this._streamsNotResponding }), index }; diff --git a/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts b/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts index 40b4fdd..bf26798 100644 --- a/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts +++ b/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts @@ -44,9 +44,7 @@ class GlobalCLI { } await this._multiDownloadEngine.addDownload(engine); - if (this._multiDownloadEngine.activeDownloads.length === 0) { - await this._multiDownloadEngine.download(); - } + this._multiDownloadEngine.download(); } private _createMultiDownloadEngine() { diff --git a/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts b/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts index 159a6cd..17b8284 100644 --- a/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts +++ b/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts @@ -64,8 +64,14 @@ export default class BaseTransferCliProgressBar implements TransferCliProgressBa return this.status.startTime < Date.now() - SKIP_ETA_START_TIME; } - protected get retryingText() { - return this.status.retrying ? `(retrying #${this.status.retryingTotalAttempts})` : ""; + protected get alertStatus() { + if (this.status.retrying) { + return `(retrying #${this.status.retryingTotalAttempts})`; + } else if (this.status.streamsNotResponding) { + return `(${this.status.streamsNotResponding} streams not responding)`; + } + + return ""; } protected getNameSize(fileName = this.status.fileName) { @@ -174,7 +180,7 @@ export default class BaseTransferCliProgressBar implements TransferCliProgressBa const {formattedPercentage, formattedSpeed, formatTransferredOfTotal, formatTotal} = this.status; const status = this.switchTransferToShortText(); - const retryingText = this.retryingText; + const alertStatus = this.alertStatus; return renderDataLine([ { type: "status", @@ -184,8 +190,8 @@ export default class BaseTransferCliProgressBar implements TransferCliProgressBa }, { type: "status", - fullText: retryingText, - size: retryingText.length, + fullText: alertStatus, + size: alertStatus.length, formatter: (text) => chalk.ansi256(196)(text) }, { diff --git a/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts b/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts index 235353c..e54648e 100644 --- a/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts +++ b/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts @@ -19,7 +19,7 @@ export default class FancyTransferCliProgressBar extends BaseTransferCliProgress const progressBarText = ` ${formattedPercentageWithPadding} (${formatTransferred}/${formatTotal}) `; const dimEta: DataLine = this.getETA(" | ", text => chalk.dim(text)); - const retryingText = this.retryingText; + const alertStatus = this.alertStatus; return renderDataLine([ { @@ -30,8 +30,8 @@ export default class FancyTransferCliProgressBar extends BaseTransferCliProgress }, { type: "status", - fullText: retryingText, - size: retryingText.length, + fullText: alertStatus, + size: alertStatus.length, formatter: (text) => chalk.ansi256(196)(text) }, { From 051ab5081680c30e2b51dd73216ef7e2e159fce8 Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 26 Feb 2025 18:53:36 +0200 Subject: [PATCH 24/37] fix(fetch): throw error --- .../download-engine-fetch-stream-fetch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts index 723bd27..9dad7e8 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts @@ -164,7 +164,7 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe if (timeoutMaxStreamWaitThrows) { return; } - throw error; + reject(error); }) .finally(() => { clearTimeout(timeoutNotResponding); From f2e979e284629907cf7cd6593e6aa4a27d15d39e Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 26 Feb 2025 18:57:15 +0200 Subject: [PATCH 25/37] ci: fix release --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0cfbfc6..d8f665c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,7 +55,7 @@ jobs: node-version: "22" - name: Install modules run: npm ci --ignore-scripts - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts - name: Move artifacts @@ -77,12 +77,12 @@ jobs: fi - name: Upload docs to GitHub Pages if: steps.set-npm-url.outputs.npm-url != '' - uses: actions/upload-pages-artifact@v2 + uses: actions/upload-pages-artifact@v3 with: name: pages-docs path: docs - name: Deploy docs to GitHub Pages if: steps.set-npm-url.outputs.npm-url != '' && github.ref == 'refs/heads/main' - uses: actions/deploy-pages@v2 + uses: actions/deploy-pages@v4 with: artifact_name: pages-docs From 8989049d45db6050f2d29ec0f52f82dfe22de3e0 Mon Sep 17 00:00:00 2001 From: ido Date: Sun, 2 Mar 2025 16:38:22 +0200 Subject: [PATCH 26/37] docs(README): automatic parallelization --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0496045..6f9e908 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ npx ipull http://example.com/file.large ## Features - Download using parallels connections +- Maximize download speed (automatic parallelization, 3+) - Pausing and resuming downloads - Node.js and browser support - Smart retry on fail From a75e6681c4953f5f13322dc0040d6c6538c6f1e9 Mon Sep 17 00:00:00 2001 From: ido Date: Sun, 2 Mar 2025 17:25:05 +0200 Subject: [PATCH 27/37] fix(fetch): abort streaming --- .../download-engine-fetch-stream-fetch.ts | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts index 9dad7e8..107a058 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts @@ -40,8 +40,16 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe this._activeController?.abort(); } + let response: Response | null = null; this._activeController = new AbortController(); - const response = await fetch(this.appendToURL(this.state.url), { + this.on("aborted", () => { + if (!response) { + this._activeController?.abort(); + } + }); + + + response = await fetch(this.appendToURL(this.state.url), { headers, signal: this._activeController.signal }); @@ -56,10 +64,6 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe throw new InvalidContentLengthError(expectedContentLength, contentLength); } - this.on("aborted", () => { - this._activeController?.abort(); - }); - const reader = response.body!.getReader(); return await this.chunkGenerator(callback, () => reader.read()); } @@ -128,23 +132,23 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe // eslint-disable-next-line no-constant-condition while (true) { - const {done, value} = await this._wrapperStreamNotResponding(getNextChunk()); + const chunkInfo = await this._wrapperStreamNotResponding(getNextChunk()); await this.paused; - if (done || this.aborted) break; + if (!chunkInfo || this.aborted || chunkInfo.done) break; - smartSplit.addChunk(value); + smartSplit.addChunk(chunkInfo.value); this.state.onProgress?.(smartSplit.savedLength); } smartSplit.sendLeftovers(); } - protected _wrapperStreamNotResponding(promise: Promise | T): Promise | T { + protected _wrapperStreamNotResponding(promise: Promise | T): Promise | T | void { if (!(promise instanceof Promise)) { return promise; } - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { let streamNotRespondedInTime = false; let timeoutMaxStreamWaitThrows = false; const timeoutNotResponding = setTimeout(() => { @@ -158,10 +162,12 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe this._activeController?.abort(); }, this.options.maxStreamWait); + this.addListener("aborted", resolve); + promise .then(resolve) .catch(error => { - if (timeoutMaxStreamWaitThrows) { + if (timeoutMaxStreamWaitThrows || this.aborted) { return; } reject(error); @@ -172,6 +178,7 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe if (streamNotRespondedInTime) { this.emit("streamNotRespondingOff"); } + this.removeListener("aborted", resolve); }); }); } From 907f5254cdbd6bcb25bd44c6f8231d776a5cba5a Mon Sep 17 00:00:00 2001 From: ido Date: Sun, 2 Mar 2025 20:03:09 +0200 Subject: [PATCH 28/37] fix(save): prevent save emit after finished --- .../download-engine/download-file/download-engine-file.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/download/download-engine/download-file/download-engine-file.ts b/src/download/download-engine/download-file/download-engine-file.ts index c3b900e..86af36d 100644 --- a/src/download/download-engine/download-file/download-engine-file.ts +++ b/src/download/download-engine/download-file/download-engine-file.ts @@ -367,7 +367,7 @@ export default class DownloadEngineFile extends EventEmitter { - if (thisProgress === this._latestProgressDate) { + if (thisProgress === this._latestProgressDate && !this._closed && this._downloadStatus !== DownloadStatus.Finished) { await this.options.onSaveProgressAsync?.(this._progress); } }); From 646cdbd4e2ceea0585283a1165dc6e0b635e1b63 Mon Sep 17 00:00:00 2001 From: ido Date: Sun, 2 Mar 2025 20:03:23 +0200 Subject: [PATCH 29/37] fix(types): export required types --- src/browser.ts | 5 ++++- src/index.ts | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/browser.ts b/src/browser.ts index 46572e0..1f81a2b 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -9,7 +9,7 @@ import FetchStreamError from "./download/download-engine/streams/download-engine import IpullError from "./errors/ipull-error.js"; import EngineError from "./download/download-engine/engine/error/engine-error.js"; import {FormattedStatus} from "./download/transfer-visualize/format-transfer-status.js"; -import DownloadEngineMultiDownload from "./download/download-engine/engine/download-engine-multi-download.js"; +import DownloadEngineMultiDownload, {DownloadEngineMultiAllowedEngines} from "./download/download-engine/engine/download-engine-multi-download.js"; import HttpError from "./download/download-engine/streams/download-engine-fetch-stream/errors/http-error.js"; import BaseDownloadEngine from "./download/download-engine/engine/base-download-engine.js"; import {InvalidOptionError} from "./download/download-engine/engine/error/InvalidOptionError.js"; @@ -32,12 +32,15 @@ export { EngineError, InvalidOptionError }; + + export type { DownloadEngineRemote, BaseDownloadEngine, DownloadFileBrowserOptions, DownloadEngineBrowser, DownloadEngineMultiDownload, + DownloadEngineMultiAllowedEngines, FormattedStatus, SaveProgressInfo, DownloadSequenceBrowserOptions diff --git a/src/index.ts b/src/index.ts index caed3df..b98a0c4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,7 @@ import FetchStreamError from "./download/download-engine/streams/download-engine import IpullError from "./errors/ipull-error.js"; import EngineError from "./download/download-engine/engine/error/engine-error.js"; import {FormattedStatus} from "./download/transfer-visualize/format-transfer-status.js"; -import DownloadEngineMultiDownload from "./download/download-engine/engine/download-engine-multi-download.js"; +import DownloadEngineMultiDownload, {DownloadEngineMultiAllowedEngines} from "./download/download-engine/engine/download-engine-multi-download.js"; import HttpError from "./download/download-engine/streams/download-engine-fetch-stream/errors/http-error.js"; import BaseDownloadEngine from "./download/download-engine/engine/base-download-engine.js"; import {InvalidOptionError} from "./download/download-engine/engine/error/InvalidOptionError.js"; @@ -46,6 +46,7 @@ export type { DownloadSequenceOptions, DownloadEngineNodejs, DownloadEngineMultiDownload, + DownloadEngineMultiAllowedEngines, SaveProgressInfo, FormattedStatus, MultiProgressBarOptions, From 0a25b7bb22a8ee756b51c0ae19cfe87b9fd363f2 Mon Sep 17 00:00:00 2001 From: ido Date: Sun, 9 Mar 2025 23:20:16 +0200 Subject: [PATCH 30/37] fix(multi-download): race condition --- .../download-engine/engine/download-engine-multi-download.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/download/download-engine/engine/download-engine-multi-download.ts b/src/download/download-engine/engine/download-engine-multi-download.ts index 077d58f..36759af 100644 --- a/src/download/download-engine/engine/download-engine-multi-download.ts +++ b/src/download/download-engine/engine/download-engine-multi-download.ts @@ -225,7 +225,9 @@ export default class DownloadEngineMultiDownload 0; - await Promise.race(this._engineWaitPromises); + if (continueIteration) { + await Promise.race(this._engineWaitPromises); + } } this._downloadEndPromise = Promise.withResolvers(); From f791886f72f670609e83ab34136f47f3a3c62b38 Mon Sep 17 00:00:00 2001 From: ido Date: Tue, 11 Mar 2025 11:09:01 +0200 Subject: [PATCH 31/37] fix(chunk-split): validate leftover --- src/download/browser-download.ts | 10 +-- .../download-file/download-engine-file.ts | 11 ++- .../base-download-program.ts | 67 ++++++++++++++----- .../download-program-stream.ts | 3 +- .../engine/base-download-engine.ts | 3 +- .../base-download-engine-fetch-stream.ts | 1 + .../download-engine-fetch-stream-fetch.ts | 2 - .../download-engine-fetch-stream-xhr.ts | 4 ++ .../utils/smart-chunk-split.ts | 51 ++++++++------ test/browser.test.ts | 21 ++++-- 10 files changed, 116 insertions(+), 57 deletions(-) diff --git a/src/download/browser-download.ts b/src/download/browser-download.ts index ac7f2c8..a6d61f0 100644 --- a/src/download/browser-download.ts +++ b/src/download/browser-download.ts @@ -6,10 +6,7 @@ import {DownloadEngineRemote} from "./download-engine/engine/DownloadEngineRemot const DEFAULT_PARALLEL_STREAMS_FOR_BROWSER = 3; -export type DownloadFileBrowserOptions = DownloadEngineOptionsBrowser & { - /** @deprecated use partURLs instead */ - partsURL?: string[]; -}; +export type DownloadFileBrowserOptions = DownloadEngineOptionsBrowser; export type DownloadSequenceBrowserOptions = DownloadEngineMultiDownloadOptions; @@ -17,11 +14,6 @@ export type DownloadSequenceBrowserOptions = DownloadEngineMultiDownloadOptions; * Download one file in the browser environment. */ export async function downloadFileBrowser(options: DownloadFileBrowserOptions) { - // TODO: Remove in the next major version - if (!("url" in options) && options.partsURL) { - options.partURLs ??= options.partsURL; - } - options.parallelStreams ??= DEFAULT_PARALLEL_STREAMS_FOR_BROWSER; return await DownloadEngineBrowser.createFromOptions(options); } diff --git a/src/download/download-engine/download-file/download-engine-file.ts b/src/download/download-engine/download-file/download-engine-file.ts index 86af36d..f9538c5 100644 --- a/src/download/download-engine/download-file/download-engine-file.ts +++ b/src/download/download-engine/download-file/download-engine-file.ts @@ -276,7 +276,8 @@ export default class DownloadEngineFile extends EventEmitter>(); - let lastChunkSize = 0; + let lastChunkSize = 0, lastInProgressIndex = startChunk; await fetchState.fetchChunks((chunks, writePosition, index) => { if (this._closed || this._progress.chunks[index] != ChunkStatus.IN_PROGRESS) { return; @@ -344,10 +345,14 @@ export default class DownloadEngineFile extends EventEmitter= this._parallelStreams) { - if (this._aborted) return; - const promiseResolvers = Promise.withResolvers(); - this._reload = promiseResolvers.resolve; + while (!this._aborted) { + if (this._activeDownloads.length >= this._parallelStreams) { + await this._waitForStreamEndWithReload(); + continue; + } - await Promise.race(this._activeDownloads.concat([promiseResolvers.promise])); + const slice = this._createOneSlice(); + if (slice == null) { + if (this._activeDownloads.length === 0) { + break; + } + await this._waitForStreamEndWithReload(); + continue; } + this._createDownload(slice); + } + } + + private async _waitForStreamEndWithReload() { + const promiseResolvers = Promise.withResolvers(); + this._reload = promiseResolvers.resolve; + return await Promise.race(this._activeDownloads.concat([promiseResolvers.promise])); + } + + private _createDownload(slice: ProgramSlice) { + const promise = this._downloadSlice(slice.start, slice.end); + this._activeDownloads.push(promise); + promise.then(() => { + this._activeDownloads.splice(this._activeDownloads.indexOf(promise), 1); + }); + } + /** + * Create all the first slices at one - make sure they will not overlap to reduce stream aborts at later stages + */ + private _createFirstSlices() { + const slices: ProgramSlice[] = []; + for (let i = 0; i < this.parallelStreams; i++) { const slice = this._createOneSlice(); - if (slice == null) break; - - if (this._aborted) return; - const promise = this._downloadSlice(slice.start, slice.end); - this._activeDownloads.push(promise); - promise.then(() => { - this._activeDownloads.splice(this._activeDownloads.indexOf(promise), 1); - }); + if (slice) { + const lastSlice = slices.find(x => x.end > slice.start && x.start < slice.start); + if (lastSlice) { + lastSlice.end = slice.start; + } + this.savedProgress.chunks[slice.start] = ChunkStatus.IN_PROGRESS; + slices.push(slice); + } else { + break; + } } - await Promise.all(this._activeDownloads); + for (const slice of slices) { + this._createDownload(slice); + } } protected abstract _createOneSlice(): ProgramSlice | null; diff --git a/src/download/download-engine/download-file/download-programs/download-program-stream.ts b/src/download/download-engine/download-file/download-programs/download-program-stream.ts index 5c90138..4d3e0b1 100644 --- a/src/download/download-engine/download-file/download-programs/download-program-stream.ts +++ b/src/download/download-engine/download-file/download-programs/download-program-stream.ts @@ -11,7 +11,8 @@ export default class DownloadProgramStream extends BaseDownloadProgram { const slice = this._findChunksSlices()[0]; if (!slice) return null; const length = slice.end - slice.start; - return {start: Math.floor(slice.start + length / 2), end: slice.end}; + const start = slice.start == 0 ? slice.start : Math.floor(slice.start + length / 2); + return {start, end: slice.end}; } private _findChunksSlices() { diff --git a/src/download/download-engine/engine/base-download-engine.ts b/src/download/download-engine/engine/base-download-engine.ts index 22fc104..eec851c 100644 --- a/src/download/download-engine/engine/base-download-engine.ts +++ b/src/download/download-engine/engine/base-download-engine.ts @@ -18,7 +18,8 @@ export type BaseDownloadEngineOptions = InputURLOptions & BaseDownloadEngineFetc parallelStreams?: number; retry?: retry.Options comment?: string; - programType?: AvailablePrograms + programType?: AvailablePrograms, + autoIncreaseParallelStreams?: boolean }; export type BaseDownloadEngineEvents = { diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts index ddb25f0..46f8593 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts @@ -53,6 +53,7 @@ export type FetchSubState = { url: string, startChunk: number, endChunk: number, + lastChunkEndsFile: boolean, totalSize: number, chunkSize: number, rangeSupport?: boolean, diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts index 107a058..6bc9257 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts @@ -139,8 +139,6 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe smartSplit.addChunk(chunkInfo.value); this.state.onProgress?.(smartSplit.savedLength); } - - smartSplit.sendLeftovers(); } protected _wrapperStreamNotResponding(promise: Promise | T): Promise | T | void { diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts index d161535..322006e 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts @@ -95,6 +95,10 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc } if (xhr.status >= 200 && xhr.status < 300) { + if (xhr.response.length != contentLength) { + throw new InvalidContentLengthError(contentLength, xhr.response.length); + } + const arrayBuffer = xhr.response; if (arrayBuffer) { resolve(new Uint8Array(arrayBuffer)); diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts b/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts index 2f9a000..653831a 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts @@ -3,19 +3,28 @@ import {WriteCallback} from "../base-download-engine-fetch-stream.js"; export type SmartChunkSplitOptions = { chunkSize: number; startChunk: number; + endChunk: number; + lastChunkEndsFile: boolean; + totalSize: number }; export default class SmartChunkSplit { private readonly _callback: WriteCallback; private readonly _options: SmartChunkSplitOptions; + private readonly _lastChunkSize: number; private _bytesWriteLocation: number; - private _bytesLeftovers: number = 0; private _chunks: Uint8Array[] = []; public constructor(_callback: WriteCallback, _options: SmartChunkSplitOptions) { this._options = _options; this._callback = _callback; this._bytesWriteLocation = _options.startChunk * _options.chunkSize; + this._lastChunkSize = _options.lastChunkEndsFile ? + this.calcLastChunkSize() : this._options.chunkSize; + } + + public calcLastChunkSize() { + return this._options.totalSize - Math.max(this._options.endChunk - 1, 0) * this._options.chunkSize; } public addChunk(data: Uint8Array) { @@ -24,32 +33,36 @@ export default class SmartChunkSplit { } public get savedLength() { - return this._bytesLeftovers + this._chunks.reduce((acc, chunk) => acc + chunk.length, 0); - } - - public sendLeftovers() { - if (this.savedLength > 0) { - this._callback(this._chunks, this._bytesWriteLocation, this._options.startChunk++); - } + return this._chunks.reduce((acc, chunk) => acc + chunk.length, 0); } private _sendChunk() { - while (this.savedLength >= this._options.chunkSize) { - if (this._chunks.length === 0) { - this._callback([], this._bytesWriteLocation, this._options.startChunk++); - this._bytesWriteLocation += this._options.chunkSize; - this._bytesLeftovers -= this._options.chunkSize; - } + const checkThreshold = () => + (this._options.endChunk - this._options.startChunk === 1 ? + this._lastChunkSize : this._options.chunkSize); - let sendLength = this._bytesLeftovers; + let calcChunkThreshold = 0; + while (this.savedLength >= (calcChunkThreshold = checkThreshold())) { + let sendLength = 0; for (let i = 0; i < this._chunks.length; i++) { - sendLength += this._chunks[i].byteLength; - if (sendLength >= this._options.chunkSize) { + const currentChunk = this._chunks[i]; + sendLength += currentChunk.length; + if (sendLength >= calcChunkThreshold) { const sendChunks = this._chunks.splice(0, i + 1); + const diffLength = sendLength - calcChunkThreshold; + + if (diffLength > 0) { + const lastChunkEnd = currentChunk.length - diffLength; + const lastChunk = currentChunk.subarray(0, lastChunkEnd); + + sendChunks.pop(); + sendChunks.push(lastChunk); + + this._chunks.unshift(currentChunk.subarray(lastChunkEnd)); + } this._callback(sendChunks, this._bytesWriteLocation, this._options.startChunk++); - this._bytesWriteLocation += sendLength - this._bytesLeftovers; - this._bytesLeftovers = sendLength - this._options.chunkSize; + this._bytesWriteLocation += calcChunkThreshold; break; } } diff --git a/test/browser.test.ts b/test/browser.test.ts index 58208d7..8e7e05a 100644 --- a/test/browser.test.ts +++ b/test/browser.test.ts @@ -11,7 +11,9 @@ globalThis.XMLHttpRequest = await import("xmlhttprequest-ssl").then(m => m.XMLHt describe("Browser Fetch API", () => { test.concurrent("Download file browser - memory", async (context) => { const downloader = await downloadFileBrowser({ - url: BIG_FILE + url: BIG_FILE, + parallelStreams: 2, + autoIncreaseParallelStreams: false }); await downloader.download(); @@ -21,10 +23,15 @@ describe("Browser Fetch API", () => { }); test.concurrent("Download file browser", async (context) => { + const response = await ensureLocalFile(BIG_FILE, BIG_FILE_EXAMPLE); + const bufferIsCorrect = Buffer.from(await fs.readFile(response)); + let buffer = Buffer.alloc(0); let lastWrite = 0; const downloader = await downloadFileBrowser({ url: BIG_FILE, + parallelStreams: 2, + autoIncreaseParallelStreams: false, onWrite(cursor, data) { buffer.set(data, cursor); if (cursor + data.length > lastWrite) { @@ -34,13 +41,17 @@ describe("Browser Fetch API", () => { }); buffer = Buffer.alloc(downloader.file.totalSize); - await downloader.download(); - context.expect(hashBuffer(buffer)) - .toMatchInlineSnapshot("\"9ae3ff19ee04fc02e9c60ce34e42858d16b46eeb88634d2035693c1ae9dbcbc9\""); + + const diff = buffer.findIndex((value, index) => value !== bufferIsCorrect[index]); + context.expect(diff) + .toBe(-1); + context.expect(lastWrite) .toBe(downloader.file.totalSize); - }); + context.expect(hashBuffer(buffer)) + .toMatchInlineSnapshot("\"9ae3ff19ee04fc02e9c60ce34e42858d16b46eeb88634d2035693c1ae9dbcbc9\""); + }, {repeats: 4, concurrent: true}); }, {timeout: 1000 * 60 * 3}); describe("Browser Fetch memory", () => { From 022ca6a2251d41b5823b672527a455154cffb117 Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 12 Mar 2025 16:10:04 +0200 Subject: [PATCH 32/37] perf(chunks): send write event only once for chunk --- README.md | 5 +-- examples/browser-log.ts | 7 ++-- src/browser.ts | 7 ++-- .../download-file/download-engine-file.ts | 15 ++++---- .../base-download-engine-write-stream.ts | 2 +- .../download-engine-write-stream-browser.ts | 22 ++++++++---- .../download-engine-write-stream-nodejs.ts | 8 ++--- .../utils/BytesWriteDebounce.ts | 36 ++++++++++--------- test/browser.test.ts | 19 ++++++---- test/download.test.ts | 2 +- 10 files changed, 71 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 6f9e908..665f1d7 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,9 @@ import {downloadFileBrowser} from "ipull/dist/browser.js"; const downloader = await downloadFileBrowser({ url: 'https://example.com/file.large', - onWrite: (cursor: number, buffer: Uint8Array, options) => { - console.log(`Writing ${buffer.length} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`); + onWrite: (cursor: number, buffers: Uint8Array[], options) => { + const totalLength = buffers.reduce((acc, buffer) => acc + buffer.length, 0); + console.log(`Writing ${totalLength} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`); } }); diff --git a/examples/browser-log.ts b/examples/browser-log.ts index fbb0c77..75b40d0 100644 --- a/examples/browser-log.ts +++ b/examples/browser-log.ts @@ -1,11 +1,12 @@ -import {downloadFileBrowser} from "ipull/dist/browser.js"; +import {downloadFileBrowser} from "ipull/browser"; const BIG_IMAGE = "https://upload.wikimedia.org/wikipedia/commons/9/9e/1_dubrovnik_pano_-_edit1.jpg"; // 40mb const downloader = await downloadFileBrowser({ url: BIG_IMAGE, - onWrite: (cursor: number, buffer: Uint8Array, options) => { - console.log(`Writing ${buffer.length} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`); + onWrite: (cursor: number, buffers: Uint8Array[], options: DownloadEngineWriteStreamOptionsBrowser) => { + const totalLength = buffers.reduce((acc, b) => acc + b.byteLength, 0); + console.log(`Writing ${totalLength} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`); } }); diff --git a/src/browser.ts b/src/browser.ts index 1f81a2b..2c5ae9e 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -15,6 +15,9 @@ import BaseDownloadEngine from "./download/download-engine/engine/base-download- import {InvalidOptionError} from "./download/download-engine/engine/error/InvalidOptionError.js"; import {DownloadFlags, DownloadStatus} from "./download/download-engine/download-file/progress-status-file.js"; import {DownloadEngineRemote} from "./download/download-engine/engine/DownloadEngineRemote.js"; +import { + DownloadEngineWriteStreamOptionsBrowser +} from "./download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.js"; export { DownloadFlags, @@ -33,7 +36,6 @@ export { InvalidOptionError }; - export type { DownloadEngineRemote, BaseDownloadEngine, @@ -43,5 +45,6 @@ export type { DownloadEngineMultiAllowedEngines, FormattedStatus, SaveProgressInfo, - DownloadSequenceBrowserOptions + DownloadSequenceBrowserOptions, + DownloadEngineWriteStreamOptionsBrowser }; diff --git a/src/download/download-engine/download-file/download-engine-file.ts b/src/download/download-engine/download-file/download-engine-file.ts index f9538c5..8963f71 100644 --- a/src/download/download-engine/download-file/download-engine-file.ts +++ b/src/download/download-engine/download-file/download-engine-file.ts @@ -317,15 +317,12 @@ export default class DownloadEngineFile extends EventEmitter { - allWrites.delete(writePromise); - }); - } + const writePromise = this.options.writeStream.write(downloadedPartsSize + writePosition, chunks); + if (writePromise) { + allWrites.add(writePromise); + writePromise.then(() => { + allWrites.delete(writePromise); + }); } // if content length is 0, we do not know how many chunks we should have diff --git a/src/download/download-engine/streams/download-engine-write-stream/base-download-engine-write-stream.ts b/src/download/download-engine/streams/download-engine-write-stream/base-download-engine-write-stream.ts index cbacdcc..ddee247 100644 --- a/src/download/download-engine/streams/download-engine-write-stream/base-download-engine-write-stream.ts +++ b/src/download/download-engine/streams/download-engine-write-stream/base-download-engine-write-stream.ts @@ -1,5 +1,5 @@ export default abstract class BaseDownloadEngineWriteStream { - abstract write(cursor: number, buffer: Uint8Array): Promise | void; + abstract write(cursor: number, buffers: Uint8Array[]): Promise | void; close(): void | Promise { } diff --git a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.ts b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.ts index daa7f9e..8099ac0 100644 --- a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.ts +++ b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.ts @@ -4,12 +4,12 @@ import BaseDownloadEngineWriteStream from "./base-download-engine-write-stream.j import WriterIsClosedError from "./errors/writer-is-closed-error.js"; import WriterNotDefineError from "./errors/writer-not-define-error.js"; -type DownloadEngineWriteStreamOptionsBrowser = { +export type DownloadEngineWriteStreamOptionsBrowser = { retry?: retry.Options file?: DownloadFile }; -export type DownloadEngineWriteStreamBrowserWriter = (cursor: number, buffer: Uint8Array, options: DownloadEngineWriteStreamOptionsBrowser) => Promise | void; +export type DownloadEngineWriteStreamBrowserWriter = (cursor: number, buffers: Uint8Array[], options: DownloadEngineWriteStreamOptionsBrowser) => Promise | void; export default class DownloadEngineWriteStreamBrowser extends BaseDownloadEngineWriteStream { protected readonly _writer?: DownloadEngineWriteStreamBrowserWriter; @@ -44,18 +44,26 @@ export default class DownloadEngineWriteStreamBrowser extends BaseDownloadEngine return this._memory = newMemory; } - public write(cursor: number, buffer: Uint8Array) { + public write(cursor: number, buffers: Uint8Array[]) { if (this.writerClosed) { throw new WriterIsClosedError(); } if (!this._writer) { - this._ensureBuffer(cursor + buffer.byteLength) - .set(buffer, cursor); - this._bytesWritten += buffer.byteLength; + const totalLength = buffers.reduce((sum, buffer) => sum + buffer.length, 0); + const bigBuffer = this._ensureBuffer(cursor + totalLength); + let writeLocation = cursor; + + for (const buffer of buffers) { + bigBuffer.set(buffer, writeLocation); + writeLocation += buffer.length; + } + + this._bytesWritten += totalLength; return; } - return this._writer(cursor, buffer, this.options); + + return this._writer(cursor, buffers, this.options); } public get result() { diff --git a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts index b57174e..12bfa73 100644 --- a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts +++ b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts @@ -52,7 +52,7 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW this._writeDebounce = new BytesWriteDebounce({ ...optionsWithDefaults.debounceWrite, - writev: (cursor, buffer) => this._writeWithoutDebounce(cursor, buffer) + writev: (cursor, buffers) => this._writeWithoutDebounce(cursor, buffers) }); } @@ -84,8 +84,8 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW }); } - async write(cursor: number, buffer: Uint8Array) { - await this._writeDebounce.addChunk(cursor, buffer); + async write(cursor: number, buffers: Uint8Array[]) { + await this._writeDebounce.addChunk(cursor, buffers); } async _writeWithoutDebounce(cursor: number, buffers: Uint8Array[]) { @@ -131,7 +131,7 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW const encoder = new TextEncoder(); const uint8Array = encoder.encode(jsonString); - await this.write(this._fileSize, uint8Array); + await this.write(this._fileSize, [uint8Array]); } async loadMetadataAfterFileWithoutRetry() { diff --git a/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts b/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts index 68810db..805dc36 100644 --- a/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts +++ b/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts @@ -19,9 +19,13 @@ export class BytesWriteDebounce { } - async addChunk(index: number, buffer: Uint8Array) { - this._writeChunks.push({index, buffer}); - this._totalSizeOfChunks += buffer.byteLength; + async addChunk(index: number, buffers: Uint8Array[]) { + let writeIndex = index; + for (const buffer of buffers) { + this._writeChunks.push({index: writeIndex, buffer}); + this._totalSizeOfChunks += buffer.length; + writeIndex += buffer.length; + } await this._writeIfNeeded(); this.checkIfWriteNeededInterval(); @@ -57,32 +61,32 @@ export class BytesWriteDebounce { const firstWrite = this._writeChunks[0]; let writeIndex = firstWrite.index; - let buffer: Uint8Array[] = [firstWrite.buffer]; - let bufferTotalLength = firstWrite.buffer.length; + let buffers: Uint8Array[] = [firstWrite.buffer]; + let buffersTotalLength = firstWrite.buffer.length; const writePromises: Promise[] = []; - for (let i = 0; i < this._writeChunks.length; i++) { - const nextWriteLocation = writeIndex + bufferTotalLength; + for (let i = 1; i < this._writeChunks.length; i++) { + const nextWriteLocation = writeIndex + buffersTotalLength; const currentWrite = this._writeChunks[i]; if (currentWrite.index < nextWriteLocation) { // overlapping, prefer the last buffer (newer data) - const lastBuffer = buffer.pop()!; - buffer.push(currentWrite.buffer); - bufferTotalLength += currentWrite.buffer.length - lastBuffer.length; + const lastBuffer = buffers.pop()!; + buffers.push(currentWrite.buffer); + buffersTotalLength += currentWrite.buffer.length - lastBuffer.length; } else if (nextWriteLocation === currentWrite.index) { - buffer.push(currentWrite.buffer); - bufferTotalLength += currentWrite.buffer.length; + buffers.push(currentWrite.buffer); + buffersTotalLength += currentWrite.buffer.length; } else { - writePromises.push(this._options.writev(writeIndex, buffer)); + writePromises.push(this._options.writev(writeIndex, buffers)); writeIndex = currentWrite.index; - buffer = [currentWrite.buffer]; - bufferTotalLength = currentWrite.buffer.length; + buffers = [currentWrite.buffer]; + buffersTotalLength = currentWrite.buffer.length; } } - writePromises.push(this._options.writev(writeIndex, buffer)); + writePromises.push(this._options.writev(writeIndex, buffers)); this._writeChunks = []; this._totalSizeOfChunks = 0; diff --git a/test/browser.test.ts b/test/browser.test.ts index 8e7e05a..251e4b5 100644 --- a/test/browser.test.ts +++ b/test/browser.test.ts @@ -26,30 +26,35 @@ describe("Browser Fetch API", () => { const response = await ensureLocalFile(BIG_FILE, BIG_FILE_EXAMPLE); const bufferIsCorrect = Buffer.from(await fs.readFile(response)); - let buffer = Buffer.alloc(0); + let bigBuffer = Buffer.alloc(0); let lastWrite = 0; const downloader = await downloadFileBrowser({ url: BIG_FILE, parallelStreams: 2, autoIncreaseParallelStreams: false, onWrite(cursor, data) { - buffer.set(data, cursor); - if (cursor + data.length > lastWrite) { - lastWrite = cursor + data.length; + let writeLocation = cursor; + for (const buffer of data) { + bigBuffer.set(buffer, writeLocation); + writeLocation += buffer.length; + } + + if (writeLocation > lastWrite) { + lastWrite = writeLocation; } } }); - buffer = Buffer.alloc(downloader.file.totalSize); + bigBuffer = Buffer.alloc(downloader.file.totalSize); await downloader.download(); - const diff = buffer.findIndex((value, index) => value !== bufferIsCorrect[index]); + const diff = bigBuffer.findIndex((value, index) => value !== bufferIsCorrect[index]); context.expect(diff) .toBe(-1); context.expect(lastWrite) .toBe(downloader.file.totalSize); - context.expect(hashBuffer(buffer)) + context.expect(hashBuffer(bigBuffer)) .toMatchInlineSnapshot("\"9ae3ff19ee04fc02e9c60ce34e42858d16b46eeb88634d2035693c1ae9dbcbc9\""); }, {repeats: 4, concurrent: true}); }, {timeout: 1000 * 60 * 3}); diff --git a/test/download.test.ts b/test/download.test.ts index 1f196f8..53099c3 100644 --- a/test/download.test.ts +++ b/test/download.test.ts @@ -47,7 +47,7 @@ describe("File Download", () => { let totalBytesWritten = 0; const fetchStream = new DownloadEngineFetchStreamFetch(); const writeStream = new DownloadEngineWriteStreamBrowser((cursor, data) => { - totalBytesWritten += data.byteLength; + totalBytesWritten += data.reduce((sum, buffer) => sum + buffer.length, 0); }); const file = await createDownloadFile(BIG_FILE); From 1024941039c514d59aea8d2d4bd39b3bc37a41d4 Mon Sep 17 00:00:00 2001 From: ido Date: Wed, 12 Mar 2025 16:10:22 +0200 Subject: [PATCH 33/37] fix(dynamic-length): send the last chunk of dynamic length --- .../download-engine-fetch-stream-fetch.ts | 2 ++ .../utils/smart-chunk-split.ts | 10 ++++++++++ .../utils/stream-response.ts | 2 +- test/daynamic-content-length.test.ts | 12 ++++++++++-- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts index 6bc9257..8888480 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts @@ -139,6 +139,8 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe smartSplit.addChunk(chunkInfo.value); this.state.onProgress?.(smartSplit.savedLength); } + + smartSplit.closeAndSendLeftoversIfLengthIsUnknown(); } protected _wrapperStreamNotResponding(promise: Promise | T): Promise | T | void { diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts b/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts index 653831a..32e14b8 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts @@ -14,6 +14,7 @@ export default class SmartChunkSplit { private readonly _lastChunkSize: number; private _bytesWriteLocation: number; private _chunks: Uint8Array[] = []; + private _closed = false; public constructor(_callback: WriteCallback, _options: SmartChunkSplitOptions) { this._options = _options; @@ -36,7 +37,16 @@ export default class SmartChunkSplit { return this._chunks.reduce((acc, chunk) => acc + chunk.length, 0); } + closeAndSendLeftoversIfLengthIsUnknown() { + if (this._chunks.length > 0 && this._options.endChunk === Infinity) { + this._callback(this._chunks, this._bytesWriteLocation, this._options.startChunk++); + } + this._closed = true; + } + private _sendChunk() { + if (this._closed) return; + const checkThreshold = () => (this._options.endChunk - this._options.startChunk === 1 ? this._lastChunkSize : this._options.chunkSize); diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/utils/stream-response.ts b/src/download/download-engine/streams/download-engine-fetch-stream/utils/stream-response.ts index b43f587..3ea036b 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/utils/stream-response.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/utils/stream-response.ts @@ -20,7 +20,7 @@ export default async function streamResponse(stream: IStreamResponse, downloadEn }); stream.on("close", () => { - smartSplit.sendLeftovers(); + smartSplit.closeAndSendLeftoversIfLengthIsUnknown(); resolve(); }); diff --git a/test/daynamic-content-length.test.ts b/test/daynamic-content-length.test.ts index ef1c232..e821d38 100644 --- a/test/daynamic-content-length.test.ts +++ b/test/daynamic-content-length.test.ts @@ -28,7 +28,11 @@ describe("Dynamic content download", async () => { const downloader = await downloadFile({ url: DYNAMIC_DOWNLOAD_FILE, directory: ".", - fileName: IPUll_FILE + fileName: IPUll_FILE, + defaultFetchDownloadInfo: { + acceptRange: false, + length: 0 + } }); await downloader.download(); @@ -41,7 +45,11 @@ describe("Dynamic content download", async () => { test.concurrent("Browser Download", async (context) => { const downloader = await downloadFileBrowser({ - url: DYNAMIC_DOWNLOAD_FILE + url: DYNAMIC_DOWNLOAD_FILE, + defaultFetchDownloadInfo: { + acceptRange: false, + length: 0 + } }); await downloader.download(); From 698ddf91fb96f61d3d03540239e87e2c6d6979c2 Mon Sep 17 00:00:00 2001 From: ido Date: Sun, 16 Mar 2025 17:27:04 +0200 Subject: [PATCH 34/37] fix(globalCLI): cli hook on non-active download --- .../transfer-cli/GlobalCLI.ts | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts b/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts index bf26798..e14e928 100644 --- a/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts +++ b/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts @@ -24,6 +24,7 @@ export type CliProgressDownloadEngineOptions = { class GlobalCLI { private _multiDownloadEngine = this._createMultiDownloadEngine(); + private _eventsRegistered = new Set(); private _transferCLI = GlobalCLI._createOptions({}); private _cliActive = false; private _downloadOptions = new WeakMap(); @@ -78,12 +79,17 @@ class GlobalCLI { }; const checkPauseCLI = () => { - if (!isDownloadActive()) { + if (this._cliActive && !isDownloadActive()) { this._transferCLI.stop(); this._cliActive = false; } }; + const checkCloseCLI = (engine: DownloadEngineMultiAllowedEngines) => { + this._eventsRegistered.delete(engine); + checkPauseCLI(); + }; + const checkResumeCLI = (engine: DownloadEngineMultiAllowedEngines) => { if (engine.status.downloadStatus === DownloadStatus.Active) { this._transferCLI.start(); @@ -91,21 +97,21 @@ class GlobalCLI { } }; - this._multiDownloadEngine.on("start", () => { - this._transferCLI.start(); - this._cliActive = true; - }); - this._multiDownloadEngine.on("finished", () => { - this._transferCLI.stop(); - this._cliActive = false; this._multiDownloadEngine = this._createMultiDownloadEngine(); + this._eventsRegistered = new Set(); }); + const eventsRegistered = this._eventsRegistered; this._multiDownloadEngine.on("childDownloadStarted", function registerEngineStatus(engine) { - engine.on("closed", checkPauseCLI); + if (eventsRegistered.has(engine)) return; + eventsRegistered.add(engine); + + checkResumeCLI(engine); engine.on("paused", checkPauseCLI); + engine.on("closed", () => checkCloseCLI(engine)); engine.on("resumed", () => checkResumeCLI(engine)); + engine.on("start", () => checkResumeCLI(engine)); if (engine instanceof DownloadEngineMultiDownload) { engine.on("childDownloadStarted", registerEngineStatus); From 903a3ab40b3fccfe6da33f0a30643c4c20b54015 Mon Sep 17 00:00:00 2001 From: ido Date: Thu, 22 May 2025 20:02:52 +0300 Subject: [PATCH 35/37] chores: Use polyfill for Promise.withResolvers to support older versions than 22 --- .github/workflows/build.yml | 2 +- .github/workflows/test.yml | 2 +- .../download-programs/base-download-program.ts | 3 ++- .../download-engine/engine/DownloadEngineRemote.ts | 3 ++- .../download-engine/engine/base-download-engine.ts | 3 ++- .../engine/download-engine-multi-download.ts | 5 +++-- src/download/download-engine/utils/concurrency.ts | 4 +++- .../download-engine/utils/promiseWithResolvers.ts | 9 +++++++++ tsconfig.json | 4 ++-- 9 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 src/download/download-engine/utils/promiseWithResolvers.ts diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8f665c..445310e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: "22" + node-version: "20" - name: Install modules run: npm ci --ignore-scripts - name: Build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88b2acd..652f105 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: "22" + node-version: "20" - name: Install modules run: npm ci --ignore-scripts - name: ESLint diff --git a/src/download/download-engine/download-file/download-programs/base-download-program.ts b/src/download/download-engine/download-file/download-programs/base-download-program.ts index 6795727..4e5141d 100644 --- a/src/download/download-engine/download-file/download-programs/base-download-program.ts +++ b/src/download/download-engine/download-file/download-programs/base-download-program.ts @@ -1,4 +1,5 @@ import {ChunkStatus, SaveProgressInfo} from "../../types.js"; +import {promiseWithResolvers} from "../../utils/promiseWithResolvers.js"; export type ProgramSlice = { start: number, @@ -71,7 +72,7 @@ export default abstract class BaseDownloadProgram { } private async _waitForStreamEndWithReload() { - const promiseResolvers = Promise.withResolvers(); + const promiseResolvers = promiseWithResolvers(); this._reload = promiseResolvers.resolve; return await Promise.race(this._activeDownloads.concat([promiseResolvers.promise])); } diff --git a/src/download/download-engine/engine/DownloadEngineRemote.ts b/src/download/download-engine/engine/DownloadEngineRemote.ts index a153b3e..e9ff5c9 100644 --- a/src/download/download-engine/engine/DownloadEngineRemote.ts +++ b/src/download/download-engine/engine/DownloadEngineRemote.ts @@ -3,12 +3,13 @@ import {EventEmitter} from "eventemitter3"; import {FormattedStatus} from "../../transfer-visualize/format-transfer-status.js"; import {DownloadStatus} from "../download-file/progress-status-file.js"; import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js"; +import {promiseWithResolvers} from "../utils/promiseWithResolvers.js"; export class DownloadEngineRemote extends EventEmitter { /** * @internal */ - _downloadEndPromise = Promise.withResolvers(); + _downloadEndPromise = promiseWithResolvers(); /** * @internal */ diff --git a/src/download/download-engine/engine/base-download-engine.ts b/src/download/download-engine/engine/base-download-engine.ts index eec851c..179f162 100644 --- a/src/download/download-engine/engine/base-download-engine.ts +++ b/src/download/download-engine/engine/base-download-engine.ts @@ -9,6 +9,7 @@ import {AvailablePrograms} from "../download-file/download-programs/switch-progr import StatusCodeError from "../streams/download-engine-fetch-stream/errors/status-code-error.js"; import {InvalidOptionError} from "./error/InvalidOptionError.js"; import {FormattedStatus} from "../../transfer-visualize/format-transfer-status.js"; +import {promiseWithResolvers} from "../utils/promiseWithResolvers.js"; const IGNORE_HEAD_STATUS_CODES = [405, 501, 404]; export type InputURLOptions = { partURLs: string[] } | { url: string }; @@ -41,7 +42,7 @@ export default class BaseDownloadEngine extends EventEmitter(); + _downloadEndPromise = promiseWithResolvers(); /** * @internal */ diff --git a/src/download/download-engine/engine/download-engine-multi-download.ts b/src/download/download-engine/engine/download-engine-multi-download.ts index 36759af..110f667 100644 --- a/src/download/download-engine/engine/download-engine-multi-download.ts +++ b/src/download/download-engine/engine/download-engine-multi-download.ts @@ -5,6 +5,7 @@ import BaseDownloadEngine, {BaseDownloadEngineEvents} from "./base-download-engi import {concurrency} from "../utils/concurrency.js"; import {DownloadFlags, DownloadStatus} from "../download-file/progress-status-file.js"; import {DownloadEngineRemote} from "./DownloadEngineRemote.js"; +import {promiseWithResolvers} from "../utils/promiseWithResolvers.js"; export type DownloadEngineMultiAllowedEngines = BaseDownloadEngine | DownloadEngineRemote | DownloadEngineMultiDownload; @@ -57,7 +58,7 @@ export default class DownloadEngineMultiDownload(); + _downloadEndPromise = promiseWithResolvers(); /** * @internal */ @@ -230,7 +231,7 @@ export default class DownloadEngineMultiDownload(array: Value[], concurrencyCount: number, callback: (value: Value) => Promise) { - const {resolve, reject, promise} = Promise.withResolvers(); + const {resolve, reject, promise} = promiseWithResolvers(); let index = 0; let activeCount = 0; diff --git a/src/download/download-engine/utils/promiseWithResolvers.ts b/src/download/download-engine/utils/promiseWithResolvers.ts new file mode 100644 index 0000000..a1fb4a5 --- /dev/null +++ b/src/download/download-engine/utils/promiseWithResolvers.ts @@ -0,0 +1,9 @@ +export function promiseWithResolvers() { + let resolve: (value: T) => void; + let reject: (reason?: any) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return {promise, resolve: resolve!, reject: reject!}; +} diff --git a/tsconfig.json b/tsconfig.json index 82085a0..aa8793d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,8 +4,8 @@ "esNext", "DOM" ], - "module": "nodeNext", - "target": "esNext", + "module": "NodeNext", + "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, "noImplicitReturns": true, From f12b2f680b6c9790cc5d04d020ca9c8f4613a7b9 Mon Sep 17 00:00:00 2001 From: ido Date: Thu, 22 May 2025 21:46:38 +0300 Subject: [PATCH 36/37] feat(redirect): do not flow redirect by default, prevent token expires --- .../engine/base-download-engine.ts | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/download/download-engine/engine/base-download-engine.ts b/src/download/download-engine/engine/base-download-engine.ts index 179f162..4b9dc90 100644 --- a/src/download/download-engine/engine/base-download-engine.ts +++ b/src/download/download-engine/engine/base-download-engine.ts @@ -14,7 +14,11 @@ import {promiseWithResolvers} from "../utils/promiseWithResolvers.js"; const IGNORE_HEAD_STATUS_CODES = [405, 501, 404]; export type InputURLOptions = { partURLs: string[] } | { url: string }; -export type BaseDownloadEngineOptions = InputURLOptions & BaseDownloadEngineFetchStreamOptions & { +export type CreateDownloadFileOptions = { + reuseRedirectURL?: boolean +}; + +export type BaseDownloadEngineOptions = CreateDownloadFileOptions & InputURLOptions & BaseDownloadEngineFetchStreamOptions & { chunkSize?: number; parallelStreams?: number; retry?: retry.Options @@ -139,7 +143,7 @@ export default class BaseDownloadEngine extends EventEmitter { try { const {length, acceptRange, newURL, fileName} = await fetchStream.fetchDownloadInfo(part); - const downloadURL = newURL ?? part; + const downloadURL = reuseRedirectURL ? (newURL ?? part) : part; const size = length || 0; downloadFile.totalSize += size; - downloadFile.parts.push({ + if (index === 0 && fileName) { + downloadFile.localFileName = fileName; + } + + return { downloadURL, size, acceptRange: size > 0 && acceptRange - }); - - if (counter++ === 0 && fileName) { - downloadFile.localFileName = fileName; - } + }; } catch (error: any) { if (error instanceof StatusCodeError && IGNORE_HEAD_STATUS_CODES.includes(error.statusCode)) { // if the server does not support HEAD request, we will skip that step - downloadFile.parts.push({ + return { downloadURL: part, size: 0, acceptRange: false - }); - continue; + }; } throw error; } - } + })); return downloadFile; } From c31e008ea7de081b7d92a61ff4e6a5b67925cc3c Mon Sep 17 00:00:00 2001 From: ido Date: Thu, 22 May 2025 22:55:39 +0300 Subject: [PATCH 37/37] chores: fix ts config --- tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index aa8793d..d8117e1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "lib": [ - "esNext", + "es2022", "DOM" ], "module": "NodeNext",