| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341 |
- "use strict";
- Object.defineProperty(exports, "__esModule", {
- value: true
- });
- exports.readBlockMap = exports.DifferentialDownloader = void 0;
- function _bluebirdLst() {
- const data = _interopRequireWildcard(require("bluebird-lst"));
- _bluebirdLst = function () {
- return data;
- };
- return data;
- }
- function _builderUtilRuntime() {
- const data = require("builder-util-runtime");
- _builderUtilRuntime = function () {
- return data;
- };
- return data;
- }
- function _fsExtraP() {
- const data = require("fs-extra-p");
- _fsExtraP = function () {
- return data;
- };
- return data;
- }
- function _DataSplitter() {
- const data = require("./DataSplitter");
- _DataSplitter = function () {
- return data;
- };
- return data;
- }
- function _downloadPlanBuilder() {
- const data = require("./downloadPlanBuilder");
- _downloadPlanBuilder = function () {
- return data;
- };
- return data;
- }
- function _multipleRangeDownloader() {
- const data = require("./multipleRangeDownloader");
- _multipleRangeDownloader = function () {
- return data;
- };
- return data;
- }
- function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { var desc = Object.defineProperty && Object.getOwnPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : {}; if (desc.get || desc.set) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } } newObj.default = obj; return newObj; } }
- const inflateRaw = _bluebirdLst().default.promisify(require("zlib").inflateRaw);
- class DifferentialDownloader {
- // noinspection TypeScriptAbstractClassConstructorCanBeMadeProtected
- constructor(blockAwareFileInfo, httpExecutor, options) {
- this.blockAwareFileInfo = blockAwareFileInfo;
- this.httpExecutor = httpExecutor;
- this.options = options;
- this.fileMetadataBuffer = null;
- this.logger = options.logger;
- this.baseRequestOptions = (0, _builderUtilRuntime().configureRequestOptionsFromUrl)(options.newUrl, {});
- }
- createRequestOptions(method = "get", newUrl) {
- return Object.assign({}, newUrl == null ? this.baseRequestOptions : (0, _builderUtilRuntime().configureRequestOptionsFromUrl)(newUrl, {}), {
- method,
- headers: Object.assign({}, this.options.requestHeaders, {
- accept: "*/*"
- })
- });
- }
- doDownload(oldBlockMap, newBlockMap) {
- // we don't check other metadata like compressionMethod - generic check that it is make sense to differentially update is suitable for it
- if (oldBlockMap.version !== newBlockMap.version) {
- throw new Error(`version is different (${oldBlockMap.version} - ${newBlockMap.version}), full download is required`);
- }
- const logger = this.logger;
- const operations = (0, _downloadPlanBuilder().computeOperations)(oldBlockMap, newBlockMap, logger);
- if (logger.debug != null) {
- logger.debug(JSON.stringify(operations, null, 2));
- }
- let downloadSize = 0;
- let copySize = 0;
- for (const operation of operations) {
- const length = operation.end - operation.start;
- if (operation.kind === _downloadPlanBuilder().OperationKind.DOWNLOAD) {
- downloadSize += length;
- } else {
- copySize += length;
- }
- }
- const newPackageSize = this.blockAwareFileInfo.size;
- if (downloadSize + copySize + (this.fileMetadataBuffer == null ? 0 : this.fileMetadataBuffer.length) !== newPackageSize) {
- throw new Error(`Internal error, size mismatch: downloadSize: ${downloadSize}, copySize: ${copySize}, newPackageSize: ${newPackageSize}`);
- }
- logger.info(`Full: ${formatBytes(newPackageSize)}, To download: ${formatBytes(downloadSize)} (${Math.round(downloadSize / (newPackageSize / 100))}%)`);
- return this.downloadFile(operations);
- }
- downloadFile(tasks) {
- const fdList = [];
- const closeFiles = () => {
- return _bluebirdLst().default.map(fdList, openedFile => {
- return (0, _fsExtraP().close)(openedFile.descriptor).catch(e => {
- this.logger.error(`cannot close file "${openedFile.path}": ${e}`);
- });
- });
- };
- return this.doDownloadFile(tasks, fdList).then(closeFiles).catch(e => {
- // then must be after catch here (since then always throws error)
- return closeFiles().catch(closeFilesError => {
- // closeFiles never throw error, but just to be sure
- try {
- this.logger.error(`cannot close files: ${closeFilesError}`);
- } catch (errorOnLog) {
- try {
- console.error(errorOnLog);
- } catch (ignored) {// ok, give up and ignore error
- }
- }
- throw e;
- }).then(() => {
- throw e;
- });
- });
- }
- doDownloadFile(tasks, fdList) {
- var _this = this;
- return (0, _bluebirdLst().coroutine)(function* () {
- const oldFileFd = yield (0, _fsExtraP().open)(_this.options.oldFile, "r");
- fdList.push({
- descriptor: oldFileFd,
- path: _this.options.oldFile
- });
- const newFileFd = yield (0, _fsExtraP().open)(_this.options.newFile, "w");
- fdList.push({
- descriptor: newFileFd,
- path: _this.options.newFile
- });
- const fileOut = (0, _fsExtraP().createWriteStream)(_this.options.newFile, {
- fd: newFileFd
- });
- yield new Promise((resolve, reject) => {
- const streams = [];
- const digestTransform = new (_builderUtilRuntime().DigestTransform)(_this.blockAwareFileInfo.sha512); // to simply debug, do manual validation to allow file to be fully written
- digestTransform.isValidateOnEnd = false;
- streams.push(digestTransform); // noinspection JSArrowFunctionCanBeReplacedWithShorthand
- fileOut.on("finish", () => {
- fileOut.close(() => {
- try {
- digestTransform.validate();
- } catch (e) {
- reject(e);
- return;
- }
- resolve();
- });
- });
- streams.push(fileOut);
- let lastStream = null;
- for (const stream of streams) {
- stream.on("error", reject);
- if (lastStream == null) {
- lastStream = stream;
- } else {
- lastStream = lastStream.pipe(stream);
- }
- }
- const firstStream = streams[0];
- let w;
- if (_this.options.useMultipleRangeRequest) {
- w = (0, _multipleRangeDownloader().executeTasks)(_this, tasks, firstStream, oldFileFd, reject);
- } else {
- let attemptCount = 0;
- let actualUrl = null;
- _this.logger.info(`Differential download: ${_this.options.newUrl}`);
- w = index => {
- if (index >= tasks.length) {
- if (_this.fileMetadataBuffer != null) {
- firstStream.write(_this.fileMetadataBuffer);
- }
- firstStream.end();
- return;
- }
- const operation = tasks[index++];
- if (operation.kind === _downloadPlanBuilder().OperationKind.COPY) {
- (0, _DataSplitter().copyData)(operation, firstStream, oldFileFd, reject, () => w(index));
- } else {
- const requestOptions = _this.createRequestOptions("get", actualUrl);
- const range = `bytes=${operation.start}-${operation.end - 1}`;
- requestOptions.headers.Range = range;
- requestOptions.redirect = "manual";
- const debug = _this.logger.debug;
- if (debug != null) {
- debug(`effective url: ${actualUrl == null ? "original" : removeQuery(actualUrl)}, range: ${range}`);
- }
- const request = _this.httpExecutor.doRequest(requestOptions, response => {
- // Electron net handles redirects automatically, our NodeJS test server doesn't use redirects - so, we don't check 3xx codes.
- if (response.statusCode >= 400) {
- reject((0, _builderUtilRuntime().createHttpError)(response));
- }
- response.pipe(firstStream, {
- end: false
- });
- response.once("end", () => {
- if (++attemptCount === 100) {
- attemptCount = 0;
- setTimeout(() => w(index), 1000);
- } else {
- w(index);
- }
- });
- });
- request.on("redirect", (statusCode, method, redirectUrl) => {
- _this.logger.info(`Redirect to ${removeQuery(redirectUrl)}`);
- actualUrl = redirectUrl;
- request.followRedirect();
- });
- _this.httpExecutor.addErrorAndTimeoutHandlers(request, reject);
- request.end();
- }
- };
- }
- w(0);
- });
- })();
- }
- readRemoteBytes(start, endInclusive) {
- var _this2 = this;
- return (0, _bluebirdLst().coroutine)(function* () {
- const buffer = Buffer.allocUnsafe(endInclusive + 1 - start);
- const requestOptions = _this2.createRequestOptions();
- requestOptions.headers.Range = `bytes=${start}-${endInclusive}`;
- let position = 0;
- yield _this2.request(requestOptions, chunk => {
- chunk.copy(buffer, position);
- position += chunk.length;
- });
- return buffer;
- })();
- }
- request(requestOptions, dataHandler) {
- return new Promise((resolve, reject) => {
- const request = this.httpExecutor.doRequest(requestOptions, response => {
- if (!(0, _multipleRangeDownloader().checkIsRangesSupported)(response, reject)) {
- return;
- }
- response.on("data", dataHandler);
- response.on("end", () => resolve());
- });
- this.httpExecutor.addErrorAndTimeoutHandlers(request, reject);
- request.end();
- });
- }
- }
- exports.DifferentialDownloader = DifferentialDownloader;
- let readBlockMap = (() => {
- var _ref = (0, _bluebirdLst().coroutine)(function* (data) {
- return JSON.parse((yield inflateRaw(data)).toString());
- });
- return function readBlockMap(_x) {
- return _ref.apply(this, arguments);
- };
- })();
- exports.readBlockMap = readBlockMap;
- function formatBytes(value, symbol = " KB") {
- return new Intl.NumberFormat("en").format((value / 1024).toFixed(2)) + symbol;
- } // safety
- function removeQuery(url) {
- const index = url.indexOf("?");
- return index < 0 ? url : url.substring(0, index);
- }
- //# sourceMappingURL=DifferentialDownloader.js.map
|