DifferentialDownloader.js 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306
  1. "use strict";
  2. Object.defineProperty(exports, "__esModule", {
  3. value: true
  4. });
  5. exports.readBlockMap = exports.DifferentialDownloader = void 0;
  6. function _bluebirdLst() {
  7. const data = _interopRequireWildcard(require("bluebird-lst"));
  8. _bluebirdLst = function () {
  9. return data;
  10. };
  11. return data;
  12. }
  13. function _builderUtilRuntime() {
  14. const data = require("builder-util-runtime");
  15. _builderUtilRuntime = function () {
  16. return data;
  17. };
  18. return data;
  19. }
  20. function _fsExtraP() {
  21. const data = require("fs-extra-p");
  22. _fsExtraP = function () {
  23. return data;
  24. };
  25. return data;
  26. }
  27. function _DataSplitter() {
  28. const data = require("./DataSplitter");
  29. _DataSplitter = function () {
  30. return data;
  31. };
  32. return data;
  33. }
  34. function _downloadPlanBuilder() {
  35. const data = require("./downloadPlanBuilder");
  36. _downloadPlanBuilder = function () {
  37. return data;
  38. };
  39. return data;
  40. }
  41. function _multipleRangeDownloader() {
  42. const data = require("./multipleRangeDownloader");
  43. _multipleRangeDownloader = function () {
  44. return data;
  45. };
  46. return data;
  47. }
  48. function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { var desc = Object.defineProperty && Object.getOwnPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : {}; if (desc.get || desc.set) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } } newObj.default = obj; return newObj; } }
  49. const inflateRaw = _bluebirdLst().default.promisify(require("zlib").inflateRaw);
  50. class DifferentialDownloader {
  51. // noinspection TypeScriptAbstractClassConstructorCanBeMadeProtected
  52. constructor(blockAwareFileInfo, httpExecutor, options) {
  53. this.blockAwareFileInfo = blockAwareFileInfo;
  54. this.httpExecutor = httpExecutor;
  55. this.options = options;
  56. this.fileMetadataBuffer = null;
  57. this.logger = options.logger;
  58. this.baseRequestOptions = (0, _builderUtilRuntime().configureRequestOptionsFromUrl)(options.newUrl, {});
  59. }
  60. createRequestOptions(method = "get", newUrl) {
  61. return Object.assign({}, newUrl == null ? this.baseRequestOptions : (0, _builderUtilRuntime().configureRequestOptionsFromUrl)(newUrl, {}), {
  62. method,
  63. headers: Object.assign({}, this.options.requestHeaders, {
  64. Accept: "*/*"
  65. })
  66. });
  67. }
  68. doDownload(oldBlockMap, newBlockMap) {
  69. // we don't check other metadata like compressionMethod - generic check that it is make sense to differentially update is suitable for it
  70. if (oldBlockMap.version !== newBlockMap.version) {
  71. throw new Error(`version is different (${oldBlockMap.version} - ${newBlockMap.version}), full download is required`);
  72. }
  73. const logger = this.logger;
  74. const operations = (0, _downloadPlanBuilder().computeOperations)(oldBlockMap, newBlockMap, logger);
  75. if (logger.debug != null) {
  76. logger.debug(JSON.stringify(operations, null, 2));
  77. }
  78. let downloadSize = 0;
  79. let copySize = 0;
  80. for (const operation of operations) {
  81. const length = operation.end - operation.start;
  82. if (operation.kind === _downloadPlanBuilder().OperationKind.DOWNLOAD) {
  83. downloadSize += length;
  84. } else {
  85. copySize += length;
  86. }
  87. }
  88. const newPackageSize = this.blockAwareFileInfo.size;
  89. if (downloadSize + copySize + (this.fileMetadataBuffer == null ? 0 : this.fileMetadataBuffer.length) !== newPackageSize) {
  90. throw new Error(`Internal error, size mismatch: downloadSize: ${downloadSize}, copySize: ${copySize}, newPackageSize: ${newPackageSize}`);
  91. }
  92. logger.info(`Full: ${formatBytes(newPackageSize)}, To download: ${formatBytes(downloadSize)} (${Math.round(downloadSize / (newPackageSize / 100))}%)`);
  93. return this.downloadFile(operations);
  94. }
  95. downloadFile(tasks) {
  96. var _this = this;
  97. return (0, _bluebirdLst().coroutine)(function* () {
  98. const oldFileFd = yield (0, _fsExtraP().open)(_this.options.oldFile, "r");
  99. const newFileFd = yield (0, _fsExtraP().open)(_this.options.newFile, "w");
  100. const fileOut = (0, _fsExtraP().createWriteStream)(_this.options.newFile, {
  101. fd: newFileFd
  102. });
  103. yield new (_bluebirdLst().default)((resolve, reject) => {
  104. const streams = [];
  105. const digestTransform = new (_builderUtilRuntime().DigestTransform)(_this.blockAwareFileInfo.sha512); // to simply debug, do manual validation to allow file to be fully written
  106. digestTransform.isValidateOnEnd = false;
  107. streams.push(digestTransform); // noinspection JSArrowFunctionCanBeReplacedWithShorthand
  108. fileOut.on("finish", () => {
  109. fileOut.close(() => {
  110. try {
  111. digestTransform.validate();
  112. } catch (e) {
  113. reject(e);
  114. return;
  115. }
  116. resolve();
  117. });
  118. });
  119. streams.push(fileOut);
  120. let lastStream = null;
  121. for (const stream of streams) {
  122. stream.on("error", reject);
  123. if (lastStream == null) {
  124. lastStream = stream;
  125. } else {
  126. lastStream = lastStream.pipe(stream);
  127. }
  128. }
  129. const firstStream = streams[0];
  130. let w;
  131. if (_this.options.useMultipleRangeRequest) {
  132. w = (0, _multipleRangeDownloader().executeTasks)(_this, tasks, firstStream, oldFileFd, reject);
  133. } else {
  134. let attemptCount = 0;
  135. let actualUrl = null;
  136. _this.logger.info(`Differential download: ${_this.options.newUrl}`);
  137. w = index => {
  138. if (index >= tasks.length) {
  139. if (_this.fileMetadataBuffer != null) {
  140. firstStream.write(_this.fileMetadataBuffer);
  141. }
  142. firstStream.end();
  143. return;
  144. }
  145. const operation = tasks[index++];
  146. if (operation.kind === _downloadPlanBuilder().OperationKind.COPY) {
  147. (0, _DataSplitter().copyData)(operation, firstStream, oldFileFd, reject, () => w(index));
  148. } else {
  149. const requestOptions = _this.createRequestOptions("get", actualUrl);
  150. const range = `bytes=${operation.start}-${operation.end - 1}`;
  151. requestOptions.headers.Range = range;
  152. requestOptions.redirect = "manual";
  153. const debug = _this.logger.debug;
  154. if (debug != null) {
  155. debug(`effective url: ${actualUrl == null ? "" : removeQuery(actualUrl)}, range: ${range}`);
  156. }
  157. const request = _this.httpExecutor.doRequest(requestOptions, response => {
  158. // Electron net handles redirects automatically, our NodeJS test server doesn't use redirects - so, we don't check 3xx codes.
  159. if (response.statusCode >= 400) {
  160. reject((0, _builderUtilRuntime().createHttpError)(response));
  161. }
  162. response.pipe(firstStream, {
  163. end: false
  164. });
  165. response.once("end", () => {
  166. if (++attemptCount === 100) {
  167. attemptCount = 0;
  168. setTimeout(() => w(index), 1000);
  169. } else {
  170. w(index);
  171. }
  172. });
  173. });
  174. request.on("redirect", (statusCode, method, redirectUrl) => {
  175. _this.logger.info(`Redirect to ${removeQuery(redirectUrl)}`);
  176. actualUrl = redirectUrl;
  177. request.followRedirect();
  178. });
  179. _this.httpExecutor.addErrorAndTimeoutHandlers(request, reject);
  180. request.end();
  181. }
  182. };
  183. }
  184. w(0);
  185. }).then(() => (0, _fsExtraP().close)(oldFileFd)).catch(error => {
  186. (0, _fsExtraP().closeSync)(oldFileFd);
  187. (0, _fsExtraP().closeSync)(newFileFd);
  188. throw error;
  189. });
  190. })();
  191. }
  192. readRemoteBytes(start, endInclusive) {
  193. var _this2 = this;
  194. return (0, _bluebirdLst().coroutine)(function* () {
  195. const buffer = Buffer.allocUnsafe(endInclusive + 1 - start);
  196. const requestOptions = _this2.createRequestOptions();
  197. requestOptions.headers.Range = `bytes=${start}-${endInclusive}`;
  198. let position = 0;
  199. yield _this2.request(requestOptions, chunk => {
  200. chunk.copy(buffer, position);
  201. position += chunk.length;
  202. });
  203. return buffer;
  204. })();
  205. }
  206. request(requestOptions, dataHandler) {
  207. return new (_bluebirdLst().default)((resolve, reject) => {
  208. const request = this.httpExecutor.doRequest(requestOptions, response => {
  209. if (!(0, _multipleRangeDownloader().checkIsRangesSupported)(response, reject)) {
  210. return;
  211. }
  212. response.on("data", dataHandler);
  213. response.on("end", () => resolve());
  214. });
  215. this.httpExecutor.addErrorAndTimeoutHandlers(request, reject);
  216. request.end();
  217. });
  218. }
  219. }
  220. exports.DifferentialDownloader = DifferentialDownloader;
  221. let readBlockMap = (() => {
  222. var _ref = (0, _bluebirdLst().coroutine)(function* (data) {
  223. return JSON.parse((yield inflateRaw(data)).toString());
  224. });
  225. return function readBlockMap(_x) {
  226. return _ref.apply(this, arguments);
  227. };
  228. })();
  229. exports.readBlockMap = readBlockMap;
  230. function formatBytes(value, symbol = " KB") {
  231. return new Intl.NumberFormat("en").format((value / 1024).toFixed(2)) + symbol;
  232. } // safety
  233. function removeQuery(url) {
  234. const index = url.indexOf("?");
  235. return index < 0 ? url : url.substring(0, index);
  236. }
  237. //# sourceMappingURL=DifferentialDownloader.js.map