DifferentialDownloader.js 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341
  1. "use strict";
  2. Object.defineProperty(exports, "__esModule", {
  3. value: true
  4. });
  5. exports.readBlockMap = exports.DifferentialDownloader = void 0;
  6. function _bluebirdLst() {
  7. const data = _interopRequireWildcard(require("bluebird-lst"));
  8. _bluebirdLst = function () {
  9. return data;
  10. };
  11. return data;
  12. }
  13. function _builderUtilRuntime() {
  14. const data = require("builder-util-runtime");
  15. _builderUtilRuntime = function () {
  16. return data;
  17. };
  18. return data;
  19. }
  20. function _fsExtraP() {
  21. const data = require("fs-extra-p");
  22. _fsExtraP = function () {
  23. return data;
  24. };
  25. return data;
  26. }
  27. function _DataSplitter() {
  28. const data = require("./DataSplitter");
  29. _DataSplitter = function () {
  30. return data;
  31. };
  32. return data;
  33. }
  34. function _downloadPlanBuilder() {
  35. const data = require("./downloadPlanBuilder");
  36. _downloadPlanBuilder = function () {
  37. return data;
  38. };
  39. return data;
  40. }
  41. function _multipleRangeDownloader() {
  42. const data = require("./multipleRangeDownloader");
  43. _multipleRangeDownloader = function () {
  44. return data;
  45. };
  46. return data;
  47. }
  48. function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { var desc = Object.defineProperty && Object.getOwnPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : {}; if (desc.get || desc.set) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } } newObj.default = obj; return newObj; } }
  49. const inflateRaw = _bluebirdLst().default.promisify(require("zlib").inflateRaw);
  50. class DifferentialDownloader {
  51. // noinspection TypeScriptAbstractClassConstructorCanBeMadeProtected
  52. constructor(blockAwareFileInfo, httpExecutor, options) {
  53. this.blockAwareFileInfo = blockAwareFileInfo;
  54. this.httpExecutor = httpExecutor;
  55. this.options = options;
  56. this.fileMetadataBuffer = null;
  57. this.logger = options.logger;
  58. this.baseRequestOptions = (0, _builderUtilRuntime().configureRequestOptionsFromUrl)(options.newUrl, {});
  59. }
  60. createRequestOptions(method = "get", newUrl) {
  61. return Object.assign({}, newUrl == null ? this.baseRequestOptions : (0, _builderUtilRuntime().configureRequestOptionsFromUrl)(newUrl, {}), {
  62. method,
  63. headers: Object.assign({}, this.options.requestHeaders, {
  64. accept: "*/*"
  65. })
  66. });
  67. }
  68. doDownload(oldBlockMap, newBlockMap) {
  69. // we don't check other metadata like compressionMethod - generic check that it is make sense to differentially update is suitable for it
  70. if (oldBlockMap.version !== newBlockMap.version) {
  71. throw new Error(`version is different (${oldBlockMap.version} - ${newBlockMap.version}), full download is required`);
  72. }
  73. const logger = this.logger;
  74. const operations = (0, _downloadPlanBuilder().computeOperations)(oldBlockMap, newBlockMap, logger);
  75. if (logger.debug != null) {
  76. logger.debug(JSON.stringify(operations, null, 2));
  77. }
  78. let downloadSize = 0;
  79. let copySize = 0;
  80. for (const operation of operations) {
  81. const length = operation.end - operation.start;
  82. if (operation.kind === _downloadPlanBuilder().OperationKind.DOWNLOAD) {
  83. downloadSize += length;
  84. } else {
  85. copySize += length;
  86. }
  87. }
  88. const newPackageSize = this.blockAwareFileInfo.size;
  89. if (downloadSize + copySize + (this.fileMetadataBuffer == null ? 0 : this.fileMetadataBuffer.length) !== newPackageSize) {
  90. throw new Error(`Internal error, size mismatch: downloadSize: ${downloadSize}, copySize: ${copySize}, newPackageSize: ${newPackageSize}`);
  91. }
  92. logger.info(`Full: ${formatBytes(newPackageSize)}, To download: ${formatBytes(downloadSize)} (${Math.round(downloadSize / (newPackageSize / 100))}%)`);
  93. return this.downloadFile(operations);
  94. }
  95. downloadFile(tasks) {
  96. const fdList = [];
  97. const closeFiles = () => {
  98. return _bluebirdLst().default.map(fdList, openedFile => {
  99. return (0, _fsExtraP().close)(openedFile.descriptor).catch(e => {
  100. this.logger.error(`cannot close file "${openedFile.path}": ${e}`);
  101. });
  102. });
  103. };
  104. return this.doDownloadFile(tasks, fdList).then(closeFiles).catch(e => {
  105. // then must be after catch here (since then always throws error)
  106. return closeFiles().catch(closeFilesError => {
  107. // closeFiles never throw error, but just to be sure
  108. try {
  109. this.logger.error(`cannot close files: ${closeFilesError}`);
  110. } catch (errorOnLog) {
  111. try {
  112. console.error(errorOnLog);
  113. } catch (ignored) {// ok, give up and ignore error
  114. }
  115. }
  116. throw e;
  117. }).then(() => {
  118. throw e;
  119. });
  120. });
  121. }
  122. doDownloadFile(tasks, fdList) {
  123. var _this = this;
  124. return (0, _bluebirdLst().coroutine)(function* () {
  125. const oldFileFd = yield (0, _fsExtraP().open)(_this.options.oldFile, "r");
  126. fdList.push({
  127. descriptor: oldFileFd,
  128. path: _this.options.oldFile
  129. });
  130. const newFileFd = yield (0, _fsExtraP().open)(_this.options.newFile, "w");
  131. fdList.push({
  132. descriptor: newFileFd,
  133. path: _this.options.newFile
  134. });
  135. const fileOut = (0, _fsExtraP().createWriteStream)(_this.options.newFile, {
  136. fd: newFileFd
  137. });
  138. yield new Promise((resolve, reject) => {
  139. const streams = [];
  140. const digestTransform = new (_builderUtilRuntime().DigestTransform)(_this.blockAwareFileInfo.sha512); // to simply debug, do manual validation to allow file to be fully written
  141. digestTransform.isValidateOnEnd = false;
  142. streams.push(digestTransform); // noinspection JSArrowFunctionCanBeReplacedWithShorthand
  143. fileOut.on("finish", () => {
  144. fileOut.close(() => {
  145. try {
  146. digestTransform.validate();
  147. } catch (e) {
  148. reject(e);
  149. return;
  150. }
  151. resolve();
  152. });
  153. });
  154. streams.push(fileOut);
  155. let lastStream = null;
  156. for (const stream of streams) {
  157. stream.on("error", reject);
  158. if (lastStream == null) {
  159. lastStream = stream;
  160. } else {
  161. lastStream = lastStream.pipe(stream);
  162. }
  163. }
  164. const firstStream = streams[0];
  165. let w;
  166. if (_this.options.useMultipleRangeRequest) {
  167. w = (0, _multipleRangeDownloader().executeTasks)(_this, tasks, firstStream, oldFileFd, reject);
  168. } else {
  169. let attemptCount = 0;
  170. let actualUrl = null;
  171. _this.logger.info(`Differential download: ${_this.options.newUrl}`);
  172. w = index => {
  173. if (index >= tasks.length) {
  174. if (_this.fileMetadataBuffer != null) {
  175. firstStream.write(_this.fileMetadataBuffer);
  176. }
  177. firstStream.end();
  178. return;
  179. }
  180. const operation = tasks[index++];
  181. if (operation.kind === _downloadPlanBuilder().OperationKind.COPY) {
  182. (0, _DataSplitter().copyData)(operation, firstStream, oldFileFd, reject, () => w(index));
  183. } else {
  184. const requestOptions = _this.createRequestOptions("get", actualUrl);
  185. const range = `bytes=${operation.start}-${operation.end - 1}`;
  186. requestOptions.headers.Range = range;
  187. requestOptions.redirect = "manual";
  188. const debug = _this.logger.debug;
  189. if (debug != null) {
  190. debug(`effective url: ${actualUrl == null ? "original" : removeQuery(actualUrl)}, range: ${range}`);
  191. }
  192. const request = _this.httpExecutor.doRequest(requestOptions, response => {
  193. // Electron net handles redirects automatically, our NodeJS test server doesn't use redirects - so, we don't check 3xx codes.
  194. if (response.statusCode >= 400) {
  195. reject((0, _builderUtilRuntime().createHttpError)(response));
  196. }
  197. response.pipe(firstStream, {
  198. end: false
  199. });
  200. response.once("end", () => {
  201. if (++attemptCount === 100) {
  202. attemptCount = 0;
  203. setTimeout(() => w(index), 1000);
  204. } else {
  205. w(index);
  206. }
  207. });
  208. });
  209. request.on("redirect", (statusCode, method, redirectUrl) => {
  210. _this.logger.info(`Redirect to ${removeQuery(redirectUrl)}`);
  211. actualUrl = redirectUrl;
  212. request.followRedirect();
  213. });
  214. _this.httpExecutor.addErrorAndTimeoutHandlers(request, reject);
  215. request.end();
  216. }
  217. };
  218. }
  219. w(0);
  220. });
  221. })();
  222. }
  223. readRemoteBytes(start, endInclusive) {
  224. var _this2 = this;
  225. return (0, _bluebirdLst().coroutine)(function* () {
  226. const buffer = Buffer.allocUnsafe(endInclusive + 1 - start);
  227. const requestOptions = _this2.createRequestOptions();
  228. requestOptions.headers.Range = `bytes=${start}-${endInclusive}`;
  229. let position = 0;
  230. yield _this2.request(requestOptions, chunk => {
  231. chunk.copy(buffer, position);
  232. position += chunk.length;
  233. });
  234. return buffer;
  235. })();
  236. }
  237. request(requestOptions, dataHandler) {
  238. return new Promise((resolve, reject) => {
  239. const request = this.httpExecutor.doRequest(requestOptions, response => {
  240. if (!(0, _multipleRangeDownloader().checkIsRangesSupported)(response, reject)) {
  241. return;
  242. }
  243. response.on("data", dataHandler);
  244. response.on("end", () => resolve());
  245. });
  246. this.httpExecutor.addErrorAndTimeoutHandlers(request, reject);
  247. request.end();
  248. });
  249. }
  250. }
  251. exports.DifferentialDownloader = DifferentialDownloader;
  252. let readBlockMap = (() => {
  253. var _ref = (0, _bluebirdLst().coroutine)(function* (data) {
  254. return JSON.parse((yield inflateRaw(data)).toString());
  255. });
  256. return function readBlockMap(_x) {
  257. return _ref.apply(this, arguments);
  258. };
  259. })();
  260. exports.readBlockMap = readBlockMap;
  261. function formatBytes(value, symbol = " KB") {
  262. return new Intl.NumberFormat("en").format((value / 1024).toFixed(2)) + symbol;
  263. } // safety
  264. function removeQuery(url) {
  265. const index = url.indexOf("?");
  266. return index < 0 ? url : url.substring(0, index);
  267. }
  268. //# sourceMappingURL=DifferentialDownloader.js.map