DataSplitter.js 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298
  1. "use strict";
  2. Object.defineProperty(exports, "__esModule", {
  3. value: true
  4. });
  5. exports.copyData = copyData;
  6. exports.DataSplitter = void 0;
  7. function _bluebirdLst() {
  8. const data = require("bluebird-lst");
  9. _bluebirdLst = function () {
  10. return data;
  11. };
  12. return data;
  13. }
  14. function _builderUtilRuntime() {
  15. const data = require("builder-util-runtime");
  16. _builderUtilRuntime = function () {
  17. return data;
  18. };
  19. return data;
  20. }
  21. function _fsExtraP() {
  22. const data = require("fs-extra-p");
  23. _fsExtraP = function () {
  24. return data;
  25. };
  26. return data;
  27. }
  28. function _stream() {
  29. const data = require("stream");
  30. _stream = function () {
  31. return data;
  32. };
  33. return data;
  34. }
  35. function _downloadPlanBuilder() {
  36. const data = require("./downloadPlanBuilder");
  37. _downloadPlanBuilder = function () {
  38. return data;
  39. };
  40. return data;
  41. }
  42. const DOUBLE_CRLF = Buffer.from("\r\n\r\n");
  43. var ReadState;
  44. (function (ReadState) {
  45. ReadState[ReadState["INIT"] = 0] = "INIT";
  46. ReadState[ReadState["HEADER"] = 1] = "HEADER";
  47. ReadState[ReadState["BODY"] = 2] = "BODY";
  48. })(ReadState || (ReadState = {}));
  49. function copyData(task, out, oldFileFd, reject, resolve) {
  50. const readStream = (0, _fsExtraP().createReadStream)("", {
  51. fd: oldFileFd,
  52. autoClose: false,
  53. start: task.start,
  54. // end is inclusive
  55. end: task.end - 1
  56. });
  57. readStream.on("error", reject);
  58. readStream.once("end", resolve);
  59. readStream.pipe(out, {
  60. end: false
  61. });
  62. }
  63. class DataSplitter extends _stream().Writable {
  64. constructor(out, options, partIndexToTaskIndex, boundary, partIndexToLength, finishHandler) {
  65. super();
  66. this.out = out;
  67. this.options = options;
  68. this.partIndexToTaskIndex = partIndexToTaskIndex;
  69. this.partIndexToLength = partIndexToLength;
  70. this.finishHandler = finishHandler;
  71. this.partIndex = -1;
  72. this.headerListBuffer = null;
  73. this.readState = ReadState.INIT;
  74. this.ignoreByteCount = 0;
  75. this.remainingPartDataCount = 0;
  76. this.actualPartLength = 0;
  77. this.boundaryLength = boundary.length + 4;
  78. /* size of \r\n-- */
  79. // first chunk doesn't start with \r\n
  80. this.ignoreByteCount = this.boundaryLength - 2;
  81. }
  82. get isFinished() {
  83. return this.partIndex === this.partIndexToLength.length;
  84. } // noinspection JSUnusedGlobalSymbols
  85. _write(data, encoding, callback) {
  86. if (this.isFinished) {
  87. console.error(`Trailing ignored data: ${data.length} bytes`);
  88. return;
  89. }
  90. this.handleData(data).then(callback).catch(callback);
  91. }
  92. handleData(chunk) {
  93. var _this = this;
  94. return (0, _bluebirdLst().coroutine)(function* () {
  95. let start = 0;
  96. if (_this.ignoreByteCount !== 0 && _this.remainingPartDataCount !== 0) {
  97. throw (0, _builderUtilRuntime().newError)("Internal error", "ERR_DATA_SPLITTER_BYTE_COUNT_MISMATCH");
  98. }
  99. if (_this.ignoreByteCount > 0) {
  100. const toIgnore = Math.min(_this.ignoreByteCount, chunk.length);
  101. _this.ignoreByteCount -= toIgnore;
  102. start = toIgnore;
  103. } else if (_this.remainingPartDataCount > 0) {
  104. const toRead = Math.min(_this.remainingPartDataCount, chunk.length);
  105. _this.remainingPartDataCount -= toRead;
  106. yield _this.processPartData(chunk, 0, toRead);
  107. start = toRead;
  108. }
  109. if (start === chunk.length) {
  110. return;
  111. }
  112. if (_this.readState === ReadState.HEADER) {
  113. const headerListEnd = _this.searchHeaderListEnd(chunk, start);
  114. if (headerListEnd === -1) {
  115. return;
  116. }
  117. start = headerListEnd;
  118. _this.readState = ReadState.BODY; // header list is ignored, we don't need it
  119. _this.headerListBuffer = null;
  120. }
  121. while (true) {
  122. if (_this.readState === ReadState.BODY) {
  123. _this.readState = ReadState.INIT;
  124. } else {
  125. _this.partIndex++;
  126. let taskIndex = _this.partIndexToTaskIndex.get(_this.partIndex);
  127. if (taskIndex == null) {
  128. if (_this.isFinished) {
  129. taskIndex = _this.options.end;
  130. } else {
  131. throw (0, _builderUtilRuntime().newError)("taskIndex is null", "ERR_DATA_SPLITTER_TASK_INDEX_IS_NULL");
  132. }
  133. }
  134. const prevTaskIndex = _this.partIndex === 0 ? _this.options.start : _this.partIndexToTaskIndex.get(_this.partIndex - 1) + 1
  135. /* prev part is download, next maybe copy */
  136. ;
  137. if (prevTaskIndex < taskIndex) {
  138. yield _this.copyExistingData(prevTaskIndex, taskIndex);
  139. } else if (prevTaskIndex > taskIndex) {
  140. throw (0, _builderUtilRuntime().newError)("prevTaskIndex must be < taskIndex", "ERR_DATA_SPLITTER_TASK_INDEX_ASSERT_FAILED");
  141. }
  142. if (_this.isFinished) {
  143. _this.onPartEnd();
  144. _this.finishHandler();
  145. return;
  146. }
  147. start = _this.searchHeaderListEnd(chunk, start);
  148. if (start === -1) {
  149. _this.readState = ReadState.HEADER;
  150. return;
  151. }
  152. }
  153. const partLength = _this.partIndexToLength[_this.partIndex];
  154. const end = start + partLength;
  155. const effectiveEnd = Math.min(end, chunk.length);
  156. yield _this.processPartStarted(chunk, start, effectiveEnd);
  157. _this.remainingPartDataCount = partLength - (effectiveEnd - start);
  158. if (_this.remainingPartDataCount > 0) {
  159. return;
  160. }
  161. start = end + _this.boundaryLength;
  162. if (start >= chunk.length) {
  163. _this.ignoreByteCount = _this.boundaryLength - (chunk.length - end);
  164. return;
  165. }
  166. }
  167. })();
  168. }
  169. copyExistingData(index, end) {
  170. return new Promise((resolve, reject) => {
  171. const w = () => {
  172. if (index === end) {
  173. resolve();
  174. return;
  175. }
  176. const task = this.options.tasks[index];
  177. if (task.kind !== _downloadPlanBuilder().OperationKind.COPY) {
  178. reject(new Error("Task kind must be COPY"));
  179. return;
  180. }
  181. copyData(task, this.out, this.options.oldFileFd, reject, () => {
  182. index++;
  183. w();
  184. });
  185. };
  186. w();
  187. });
  188. }
  189. searchHeaderListEnd(chunk, readOffset) {
  190. const headerListEnd = chunk.indexOf(DOUBLE_CRLF, readOffset);
  191. if (headerListEnd !== -1) {
  192. return headerListEnd + DOUBLE_CRLF.length;
  193. } // not all headers data were received, save to buffer
  194. const partialChunk = readOffset === 0 ? chunk : chunk.slice(readOffset);
  195. if (this.headerListBuffer == null) {
  196. this.headerListBuffer = partialChunk;
  197. } else {
  198. this.headerListBuffer = Buffer.concat([this.headerListBuffer, partialChunk]);
  199. }
  200. return -1;
  201. }
  202. onPartEnd() {
  203. const expectedLength = this.partIndexToLength[this.partIndex - 1];
  204. if (this.actualPartLength !== expectedLength) {
  205. throw (0, _builderUtilRuntime().newError)(`Expected length: ${expectedLength} differs from actual: ${this.actualPartLength}`, "ERR_DATA_SPLITTER_LENGTH_MISMATCH");
  206. }
  207. this.actualPartLength = 0;
  208. }
  209. processPartStarted(data, start, end) {
  210. if (this.partIndex !== 0) {
  211. this.onPartEnd();
  212. }
  213. return this.processPartData(data, start, end);
  214. }
  215. processPartData(data, start, end) {
  216. this.actualPartLength += end - start;
  217. const out = this.out;
  218. if (out.write(start === 0 && data.length === end ? data : data.slice(start, end))) {
  219. return Promise.resolve();
  220. } else {
  221. return new Promise((resolve, reject) => {
  222. out.on("error", reject);
  223. out.once("drain", () => {
  224. out.removeListener("error", reject);
  225. resolve();
  226. });
  227. });
  228. }
  229. }
  230. } exports.DataSplitter = DataSplitter;
  231. //# sourceMappingURL=DataSplitter.js.map