first commit

app_vue/node_modules/thread-loader/dist/WorkerPool.js

@@ -0,0 +1,468 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+var _child_process = _interopRequireDefault(require("child_process"));
+
+var _queue = _interopRequireDefault(require("neo-async/queue"));
+
+var _mapSeries = _interopRequireDefault(require("neo-async/mapSeries"));
+
+var _readBuffer = _interopRequireDefault(require("./readBuffer"));
+
+var _WorkerError = _interopRequireDefault(require("./WorkerError"));
+
+var _serializer = require("./serializer");
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+/* eslint-disable no-console */
+const workerPath = require.resolve('./worker');
+
+let workerId = 0;
+
+class PoolWorker {
+  constructor(options, onJobDone) {
+    this.disposed = false;
+    this.nextJobId = 0;
+    this.jobs = Object.create(null);
+    this.activeJobs = 0;
+    this.onJobDone = onJobDone;
+    this.id = workerId;
+    workerId += 1; // Empty or invalid node args would break the child process
+
+    const sanitizedNodeArgs = (options.nodeArgs || []).filter(opt => !!opt);
+    this.worker = _child_process.default.spawn(process.execPath, [].concat(sanitizedNodeArgs).concat(workerPath, options.parallelJobs), {
+      detached: true,
+      stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe']
+    });
+    this.worker.unref(); // This prevents a problem where the worker stdio can be undefined
+    // when the kernel hits the limit of open files.
+    // More info can be found on: https://github.com/webpack-contrib/thread-loader/issues/2
+
+    if (!this.worker.stdio) {
+      throw new Error(`Failed to create the worker pool with workerId: ${workerId} and ${''}configuration: ${JSON.stringify(options)}. Please verify if you hit the OS open files limit.`);
+    }
+
+    const [,,, readPipe, writePipe] = this.worker.stdio;
+    this.readPipe = readPipe;
+    this.writePipe = writePipe;
+    this.listenStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr);
+    this.readNextMessage();
+  }
+
+  listenStdOutAndErrFromWorker(workerStdout, workerStderr) {
+    if (workerStdout) {
+      workerStdout.on('data', this.writeToStdout);
+    }
+
+    if (workerStderr) {
+      workerStderr.on('data', this.writeToStderr);
+    }
+  }
+
+  ignoreStdOutAndErrFromWorker(workerStdout, workerStderr) {
+    if (workerStdout) {
+      workerStdout.removeListener('data', this.writeToStdout);
+    }
+
+    if (workerStderr) {
+      workerStderr.removeListener('data', this.writeToStderr);
+    }
+  }
+
+  writeToStdout(data) {
+    if (!this.disposed) {
+      process.stdout.write(data);
+    }
+  }
+
+  writeToStderr(data) {
+    if (!this.disposed) {
+      process.stderr.write(data);
+    }
+  }
+
+  run(data, callback) {
+    const jobId = this.nextJobId;
+    this.nextJobId += 1;
+    this.jobs[jobId] = {
+      data,
+      callback
+    };
+    this.activeJobs += 1;
+    this.writeJson({
+      type: 'job',
+      id: jobId,
+      data
+    });
+  }
+
+  warmup(requires) {
+    this.writeJson({
+      type: 'warmup',
+      requires
+    });
+  }
+
+  writeJson(data) {
+    const lengthBuffer = Buffer.alloc(4);
+    const messageBuffer = Buffer.from(JSON.stringify(data, _serializer.replacer), 'utf-8');
+    lengthBuffer.writeInt32BE(messageBuffer.length, 0);
+    this.writePipe.write(lengthBuffer);
+    this.writePipe.write(messageBuffer);
+  }
+
+  writeEnd() {
+    const lengthBuffer = Buffer.alloc(4);
+    lengthBuffer.writeInt32BE(0, 0);
+    this.writePipe.write(lengthBuffer);
+  }
+
+  readNextMessage() {
+    this.state = 'read length';
+    this.readBuffer(4, (lengthReadError, lengthBuffer) => {
+      if (lengthReadError) {
+        console.error(`Failed to communicate with worker (read length) ${lengthReadError}`);
+        return;
+      }
+
+      this.state = 'length read';
+      const length = lengthBuffer.readInt32BE(0);
+      this.state = 'read message';
+      this.readBuffer(length, (messageError, messageBuffer) => {
+        if (messageError) {
+          console.error(`Failed to communicate with worker (read message) ${messageError}`);
+          return;
+        }
+
+        this.state = 'message read';
+        const messageString = messageBuffer.toString('utf-8');
+        const message = JSON.parse(messageString, _serializer.reviver);
+        this.state = 'process message';
+        this.onWorkerMessage(message, err => {
+          if (err) {
+            console.error(`Failed to communicate with worker (process message) ${err}`);
+            return;
+          }
+
+          this.state = 'soon next';
+          setImmediate(() => this.readNextMessage());
+        });
+      });
+    });
+  }
+
+  onWorkerMessage(message, finalCallback) {
+    const {
+      type,
+      id
+    } = message;
+
+    switch (type) {
+      case 'job':
+        {
+          const {
+            data,
+            error,
+            result
+          } = message;
+          (0, _mapSeries.default)(data, (length, callback) => this.readBuffer(length, callback), (eachErr, buffers) => {
+            const {
+              callback: jobCallback
+            } = this.jobs[id];
+
+            const callback = (err, arg) => {
+              if (jobCallback) {
+                delete this.jobs[id];
+                this.activeJobs -= 1;
+                this.onJobDone();
+
+                if (err) {
+                  jobCallback(err instanceof Error ? err : new Error(err), arg);
+                } else {
+                  jobCallback(null, arg);
+                }
+              }
+
+              finalCallback();
+            };
+
+            if (eachErr) {
+              callback(eachErr);
+              return;
+            }
+
+            let bufferPosition = 0;
+
+            if (result.result) {
+              result.result = result.result.map(r => {
+                if (r.buffer) {
+                  const buffer = buffers[bufferPosition];
+                  bufferPosition += 1;
+
+                  if (r.string) {
+                    return buffer.toString('utf-8');
+                  }
+
+                  return buffer;
+                }
+
+                return r.data;
+              });
+            }
+
+            if (error) {
+              callback(this.fromErrorObj(error), result);
+              return;
+            }
+
+            callback(null, result);
+          });
+          break;
+        }
+
+      case 'loadModule':
+        {
+          const {
+            request,
+            questionId
+          } = message;
+          const {
+            data
+          } = this.jobs[id]; // eslint-disable-next-line no-unused-vars
+
+          data.loadModule(request, (error, source, sourceMap, module) => {
+            this.writeJson({
+              type: 'result',
+              id: questionId,
+              error: error ? {
+                message: error.message,
+                details: error.details,
+                missing: error.missing
+              } : null,
+              result: [source, sourceMap // TODO: Serialize module?
+              // module,
+              ]
+            });
+          });
+          finalCallback();
+          break;
+        }
+
+      case 'resolve':
+        {
+          const {
+            context,
+            request,
+            options,
+            questionId
+          } = message;
+          const {
+            data
+          } = this.jobs[id];
+
+          if (options) {
+            data.getResolve(options)(context, request, (error, result) => {
+              this.writeJson({
+                type: 'result',
+                id: questionId,
+                error: error ? {
+                  message: error.message,
+                  details: error.details,
+                  missing: error.missing
+                } : null,
+                result
+              });
+            });
+          } else {
+            data.resolve(context, request, (error, result) => {
+              this.writeJson({
+                type: 'result',
+                id: questionId,
+                error: error ? {
+                  message: error.message,
+                  details: error.details,
+                  missing: error.missing
+                } : null,
+                result
+              });
+            });
+          }
+
+          finalCallback();
+          break;
+        }
+
+      case 'emitWarning':
+        {
+          const {
+            data
+          } = message;
+          const {
+            data: jobData
+          } = this.jobs[id];
+          jobData.emitWarning(this.fromErrorObj(data));
+          finalCallback();
+          break;
+        }
+
+      case 'emitError':
+        {
+          const {
+            data
+          } = message;
+          const {
+            data: jobData
+          } = this.jobs[id];
+          jobData.emitError(this.fromErrorObj(data));
+          finalCallback();
+          break;
+        }
+
+      default:
+        {
+          console.error(`Unexpected worker message ${type} in WorkerPool.`);
+          finalCallback();
+          break;
+        }
+    }
+  }
+
+  fromErrorObj(arg) {
+    let obj;
+
+    if (typeof arg === 'string') {
+      obj = {
+        message: arg
+      };
+    } else {
+      obj = arg;
+    }
+
+    return new _WorkerError.default(obj, this.id);
+  }
+
+  readBuffer(length, callback) {
+    (0, _readBuffer.default)(this.readPipe, length, callback);
+  }
+
+  dispose() {
+    if (!this.disposed) {
+      this.disposed = true;
+      this.ignoreStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr);
+      this.writeEnd();
+    }
+  }
+
+}
+
+class WorkerPool {
+  constructor(options) {
+    this.options = options || {};
+    this.numberOfWorkers = options.numberOfWorkers;
+    this.poolTimeout = options.poolTimeout;
+    this.workerNodeArgs = options.workerNodeArgs;
+    this.workerParallelJobs = options.workerParallelJobs;
+    this.workers = new Set();
+    this.activeJobs = 0;
+    this.timeout = null;
+    this.poolQueue = (0, _queue.default)(this.distributeJob.bind(this), options.poolParallelJobs);
+    this.terminated = false;
+    this.setupLifeCycle();
+  }
+
+  isAbleToRun() {
+    return !this.terminated;
+  }
+
+  terminate() {
+    if (this.terminated) {
+      return;
+    }
+
+    this.terminated = true;
+    this.poolQueue.kill();
+    this.disposeWorkers(true);
+  }
+
+  setupLifeCycle() {
+    process.on('exit', () => {
+      this.terminate();
+    });
+  }
+
+  run(data, callback) {
+    if (this.timeout) {
+      clearTimeout(this.timeout);
+      this.timeout = null;
+    }
+
+    this.activeJobs += 1;
+    this.poolQueue.push(data, callback);
+  }
+
+  distributeJob(data, callback) {
+    // use worker with the fewest jobs
+    let bestWorker;
+
+    for (const worker of this.workers) {
+      if (!bestWorker || worker.activeJobs < bestWorker.activeJobs) {
+        bestWorker = worker;
+      }
+    }
+
+    if (bestWorker && (bestWorker.activeJobs === 0 || this.workers.size >= this.numberOfWorkers)) {
+      bestWorker.run(data, callback);
+      return;
+    }
+
+    const newWorker = this.createWorker();
+    newWorker.run(data, callback);
+  }
+
+  createWorker() {
+    // spin up a new worker
+    const newWorker = new PoolWorker({
+      nodeArgs: this.workerNodeArgs,
+      parallelJobs: this.workerParallelJobs
+    }, () => this.onJobDone());
+    this.workers.add(newWorker);
+    return newWorker;
+  }
+
+  warmup(requires) {
+    while (this.workers.size < this.numberOfWorkers) {
+      this.createWorker().warmup(requires);
+    }
+  }
+
+  onJobDone() {
+    this.activeJobs -= 1;
+
+    if (this.activeJobs === 0 && isFinite(this.poolTimeout)) {
+      this.timeout = setTimeout(() => this.disposeWorkers(), this.poolTimeout);
+    }
+  }
+
+  disposeWorkers(fromTerminate) {
+    if (!this.options.poolRespawn && !fromTerminate) {
+      this.terminate();
+      return;
+    }
+
+    if (this.activeJobs === 0 || fromTerminate) {
+      for (const worker of this.workers) {
+        worker.dispose();
+      }
+
+      this.workers.clear();
+    }
+  }
+
+}
+
+exports.default = WorkerPool;