Feat/network

front_end/ndb.json

@@ -1,15 +1,21 @@
 {
-  "modules" : [
-     { "name": "ndb_sdk", "type": "autostart" },
-     { "name": "ndb", "type": "autostart" },
-     { "name": "layer_viewer" },
-     { "name": "timeline_model" },
-     { "name": "timeline" },
-     { "name": "product_registry" },
-     { "name": "mobile_throttling" },
-     { "name": "ndb_ui" },
-     { "name": "xterm" }
- ],
+  "modules": [
+    { "name": "ndb_sdk", "type": "autostart" },
+    { "name": "ndb", "type": "autostart" },
+    { "name": "layer_viewer" },
+    { "name": "timeline_model" },
+    { "name": "timeline" },
+    { "name": "product_registry" },
+    { "name": "mobile_throttling" },
+    { "name": "ndb_ui" },
+    { "name": "xterm" },
+    { "name": "emulation", "type": "autostart" },
+    { "name": "inspector_main", "type": "autostart" },
+    { "name": "mobile_throttling", "type": "autostart" },
+    { "name": "cookie_table" },
+    { "name": "har_importer" },
+    { "name": "network" }
+  ],
   "extends": "shell",
   "has_html": true
 }

front_end/ndb/NdbMain.js

@@ -172,9 +172,39 @@ Ndb.NodeProcessManager = class extends Common.Object {
   static async create(targetManager) {
     const manager = new Ndb.NodeProcessManager(targetManager);
     manager._service = await Ndb.backend.createService('ndd_service.js', rpc.handle(manager));
+    InspectorFrontendHost.sendMessageToBackend = manager.sendMessageToBackend.bind(manager);
     return manager;
   }
 
+  async sendMessageToBackend(message) {
+    if (this._service && this._service.sendMessage)
+      return this._service.sendMessage(message);
+  }
+
+  sendLoadingFinished({ type, payload }) {
+    SDK._mainConnection._onMessage(JSON.stringify({
+      method: 'Network.loadingFinished',
+      params: payload
+    }));
+  }
+
+  responseToFrontEnd(id, result) {
+    InspectorFrontendHost.events.dispatchEventToListeners(
+        InspectorFrontendHostAPI.Events.DispatchMessage,
+        {
+          id,
+          result
+        }
+    );
+  }
+
+  sendNetworkData({ type, payload }) {
+    SDK._mainConnection._onMessage(JSON.stringify({
+      method: type,
+      params: payload
+    }));
+  }
+
   env() {
     return this._service.env();
   }
@@ -208,6 +238,26 @@ Ndb.NodeProcessManager = class extends Common.Object {
         info.id, userFriendlyName(info), SDK.Target.Type.Node,
         this._targetManager.targetById(info.ppid) || this._targetManager.mainTarget(), undefined, false, connection);
     target[NdbSdk.connectionSymbol] = connection;
+
+    try {
+      // this doesnt work
+      // target.runtimeAgent().invoke_evaluate({
+      //   expression: `
+      //     const zlib = require('http');
+      //     console.log('foo');
+      //   `
+      // });
+      //
+      // but this does
+      // target.runtimeAgent().invoke_evaluate({
+      //   expression: `
+      //     console.log('foo');
+      //   `
+      // });
+    } catch(err) {
+      console.log(err);
+    }
+
     await this.addFileSystem(info.cwd, info.scriptName);
     if (info.scriptName) {
       const scriptURL = Common.ParsedURL.platformPathToURL(info.scriptName);

front_end/ndb/module.json

@@ -52,7 +52,14 @@
             "className": "Ndb.ContextMenuProvider"
         }
     ],
-    "dependencies": ["common", "sdk", "ndb_sdk", "bindings", "persistence", "components"],
+    "dependencies": [
+        "common",
+        "sdk",
+        "ndb_sdk",
+        "bindings",
+        "persistence",
+        "components"
+    ],
     "scripts": [
         "InspectorFrontendHostOverrides.js",
         "Connection.js",

lib/backend.js

@@ -48,6 +48,12 @@ class Backend {
     opn(url);
   }
 
+  async httpMonkeyPatchingSource() {
+    const pathToHttpPatch = path.resolve(__dirname, '..', './lib/preload/ndb/httpMonkeyPatching.js');
+    const content = await fsReadFile(pathToHttpPatch, 'utf8');
+    return content;
+  }
+
   pkg() {
     // TODO(ak239spb): implement it as decorations over package.json file.
     try {

lib/preload/ndb/httpMonkeyPatching.js

@@ -0,0 +1,153 @@
+const zlib = require('zlib');
+const http = require('http');
+const https = require('https');
+
+const initTime = process.hrtime();
+
+// DT requires us to use relative time in a strange format (xxx.xxx)
+const getTime = () => {
+  const diff = process.hrtime(initTime);
+
+  return diff[0] + diff[1] / 1e9;
+};
+
+const formatRequestHeaders = req => {
+  if (!req.headers) return {};
+  return Object.keys(req.headers).reduce((acc, k) => {
+    if (typeof req.headers[k] === 'string') acc[k] = req.headers[k];
+    return acc;
+  }, {});
+};
+
+const formatResponseHeaders = res => {
+  if (!res.headers) return {};
+  return Object.keys(res.headers).reduce((acc, k) => {
+    if (typeof res.headers[k] === 'string') acc[k] = res.headers[k];
+    return acc;
+  }, {});
+};
+
+const getMineType = mimeType => {
+  // nasty hack for ASF
+  if (mimeType === 'OPENJSON')
+    return 'application/json;charset=UTF-8';
+
+
+  return mimeType;
+};
+
+const cacheRequests = {};
+let id = 1;
+const getId = () => id++;
+
+const callbackWrapper = (callback, req) => res => {
+  const requestId = getId();
+  res.req.__requestId = requestId;
+
+  process.send({
+    payload: {
+      requestId: requestId,
+      loaderId: requestId,
+      documentURL: req.href,
+      request: {
+        url: req.href,
+        method: req.method,
+        headers: formatRequestHeaders(req),
+        mixedContentType: 'none',
+        initialPriority: 'VeryHigh',
+        referrerPolicy: 'no-referrer-when-downgrade',
+        postData: req.body
+      },
+      timestamp: getTime(),
+      wallTime: Date.now(),
+      initiator: {
+        type: 'other'
+      },
+      type: 'Document'
+    },
+    type: 'Network.requestWillBeSent'
+  });
+
+  const encoding = res.headers['content-encoding'];
+  let rawData = [];
+
+  const onEnd = function() {
+    rawData = Buffer.concat(rawData);
+    rawData = rawData.toString('base64');
+
+    cacheRequests[res.req.__requestId] = {
+      ...res,
+      __rawData: rawData,
+      base64Encoded: true
+    };
+    const payload = {
+      id: res.req.__requestId,
+      requestId: res.req.__requestId,
+      loaderId: res.req.__requestId,
+      base64Encoded: true,
+      data: cacheRequests[res.req.__requestId].__rawData,
+      timestamp: getTime(),
+      type: 'XHR',
+      encodedDataLength: 100,
+      response: {
+        url: req.href,
+        status: res.statusCode,
+        statusText: res.statusText,
+        // set-cookie prop in the header has value as an array
+        // for example: ["__cfduid=dbfe006ef71658bf4dba321343c227f9a15449556…20:29 GMT; path=/; domain=.typicode.com; HttpOnly"]
+        headers: formatResponseHeaders(res),
+        mimeType: getMineType(
+            res.headers['content-encoding'] ||
+                        res.headers['content-type']
+        ),
+        requestHeaders: formatRequestHeaders(req)
+      }
+    };
+
+    // Send the response back.
+    process.send({ payload: payload, type: 'Network.responseReceived' });
+    process.send({ payload: payload, type: 'Network.loadingFinished' });
+  };
+
+  if (encoding === 'gzip' || encoding === 'x-gzip') {
+    const gunzip = zlib.createGunzip();
+    res.pipe(gunzip);
+
+    gunzip.on('data', function(data) {
+      rawData.push(data);
+    });
+    gunzip.on('end', onEnd);
+  } else {
+    res.on('data', chunk => {
+      rawData.push(chunk);
+    });
+    res.on('end', onEnd);
+  }
+
+  callback && callback(res);
+};
+
+const originHTTPRequest = http.request;
+http.request = function wrapMethodRequest(req, callback) {
+  const request = originHTTPRequest.call(
+      this,
+      req,
+      callbackWrapper(callback, req)
+  );
+  return request;
+};
+
+const originHTTPSRequest = https.request;
+https.request = function wrapMethodRequest(req, callback) {
+  const request = originHTTPSRequest.call(
+      this,
+      req,
+      callbackWrapper(callback, req)
+  );
+  const originWrite = request.write.bind(request);
+  request.write = data => {
+    req.body = data.toString();
+    originWrite(data);
+  };
+  return request;
+};

services/ndd_service.js

@@ -23,6 +23,8 @@ function silentRpcErrors(error) {
 process.on('uncaughtException', silentRpcErrors);
 process.on('unhandledRejection', silentRpcErrors);
 
+const catchedRequests = {};
+
 const DebugState = {
   WS_OPEN: 1,
   WS_ERROR: 2,
@@ -141,17 +143,31 @@ class NddService {
     };
   }
 
+  sendMessage(rawMessage) {
+    const message = JSON.parse(rawMessage);
+    // send message to frontend directly
+    // (eg: getResponseBody)
+    const { base64Encoded, data } = catchedRequests[message.params.requestId];
+    this._frontend.responseToFrontEnd(message.id, { base64Encoded, body: data });
+  }
+
   async debug(execPath, args, options) {
     const env = this.env();
     if (options.data)
       env.NDD_DATA = options.data;
+
     const p = spawn(execPath, args, {
       cwd: options.cwd,
       env: { ...process.env, ...env },
-      stdio: options.ignoreOutput ? 'ignore' : ['inherit', 'pipe', 'pipe'],
+      stdio: options.ignoreOutput ? ['ignore', 'ignore', 'ignore', 'ipc'] : ['pipe', 'pipe', 'pipe', 'ipc'],
       windowsHide: true
     });
     if (!options.ignoreOutput) {
+      p.on('message', ({ type, payload }) => {
+        if (!(type && payload)) return;
+        catchedRequests[payload.id] = payload;
+        this._frontend.sendNetworkData({ type, payload });
+      });
       p.stderr.on('data', data => {
         if (process.connected)
           this._frontend.terminalData('stderr', data.toString('base64'));