Feat/network

front_end/ndb.json

@@ -1,15 +1,21 @@
 {
-  "modules" : [
-     { "name": "ndb_sdk", "type": "autostart" },
-     { "name": "ndb", "type": "autostart" },
-     { "name": "layer_viewer" },
-     { "name": "timeline_model" },
-     { "name": "timeline" },
-     { "name": "product_registry" },
-     { "name": "mobile_throttling" },
-     { "name": "ndb_ui" },
-     { "name": "xterm" }
- ],
+  "modules": [
+    { "name": "ndb_sdk", "type": "autostart" },
+    { "name": "ndb", "type": "autostart" },
+    { "name": "layer_viewer" },
+    { "name": "timeline_model" },
+    { "name": "timeline" },
+    { "name": "product_registry" },
+    { "name": "mobile_throttling" },
+    { "name": "ndb_ui" },
+    { "name": "xterm" },
+    { "name": "emulation", "type": "autostart" },
+    { "name": "inspector_main", "type": "autostart" },
+    { "name": "mobile_throttling", "type": "autostart" },
+    { "name": "cookie_table" },
+    { "name": "har_importer" },
+    { "name": "network" }
+  ],
   "extends": "shell",
   "has_html": true
 }

front_end/ndb/InspectorFrontendHostOverrides.js

@@ -4,7 +4,7 @@
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  */
 
-(function(){
+(async function(){
   InspectorFrontendHost.getPreferences = async function(callback) {
     [Ndb.backend] = await carlo.loadParams();
     const prefs = {
@@ -30,4 +30,53 @@
       callback({statusCode: 404});
     }
   };
+
+  InspectorFrontendHost.sendMessageToBackend = async rawMessage => {
+    const parsedMes = JSON.parse(rawMessage);
+    if (parsedMes.method !== 'Network.getResponseBody')
+      return;
+
+    const mes = await target.runtimeAgent().invoke_evaluate({
+      expression: `process._sendMessage(${JSON.stringify(JSON.parse(rawMessage))})`,
+      awaitPromise: true
+    });
+
+    if (!mes.result) return;
+    try {
+      const [id, result] = mes.result.value;
+      if (result) {
+        InspectorFrontendHost.events.dispatchEventToListeners(
+            InspectorFrontendHostAPI.Events.DispatchMessage,
+            {
+              id,
+              result
+            }
+        );
+      }
+    } catch (err) {
+      console.log(err);
+    }
+  };
+
+  while (true) {
+    const message = await target.runtimeAgent().invoke_evaluate({
+      expression: 'process._getNetworkMessages()',
+      awaitPromise: true
+    });
+
+    if (!message.result) return;
+    const arrMessages = JSON.parse(message.result.value);
+
+    for (const mes of arrMessages) {
+      const { type, payload } = mes;
+
+      if (type) {
+        SDK._mainConnection._onMessage(JSON.stringify({
+          method: type,
+          params: payload
+        }));
+      }
+    }
+  }
+
 })();

front_end/ndb/NdbMain.js

@@ -208,6 +208,12 @@ Ndb.NodeProcessManager = class extends Common.Object {
         info.id, userFriendlyName(info), SDK.Target.Type.Node,
         this._targetManager.targetById(info.ppid) || this._targetManager.mainTarget(), undefined, false, connection);
     target[NdbSdk.connectionSymbol] = connection;
+
+    target.runtimeAgent().invoke_evaluate({
+      expression: await Ndb.backend.httpMonkeyPatchingSource(),
+      includeCommandLineAPI: true
+    });
+
     await this.addFileSystem(info.cwd, info.scriptName);
     if (info.scriptName) {
       const scriptURL = Common.ParsedURL.platformPathToURL(info.scriptName);

lib/backend.js

@@ -48,6 +48,12 @@ class Backend {
     opn(url);
   }
 
+  async httpMonkeyPatchingSource() {
+    const pathToHttpPatch = path.resolve(__dirname, '..', './lib/preload/ndb/httpMonkeyPatching.js');
+    const content = await fsReadFile(pathToHttpPatch, 'utf8');
+    return content;
+  }
+
   pkg() {
     // TODO(ak239spb): implement it as decorations over package.json file.
     try {

lib/preload/ndb/httpMonkeyPatching.js

@@ -0,0 +1,188 @@
+const zlib = require('zlib');
+const http = require('http');
+const https = require('https');
+
+const initTime = process.hrtime();
+
+// DT requires us to use relative time in a strange format (xxx.xxx)
+const getTime = () => {
+  const diff = process.hrtime(initTime);
+
+  return diff[0] + diff[1] / 1e9;
+};
+
+const formatRequestHeaders = req => {
+  if (!req.headers) return {};
+  return Object.keys(req.headers).reduce((acc, k) => {
+    if (typeof req.headers[k] === 'string') acc[k] = req.headers[k];
+    return acc;
+  }, {});
+};
+
+const formatResponseHeaders = res => {
+  if (!res.headers) return {};
+  return Object.keys(res.headers).reduce((acc, k) => {
+    if (typeof res.headers[k] === 'string') acc[k] = res.headers[k];
+    return acc;
+  }, {});
+};
+
+const getMineType = mimeType => {
+  // nasty hack for ASF
+  if (mimeType === 'OPENJSON')
+    return 'application/json;charset=UTF-8';
+
+
+  return mimeType;
+};
+
+let cacheRequests = {};
+let id = 1;
+const getId = () => id++;
+
+let messages = [];
+let messageAdded = null;
+
+function reportMessage(message) {
+  messages.push(message);
+  if (messageAdded) {
+    setTimeout(messageAdded, 0);
+    messageAdded = null;
+  }
+}
+
+process._getNetworkMessages = async function() {
+  if (!messages.length)
+    await new Promise(resolve => messageAdded = resolve);
+  return JSON.stringify(messages.splice(0));
+};
+
+process._sendMessage = async function(rawMessage) {
+  return new Promise(resolve => {
+    const message = rawMessage;
+    if (!cacheRequests[message.params.requestId]) {
+      resolve(JSON.stringify({}));
+    } else {
+      if (message.method === 'Network.getResponseBody') {
+        const { base64Encoded, data } = cacheRequests[message.params.requestId];
+
+        console.log({ cacheRequests });
+        console.log({ data });
+        resolve(JSON.stringify([message.id, { base64Encoded, body: data }]));
+      }
+    }
+  });
+};
+
+const callbackWrapper = (callback, req) => res => {
+  const requestId = getId();
+  res.req.__requestId = requestId;
+
+  reportMessage({
+    payload: {
+      requestId: requestId,
+      loaderId: requestId,
+      documentURL: req.href,
+      request: {
+        url: req.href,
+        method: req.method,
+        headers: formatRequestHeaders(req),
+        mixedContentType: 'none',
+        initialPriority: 'VeryHigh',
+        referrerPolicy: 'no-referrer-when-downgrade',
+        postData: req.body
+      },
+      timestamp: getTime(),
+      wallTime: Date.now(),
+      initiator: {
+        type: 'other'
+      },
+      type: 'Document'
+    },
+    type: 'Network.requestWillBeSent'
+  });
+
+  const encoding = res.headers['content-encoding'];
+  let rawData = [];
+
+  const onEnd = function() {
+    rawData = Buffer.concat(rawData);
+    rawData = rawData.toString('base64');
+
+    cacheRequests[res.req.__requestId] = {
+      ...res,
+      __rawData: rawData,
+      base64Encoded: true
+    };
+
+    const payload = {
+      id: res.req.__requestId,
+      requestId: res.req.__requestId,
+      loaderId: res.req.__requestId,
+      base64Encoded: true,
+      data: cacheRequests[res.req.__requestId].__rawData,
+      timestamp: getTime(),
+      type: 'XHR',
+      encodedDataLength: 100,
+      response: {
+        url: req.href,
+        status: res.statusCode,
+        statusText: res.statusText,
+        // set-cookie prop in the header has value as an array
+        // for example: ["__cfduid=dbfe006ef71658bf4dba321343c227f9a15449556…20:29 GMT; path=/; domain=.typicode.com; HttpOnly"]
+        headers: formatResponseHeaders(res),
+        mimeType: getMineType(
+            res.headers['content-encoding'] ||
+                        res.headers['content-type']
+        ),
+        requestHeaders: formatRequestHeaders(req)
+      }
+    };
+
+    // Send the response back.
+    reportMessage({ payload: payload, type: 'Network.responseReceived' });
+    reportMessage({ payload: payload, type: 'Network.loadingFinished' });
+  };
+
+  if (encoding === 'gzip' || encoding === 'x-gzip') {
+    const gunzip = zlib.createGunzip();
+    res.pipe(gunzip);
+
+    gunzip.on('data', function(data) {
+      rawData.push(data);
+    });
+    gunzip.on('end', onEnd);
+  } else {
+    res.on('data', chunk => {
+      rawData.push(chunk);
+    });
+    res.on('end', onEnd);
+  }
+
+  callback && callback(res);
+};
+
+const originHTTPRequest = http.request;
+http.request = function wrapMethodRequest(req, callback) {
+  const request = originHTTPRequest.call(
+      this,
+      req,
+      callbackWrapper(callback, req)
+  );
+  return request;
+};
+
+const originHTTPSRequest = https.request;
+https.request = function wrapMethodRequest(req, callback) {
+  const request = originHTTPSRequest.call(
+      this,
+      req,
+      callbackWrapper(callback, req)
+  );
+  const originWrite = request.write.bind(request);
+  request.write = data => {
+    req.body = data.toString();
+    originWrite(data);
+  };
+  return request;
+};