feat: Start Botproject runtime in bot folder

BotProject/Templates/CSharp/Program.cs

@@ -2,6 +2,8 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Diagnostics;
+using System.IO;
 using Microsoft.AspNetCore;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Configuration;
@@ -24,8 +26,20 @@ public static IWebHost BuildWebHost(string[] args) =>
                 config
                     .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
                     .AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true, reloadOnChange: true)
+                    .AddJsonFile($"ComposerDialogs/settings/appsettings.json", optional: true, reloadOnChange: true)
                     .AddJsonFile($"luis.settings.{env.EnvironmentName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true)
                     .AddJsonFile($"luis.settings.{Environment.UserName}.{luisAuthoringRegion}.json", optional: true, reloadOnChange: true);
+                try
+                {
+                    foreach (string filePath in Directory.GetFiles($"ComposerDialogs", "generated/luis.settings.*.json"))
+                    {
+                        config.AddJsonFile(filePath, optional: true, reloadOnChange: true);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Trace.WriteLine(ex.Message);
+                }
 
                 if (env.IsDevelopment())
                 {

BotProject/Templates/CSharp/Scripts/build_runtime.ps1

@@ -0,0 +1,25 @@
+Param(
+	[object] $config,
+	[string] $customSettingFolder,
+    [string] $luisAuthroingKey,
+    [SecureString] $appPassword,
+    [string] $projFolder = $(Join-Path $(Get-Location) BotProject CSharp)
+)
+
+if ($PSVersionTable.PSVersion.Major -lt 6){
+	Write-Host "! Powershell 6 is required, current version is $($PSVersionTable.PSVersion.Major), please refer following documents for help."
+	Write-Host "For Windows - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-windows?view=powershell-6"
+	Write-Host "For Mac - https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-macos?view=powershell-6"
+	Break
+}
+
+if ((dotnet --version) -lt 3) {
+	Write-Host "! dotnet core 3.0 is required, please refer following documents for help."
+	Write-Host "https://dotnet.microsoft.com/download/dotnet-core/3.0"
+	Break
+}
+
+# Init user secret id
+dotnet user-secrets init
+
+dotnet build

Composer/packages/client/src/store/action/setting.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 import get from 'lodash/get';
-import { SensitiveProperties } from '@bfc/shared';
+import { SensitiveProperties as defaultSensitiveProperties } from '@bfc/shared';
 
 import { ActionCreator, DialogSetting } from '../types';
 import settingsStorage from '../../utils/dialogSettingStorage';
@@ -26,6 +26,9 @@ export const setSettings: ActionCreator = async (
       },
     });
     // set value in local storage
+    const SensitiveProperties = settings.SensitiveProperties
+      ? settings.SensitiveProperties
+      : defaultSensitiveProperties;
     for (const property of SensitiveProperties) {
       const propertyValue = get(settings, property);
       settingsStorage.setField(botName, property, propertyValue ? propertyValue : '');

Composer/packages/client/src/store/reducer/index.ts

@@ -4,7 +4,7 @@
 import get from 'lodash/get';
 import set from 'lodash/set';
 import { dialogIndexer } from '@bfc/indexers/lib/dialogIndexer';
-import { SensitiveProperties } from '@bfc/shared';
+import { SensitiveProperties as defaultSensitiveProperties } from '@bfc/shared';
 
 import { ActionTypes, FileTypes } from '../../constants';
 import { DialogSetting, ReducerFunc } from '../types';
@@ -18,6 +18,8 @@ const projectFiles = ['bot', 'botproj'];
 
 // if user set value in terminal or appsetting.json, it should update the value in localStorage
 const refreshLocalStorage = (botName: string, settings: DialogSetting) => {
+  const SensitiveProperties = settings.SensitiveProperties ? settings.SensitiveProperties : defaultSensitiveProperties;
+  console.log(SensitiveProperties);
   for (const property of SensitiveProperties) {
     const value = get(settings, property);
     if (value) {
@@ -29,6 +31,7 @@ const refreshLocalStorage = (botName: string, settings: DialogSetting) => {
 // merge sensitive values in localStorage
 const mergeLocalStorage = (botName: string, settings: DialogSetting) => {
   const localSetting = settingStorage.get(botName);
+  const SensitiveProperties = settings.SensitiveProperties ? settings.SensitiveProperties : defaultSensitiveProperties;
   if (localSetting) {
     for (const property of SensitiveProperties) {
       const value = get(localSetting, property);

Composer/packages/server/src/controllers/connector.ts

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
+import merge from 'lodash/merge';
 
 import { BotEnvironments } from '../models/connector';
 import { EnvironmentProvider } from '../models/environment';
@@ -33,7 +34,8 @@ async function getPublishHistory(req: any, res: any) {
 async function sync(req: any, res: any) {
   try {
     const environment = EnvironmentProvider.getCurrent();
-    await environment.getBotConnector().sync({ ...req.body, user: req.user });
+    const settingsInDisk = await environment.getSettingsManager().get('', false);
+    await environment.getBotConnector().sync(merge(settingsInDisk, req.body, { user: req.user }));
     res.send('OK');
   } catch (error) {
     res.status(400).json({

Composer/packages/server/src/models/connector/csharpBotConnector.ts

@@ -2,74 +2,142 @@
 // Licensed under the MIT License.
 
 import fs from 'fs';
-import Path from 'path';
+import { ChildProcess, spawn } from 'child_process';
 
-import axios from 'axios';
 import archiver from 'archiver';
-import FormData from 'form-data';
 
 import { BotProjectService } from '../../services/project';
 import { DialogSetting } from '../bot/interface';
+import { Path } from '../../utility/path';
 
 import { BotConfig, BotEnvironments, BotStatus, IBotConnector, IPublishHistory } from './interface';
-
+let runtime: ChildProcess | null = null;
 export class CSharpBotConnector implements IBotConnector {
-  private adminEndpoint: string;
+  public status: BotStatus = BotStatus.NotConnected;
   private endpoint: string;
-  constructor(adminEndpoint: string, endpoint: string) {
-    this.adminEndpoint = adminEndpoint;
+  constructor(endpoint: string) {
     this.endpoint = endpoint;
   }
 
-  public status: BotStatus = BotStatus.NotConnected;
+  private stop = () => {
+    shutdown();
+    this.status = BotStatus.NotConnected;
+  };
 
-  connect = async (_: BotEnvironments, __: string) => {
-    // confirm bot runtime is listening here
-    try {
-      await axios.get(this.adminEndpoint + '/api/admin');
-    } catch (err) {
-      throw new Error(err);
+  private buildProcess = async (dir: string): Promise<number | null> => {
+    return new Promise((resolve, reject) => {
+      const build = spawn(`pwsh ./Scripts/build_runtime.ps1`, {
+        cwd: dir,
+        detached: true,
+        shell: true,
+        stdio: ['ignore', 'ignore', 'inherit'],
+      });
+      console.log(`build pid : ${build.pid}`);
+
+      build.stderr &&
+        build.stderr.on('data', function(err) {
+          reject(err.toString());
+        });
+
+      build.on('exit', function(code) {
+        resolve(code);
+      });
+    });
+  };
+
+  private getConnectorConfig = (config: DialogSetting) => {
+    const configList: string[] = [];
+    if (config.MicrosoftAppPassword) {
+      configList.push('--MicrosoftAppPassword');
+      configList.push(config.MicrosoftAppPassword);
+    }
+    if (config.luis) {
+      if (config.luis.authoringKey) {
+        configList.push('--luis:endpointKey');
+        configList.push(config.luis.authoringKey);
+      }
+      if (config.luis.authoringRegion) {
+        configList.push('--luis:endpoint');
+        configList.push(`https://${config.luis.authoringRegion}.api.cognitive.microsoft.com`);
+      }
     }
 
-    this.status = BotStatus.NotConnected;
+    return configList;
+  };
+  private addListeners = (child: ChildProcess, handler: Function) => {
+    if (child.stdout !== null) {
+      child.stdout.on('data', (data: any) => {
+        console.log(`stdout: ${data}`);
+      });
+    }
+
+    if (child.stderr !== null) {
+      child.stderr.on('data', (data: any) => {
+        console.log(`stderr: ${data}`);
+      });
+    }
+
+    child.on('close', code => {
+      console.log(`close ${code}`);
+      handler();
+    });
+
+    child.on('error', (err: any) => {
+      console.log(`stderr: ${err}`);
+    });
+
+    child.on('exit', code => {
+      console.log(`exit: ${code}`);
+      handler();
+    });
 
-    return `${this.endpoint}/api/messages`;
+    child.on('message', msg => {
+      console.log(msg);
+    });
+
+    child.on('disconnect', code => {
+      console.log(`disconnect: ${code}`);
+      handler();
+    });
   };
 
-  sync = async (config: DialogSetting) => {
-    // archive the project
-    // send to bot runtime service
+  private getBotPath = () => {
     const currentProject = BotProjectService.getCurrentBotProject();
     if (currentProject === undefined) {
       throw new Error('no project is opened, nothing to sync');
     }
-    const dir = Path.join(currentProject.dataDir);
-    const luisConfig = currentProject.luPublisher.getLuisConfig();
-    await this.archiveDirectory(dir, './tmp.zip');
-    const content = fs.readFileSync('./tmp.zip');
-
-    const form = new FormData();
-    form.append('file', content, 'bot.zip');
+    return Path.join(currentProject.dir);
+  };
 
-    if (luisConfig && luisConfig.authoringKey !== null && !currentProject.checkLuisPublished()) {
-      throw new Error('Please publish your Luis models');
-    }
+  private start = async (dir: string, config: DialogSetting) => {
+    runtime = spawn(
+      'dotnet',
+      ['bin/Debug/netcoreapp2.1/BotProject.dll', `--urls`, this.endpoint, ...this.getConnectorConfig(config)],
+      {
+        detached: true,
+        cwd: dir,
+        stdio: ['ignore', 'ignore', 'inherit'],
+      }
+    );
+    console.log(`start runtime at ${runtime.pid}`);
+    this.addListeners(runtime, this.stop);
+    this.status = BotStatus.Connected;
+  };
 
-    if (luisConfig) {
-      form.append('endpointKey', luisConfig.endpointKey || luisConfig.authoringKey || '');
-    }
+  connect = async (_: BotEnvironments, __: string) => {
+    // confirm bot runtime can listening here
+    return Promise.resolve(`${this.endpoint}/api/messages`);
+  };
 
-    config = {
-      ...(await currentProject.settingManager.get(currentProject.environment.getDefaultSlot(), false)),
-      ...config,
-    };
-    if (config.MicrosoftAppPassword) {
-      form.append('microsoftAppPassword', config.MicrosoftAppPassword);
-    }
+  sync = async (config: DialogSetting) => {
     try {
-      await axios.post(this.adminEndpoint + '/api/admin', form, { headers: form.getHeaders() });
+      this.stop();
+      const dir = this.getBotPath();
+      await this.buildProcess(dir);
+      await this.start(dir, config);
     } catch (err) {
-      throw new Error('Unable to sync content to bot runtime');
+      this.stop();
+      throw new Error(err);
     }
   };
 
@@ -109,3 +177,26 @@ export class CSharpBotConnector implements IBotConnector {
     });
   };
 }
+
+process.on('SIGINT', () => {
+  console.log('[SIGINT] start graceful shutdown');
+  shutdown();
+  process.exit(1);
+});
+process.on('SIGTERM', () => {
+  console.log('[SIGTERM] start graceful shutdown');
+  shutdown();
+  process.exit(1);
+});
+process.on('SIGQUIT', () => {
+  console.log('[SIGQUIT] start graceful shutdown');
+  shutdown();
+  process.exit(1);
+});
+function shutdown() {
+  if (runtime) {
+    console.log(`kill runtime before exit at ${runtime.pid}`);
+    runtime.kill('SIGKILL');
+    runtime = null;
+  }
+}

Composer/packages/server/src/models/environment/defaultEnvironment.ts

@@ -18,7 +18,7 @@ export class DefaultEnvironment implements IEnvironment {
   public constructor(config: IEnvironmentConfig) {
     this.config = config;
     this.settingManager = new DefaultSettingManager(this.config.basePath);
-    this.botConnector = new CSharpBotConnector(this.config.adminEndpoint, this.config.endpoint);
+    this.botConnector = new CSharpBotConnector(this.config.endpoint);
   }
 
   public getEnvironmentName(_: string): string | undefined {

Composer/packages/server/src/models/settings/defaultSettingManager.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 import omit from 'lodash/omit';
-import { SensitiveProperties } from '@bfc/shared';
+import { SensitiveProperties as defaultSensitiveProperties } from '@bfc/shared';
 
 import { Path } from '../../utility/path';
 
@@ -34,6 +34,7 @@ export class DefaultSettingManager extends FileSettingManager {
   };
 
   private filterOutSensitiveValue = (obj: any) => {
+    const SensitiveProperties = obj.SensitiveProperties ? obj.SensitiveProperties : defaultSensitiveProperties;
     if (obj && typeof obj === 'object') {
       return omit(obj, SensitiveProperties);
     }