Feat/tests russian (#90)

* feat: runtests russian * fix: test file and elements * feat: tests russian in jenkinsfile * feat: files for tests * fix; program-y name * fix: change to dff-intent-responder-skill * fix: sync with agent updates * fix: cleanup for both runtests * fix: fix path to pipeline conf * fix: remove ner from tests * fix: unbuild english bot before russian tests * fix: codestyle Co-authored-by: Fedor Ignatov <[email protected]> Co-authored-by: Fedor Ignatov <[email protected]>
deeppavlov · Feb 2, 2022 · e394595 · e394595
1 parent 3e505d1
commit e394595
Show file tree

Hide file tree

Showing 8 changed files with 915 additions and 16 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -157,6 +157,105 @@ pipeline {
         }
       }
     }
+
+    stage('Build-RU') {
+      steps {
+        script{
+          startTime = currentBuild.duration
+          Exception ex = null
+          catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+            try {
+              sh '''
+                tests/runtests.sh MODE=clean
+                tests/runtests_russian.sh MODE=build
+              '''
+            }
+            catch (Exception e) {
+              int duration = (currentBuild.duration - startTime) / 1000
+              throw e
+            }
+          }
+        }
+      }
+      post {
+        failure {
+          script {
+            sh 'tests/runtests_russian.sh MODE=clean'
+          }
+        }
+        success {
+          script {
+            int duration = (currentBuild.duration - startTime) / 1000
+          }
+        }
+      }
+    }
+
+    stage('Start-RU') {
+      steps {
+        script {
+          startTime = currentBuild.duration
+          Exception ex = null
+          catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+            try {
+              sh 'tests/runtests_russian.sh MODE=clean && tests/runtests_russian.sh MODE=start'
+            }
+            catch (Exception e) {
+              int duration = (currentBuild.duration - startTime) / 1000
+              throw e
+            }
+          }
+        }
+      }
+      post {
+        failure {
+          script {
+            sh 'tests/runtests_russian.sh MODE=clean'
+          }
+        }
+        success {
+          script {
+            started = true
+            int duration = (currentBuild.duration - startTime) / 1000
+          }
+        }
+        aborted {
+          script {
+            sh 'tests/runtests_russian.sh MODE=clean'
+          }
+        }
+      }
+    }
+
+    stage('Test skills-RU') {
+      steps {
+        script {
+          startTime = currentBuild.duration
+          Exception ex = null
+          catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+            try {
+              sh label: 'test skills', script: 'tests/runtests_russian.sh MODE=test_skills'
+            }
+            catch (Exception e) {
+              int duration = (currentBuild.duration - startTime) / 1000
+              throw e
+            }
+          }
+        }
+      }
+      post {
+        success {
+          script {
+            int duration = (currentBuild.duration - startTime) / 1000
+          }
+        }
+        aborted {
+          script {
+            sh 'tests/runtests_russian.sh MODE=clean'
+          }
+        }
+      }
+    }
   }
   post {
     aborted {
@@ -168,6 +267,7 @@ pipeline {
       script {
         if (started) {
           sh './tests/runtests.sh MODE=clean'
+          sh './tests/runtests_russian.sh MODE=clean'
         }
       }
     }

diff --git a/assistant_dists/dream_russian/cpu.yml b/assistant_dists/dream_russian/cpu.yml
@@ -0,0 +1,10 @@
+version: '3.7'
+services:
+  ner:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""
+  entity-detection:
+    environment:
+      CUDA_VISIBLE_DEVICES: ""
+
diff --git a/assistant_dists/dream_russian/db_conf.json b/assistant_dists/dream_russian/db_conf.json
@@ -0,0 +1,6 @@
+{
+    "host": "DB_HOST",
+    "port": "DB_PORT",
+    "name": "DB_NAME",
+    "env": true
+}
diff --git a/assistant_dists/dream_russian/dev.yml b/assistant_dists/dream_russian/dev.yml
@@ -0,0 +1,108 @@
+# С такими volumes удобно дебажить, не нужно пересобирать контейнер каждый раз при изменении кода
+services:
+  agent:
+    volumes:
+      - ".:/dp-agent"
+    ports:
+      - 4242:4242
+  dff-program-y-skill:
+    volumes:
+      - "./skills/dff_program_y_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8008:8008
+  sentseg:
+    volumes:
+      - "./annotators/SentSeg:/src"
+    ports:
+      - 8011:8011
+  convers-evaluation-selector:
+    volumes:
+      - "./response_selectors/convers_evaluation_based_selector:/src"
+      - "./common:/src/common"
+    ports:
+      - 8009:8009
+  dff-intent-responder-skill:
+    volumes:
+      - "./skills/dff_intent_responder_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8012:8012
+  intent-catcher:
+    volumes:
+      - "./annotators/IntentCatcher/src:/src"
+      - "./common:/src/common"
+    ports:
+      - 8014:8014
+  badlisted-words:
+    volumes:
+      - "./annotators/BadlistedWordsDetector:/src"
+      - "./common:/src/common"
+    ports:
+      - 8018:8018
+  ner:
+    volumes:
+      - './annotators/NER:/src'
+      - "~/.deeppavlov:/root/.deeppavlov"
+    ports:
+      - 8021:8021
+  convert-reddit:
+    volumes:
+      - "./skills/convert_reddit:/app"
+      - "./common:/src/common"
+    ports:
+      - 8029:8029
+  personal-info-skill:
+    volumes:
+      - "./skills/personal_info_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8030:8030
+  entity-linking:
+    volumes:
+      - "./annotators/entity_linking:/src"
+      - "~/.deeppavlov:/root/.deeppavlov"
+    ports:
+      - 8075:8075
+  wiki-parser:
+    volumes:
+      - "./annotators/wiki_parser:/src"
+      - "./common:/src/common"
+    ports:
+      - 8077:8077
+  mongo:
+    ports:
+      - 27017:27017
+    # # you can use persistent local volume if you need
+    # volumes:
+    #   - ./venv/data/db_data:/root/data/db
+  spelling-preprocessing:
+    volumes:
+      - "./annotators/spelling_preprocessing:/src"
+    ports:
+      - 8074:8074
+  dff-friendship-skill:
+    volumes:
+      - "./skills/dff_friendship_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8086:8086
+  dff-wiki-skill:
+    volumes:
+      - "./skills/dff_wiki_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8111:8111
+  entity-detection:
+    volumes:
+      - "./annotators/entity_detection:/src"
+      - "~/.deeppavlov:/root/.deeppavlov"
+    ports:
+      - 8103:8103
+  dff-template-skill:
+    volumes:
+      - "./skills/dff_template_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8120:8120
+version: "3.7"
diff --git a/assistant_dists/dream_russian/docker-compose.override.yml b/assistant_dists/dream_russian/docker-compose.override.yml
@@ -1,28 +1,23 @@
 services:
   agent:
-    volumes:
-      - "./assistant_dists/dream:/dp-agent/"
-      - "./skill_selectors/:/dp-agent/skill_selectors/"
-      - "./state_formatters/:/dp-agent/state_formatters/"
-      - "./skills/:/dp-agent/skills/"
-      - "./common/:/dp-agent/common/"
+    command: sh -c 'bin/wait && python -m deeppavlov_agent.run -ch http_client -pl assistant_dists/dream_russian/pipeline_conf.json --cors'
     environment:
-      WAIT_HOSTS: "dff-program-y:8008, sentseg:8011, convers-evaluation-selector:8009,
-          intent-responder:8012, intent-catcher:8014, badlisted-words:8018,
-          ner:8021, convert-reddit:8029, personal-info-skill:8030,
+      WAIT_HOSTS: "dff-program-y-skill:8008, sentseg:8011, convers-evaluation-selector:8009, 
+          dff-intent-responder-skill:8012, intent-catcher:8014, badlisted-words:8018,
+          sentrewrite:8017, ner:8021, convert-reddit:8029, personal-info-skill:8030,
           spelling-preprocessing:8074, entity-linking:8075, wiki-parser:8077,
           dff-friendship-skill:8086, dff-wiki-skill:8111, entity-detection:8103,
           dff-template-skill:8120"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-480}
 
-  dff-program-y:
+  dff-program-y-skill:
     env_file: [.env]
     build:
       args:
         SERVICE_PORT: 8008
-        SERVICE_NAME: dff_program_y
+        SERVICE_NAME: dff_program_y_skill
       context: .
-      dockerfile: ./skills/dff_program_y/Dockerfile
+      dockerfile: ./skills/dff_program_y_skill/Dockerfile
     command: gunicorn --workers=1 server:app -b 0.0.0.0:8008 --reload
     deploy:
       resources:
@@ -60,14 +55,29 @@ services:
         reservations:
           memory: 256M
 
-  intent-responder:
+  sentrewrite:
     env_file: [.env]
     build:
-      context: .
-      dockerfile: ./skills/IntentResponder/Dockerfile
-    command: flask run -h 0.0.0.0 -p 8012
+      context: ./annotators/SentRewrite/
+    command: flask run -h 0.0.0.0 -p 8017
     environment:
       - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 512M
+        reservations:
+          memory: 512M
+
+  dff-intent-responder-skill:
+    env_file: [ .env ]
+    build:
+      args:
+        SERVICE_PORT: 8012
+        SERVICE_NAME: dff_intent_responder_skill
+      context: .
+      dockerfile: ./skills/dff_intent_responder_skill/Dockerfile
+    command: gunicorn --workers=1 server:app -b 0.0.0.0:8012 --reload
     deploy:
       resources:
         limits: