Image captioning

README.md

@@ -195,7 +195,7 @@ Dream Architecture is presented in the following image:
 | Entity linking              | 16 GiB RAM, 1.5 GiB GPU  | finds Wikidata entity ids for the entities detected with Entity Detection                                                                                                                                                      |
 | Entity Storer               | 220 MiB RAM              | a rule-based component, which stores entities from the user's and socialbot's utterances if opinion expression is detected with patterns or MIDAS Classifier and saves them along with the detected attitude to dialogue state |
 | Fact random                 | 50 MiB RAM               | returns random facts for the given entity (for entities from user utterance)                                                                                                                                                   |
-| Fact retrieval              | 400 MiB GPU              | extracts facts from Wikipedia and wikiHow                                                                                                                                                                                      |
+| Fact retrieval              | 400 MiB GPU              | extracts facts from Wikipedia and wikiHow                                                                                                                                                                                      |                                                                                                                                                                            |
 | Image captioning            | 4 GiB RAM, 5.4 GiB GPU   | creates text representation of a received image                                                                                                                                                                                                                                 |
 | Intent catcher              | 2.7 GiB RAM              | classifies user utterances into a number of predefined intents which are trained on a set of phrases and regexps                                                                                                               |
 | KBQA                        | 360 MiB GPU              | answers user's factoid questions based on Wikidata KB                                                                                                                                                                          |

assistant_dists/dream/docker-compose.override.yml

@@ -19,7 +19,7 @@ services:
           dff-funfact-skill:8104, dff-bot-persona-skill:8105, news-api-annotator:8112,
           dff-gossip-skill:8109, dff-wiki-skill:8111, dff-gaming-skill:8115, topic-recommendation:8113,
           user-persona-extractor:8114, wiki-facts:8116, dff-music-skill:8099, entity-detection:8103, dff-art-skill:8117,
-          midas-predictor:8121, dialogpt:8125, infilling:8122, dff-template-skill:8120, image-captioning:8123"
+          midas-predictor:8121, dialogpt:8125, infilling:8122, dff-template-skill:8120"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-480}
   convers-evaluator-annotator:
     env_file: [.env]
@@ -1223,22 +1223,4 @@ services:
         reservations:
           memory: 128M
 
-  image-captioning:
-    env_file: [ .env ]
-    build:
-      args:
-        SERVICE_PORT: 8123
-        SERVICE_NAME: image_captioning
-      context: ./services/image_captioning/
-    command: flask run -h 0.0.0.0 -p 8123
-    environment:
-      - CUDA_VISIBLE_DEVICES=0
-      - FLASK_APP=server
-    deploy:
-      resources:
-        limits:
-          memory: 5G
-        reservations:
-          memory: 5G
-
 version: '3.7'

assistant_dists/dream_mini/cpu.yml

@@ -1,9 +1,5 @@
 version: '3.7'
 services:
-  convers-evaluator-annotator:
-    environment:
-      DEVICE: cpu
-      CUDA_VISIBLE_DEVICES: ""
   dialogpt:
     environment:
       DEVICE: cpu
@@ -12,3 +8,7 @@ services:
     environment:
       DEVICE: cpu
       CUDA_VISIBLE_DEVICES: ""
+  sentence-ranker:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""

assistant_dists/dream_mini/dev.yml

@@ -5,12 +5,6 @@ services:
       - ".:/dp-agent"
     ports:
       - 4242:4242
-  convers-evaluator-annotator:
-    volumes:
-      - "./annotators/ConversationEvaluator:/src"
-      - "~/.deeppavlov:/root/.deeppavlov"
-    ports:
-      - 8004:8004
   dff-program-y-skill:
     volumes:
       - "./skills/dff_program_y_skill:/src"
@@ -57,9 +51,9 @@ services:
       - "./services/dialogpt:/src"
     ports:
       - 8125:8125
-  image-captioning:
+  sentence-ranker:
     volumes:
-      - "./services/image_captioning:/src"
+      - "./services/sentence_ranker:/src"
     ports:
-      - 8123:8123
+      - 8128:8128
 version: "3.7"

assistant_dists/dream_mini/docker-compose.override.yml

@@ -2,29 +2,10 @@ services:
   agent:
     command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_mini/pipeline_conf.json'
     environment:
-      WAIT_HOSTS: "convers-evaluator-annotator:8004, dff-program-y-skill:8008, sentseg:8011, convers-evaluation-selector:8009, 
+      WAIT_HOSTS: "dff-program-y-skill:8008, sentseg:8011, convers-evaluation-selector:8009, 
           dff-intent-responder-skill:8012, intent-catcher:8014, badlisted-words:8018,
-          spelling-preprocessing:8074, dialogpt:8125, image-captioning:8123"
+          spelling-preprocessing:8074, dialogpt:8125, sentence-ranker:8128"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-480}
-  convers-evaluator-annotator:
-    env_file: [.env]
-    build:
-      args:
-        CONFIG: conveval.json
-        PORT: 8004
-        DATA_URL: https://files.deeppavlov.ai/alexaprize_data/cobot_conveval2.tar.gz
-      context: .
-      dockerfile: ./annotators/ConversationEvaluator/Dockerfile
-    environment:
-      - CUDA_VISIBLE_DEVICES=0
-    deploy:
-      mode: replicated
-      replicas: 1
-      resources:
-        limits:
-          memory: 2G
-        reservations:
-          memory: 2G
 
   dff-program-y-skill:
     env_file: [.env]
@@ -173,22 +154,22 @@ services:
         reservations:
           memory: 2G
 
-  image-captioning:
+  sentence-ranker:
     env_file: [ .env ]
     build:
       args:
-        SERVICE_PORT: 8123
-        SERVICE_NAME: image_captioning
-      context: ./services/image_captioning/
-    command: flask run -h 0.0.0.0 -p 8123
+        SERVICE_PORT: 8128
+        PRETRAINED_MODEL_NAME_OR_PATH: sentence-transformers/bert-base-nli-mean-tokens
+      context: ./services/sentence_ranker/
+    command: flask run -h 0.0.0.0 -p 8128
     environment:
       - CUDA_VISIBLE_DEVICES=0
       - FLASK_APP=server
     deploy:
       resources:
         limits:
-          memory: 5G
+          memory: 3G
         reservations:
-          memory: 5G
+          memory: 3G
 
 version: '3.7'

assistant_dists/dream_mini/pipeline_conf.json

@@ -117,19 +117,6 @@
                     "annotators.spelling_preprocessing",
                     "annotators.sentseg"
                 ]
-            },
-            "image_captioning": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 5,
-                    "url": "http://image_captioning:8123/respond"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:cobot_classifiers_formatter_service",
-                "response_formatter": "state_formatters.dp_formatters:skill_with_attributes_formatter_service",
-                "previous_services": [
-                    "skill_selectors"
-                ],
-                "state_manager_method": "add_hypothesis"
             }
         },
         "skill_selectors": {
@@ -215,13 +202,13 @@
                 ],
                 "state_manager_method": "add_hypothesis_annotation_batch"
             },
-            "convers_evaluator_annotator": {
+            "sentence_ranker": {
                 "connector": {
                     "protocol": "http",
                     "timeout": 1,
-                    "url": "http://convers-evaluator-annotator:8004/batch_model"
+                    "url": "http://sentence-ranker:8128/respond"
                 },
-                "dialog_formatter": "state_formatters.dp_formatters:convers_evaluator_annotator_formatter",
+                "dialog_formatter": "state_formatters.dp_formatters:sentence_ranker_formatter",
                 "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
                 "previous_services": ["skills"],
                 "state_manager_method": "add_hypothesis_annotation_batch"

assistant_dists/dream_mini/proxy.yml

@@ -1,12 +1,4 @@
 services:
-  convers-evaluator-annotator:
-    command: ["nginx", "-g", "daemon off;"]
-    build:
-      context: dp/proxy/
-      dockerfile: Dockerfile
-    environment:
-      - PROXY_PASS=dream.deeppavlov.ai:8004
-      - PORT=8004
 
   dff-program-y-skill:
     command: ["nginx", "-g", "daemon off;"]
@@ -80,4 +72,12 @@ services:
       - PROXY_PASS=dream.deeppavlov.ai:8125
       - PORT=8125
 
+  sentence-ranker:
+    command: [ "nginx", "-g", "daemon off;" ]
+    build:
+      context: dp/proxy/
+      dockerfile: Dockerfile
+    environment:
+      - PROXY_PASS=dream.deeppavlov.ai:8128
+      - PORT=8128
 version: '3.7'

assistant_dists/dream_multimodal/cpu.yml

@@ -0,0 +1,14 @@
+version: '3.7'
+services:
+  dialogpt:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""
+  intent-catcher:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""
+  sentence-ranker:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""

assistant_dists/dream_multimodal/db_conf.json

@@ -0,0 +1,6 @@
+{
+    "host": "DB_HOST",
+    "port": "DB_PORT",
+    "name": "DB_NAME",
+    "env": true
+}

assistant_dists/dream_multimodal/dev.yml

@@ -0,0 +1,64 @@
+# С такими volumes удобно дебажить, не нужно пересобирать контейнер каждый раз при изменении кода
+services:
+  agent:
+    volumes:
+      - ".:/dp-agent"
+    ports:
+      - 4242:4242
+  dff-program-y-skill:
+    volumes:
+      - "./skills/dff_program_y_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8008:8008
+  sentseg:
+    volumes:
+      - "./annotators/SentSeg:/src"
+    ports:
+      - 8011:8011
+  convers-evaluation-selector:
+    volumes:
+      - "./response_selectors/convers_evaluation_based_selector:/src"
+      - "./common:/src/common"
+    ports:
+      - 8009:8009
+  dff-intent-responder-skill:
+    volumes:
+      - "./skills/dff_intent_responder_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8012:8012
+  intent-catcher:
+    volumes:
+      - "./annotators/IntentCatcherTransformers:/src"
+      - "./common:/src/common"
+      - "~/.deeppavlov:/root/.deeppavlov"
+    ports:
+      - 8014:8014
+  badlisted-words:
+    volumes:
+      - "./annotators/BadlistedWordsDetector:/src"
+      - "./common:/src/common"
+    ports:
+      - 8018:8018
+  spelling-preprocessing:
+    volumes:
+      - "./annotators/spelling_preprocessing:/src"
+    ports:
+      - 8074:8074
+  dialogpt:
+    volumes:
+      - "./services/dialogpt:/src"
+    ports:
+      - 8125:8125
+  sentence-ranker:
+    volumes:
+      - "./services/sentence_ranker:/src"
+    ports:
+      - 8128:8128
+  image-captioning:
+    volumes:
+      - "./services/image_captioning:/src"
+    ports:
+      - 8123:8123
+version: "3.7"