updated image captioning

README.md

@@ -217,9 +217,11 @@ Dream Architecture is presented in the following image:
 | Wiki Facts                  | 1.7 GiB RAM              |                                                                                                                                                                                                                                |
 
 ## Services
+
 | Name                | Requirements               | Description                                                                                                                                                                                              |
 |---------------------|----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | DialoGPT            | 1.2 GiB RAM, 2.1 GiB GPU   | generative service based on Transformers generative model, the model is set in docker compose argument `PRETRAINED_MODEL_NAME_OR_PATH` (for example, `microsoft/DialoGPT-small` with 0.2-0.5 sec on GPU) |
+| Image captioning    | 4 GiB RAM, 5.4 GiB GPU   | creates text representation of a received image                                                                                                                                                          |
 | Infilling           | 1  GiB RAM, 1.2 GiB GPU    | generative service based on Infilling model, for the given utterance returns utterance where `_` from original text is replaced with generated tokens                                                    |
 | Knowledge Grounding | 2 GiB RAM, 2.1 GiB GPU     | generative service based on BlenderBot architecture providing a response to the context taking into account an additional text paragraph                                                                 |
 | Masked LM           | 1.1 GiB RAM, 1 GiB GPU     |                                                                                                                                                                                                          |

annotators/combined_classification/requirements.txt

@@ -1,12 +1,12 @@
-gunicorn==19.9.0
-sentry-sdk[flask]==0.14.1
-flask==1.1.1
-itsdangerous==2.0.1
-requests==2.22.0
-uvicorn==0.11.7
-prometheus-client==0.7.1
-filelock==3.0.12
-torch==1.5.1
-transformers==4.6.0
-jinja2<=3.0.3
-Werkzeug<=2.0.3
+gunicorn==19.9.0
+sentry-sdk[flask]==0.14.1
+flask==1.1.1
+itsdangerous==2.0.1
+requests==2.22.0
+uvicorn==0.11.7
+prometheus-client==0.7.1
+filelock==3.0.12
+torch==1.5.1
+transformers==4.6.0
+jinja2<=3.0.3
+Werkzeug<=2.0.3

assistant_dists/dream_multimodal/cpu.yml

@@ -0,0 +1,14 @@
+version: '3.7'
+services:
+  dialogpt:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""
+  intent-catcher:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""
+  sentence-ranker:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""

assistant_dists/dream_multimodal/db_conf.json

@@ -0,0 +1,6 @@
+{
+    "host": "DB_HOST",
+    "port": "DB_PORT",
+    "name": "DB_NAME",
+    "env": true
+}

assistant_dists/dream_multimodal/dev.yml

@@ -0,0 +1,64 @@
+# С такими volumes удобно дебажить, не нужно пересобирать контейнер каждый раз при изменении кода
+services:
+  agent:
+    volumes:
+      - ".:/dp-agent"
+    ports:
+      - 4242:4242
+  dff-program-y-skill:
+    volumes:
+      - "./skills/dff_program_y_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8008:8008
+  sentseg:
+    volumes:
+      - "./annotators/SentSeg:/src"
+    ports:
+      - 8011:8011
+  convers-evaluation-selector:
+    volumes:
+      - "./response_selectors/convers_evaluation_based_selector:/src"
+      - "./common:/src/common"
+    ports:
+      - 8009:8009
+  dff-intent-responder-skill:
+    volumes:
+      - "./skills/dff_intent_responder_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8012:8012
+  intent-catcher:
+    volumes:
+      - "./annotators/IntentCatcherTransformers:/src"
+      - "./common:/src/common"
+      - "~/.deeppavlov:/root/.deeppavlov"
+    ports:
+      - 8014:8014
+  badlisted-words:
+    volumes:
+      - "./annotators/BadlistedWordsDetector:/src"
+      - "./common:/src/common"
+    ports:
+      - 8018:8018
+  spelling-preprocessing:
+    volumes:
+      - "./annotators/spelling_preprocessing:/src"
+    ports:
+      - 8074:8074
+  dialogpt:
+    volumes:
+      - "./services/dialogpt:/src"
+    ports:
+      - 8125:8125
+  sentence-ranker:
+    volumes:
+      - "./services/sentence_ranker:/src"
+    ports:
+      - 8128:8128
+  image-captioning:
+    volumes:
+      - "./services/image_captioning:/src"
+    ports:
+      - 8123:8123
+version: "3.7"

assistant_dists/dream_multimodal/docker-compose.override.yml

@@ -0,0 +1,193 @@
+services:
+  agent:
+    command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_mini/pipeline_conf.json'
+    environment:
+      WAIT_HOSTS: "dff-program-y-skill:8008, sentseg:8011, convers-evaluation-selector:8009, 
+          dff-intent-responder-skill:8012, intent-catcher:8014, badlisted-words:8018,
+          spelling-preprocessing:8074, dialogpt:8125, sentence-ranker:8128, image-captioning:8123"
+      WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-480}
+
+  dff-program-y-skill:
+    env_file: [.env]
+    build:
+      args:
+        SERVICE_PORT: 8008
+        SERVICE_NAME: dff_program_y_skill
+        LANGUAGE: EN
+      context: .
+      dockerfile: ./skills/dff_program_y_skill/Dockerfile
+    command: gunicorn --workers=1 server:app -b 0.0.0.0:8008 --reload
+    deploy:
+      resources:
+        limits:
+          memory: 1024M
+        reservations:
+          memory: 1024M
+
+
+  sentseg:
+    env_file: [.env]
+    build:
+      context: ./annotators/SentSeg/
+    command: flask run -h 0.0.0.0 -p 8011
+    environment:
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 1.5G
+        reservations:
+          memory: 1.5G
+
+  convers-evaluation-selector:
+    env_file: [.env]
+    build:
+      args:
+        TAG_BASED_SELECTION: 1
+        CALL_BY_NAME_PROBABILITY: 0.5
+        PROMPT_PROBA: 0.3
+        ACKNOWLEDGEMENT_PROBA: 0.3
+        PRIORITIZE_WITH_REQUIRED_ACT: 1
+        PRIORITIZE_NO_DIALOG_BREAKDOWN: 0
+        PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 1
+        IGNORE_DISLIKED_SKILLS: 0
+        GREETING_FIRST: 1
+        RESTRICTION_FOR_SENSITIVE_CASE: 1
+        PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0
+        ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 1
+        PRIORITIZE_SCRIPTED_SKILLS: 0
+      context: .
+      dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile
+    command: flask run -h 0.0.0.0 -p 8009
+    environment:
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+        reservations:
+          memory: 256M
+
+  dff-intent-responder-skill:
+    env_file: [ .env ]
+    build:
+      args:
+        SERVICE_PORT: 8012
+        SERVICE_NAME: dff_intent_responder_skill
+        INTENT_RESPONSE_PHRASES_FNAME: intent_response_phrases.json
+      context: .
+      dockerfile: ./skills/dff_intent_responder_skill/Dockerfile
+    command: gunicorn --workers=1 server:app -b 0.0.0.0:8012 --reload
+    deploy:
+      resources:
+        limits:
+          memory: 128M
+        reservations:
+          memory: 128M
+
+  intent-catcher:
+    env_file: [.env]
+    build:
+      context: .
+      dockerfile: ./annotators/IntentCatcherTransformers/Dockerfile
+      args:
+        SERVICE_PORT: 8014
+        CONFIG_NAME: intents_model_dp_config.json
+        INTENT_PHRASES_PATH: intent_phrases.json
+    command:  python -m flask run -h 0.0.0.0 -p 8014
+    environment:
+      - FLASK_APP=server
+      - CUDA_VISIBLE_DEVICES=0
+    deploy:
+      resources:
+        limits:
+          memory: 3.5G
+        reservations:
+          memory: 3.5G
+
+  badlisted-words:
+    env_file: [.env]
+    build:
+      context: annotators/BadlistedWordsDetector/
+    command: flask run -h 0.0.0.0 -p 8018
+    environment:
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+        reservations:
+          memory: 256M
+
+  spelling-preprocessing:
+    env_file: [.env]
+    build:
+      context: ./annotators/spelling_preprocessing/
+    command: flask run -h 0.0.0.0 -p 8074
+    environment:
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 50M
+        reservations:
+          memory: 50M
+
+  dialogpt:
+    env_file: [ .env ]
+    build:
+      args:
+        SERVICE_PORT: 8125
+        SERVICE_NAME: dialogpt
+        PRETRAINED_MODEL_NAME_OR_PATH: microsoft/DialoGPT-medium
+        N_HYPOTHESES_TO_GENERATE: 5
+        CONFIG_NAME: dialogpt_en.json
+      context: ./services/dialogpt/
+    command: flask run -h 0.0.0.0 -p 8125
+    environment:
+      - CUDA_VISIBLE_DEVICES=0
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+        reservations:
+          memory: 2G
+
+  sentence-ranker:
+    env_file: [ .env ]
+    build:
+      args:
+        SERVICE_PORT: 8128
+        PRETRAINED_MODEL_NAME_OR_PATH: sentence-transformers/bert-base-nli-mean-tokens
+      context: ./services/sentence_ranker/
+    command: flask run -h 0.0.0.0 -p 8128
+    environment:
+      - CUDA_VISIBLE_DEVICES=0
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 3G
+        reservations:
+          memory: 3G
+
+  image-captioning:
+    env_file: [ .env ]
+    build:
+      args:
+        SERVICE_PORT: 8123
+        SERVICE_NAME: image_captioning
+      context: ./services/image_captioning/
+    command: flask run -h 0.0.0.0 -p 8123
+    environment:
+      - CUDA_VISIBLE_DEVICES=0
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 5G
+        reservations:
+          memory: 5G
+
+version: '3.7'