Add compose definitions to component cards (#384)

* Fix requirements.txt (#84)

* fix itsdangerous requirements

* pin itsdangerous requirements for all flask==1.1.1 servers

* add compose definitions to component cards

* add host, port, endpoint to pipeline cards

* fix authors, add missing pipelines, add template

* add prompt-selector template, fix templates

* fix template keys

* remove unused comet-atomic component

* rework service, component cards

* move build args and env vars to environment.yml file

* fix empty proxy values

* fix pipeline confs

* fix component connector.annotations

* fix wait hosts in response_annotator_selectors service

* fix wait hosts in other services, environments

* fix response selectors, pipelines

* fix ports in response selector

* fix generative components

* fix: remove spelling and badlists from dream_persona_prompted

* fix: ignore all env_secret

* fix: new formatters for dff

* fix: universal dist does not utilize env_secret

* fix: multiskill components and timeouts

* fix: remove template_template distribution

* fix: deeppavlov assistant

* fix: formatters in components

* fix: volumes for resp selectors

* fix: correct filenames for multiskill

* fix: rullama7bru distribution

* fix pipelines after merge

* fix sentseg annotator source in pipelines

* fix agent services, components

* fix: sentseg and prompt selector do not wait for spelling

* fix: response_annotator_selectors card

* fix: timeouts

* fix: build args

* create services, components for prompt selector

* fix prompt selectors environment

* fix: flask server

* fix: path to prompt selector

* fix: required groups

* fix: required group skill_selectors

* fix: required elements

* fix: previous services

* fix: correct link to sentseg components card

* fix: correct link to sentseg components card

* remove unused prompt selector component

* remove old configs

* fix: rename files without dashes

---------

Co-authored-by: Andrii.Hura <[email protected]>
Co-authored-by: Dilyara Baymurzina <[email protected]>

.gitignore

@@ -144,4 +144,4 @@ docker-compose-one-replica.yml
 *.pem
 
 # personal env keys and tokens
-.env_secret
+*.env_secret

annotators/BadlistedWordsDetector/service_configs/badlisted-words/environment.yml

@@ -0,0 +1,3 @@
+SERVICE_PORT: 8018
+SERVICE_NAME: badlisted_words
+FLASK_APP: server

annotators/BadlistedWordsDetector/service_configs/badlisted-words/service.yml

@@ -0,0 +1,38 @@
+name: badlisted-words
+endpoints:
+- badlisted_words
+- badlisted_words_batch
+compose:
+  env_file:
+  - .env
+  build:
+    args:
+      SERVICE_PORT: 8018
+      SERVICE_NAME: badlisted_words
+      FLASK_APP: server
+    context: annotators/BadlistedWordsDetector/
+  command: flask run -h 0.0.0.0 -p 8018
+  environment:
+  - FLASK_APP=server
+  deploy:
+    resources:
+      limits:
+        memory: 256M
+      reservations:
+        memory: 256M
+  volumes:
+  - ./annotators/BadlistedWordsDetector:/src
+  - ./common:/src/common
+  ports:
+  - 8018:8018
+proxy:
+  command:
+  - nginx
+  - -g
+  - daemon off;
+  build:
+    context: dp/proxy/
+    dockerfile: Dockerfile
+  environment:
+  - PROXY_PASS=dream.deeppavlov.ai:8018
+  - PORT=8018

annotators/BadlistedWordsDetector_ru/service_configs/badlisted-words-ru/environment.yml

@@ -0,0 +1,3 @@
+SERVICE_PORT: 8018
+SERVICE_NAME: badlisted_words
+FLASK_APP: server

annotators/BadlistedWordsDetector_ru/service_configs/badlisted-words-ru/service.yml

@@ -0,0 +1,28 @@
+name: badlisted-words-ru
+endpoints:
+- badlisted_words
+- badlisted_words_batch
+compose:
+  env_file:
+  - .env_ru
+  build:
+    args:
+      SERVICE_PORT: 8018
+      SERVICE_NAME: badlisted_words
+      FLASK_APP: server
+    context: annotators/BadlistedWordsDetector_ru/
+  command: flask run -h 0.0.0.0 -p 8018
+  environment:
+  - FLASK_APP=server
+  deploy:
+    resources:
+      limits:
+        memory: 128M
+      reservations:
+        memory: 128M
+  volumes:
+  - ./annotators/BadlistedWordsDetector_ru:/src
+  - ./common:/src/common
+  ports:
+  - 8018:8018
+proxy: null

annotators/COMeT/service_configs/comet-conceptnet/environment.yml

@@ -0,0 +1,8 @@
+GRAPH: conceptnet
+SERVICE_HOME: ./annotators/COMeT/
+SERVICE_NAME: comet_conceptnet
+SERVICE_PORT: 8065
+PRETRAINED_MODEL: http://lnsigo.mipt.ru/export/alexaprize_data/conceptnet/conceptnet_pretrained_model.pickle
+PREPROCESS_DATA: http://lnsigo.mipt.ru/export/alexaprize_data/conceptnet/rel_language-trainsize_100-devversion_12-maxe1_10-maxe2_15-maxr_5.pickle
+DECODING_ALGO: beam-3
+CUDA_VISIBLE_DEVICES: '0'

annotators/COMeT/service_configs/comet-conceptnet/service.yml

@@ -0,0 +1,43 @@
+name: comet-conceptnet
+endpoints:
+- comet_annotator
+- comet
+compose:
+  env_file:
+  - .env
+  build:
+    context: .
+    dockerfile: ./annotators/COMeT/Dockerfile
+    args:
+      GRAPH: conceptnet
+      SERVICE_HOME: ./annotators/COMeT/
+      SERVICE_NAME: comet_conceptnet
+      SERVICE_PORT: 8065
+      PRETRAINED_MODEL: http://lnsigo.mipt.ru/export/alexaprize_data/conceptnet/conceptnet_pretrained_model.pickle
+      PREPROCESS_DATA: http://lnsigo.mipt.ru/export/alexaprize_data/conceptnet/rel_language-trainsize_100-devversion_12-maxe1_10-maxe2_15-maxr_5.pickle
+      DECODING_ALGO: beam-3
+      CUDA_VISIBLE_DEVICES: '0'
+  environment:
+  - CUDA_VISIBLE_DEVICES=0
+  deploy:
+    resources:
+      limits:
+        memory: 3.5G
+      reservations:
+        memory: 3.5G
+  volumes:
+  - ./annotators/COMeT:/src
+  - ./common:/src/common
+  ports:
+  - 8065:8065
+proxy:
+  command:
+  - nginx
+  - -g
+  - daemon off;
+  build:
+    context: dp/proxy/
+    dockerfile: Dockerfile
+  environment:
+  - PROXY_PASS=dream.deeppavlov.ai:8065
+  - PORT=8065

annotators/ConversationEvaluator/service_configs/convers-evaluator-annotator/environment.yml

@@ -0,0 +1,4 @@
+CONFIG: conveval.json
+SERVICE_PORT: 8004
+DATA_URL: https://files.deeppavlov.ai/alexaprize_data/cobot_conveval2.tar.gz
+CUDA_VISIBLE_DEVICES: '0'

annotators/ConversationEvaluator/service_configs/convers-evaluator-annotator/service.yml

@@ -0,0 +1,40 @@
+name: convers-evaluator-annotator
+endpoints:
+- batch_model
+compose:
+  env_file:
+  - .env
+  build:
+    args:
+      CONFIG: conveval.json
+      SERVICE_PORT: 8004
+      DATA_URL: https://files.deeppavlov.ai/alexaprize_data/cobot_conveval2.tar.gz
+      CUDA_VISIBLE_DEVICES: '0'
+    context: .
+    dockerfile: ./annotators/ConversationEvaluator/Dockerfile
+  environment:
+  - CUDA_VISIBLE_DEVICES=0
+  deploy:
+    mode: replicated
+    replicas: 1
+    resources:
+      limits:
+        memory: 2G
+      reservations:
+        memory: 2G
+  volumes:
+  - ./annotators/ConversationEvaluator:/src
+  - ~/.deeppavlov:/root/.deeppavlov
+  ports:
+  - 8004:8004
+proxy:
+  command:
+  - nginx
+  - -g
+  - daemon off;
+  build:
+    context: dp/proxy/
+    dockerfile: Dockerfile
+  environment:
+  - PROXY_PASS=dream.deeppavlov.ai:8004
+  - PORT=8004