fix: remove badlists from prompted dists (#431)

deeppavlov · Apr 26, 2023 · d3a4619 · d3a4619
1 parent 6b36da9
commit d3a4619
Show file tree

Hide file tree

Showing 43 changed files with 11 additions and 762 deletions.
diff --git a/assistant_dists/ai_faq_assistant/dev.yml b/assistant_dists/ai_faq_assistant/dev.yml
@@ -16,12 +16,6 @@ services:
       - "./common:/src/common"
     ports:
       - 8002:8002
-  badlisted-words:
-    volumes:
-      - "./annotators/BadlistedWordsDetector:/src"
-      - "./common:/src/common"
-    ports:
-      - 8018:8018
   combined-classification:
     volumes:
       - "./common:/src/common"

diff --git a/assistant_dists/ai_faq_assistant/docker-compose.override.yml b/assistant_dists/ai_faq_assistant/docker-compose.override.yml
@@ -2,7 +2,7 @@ services:
   agent:
     command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/ai_faq_assistant/pipeline_conf.json'
     environment:
-      WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, 
+      WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, combined-classification:8087, 
         sentence-ranker:8128, prompt-selector:8135, transformers-lm-oasst12b:8158, 
         dff-ai-faq-prompted-skill:8143"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000}
@@ -62,23 +62,6 @@ services:
         reservations:
           memory: 100M
 
-  badlisted-words:
-    env_file: [ .env ]
-    build:
-      args:
-        SERVICE_PORT: 8018
-        SERVICE_NAME: badlisted_words
-      context: annotators/BadlistedWordsDetector/
-    command: flask run -h 0.0.0.0 -p 8018
-    environment:
-      - FLASK_APP=server
-    deploy:
-      resources:
-        limits:
-          memory: 256M
-        reservations:
-          memory: 256M
-
   prompt-selector:
     env_file: [ .env ]
     build:

diff --git a/assistant_dists/ai_faq_assistant/pipeline_conf.json b/assistant_dists/ai_faq_assistant/pipeline_conf.json
@@ -113,24 +113,6 @@
                     "endpoint": "sentseg"
                 }
             },
-            "badlisted_words": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 1.0,
-                    "url": "http://badlisted-words:8018/badlisted_words"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:preproc_last_human_utt_dialog",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                ],
-                "state_manager_method": "add_annotation",
-                "is_enabled": true,
-                "source": {
-                    "directory": "annotators/BadlistedWordsDetector",
-                    "container": "badlisted-words",
-                    "endpoint": "badlisted_words"
-                }
-            },
             "prompt_selector": {
                 "connector": {
                     "protocol": "http",
@@ -210,25 +192,6 @@
             }
         },
         "candidate_annotators": {
-            "badlisted_words": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 1.0,
-                    "url": "http://badlisted-words:8018/badlisted_words_batch"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                    "skills"
-                ],
-                "state_manager_method": "add_hypothesis_annotation_batch",
-                "is_enabled": true,
-                "source": {
-                    "directory": "annotators/BadlistedWordsDetector",
-                    "container": "badlisted-words",
-                    "endpoint": "badlisted_words_batch"
-                }
-            },
             "combined_classification": {
                 "connector": {
                     "protocol": "http",

diff --git a/assistant_dists/ai_faq_assistant/proxy.yml b/assistant_dists/ai_faq_assistant/proxy.yml
@@ -9,15 +9,6 @@ services:
       - PROXY_PASS=proxy.deeppavlov.ai:8011
       - PORT=8011
 
-  badlisted-words:
-    command: ["nginx", "-g", "daemon off;"]
-    build:
-      context: dp/proxy/
-      dockerfile: Dockerfile
-    environment:
-      - PROXY_PASS=proxy.deeppavlov.ai:8018
-      - PORT=8018
-
   combined-classification:
     command: ["nginx", "-g", "daemon off;"]
     build:

diff --git a/assistant_dists/dream_persona_openai_prompted/dev.yml b/assistant_dists/dream_persona_openai_prompted/dev.yml
@@ -16,12 +16,6 @@ services:
       - "./common:/src/common"
     ports:
       - 8002:8002
-  badlisted-words:
-    volumes:
-      - "./annotators/BadlistedWordsDetector:/src"
-      - "./common:/src/common"
-    ports:
-      - 8018:8018
   combined-classification:
     volumes:
       - "./common:/src/common"

diff --git a/assistant_dists/dream_persona_openai_prompted/docker-compose.override.yml b/assistant_dists/dream_persona_openai_prompted/docker-compose.override.yml
@@ -2,7 +2,7 @@ services:
   agent:
     command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_persona_openai_prompted/pipeline_conf.json'
     environment:
-      WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, 
+      WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, combined-classification:8087, 
                    sentence-ranker:8128, prompt-selector:8135, openai-api-davinci3:8131,
                    dff-dream-persona-davinci3-prompted-skill:8137"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000}
@@ -62,23 +62,6 @@ services:
         reservations:
           memory: 100M
 
-  badlisted-words:
-    env_file: [ .env ]
-    build:
-      args:
-        SERVICE_PORT: 8018
-        SERVICE_NAME: badlisted_words
-      context: annotators/BadlistedWordsDetector/
-    command: flask run -h 0.0.0.0 -p 8018
-    environment:
-      - FLASK_APP=server
-    deploy:
-      resources:
-        limits:
-          memory: 256M
-        reservations:
-          memory: 256M
-
   prompt-selector:
     env_file: [ .env ]
     build:

diff --git a/assistant_dists/dream_persona_openai_prompted/pipeline_conf.json b/assistant_dists/dream_persona_openai_prompted/pipeline_conf.json
@@ -113,24 +113,6 @@
                     "endpoint": "sentseg"
                 }
             },
-            "badlisted_words": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 1.0,
-                    "url": "http://badlisted-words:8018/badlisted_words"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:preproc_last_human_utt_dialog",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                ],
-                "state_manager_method": "add_annotation",
-                "is_enabled": true,
-                "source": {
-                    "directory": "annotators/BadlistedWordsDetector",
-                    "container": "badlisted-words",
-                    "endpoint": "badlisted_words"
-                }
-            },
             "prompt_selector": {
                 "connector": {
                     "protocol": "http",
@@ -210,25 +192,6 @@
             }
         },
         "candidate_annotators": {
-            "badlisted_words": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 1.0,
-                    "url": "http://badlisted-words:8018/badlisted_words_batch"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                    "skills"
-                ],
-                "state_manager_method": "add_hypothesis_annotation_batch",
-                "is_enabled": true,
-                "source": {
-                    "directory": "annotators/BadlistedWordsDetector",
-                    "container": "badlisted-words",
-                    "endpoint": "badlisted_words_batch"
-                }
-            },
             "combined_classification": {
                 "connector": {
                     "protocol": "http",

diff --git a/assistant_dists/dream_persona_openai_prompted/proxy.yml b/assistant_dists/dream_persona_openai_prompted/proxy.yml
@@ -9,15 +9,6 @@ services:
       - PROXY_PASS=proxy.deeppavlov.ai:8011
       - PORT=8011
 
-  badlisted-words:
-    command: ["nginx", "-g", "daemon off;"]
-    build:
-      context: dp/proxy/
-      dockerfile: Dockerfile
-    environment:
-      - PROXY_PASS=proxy.deeppavlov.ai:8018
-      - PORT=8018
-
   combined-classification:
     command: ["nginx", "-g", "daemon off;"]
     build:

diff --git a/assistant_dists/dream_persona_prompted/dev.yml b/assistant_dists/dream_persona_prompted/dev.yml
@@ -16,12 +16,6 @@ services:
       - "./common:/src/common"
     ports:
       - 8002:8002
-  badlisted-words:
-    volumes:
-      - "./annotators/BadlistedWordsDetector:/src"
-      - "./common:/src/common"
-    ports:
-      - 8018:8018
   combined-classification:
     volumes:
       - "./common:/src/common"

diff --git a/assistant_dists/dream_persona_prompted/docker-compose.override.yml b/assistant_dists/dream_persona_prompted/docker-compose.override.yml
@@ -2,7 +2,7 @@ services:
   agent:
     command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_persona_prompted/pipeline_conf.json'
     environment:
-      WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, 
+      WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, combined-classification:8087, 
         sentence-ranker:8128, prompt-selector:8135, transformers-lm-gptj:8130,
         dff-dream-persona-gpt-j-prompted-skill:8134"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000}
@@ -62,23 +62,6 @@ services:
         reservations:
           memory: 100M
 
-  badlisted-words:
-    env_file: [ .env ]
-    build:
-      args:
-        SERVICE_PORT: 8018
-        SERVICE_NAME: badlisted_words
-      context: annotators/BadlistedWordsDetector/
-    command: flask run -h 0.0.0.0 -p 8018
-    environment:
-      - FLASK_APP=server
-    deploy:
-      resources:
-        limits:
-          memory: 256M
-        reservations:
-          memory: 256M
-
   prompt-selector:
     env_file: [ .env ]
     build:

diff --git a/assistant_dists/dream_persona_prompted/pipeline_conf.json b/assistant_dists/dream_persona_prompted/pipeline_conf.json
@@ -113,24 +113,6 @@
                     "endpoint": "sentseg"
                 }
             },
-            "badlisted_words": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 1.0,
-                    "url": "http://badlisted-words:8018/badlisted_words"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:preproc_last_human_utt_dialog",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                ],
-                "state_manager_method": "add_annotation",
-                "is_enabled": true,
-                "source": {
-                    "directory": "annotators/BadlistedWordsDetector",
-                    "container": "badlisted-words",
-                    "endpoint": "badlisted_words"
-                }
-            },
             "prompt_selector": {
                 "connector": {
                     "protocol": "http",
@@ -210,25 +192,6 @@
             }
         },
         "candidate_annotators": {
-            "badlisted_words": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 1.0,
-                    "url": "http://badlisted-words:8018/badlisted_words_batch"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                    "skills"
-                ],
-                "state_manager_method": "add_hypothesis_annotation_batch",
-                "is_enabled": true,
-                "source": {
-                    "directory": "annotators/BadlistedWordsDetector",
-                    "container": "badlisted-words",
-                    "endpoint": "badlisted_words_batch"
-                }
-            },
             "combined_classification": {
                 "connector": {
                     "protocol": "http",

diff --git a/assistant_dists/dream_persona_prompted/proxy.yml b/assistant_dists/dream_persona_prompted/proxy.yml
@@ -9,15 +9,6 @@ services:
       - PROXY_PASS=proxy.deeppavlov.ai:8011
       - PORT=8011
 
-  badlisted-words:
-    command: ["nginx", "-g", "daemon off;"]
-    build:
-      context: dp/proxy/
-      dockerfile: Dockerfile
-    environment:
-      - PROXY_PASS=proxy.deeppavlov.ai:8018
-      - PORT=8018
-
   combined-classification:
     command: ["nginx", "-g", "daemon off;"]
     build:

diff --git a/assistant_dists/dream_persona_rullama_prompted/dev.yml b/assistant_dists/dream_persona_rullama_prompted/dev.yml
@@ -11,12 +11,6 @@ services:
       - "./common:/src/common"
     ports:
       - 8002:8002
-  badlisted-words-ru:
-    volumes:
-      - "./annotators/BadlistedWordsDetector_ru:/src"
-      - "./common:/src/common"
-    ports:
-      - 8018:8018
   dialogrpt-ru:
     volumes:
       - "./services/dialogrpt_ru:/src"

diff --git a/assistant_dists/dream_persona_rullama_prompted/docker-compose.override.yml b/assistant_dists/dream_persona_rullama_prompted/docker-compose.override.yml
@@ -2,7 +2,7 @@ services:
   agent:
     command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_persona_rullama_prompted/pipeline_conf.json'
     environment:
-      WAIT_HOSTS: "ranking-based-response-selector-ru:8002, badlisted-words-ru:8018,
+      WAIT_HOSTS: "ranking-based-response-selector-ru:8002,
           dialogrpt-ru:8122, prompt-selector-ru:8135, transformers-lm-llama7bru:8149, 
           dff-dream-persona-llama7bru-prompted-skill:8151"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1800}
@@ -34,23 +34,6 @@ services:
         reservations:
           memory: 100M
 
-  badlisted-words-ru:
-    env_file: [ .env_ru ]
-    build:
-      args:
-        SERVICE_PORT: 8018
-        SERVICE_NAME: badlisted_words
-      context: annotators/BadlistedWordsDetector_ru/
-    command: flask run -h 0.0.0.0 -p 8018
-    environment:
-      - FLASK_APP=server
-    deploy:
-      resources:
-        limits:
-          memory: 128M
-        reservations:
-          memory: 128M
-
   dialogrpt-ru:
     env_file: [ .env_ru ]
     build: