feat: remove combined cls (#166)

deeppavlov · Dec 20, 2023 · f44a6f5 · f44a6f5
1 parent 62fb573
commit f44a6f5
Show file tree

Hide file tree

Showing 106 changed files with 22 additions and 1,182 deletions.
diff --git a/assistant_dists/ai_faq_assistant/cpu.yml b/assistant_dists/ai_faq_assistant/cpu.yml
@@ -1,8 +1,5 @@
 version: '3.7'
 services:
-  combined-classification:
-    environment:
-      CUDA_VISIBLE_DEVICES: ""
   sentence-ranker:
     environment:
       CUDA_VISIBLE_DEVICES: ""
diff --git a/assistant_dists/ai_faq_assistant/dev.yml b/assistant_dists/ai_faq_assistant/dev.yml
@@ -11,12 +11,6 @@ services:
       - "./common:/src/common"
     ports:
       - 8002:8002
-  combined-classification:
-    volumes:
-      - "./common:/src/common"
-      - "./annotators/combined_classification:/src"
-    ports:
-      - 8087:8087
   sentence-ranker:
     volumes:
       - "./services/sentence_ranker:/src"

diff --git a/assistant_dists/ai_faq_assistant/docker-compose.override.yml b/assistant_dists/ai_faq_assistant/docker-compose.override.yml
@@ -2,7 +2,7 @@ services:
   agent:
     command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/ai_faq_assistant/pipeline_conf.json agent.channel=$AGENT_CHANNEL'
     environment:
-      WAIT_HOSTS: "ranking-based-response-selector:8002, combined-classification:8087, 
+      WAIT_HOSTS: "ranking-based-response-selector:8002,
         sentence-ranker:8128, prompt-selector:8135, openai-api-chatgpt:8145, 
         dff-ai-faq-prompted-skill:8143"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000}
@@ -12,24 +12,6 @@ services:
       LANGUAGE: EN
       FALLBACK_FILE: fallbacks_dream_en.json
 
-  combined-classification:
-    env_file: [ .env ]
-    build:
-      args:
-        CONFIG: combined_classifier.json
-        SERVICE_PORT: 8087
-      context: .
-      dockerfile: ./annotators/combined_classification/Dockerfile
-    command: gunicorn --workers=1 server:app -b 0.0.0.0:8087 --timeout 600
-    environment:
-      - CUDA_VISIBLE_DEVICES=0
-    deploy:
-      resources:
-        limits:
-          memory: 2G
-        reservations:
-          memory: 2G
-
   ranking-based-response-selector:
     env_file: [ .env ]
     build:

diff --git a/assistant_dists/ai_faq_assistant/pipeline_conf.json b/assistant_dists/ai_faq_assistant/pipeline_conf.json
@@ -135,24 +135,6 @@
             }
         },
         "candidate_annotators": {
-            "combined_classification": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 2.0,
-                    "url": "http://combined-classification:8087/batch_model"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:hypothesis_histories_list",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                    "skills"
-                ],
-                "state_manager_method": "add_hypothesis_annotation_batch",
-                "is_enabled": true,
-                "source": {
-                    "component": "components/PbLNvh4hrvs47rPaf2bfYQ.yml",
-                    "service": "annotators/combined_classification/service_configs/combined-classification"
-                }
-            },
             "sentence_ranker": {
                 "connector": {
                     "protocol": "http",

diff --git a/assistant_dists/ai_faq_assistant/proxy.yml b/assistant_dists/ai_faq_assistant/proxy.yml
@@ -1,14 +1,5 @@
 services:
 
-  combined-classification:
-    command: ["nginx", "-g", "daemon off;"]
-    build:
-      context: dp/proxy/
-      dockerfile: Dockerfile
-    environment:
-      - PROXY_PASS=44.211.138.94:8087
-      - PORT=8087
-
   sentence-ranker:
     command: [ "nginx", "-g", "daemon off;" ]
     build:

diff --git a/assistant_dists/document_based_qa/cpu.yml b/assistant_dists/document_based_qa/cpu.yml
@@ -1,9 +1,5 @@
 version: '3.7'
 services:
-  combined-classification:
-    environment:
-      DEVICE: cpu
-      CUDA_VISIBLE_DEVICES: ""
   sentence-ranker:
     environment:
       DEVICE: cpu

diff --git a/assistant_dists/document_based_qa/dev.yml b/assistant_dists/document_based_qa/dev.yml
@@ -15,12 +15,6 @@ services:
       - "./annotators/SentSeg:/src"
     ports:
       - 8011:8011
-  combined-classification:
-    volumes:
-      - "./common:/src/common"
-      - "./annotators/combined_classification:/src"
-    ports:
-      - 8087:8087
   sentence-ranker:
     volumes:
       - "./services/sentence_ranker:/src"

diff --git a/assistant_dists/document_based_qa/docker-compose.override.yml b/assistant_dists/document_based_qa/docker-compose.override.yml
@@ -2,7 +2,7 @@ services:
   agent:
     command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/document_based_qa/pipeline_conf.json agent.channel=$AGENT_CHANNEL'
     environment:
-      WAIT_HOSTS: "sentseg:8011, combined-classification:8087, ranking-based-response-selector:8002, 
+      WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, 
         sentence-ranker:8128, openai-api-chatgpt:8145, doc-processor-from-args:8188, doc-retriever:8165, dff-document-qa-llm-skill:8166"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000}
       HIGH_PRIORITY_INTENTS: 1
@@ -74,24 +74,6 @@ services:
         reservations:
           memory: 5G
 
-  combined-classification:
-    env_file: [ .env ]
-    build:
-      args:
-        CONFIG: combined_classifier.json
-        SERVICE_PORT: 8087
-      context: .
-      dockerfile: ./annotators/combined_classification/Dockerfile
-    command: gunicorn --workers=1 server:app -b 0.0.0.0:8087 --timeout 600
-    environment:
-      - CUDA_VISIBLE_DEVICES=0
-    deploy:
-      resources:
-        limits:
-          memory: 2G
-        reservations:
-          memory: 2G
-
   ranking-based-response-selector:
     env_file: [ .env ]
     build:

diff --git a/assistant_dists/document_based_qa/pipeline_conf.json b/assistant_dists/document_based_qa/pipeline_conf.json
@@ -236,24 +236,6 @@
             }
         },
         "candidate_annotators": {
-            "combined_classification": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 2.0,
-                    "url": "http://combined-classification:8087/batch_model"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:hypothesis_histories_list",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                    "skills"
-                ],
-                "state_manager_method": "add_hypothesis_annotation_batch",
-                "is_enabled": true,
-                "source": {
-                    "component": "components/PbLNvh4hrvs47rPaf2bfYQ.yml",
-                    "service": "annotators/combined_classification/service_configs/combined-classification"
-                }
-            },
             "sentence_ranker": {
                 "connector": {
                     "protocol": "http",

diff --git a/assistant_dists/document_based_qa/proxy.yml b/assistant_dists/document_based_qa/proxy.yml
@@ -1,12 +1,4 @@
 services:
-  combined-classification:
-    command: ["nginx", "-g", "daemon off;"]
-    build:
-      context: dp/proxy/
-      dockerfile: Dockerfile
-    environment:
-      - PROXY_PASS=44.211.138.94:8087
-      - PORT=8087
 
   sentseg:
     command: ["nginx", "-g", "daemon off;"]

diff --git a/assistant_dists/document_based_qa/test.yml b/assistant_dists/document_based_qa/test.yml
@@ -14,12 +14,6 @@ services:
     volumes:
       - "./response_selectors/ranking_based_response_selector:/src"
       - "./common:/src/common"
-  combined-classification:
-    volumes:
-      - "~/.deeppavlov:/root/.deeppavlov"
-      - "~/.deeppavlov/cache:/root/.cache"
-    environment:
-      - CUDA_VISIBLE_DEVICES=8
   sentseg:
   sentence-ranker:
     volumes:

diff --git a/assistant_dists/dream_browser_agent/cpu.yml b/assistant_dists/dream_browser_agent/cpu.yml
@@ -1,8 +1,5 @@
 version: '3.7'
 services:
-  combined-classification:
-    environment:
-      CUDA_VISIBLE_DEVICES: ""
   sentence-ranker:
     environment:
       CUDA_VISIBLE_DEVICES: ""
diff --git a/assistant_dists/dream_browser_agent/dev.yml b/assistant_dists/dream_browser_agent/dev.yml
@@ -17,14 +17,6 @@ services:
     # # you can use persistent local volume if you need
     # volumes:
     #   - ./venv/data/db_data:/root/data/db
-  combined-classification:
-    volumes:
-      - "./common:/src/common"
-      - "./annotators/combined_classification:/src"
-      - "~/.deeppavlov:/root/.deeppavlov"
-      - "~/.deeppavlov/cache:/root/.cache"
-    ports:
-      - 8087:8087
   openai-api-chatgpt-16k:
     volumes:
       - "./services/openai_api_lm:/src"

diff --git a/assistant_dists/dream_browser_agent/docker-compose.override.yml b/assistant_dists/dream_browser_agent/docker-compose.override.yml
@@ -2,8 +2,7 @@ services:
   agent:
     command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_browser_agent/pipeline_conf.json'
     environment:
-      WAIT_HOSTS: "combined-classification:8087,
-          openai-api-chatgpt-16k:8167, openai-api-gpt4:8159, dff-browser-skill:8180, ranking-based-response-selector:8002,
+      WAIT_HOSTS: "openai-api-chatgpt-16k:8167, openai-api-gpt4:8159, dff-browser-skill:8180, ranking-based-response-selector:8002,
           sentence-ranker:8128"
       WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-480}
       HIGH_PRIORITY_INTENTS: 1
@@ -55,24 +54,6 @@ services:
         reservations:
           memory: 100M
 
-  combined-classification:
-    env_file: [ .env ]
-    build:
-      args:
-        SERVICE_PORT: 8087
-        SERVICE_NAME: combined_classification
-        CONFIG: combined_classifier.json
-      context: .
-      dockerfile: ./annotators/combined_classification/Dockerfile
-    environment:
-      - CUDA_VISIBLE_DEVICES=0
-    deploy:
-      resources:
-        limits:
-          memory: 2G
-        reservations:
-          memory: 2G
-
   openai-api-chatgpt-16k:
     env_file: [ .env,.env_azure ]
     build:

diff --git a/assistant_dists/dream_browser_agent/gpu1.yml b/assistant_dists/dream_browser_agent/gpu1.yml
@@ -6,10 +6,6 @@ services:
       - ".:/dp-agent"
     ports:
       - ${AGENT_PORT}:4242
-  combined-classification:
-    restart: unless-stopped
-    environment:
-      - CUDA_VISIBLE_DEVICES=7
   mongo:
     restart: unless-stopped
     command: mongod

diff --git a/assistant_dists/dream_browser_agent/pipeline_conf.json b/assistant_dists/dream_browser_agent/pipeline_conf.json
@@ -35,24 +35,7 @@
                 "service": "services/agent_services/service_configs/dream_browser_agent"
             }
         },
-        "annotators": {
-            "combined_classification": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 3.0,
-                    "url": "http://combined-classification:8087/model"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:preproc_last_human_utt_dialog_w_hist",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [],
-                "state_manager_method": "add_annotation",
-                "is_enabled": true,
-                "source": {
-                    "component": "components/PbLNvh4hrvs47rPaf2bfYQ.yml",
-                    "service": "annotators/combined_classification/service_configs/combined-classification"
-                }
-            }
-        },
+        "annotators": {},
         "response_annotators": {},
         "response_annotator_selectors": {
             "connector": {
@@ -88,24 +71,6 @@
                     "component": "components/XGwmAHtAOu0NDqqG3QCJw.yml",
                     "service": "services/sentence_ranker/service_configs/sentence-ranker"
                 }
-            },
-            "combined_classification": {
-                "connector": {
-                    "protocol": "http",
-                    "timeout": 2.0,
-                    "url": "http://combined-classification:8087/batch_model"
-                },
-                "dialog_formatter": "state_formatters.dp_formatters:hypothesis_histories_list",
-                "response_formatter": "state_formatters.dp_formatters:simple_formatter_service",
-                "previous_services": [
-                    "skills"
-                ],
-                "state_manager_method": "add_hypothesis_annotation_batch",
-                "is_enabled": true,
-                "source": {
-                    "component": "components/PbLNvh4hrvs47rPaf2bfYQ.yml",
-                    "service": "annotators/combined_classification/service_configs/combined-classification"
-                }
             }
         },
         "skill_selectors": {

diff --git a/assistant_dists/dream_browser_agent/proxy.yml b/assistant_dists/dream_browser_agent/proxy.yml
@@ -1,14 +1,5 @@
 services:
 
-  combined-classification:
-    command: ["nginx", "-g", "daemon off;"]
-    build:
-      context: dp/proxy/
-      dockerfile: Dockerfile
-    environment:
-      - PROXY_PASS=44.211.138.94:8087
-      - PORT=8087
-
   sentence-ranker:
     command: [ "nginx", "-g", "daemon off;" ]
     build:

diff --git a/assistant_dists/dream_browser_agent/test.yml b/assistant_dists/dream_browser_agent/test.yml
@@ -4,14 +4,6 @@ services:
       - "/cephfs/home/ignatov/artifacts:/output"
     ports:
       - ${AGENT_PORT}:4242
-  combined-classification:
-    volumes:
-      - "./common:/src/common"
-      - "./annotators/combined_classification:/src"
-      - "~/.deeppavlov:/root/.deeppavlov"
-      - "~/.deeppavlov/cache:/root/.cache"
-    environment:
-      - CUDA_VISIBLE_DEVICES=7
   mongo:
     command: mongod
     image: mongo:4.0.0

diff --git a/assistant_dists/dream_from_scratch_prompted/cpu.yml b/assistant_dists/dream_from_scratch_prompted/cpu.yml
@@ -1,9 +1,5 @@
 version: '3.7'
 services:
-  combined-classification:
-    environment:
-      DEVICE: cpu
-      CUDA_VISIBLE_DEVICES: ""
   sentence-ranker:
     environment:
       DEVICE: cpu

diff --git a/assistant_dists/dream_from_scratch_prompted/dev.yml b/assistant_dists/dream_from_scratch_prompted/dev.yml
@@ -18,12 +18,6 @@ services:
       - "./common:/src/common"
     ports:
       - 8003:8003
-  combined-classification:
-    volumes:
-      - "./common:/src/common"
-      - "./annotators/combined_classification:/src"
-    ports:
-      - 8087:8087
   sentence-ranker:
     volumes:
       - "./services/sentence_ranker:/src"