configurable shards

ENCODE-DCC · Oct 20, 2020 · f8dcb87 · f8dcb87
1 parent d4e9c77
commit f8dcb87
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 6 deletions.
diff --git a/base.ini b/base.ini
@@ -5,6 +5,8 @@ retry.attempts = 3
 file_upload_bucket = snowflakes-files-dev
 file_upload_profile_name = snowflakes-files-upload
 elasticsearch.server = localhost:9201
+elasticsearch.shards.primary = 1
+elasticsearch.shards.replicate = 0
 aws_ip_ranges_path = %(here)s/aws-ip-ranges.json
 download_proxy = https://download.encodeproject.org/
 annotations_path = %(here)s/annotations.json

diff --git a/src/snovault/elasticsearch/create_mapping.py b/src/snovault/elasticsearch/create_mapping.py
@@ -162,13 +162,13 @@ def schema_mapping(name, schema):
         }
 
 
-def index_settings():
+def index_settings(primary_shards=3, replicate_shards=1):
     return {
         'settings': {
             'index.max_result_window': 99999,
             'index.mapping.total_fields.limit': 5000,
-            'index.number_of_shards': 5,
-            'index.number_of_replicas': 2,
+            'index.number_of_shards': primary_shards,
+            'index.number_of_replicas': replicate_shards,
             'analysis': {
                 'filter': {
                     'substring': {
@@ -325,7 +325,7 @@ def es_mapping(mapping):
                         }
                     },
                 },
-            }
+            },
         ],
         'properties': {
             'uuid': {
@@ -487,6 +487,8 @@ def create_snovault_index_alias(es, indices):
 def run(app, collections=None, dry_run=False):
     index = app.registry.settings['snovault.elasticsearch.index']
     registry = app.registry
+    primary_shards = registry.settings.get('elasticsearch.shards.primary', 3)
+    replicate_shards = registry.settings.get('elasticsearch.shards.replicate', 1)
     if not dry_run:
         es = app.registry[ELASTIC_SEARCH]
         print('CREATE MAPPING RUNNING')
@@ -507,9 +509,9 @@ def run(app, collections=None, dry_run=False):
         if mapping is None:
             continue  # Testing collections
         if dry_run:
-            print(json.dumps(sorted_dict({index: {doc_type: mapping}}), indent=4))
+            print(json.dumps(sorted_dict({index: {collection_name: mapping}}), indent=4))
             continue
-        create_elasticsearch_index(es, index, index_settings())
+        create_elasticsearch_index(es, index, index_settings(primary_shards, replicate_shards))
         set_index_mapping(es, index, doc_type, {doc_type: mapping})
         if collection_name != 'meta':
             indices.append(index)