Merge bytebuffer and memory stores into a single memory store options, …

…closes #22.
elastic · Feb 17, 2010 · 8727815 · 8727815
1 parent 042d710
commit 8727815
Show file tree

Hide file tree

Showing 17 changed files with 77 additions and 103 deletions.
diff --git a/...k/micro/src/main/java/org/elasticsearch/benchmark/index/engine/SimpleEngineBenchmark.java b/...k/micro/src/main/java/org/elasticsearch/benchmark/index/engine/SimpleEngineBenchmark.java
@@ -36,7 +36,7 @@
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.Store;
-import org.elasticsearch.index.store.memory.MemoryStore;
+import org.elasticsearch.index.store.memory.ByteBufferStore;
 import org.elasticsearch.index.translog.memory.MemoryTranslog;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.dynamic.DynamicThreadPool;
@@ -277,7 +277,8 @@ public static void main(String[] args) throws Exception {
         Settings settings = EMPTY_SETTINGS;
 
 //        Store store = new RamStore(shardId, settings);
-        Store store = new MemoryStore(shardId, settings);
+        Store store = new ByteBufferStore(shardId, settings);
+//        Store store = new HeapStore(shardId, settings);
 //        Store store = new NioFsStore(shardId, settings);
 
         store.deleteContent();

diff --git a/...ark/micro/src/main/java/org/elasticsearch/benchmark/index/store/SimpleStoreBenchmark.java b/...ark/micro/src/main/java/org/elasticsearch/benchmark/index/store/SimpleStoreBenchmark.java
@@ -25,11 +25,11 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.Store;
-import org.elasticsearch.index.store.bytebuffer.ByteBufferStore;
 import org.elasticsearch.index.store.fs.MmapFsStore;
 import org.elasticsearch.index.store.fs.NioFsStore;
 import org.elasticsearch.index.store.fs.SimpleFsStore;
-import org.elasticsearch.index.store.memory.MemoryStore;
+import org.elasticsearch.index.store.memory.ByteBufferStore;
+import org.elasticsearch.index.store.memory.HeapStore;
 import org.elasticsearch.index.store.ram.RamStore;
 import org.elasticsearch.util.SizeUnit;
 import org.elasticsearch.util.SizeValue;
@@ -276,23 +276,17 @@ public static void main(String[] args) throws Exception {
             store = new NioFsStore(shardId, settings, environment, localNodeId);
         } else if (type.equalsIgnoreCase("nio-fs")) {
             store = new MmapFsStore(shardId, settings, environment, localNodeId);
-        } else if (type.equalsIgnoreCase("bb")) {
-            Settings byteBufferSettings = settingsBuilder()
-                    .putAll(settings)
-                    .putBoolean("index.store.bytebuffer.direct", false)
-                    .build();
-            store = new ByteBufferStore(shardId, byteBufferSettings);
-        } else if (type.equalsIgnoreCase("bb-direct")) {
+        } else if (type.equalsIgnoreCase("memory-direct")) {
             Settings byteBufferSettings = settingsBuilder()
                     .putAll(settings)
                     .putBoolean("index.store.bytebuffer.direct", true)
                     .build();
             store = new ByteBufferStore(shardId, byteBufferSettings);
-        } else if (type.equalsIgnoreCase("mem")) {
+        } else if (type.equalsIgnoreCase("memory-heap")) {
             Settings memorySettings = settingsBuilder()
                     .putAll(settings)
                     .build();
-            store = new MemoryStore(shardId, memorySettings);
+            store = new HeapStore(shardId, memorySettings);
         } else {
             throw new IllegalArgumentException("No type store [" + type + "]");
         }

diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/StoreModule.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/StoreModule.java
@@ -21,7 +21,6 @@
 
 import com.google.inject.AbstractModule;
 import com.google.inject.Module;
-import org.elasticsearch.index.store.bytebuffer.ByteBufferStoreModule;
 import org.elasticsearch.index.store.fs.MmapFsStoreModule;
 import org.elasticsearch.index.store.fs.NioFsStoreModule;
 import org.elasticsearch.index.store.fs.SimpleFsStoreModule;
@@ -52,8 +51,6 @@ public StoreModule(Settings settings) {
             storeModule = RamStoreModule.class;
         } else if ("memory".equalsIgnoreCase(storeType)) {
             storeModule = MemoryStoreModule.class;
-        } else if ("bytebuffer".equalsIgnoreCase(storeType)) {
-            storeModule = ByteBufferStoreModule.class;
         } else if ("fs".equalsIgnoreCase(storeType)) {
             // nothing to set here ... (we default to fs)
         } else if ("simplefs".equalsIgnoreCase(storeType)) {

diff --git a/...csearch/src/main/java/org/elasticsearch/index/store/bytebuffer/ByteBufferStoreModule.java b/...csearch/src/main/java/org/elasticsearch/index/store/bytebuffer/ByteBufferStoreModule.java
diff --git a/...store/bytebuffer/ByteBufferDirectory.java → ...dex/store/memory/ByteBufferDirectory.java b/...store/bytebuffer/ByteBufferDirectory.java → ...dex/store/memory/ByteBufferDirectory.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.store.bytebuffer;
+package org.elasticsearch.index.store.memory;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
@@ -197,14 +197,14 @@ ByteBuffer acquireBuffer() {
         return byteBuffer;
     }
 
-    ByteBuffer createBuffer() {
+    private ByteBuffer createBuffer() {
         if (isDirect()) {
             return ByteBuffer.allocateDirect(bufferSizeInBytes());
         }
         return ByteBuffer.allocate(bufferSizeInBytes());
     }
 
-    void closeBuffer(ByteBuffer byteBuffer) {
+    private void closeBuffer(ByteBuffer byteBuffer) {
         if (isDirect()) {
             ((DirectBuffer) byteBuffer).cleaner().clean();
         }

diff --git a/...ndex/store/bytebuffer/ByteBufferFile.java → ...ch/index/store/memory/ByteBufferFile.java b/...ndex/store/bytebuffer/ByteBufferFile.java → ...ch/index/store/memory/ByteBufferFile.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.store.bytebuffer;
+package org.elasticsearch.index.store.memory;
 
 import java.nio.ByteBuffer;
 

diff --git a/...tore/bytebuffer/ByteBufferIndexInput.java → ...ex/store/memory/ByteBufferIndexInput.java b/...tore/bytebuffer/ByteBufferIndexInput.java → ...ex/store/memory/ByteBufferIndexInput.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.store.bytebuffer;
+package org.elasticsearch.index.store.memory;
 
 import org.apache.lucene.store.IndexInput;
 

diff --git a/...ore/bytebuffer/ByteBufferIndexOutput.java → ...x/store/memory/ByteBufferIndexOutput.java b/...ore/bytebuffer/ByteBufferIndexOutput.java → ...x/store/memory/ByteBufferIndexOutput.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.store.bytebuffer;
+package org.elasticsearch.index.store.memory;
 
 import org.apache.lucene.store.IndexOutput;
 

diff --git a/...dex/store/bytebuffer/ByteBufferStore.java → ...h/index/store/memory/ByteBufferStore.java b/...dex/store/bytebuffer/ByteBufferStore.java → ...h/index/store/memory/ByteBufferStore.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.store.bytebuffer;
+package org.elasticsearch.index.store.memory;
 
 import com.google.inject.Inject;
 import org.elasticsearch.index.settings.IndexSettings;

diff --git a/...h/index/store/memory/MemoryDirectory.java → ...rch/index/store/memory/HeapDirectory.java b/...h/index/store/memory/MemoryDirectory.java → ...rch/index/store/memory/HeapDirectory.java
@@ -37,9 +37,9 @@
 /**
  * @author kimchy (Shay Banon)
  */
-public class MemoryDirectory extends Directory {
+public class HeapDirectory extends Directory {
 
-    private final Map<String, MemoryFile> files = newConcurrentMap();
+    private final Map<String, HeapRamFile> files = newConcurrentMap();
 
     private final Queue<byte[]> cache;
 
@@ -51,11 +51,11 @@ public class MemoryDirectory extends Directory {
 
     private final boolean disableCache;
 
-    public MemoryDirectory() {
+    public HeapDirectory() {
         this(new SizeValue(1, SizeUnit.KB), new SizeValue(20, SizeUnit.MB), false);
     }
 
-    public MemoryDirectory(SizeValue bufferSize, SizeValue cacheSize, boolean warmCache) {
+    public HeapDirectory(SizeValue bufferSize, SizeValue cacheSize, boolean warmCache) {
         disableCache = cacheSize.bytes() == 0;
         if (!disableCache && cacheSize.bytes() < bufferSize.bytes()) {
             throw new IllegalArgumentException("Cache size [" + cacheSize + "] is smaller than buffer size [" + bufferSize + "]");
@@ -94,14 +94,14 @@ int bufferSizeInBytes() {
     }
 
     @Override public long fileModified(String name) throws IOException {
-        MemoryFile file = files.get(name);
+        HeapRamFile file = files.get(name);
         if (file == null)
             throw new FileNotFoundException(name);
         return file.lastModified();
     }
 
     @Override public void touchFile(String name) throws IOException {
-        MemoryFile file = files.get(name);
+        HeapRamFile file = files.get(name);
         if (file == null)
             throw new FileNotFoundException(name);
 
@@ -122,33 +122,33 @@ int bufferSizeInBytes() {
     }
 
     @Override public void deleteFile(String name) throws IOException {
-        MemoryFile file = files.remove(name);
+        HeapRamFile file = files.remove(name);
         if (file == null)
             throw new FileNotFoundException(name);
         file.clean();
     }
 
     @Override public long fileLength(String name) throws IOException {
-        MemoryFile file = files.get(name);
+        HeapRamFile file = files.get(name);
         if (file == null)
             throw new FileNotFoundException(name);
         return file.length();
     }
 
     @Override public IndexOutput createOutput(String name) throws IOException {
-        MemoryFile file = new MemoryFile(this);
-        MemoryFile existing = files.put(name, file);
+        HeapRamFile file = new HeapRamFile(this);
+        HeapRamFile existing = files.put(name, file);
         if (existing != null) {
             existing.clean();
         }
-        return new MemoryIndexOutput(this, file);
+        return new HeapIndexOutput(this, file);
     }
 
     @Override public IndexInput openInput(String name) throws IOException {
-        MemoryFile file = files.get(name);
+        HeapRamFile file = files.get(name);
         if (file == null)
             throw new FileNotFoundException(name);
-        return new MemoryIndexInput(this, file);
+        return new HeapIndexInput(this, file);
     }
 
     @Override public void close() throws IOException {

diff --git a/.../index/store/memory/MemoryIndexInput.java → ...ch/index/store/memory/HeapIndexInput.java b/.../index/store/memory/MemoryIndexInput.java → ...ch/index/store/memory/HeapIndexInput.java
@@ -26,10 +26,10 @@
 /**
  * @author kimchy (Shay Banon)
  */
-public class MemoryIndexInput extends IndexInput {
+public class HeapIndexInput extends IndexInput {
 
     private final int bufferSize;
-    private final MemoryFile file;
+    private final HeapRamFile file;
 
     private long length;
 
@@ -40,7 +40,7 @@ public class MemoryIndexInput extends IndexInput {
     private long bufferStart;
     private int bufferLength;
 
-    public MemoryIndexInput(MemoryDirectory dir, MemoryFile file) throws IOException {
+    public HeapIndexInput(HeapDirectory dir, HeapRamFile file) throws IOException {
         this.bufferSize = dir.bufferSizeInBytes();
         this.file = file;
 

diff --git a/...index/store/memory/MemoryIndexOutput.java → ...h/index/store/memory/HeapIndexOutput.java b/...index/store/memory/MemoryIndexOutput.java → ...h/index/store/memory/HeapIndexOutput.java
@@ -27,10 +27,10 @@
 /**
  * @author kimchy (Shay Banon)
  */
-public class MemoryIndexOutput extends IndexOutput {
+public class HeapIndexOutput extends IndexOutput {
 
-    private final MemoryDirectory dir;
-    private final MemoryFile file;
+    private final HeapDirectory dir;
+    private final HeapRamFile file;
 
     private ArrayList<byte[]> buffers = new ArrayList<byte[]>();
 
@@ -41,7 +41,7 @@ public class MemoryIndexOutput extends IndexOutput {
     private long bufferStart;
     private int bufferLength;
 
-    public MemoryIndexOutput(MemoryDirectory dir, MemoryFile file) {
+    public HeapIndexOutput(HeapDirectory dir, HeapRamFile file) {
         this.dir = dir;
         this.file = file;
 

diff --git a/...search/index/store/memory/MemoryFile.java → ...earch/index/store/memory/HeapRamFile.java b/...search/index/store/memory/MemoryFile.java → ...earch/index/store/memory/HeapRamFile.java
@@ -22,17 +22,17 @@
 /**
  * @author kimchy (Shay Banon)
  */
-public class MemoryFile {
+public class HeapRamFile {
 
-    private final MemoryDirectory dir;
+    private final HeapDirectory dir;
 
     private volatile long lastModified = System.currentTimeMillis();
 
     private volatile long length;
 
     private volatile byte[][] buffers;
 
-    public MemoryFile(MemoryDirectory dir) {
+    public HeapRamFile(HeapDirectory dir) {
         this.dir = dir;
     }
 

diff --git a/...earch/index/store/memory/MemoryStore.java → ...csearch/index/store/memory/HeapStore.java b/...earch/index/store/memory/MemoryStore.java → ...csearch/index/store/memory/HeapStore.java
@@ -30,29 +30,29 @@
 /**
  * @author kimchy (Shay Banon)
  */
-public class MemoryStore extends AbstractStore<MemoryDirectory> {
+public class HeapStore extends AbstractStore<HeapDirectory> {
 
     private final SizeValue bufferSize;
 
     private final SizeValue cacheSize;
 
     private final boolean warmCache;
 
-    private MemoryDirectory directory;
+    private HeapDirectory directory;
 
-    @Inject public MemoryStore(ShardId shardId, @IndexSettings Settings indexSettings) {
+    @Inject public HeapStore(ShardId shardId, @IndexSettings Settings indexSettings) {
         super(shardId, indexSettings);
 
         this.bufferSize = componentSettings.getAsSize("bufferSize", new SizeValue(1, SizeUnit.KB));
         this.cacheSize = componentSettings.getAsSize("cacheSize", new SizeValue(20, SizeUnit.MB));
         this.warmCache = componentSettings.getAsBoolean("warmCache", true);
 
-        this.directory = new MemoryDirectory(bufferSize, cacheSize, warmCache);
+        this.directory = new HeapDirectory(bufferSize, cacheSize, warmCache);
         logger.debug("Using [Memory] Store with bufferSize[{}], cacheSize[{}], warmCache[{}]",
                 new Object[]{directory.bufferSize(), directory.cacheSize(), warmCache});
     }
 
-    @Override public MemoryDirectory directory() {
+    @Override public HeapDirectory directory() {
         return directory;
     }
 

diff --git a/...s/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/MemoryStoreModule.java b/...s/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/MemoryStoreModule.java
@@ -20,14 +20,29 @@
 package org.elasticsearch.index.store.memory;
 
 import com.google.inject.AbstractModule;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.index.store.Store;
+import org.elasticsearch.util.settings.Settings;
 
 /**
  * @author kimchy (Shay Banon)
  */
 public class MemoryStoreModule extends AbstractModule {
 
+    private final Settings settings;
+
+    public MemoryStoreModule(Settings settings) {
+        this.settings = settings;
+    }
+
     @Override protected void configure() {
-        bind(Store.class).to(MemoryStore.class).asEagerSingleton();
+        String location = settings.get("index.store.memory.location", "direct");
+        if ("direct".equalsIgnoreCase(location)) {
+            bind(Store.class).to(ByteBufferStore.class).asEagerSingleton();
+        } else if ("heap".equalsIgnoreCase(location)) {
+            bind(Store.class).to(HeapStore.class).asEagerSingleton();
+        } else {
+            throw new ElasticSearchIllegalArgumentException("Memory location [" + location + "] is invalid, can be one of [direct,heap]");
+        }
     }
 }