diff --git a/org.eclipse.lemminx/src/main/java/org/eclipse/lemminx/extensions/contentmodel/uriresolver/XMLCacheResolverExtension.java b/org.eclipse.lemminx/src/main/java/org/eclipse/lemminx/extensions/contentmodel/uriresolver/XMLCacheResolverExtension.java
index 1583a6789..b79ad2b87 100644
--- a/org.eclipse.lemminx/src/main/java/org/eclipse/lemminx/extensions/contentmodel/uriresolver/XMLCacheResolverExtension.java
+++ b/org.eclipse.lemminx/src/main/java/org/eclipse/lemminx/extensions/contentmodel/uriresolver/XMLCacheResolverExtension.java
@@ -13,6 +13,7 @@
 package org.eclipse.lemminx.extensions.contentmodel.uriresolver;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
@@ -32,6 +33,30 @@
  */
 public class XMLCacheResolverExtension implements URIResolverExtension {
 
+	private static class XMLFileInputSource extends XMLInputSource {
+
+		private final Path file;
+
+		public XMLFileInputSource(XMLResourceIdentifier resourceIdentifier, Path file) {
+			super(resourceIdentifier);
+			this.file = file;
+		}
+
+		@Override
+		public InputStream getByteStream() {
+			// Load the file input stream only if it is used
+			InputStream input = super.getByteStream();
+			if (input == null) {
+				try {
+					super.setByteStream(Files.newInputStream(file));
+				} catch (IOException e) {
+					throw new RuntimeException(e);
+				}
+			}
+			return super.getByteStream();
+		}
+	}
+
 	private final CacheResourcesManager cacheResourcesManager;
 
 	public XMLCacheResolverExtension() {
@@ -53,9 +78,7 @@ public XMLInputSource resolveEntity(XMLResourceIdentifier resourceIdentifier) th
 		Path file = getCachedResource(url);
 		if (file != null) {
 			// The resource was downloaded locally, use it.
-			XMLInputSource source = new XMLInputSource(resourceIdentifier);
-			source.setByteStream(Files.newInputStream(file));
-			return source;
+			return new XMLFileInputSource(resourceIdentifier, file);
 		}
 		return null;
 	}