Zip Archive Abstraction plus Minor tweaks for factor file generator

Common/Data/Auxiliary/FactorFile.cs

@@ -49,6 +49,12 @@ public class FactorFile : IEnumerable<FactorFileRow>
         /// </remarks>
         public DateTime? FactorFileMinimumDate { get; set; }
 
+        /// <summary>
+        /// Gets the most recent factor change in the factor file
+        /// </summary>
+        public DateTime MostRecentFactorChange => SortedFactorFileData.Reverse()
+            .FirstOrDefault(kvp => kvp.Key != Time.EndOfTime).Key;
+
         /// <summary>
         /// Gets the symbol this factor file represents
         /// </summary>
@@ -257,6 +263,12 @@ public void WriteToCsv(Symbol symbol)
         public List<BaseData> GetSplitsAndDividends(Symbol symbol, SecurityExchangeHours exchangeHours)
         {
             var dividendsAndSplits = new List<BaseData>();
+            if (SortedFactorFileData.Count == 0)
+            {
+                Log.Trace($"{symbol} has no factors!");
+                return dividendsAndSplits;
+            }
+
             var futureFactorFileRow = SortedFactorFileData.Last().Value;
             for (var i = SortedFactorFileData.Count - 2; i >= 0 ; i--)
             {

Common/Data/Auxiliary/FactorFileRow.cs

@@ -180,15 +180,15 @@ public FactorFileRow Apply(Split split, SecurityExchangeHours exchangeHours)
                 throw new ArgumentException("Unable to apply split with reference price of zero.");
             }
 
+            var previousTradingDay = exchangeHours.GetPreviousTradingDay(split.Time);
+
             // this instance must be chronologically at or in front of the split
             // this is because the factors are defined working from current to past
-            if (Date < split.Time.Date)
+            if (Date < previousTradingDay)
             {
                 throw new ArgumentException($"Factor file row date '{Date:yyy-MM-dd}' is before split date '{split.Time.Date:yyyy-MM-dd}'.");
             }
 
-            var previousTradingDay = exchangeHours.GetPreviousTradingDay(split.Time);
-
             return new FactorFileRow(
                 previousTradingDay,
                 PriceFactor,
@@ -267,9 +267,10 @@ public static FactorFileRow Parse(string line)
         /// <summary>
         /// Writes this row to csv format
         /// </summary>
-        public string ToCsv()
+        public string ToCsv(string source = null)
         {
-            return $"{Date.ToString(DateFormat.EightCharacter)},{PriceFactor.Normalize()},{SplitFactor.Normalize()},{ReferencePrice.Normalize()}";
+            source = source == null ? "" : $",{source}";
+            return $"{Date.ToString(DateFormat.EightCharacter)},{PriceFactor.Normalize()},{SplitFactor.Normalize()},{ReferencePrice.Normalize()}{source}";
         }
 
         /// <summary>

Common/Data/Auxiliary/MapFile.cs

@@ -36,7 +36,7 @@ public class MapFile : IEnumerable<MapFileRow>
         /// <summary>
         /// Gets the entity's unique symbol, i.e OIH.1
         /// </summary>
-        public string Permtick { get; private set; }
+        public string Permtick { get; }
 
         /// <summary>
         /// Gets the last date in the map file which is indicative of a delisting event
@@ -54,13 +54,35 @@ public DateTime FirstDate
             get { return _data.Keys.Count == 0 ? Time.BeginningOfTime : _data.Keys.First(); }
         }
 
+        /// <summary>
+        /// Gets the first ticker for the security represented by this map file
+        /// </summary>
+        public string FirstTicker { get; }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="MapFile"/> class.
         /// </summary>
         public MapFile(string permtick, IEnumerable<MapFileRow> data)
         {
             Permtick = permtick.ToUpper();
             _data = new SortedDictionary<DateTime, MapFileRow>(data.Distinct().ToDictionary(x => x.Date));
+
+            var firstTicker = GetMappedSymbol(FirstDate, Permtick);
+            if (char.IsDigit(firstTicker.Last()))
+            {
+                var dotIndex = firstTicker.LastIndexOf(".", StringComparison.Ordinal);
+                if (dotIndex > 0)
+                {
+                    int value;
+                    var number = firstTicker.Substring(dotIndex, firstTicker.Length - dotIndex - 1);
+                    if (int.TryParse(number, out value))
+                    {
+                        firstTicker = firstTicker.Substring(0, dotIndex);
+                    }
+                }
+            }
+
+            FirstTicker = firstTicker;
         }
 
         /// <summary>

Common/Data/Market/Dividend.cs

@@ -39,7 +39,7 @@ public decimal Distribution
         public decimal ReferencePrice
         {
             get;
-            private set;
+            set;
         }
 
         /// <summary>

Common/Util/StreamReaderEnumerable.cs

@@ -24,7 +24,7 @@ namespace QuantConnect.Util
     /// <summary>
     /// Converts a <see cref="StreamReader"/> into an enumerable of string
     /// </summary>
-    public class StreamReaderEnumerable : IEnumerable<string>
+    public class StreamReaderEnumerable : IEnumerable<string>, IDisposable
     {
         private int _createdEnumerator;
         private readonly StreamReader _reader;
@@ -71,7 +71,6 @@ IEnumerator IEnumerable.GetEnumerator()
 
         private class Enumerator : IEnumerator<string>
         {
-            private string _current;
             private readonly StreamReader _reader;
 
             public Enumerator(StreamReader reader)
@@ -92,7 +91,7 @@ public bool MoveNext()
                     return false;
                 }
 
-                _current = line;
+                Current = line;
                 return true;
             }
 
@@ -106,13 +105,16 @@ public void Reset()
                 _reader.BaseStream.Seek(0, SeekOrigin.Begin);
             }
 
-            public string Current
-            {
-                get { return _current; }
-                private set { _current = value; }
-            }
+            public string Current { get; private set; }
 
             object IEnumerator.Current => Current;
         }
+
+        /// <summary>Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.</summary>
+        /// <filterpriority>2</filterpriority>
+        public void Dispose()
+        {
+            _reader.DisposeSafely();
+        }
     }
 }

Compression/Archives/Archive.cs

@@ -0,0 +1,125 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.IO;
+using System.IO.Compression;
+using SharpZipLibZipFile = ICSharpCode.SharpZipLib.Zip.ZipFile;
+using IonicZipFile = Ionic.Zip.ZipFile;
+using SharpZipLibZipOutputStream = ICSharpCode.SharpZipLib.Zip.ZipOutputStream;
+
+namespace QuantConnect.Archives
+{
+    /// <summary>
+    /// Provides methods for easily creating archive instances using different underlying implementations
+    /// </summary>
+    public static class Archive
+    {
+        /// <summary>
+        /// Opens the archive at the specified path
+        /// </summary>
+        /// <param name="path">The file path to the archive</param>
+        /// <param name="impl">The archive implementation to use</param>
+        /// <returns>The archive</returns>
+        public static IArchive OpenReadOnly(string path, ArchiveImplementation impl = ArchiveImplementation.DotNetFramework)
+        {
+            var extension = Path.GetExtension(path);
+            if (!string.Equals(extension, ".zip", StringComparison.InvariantCultureIgnoreCase))
+            {
+                throw new NotImplementedException($"Archive file with extension '{extension}' is not implemented.");
+            }
+
+            if (!File.Exists(path))
+            {
+                throw new FileNotFoundException($"Archive file was not found at {new FileInfo(path).FullName}");
+            }
+
+            var fileStream = new Lazy<FileStream>(() => new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read));
+
+            switch (impl)
+            {
+                case ArchiveImplementation.DotNetFramework:
+                    return new DotNetFrameworkZipArchive(new ZipArchive(fileStream.Value));
+
+                case ArchiveImplementation.SharpZipLib:
+                    return new SharpZipLibArchive(new SharpZipLibZipFile(fileStream.Value));
+
+                case ArchiveImplementation.Ionic:
+                    return new IonicZipArchive(new IonicZipFile(path));
+
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(impl), impl, null);
+            }
+        }
+
+        /// <summary>
+        /// Opens the archive at the specified path for read/write
+        /// </summary>
+        /// <param name="path">The file path to the archive</param>
+        /// <param name="impl">The archive implementation to use</param>
+        /// <returns>The archive</returns>
+        public static IArchive OpenWrite(string path, ArchiveImplementation impl = ArchiveImplementation.DotNetFramework)
+        {
+            var extension = Path.GetExtension(path);
+            if (!string.Equals(extension, ".zip", StringComparison.InvariantCultureIgnoreCase))
+            {
+                throw new NotImplementedException($"Archive file with extension '{extension}' is not implemented.");
+            }
+
+            if (File.Exists(path))
+            {
+                switch (impl)
+                {
+                    case ArchiveImplementation.DotNetFramework:
+                        return new DotNetFrameworkZipArchive(new ZipArchive(
+                            new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None), ZipArchiveMode.Update)
+                        );
+
+                    case ArchiveImplementation.SharpZipLib:
+                        return new SharpZipLibArchive(new SharpZipLibZipOutputStream(
+                            new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None))
+                        );
+
+                    case ArchiveImplementation.Ionic:
+                        return new IonicZipArchive(IonicZipFile.Read(path));
+
+                    default:
+                        throw new ArgumentOutOfRangeException(nameof(impl), impl, null);
+                }
+            }
+            else
+            {
+                switch (impl)
+                {
+                    case ArchiveImplementation.DotNetFramework:
+                        return new DotNetFrameworkZipArchive(new ZipArchive(
+                            new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None), ZipArchiveMode.Update)
+                        );
+
+                    case ArchiveImplementation.SharpZipLib:
+                        return new SharpZipLibArchive(new SharpZipLibZipOutputStream(
+                            new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
+                        );
+
+                    case ArchiveImplementation.Ionic:
+                        return new IonicZipArchive(new IonicZipFile(path));
+
+                    default:
+                        throw new ArgumentOutOfRangeException(nameof(impl), impl, null);
+                }
+            }
+        }
+    }
+}