From a9571d443fb515c81d8872d565b87df5fb32dd82 Mon Sep 17 00:00:00 2001 From: Killer Date: Wed, 10 Jun 2020 16:59:32 +0800 Subject: [PATCH 1/9] add DataBlock add DbConfig --- maker/csharp/.gitignore | 341 ++++++++++++++++++ maker/csharp/DbMaker/DataBlock.cs | 47 +++ maker/csharp/DbMaker/DbConfig.cs | 49 +++ maker/csharp/DbMaker/DbMaker.csproj | 8 + .../csharp/DbMaker/DbMakerConfigException.cs | 14 + maker/csharp/DbMaker/Program.cs | 12 + 6 files changed, 471 insertions(+) create mode 100644 maker/csharp/.gitignore create mode 100644 maker/csharp/DbMaker/DataBlock.cs create mode 100644 maker/csharp/DbMaker/DbConfig.cs create mode 100644 maker/csharp/DbMaker/DbMaker.csproj create mode 100644 maker/csharp/DbMaker/DbMakerConfigException.cs create mode 100644 maker/csharp/DbMaker/Program.cs diff --git a/maker/csharp/.gitignore b/maker/csharp/.gitignore new file mode 100644 index 00000000..3e8a1553 --- /dev/null +++ b/maker/csharp/.gitignore @@ -0,0 +1,341 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ +# ASP.NET Core default setup: bower directory is configured as wwwroot/lib/ and bower restore is true +**/wwwroot/lib/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# BeatPulse healthcheck temp database +healthchecksdb diff --git a/maker/csharp/DbMaker/DataBlock.cs b/maker/csharp/DbMaker/DataBlock.cs new file mode 100644 index 00000000..3c81edb6 --- /dev/null +++ b/maker/csharp/DbMaker/DataBlock.cs @@ -0,0 +1,47 @@ +using System; + +namespace DbMaker +{ + /// + /// data block class + /// + public class DataBlock + { + /// + /// construct method + /// + /// + /// + /// + public DataBlock(int cityId, string region, int dataPtr) + { + CityId = cityId; + Region = region; + DataPtr = dataPtr; + } + + public DataBlock(int cityId, string region) : this(cityId, region, 0) + { + } + + /// + /// city id + /// + public int CityId { get; set; } + + /// + /// region address + /// + public string Region { get; set; } + + /// + /// region ptr in the db file + /// + public int DataPtr { get; set; } + + public override string ToString() + { + return String.Join("|", this.CityId, this.Region, this.DataPtr); + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/DbConfig.cs b/maker/csharp/DbMaker/DbConfig.cs new file mode 100644 index 00000000..0f14752a --- /dev/null +++ b/maker/csharp/DbMaker/DbConfig.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace DbMaker +{ + /// + /// database configuration class + /// + public class DbConfig + { + public DbConfig(int totalHeaderSize) + { + if (totalHeaderSize % 8 != 0) + { + throw new DbMakerConfigException("totalHeaderSize must be times of 8"); + } + + TotalHeaderSize = totalHeaderSize; + IndexBlockSize = 8192; //4*2048 + } + + public DbConfig() : this(8 * 2048) + { + + } + + /// + /// total header data block size + /// + public int TotalHeaderSize { get; set; } + /// + /// max index data block size + /// u should always choice the fastest read block size + /// + public int IndexBlockSize { get; set; } + + public DbConfig SetTotalHeaderSize(int totalHeaderSize) + { + this.TotalHeaderSize = totalHeaderSize; + return this; + } + public DbConfig SetIndexBlockSize(int dataBlockSize) + { + this.IndexBlockSize = dataBlockSize; + return this; + } + } +} diff --git a/maker/csharp/DbMaker/DbMaker.csproj b/maker/csharp/DbMaker/DbMaker.csproj new file mode 100644 index 00000000..c73e0d16 --- /dev/null +++ b/maker/csharp/DbMaker/DbMaker.csproj @@ -0,0 +1,8 @@ + + + + Exe + netcoreapp3.1 + + + diff --git a/maker/csharp/DbMaker/DbMakerConfigException.cs b/maker/csharp/DbMaker/DbMakerConfigException.cs new file mode 100644 index 00000000..5bc49794 --- /dev/null +++ b/maker/csharp/DbMaker/DbMakerConfigException.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace DbMaker +{ + class DbMakerConfigException : Exception + { + public DbMakerConfigException(String message) : base(message) + { + + } + } +} diff --git a/maker/csharp/DbMaker/Program.cs b/maker/csharp/DbMaker/Program.cs new file mode 100644 index 00000000..97138e2d --- /dev/null +++ b/maker/csharp/DbMaker/Program.cs @@ -0,0 +1,12 @@ +using System; + +namespace DbMaker +{ + class Program + { + static void Main(string[] args) + { + Console.WriteLine("Hello World!"); + } + } +} From bf077253425ea715f7da80427e62eccc518fc886 Mon Sep 17 00:00:00 2001 From: Killer Date: Wed, 10 Jun 2020 17:23:01 +0800 Subject: [PATCH 2/9] add HeaderBlock.cs add IndexBlock.cs add Util.cs --- maker/csharp/DbMaker/HeaderBlock.cs | 57 +++++++++++ maker/csharp/DbMaker/IndexBlock.cs | 92 ++++++++++++++++++ maker/csharp/DbMaker/Util.cs | 143 ++++++++++++++++++++++++++++ 3 files changed, 292 insertions(+) create mode 100644 maker/csharp/DbMaker/HeaderBlock.cs create mode 100644 maker/csharp/DbMaker/IndexBlock.cs create mode 100644 maker/csharp/DbMaker/Util.cs diff --git a/maker/csharp/DbMaker/HeaderBlock.cs b/maker/csharp/DbMaker/HeaderBlock.cs new file mode 100644 index 00000000..b561e0f3 --- /dev/null +++ b/maker/csharp/DbMaker/HeaderBlock.cs @@ -0,0 +1,57 @@ +namespace DbMaker +{ + /// + /// header block class + /// + public class HeaderBlock + { + public HeaderBlock(long indexStartIp, int indexPtr) + { + IndexStartIp = indexStartIp; + IndexPtr = indexPtr; + } + + /// + /// index block start ip address + /// + public long IndexStartIp { get; set; } + + /// + /// ip address + /// + public int IndexPtr { get; set; } + + public HeaderBlock SetIndexStartIp(long indexStartIp) + { + IndexStartIp = indexStartIp; + return this; + } + + public HeaderBlock SetIndexPtr(int indexPtr) + { + IndexPtr = indexPtr; + return this; + } + + /// + /// get the bytes for db storage + /// + /// + public byte[] GetBytes() + { + /* + * +------------+-----------+ + * | 4bytes | 4bytes | + * +------------+-----------+ + * start ip index ptr + */ + + var b = new byte[8]; + + Util.writeIntLong(b, 0, IndexStartIp); + Util.writeIntLong(b, 4, IndexPtr); + + return b; + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/IndexBlock.cs b/maker/csharp/DbMaker/IndexBlock.cs new file mode 100644 index 00000000..b456126e --- /dev/null +++ b/maker/csharp/DbMaker/IndexBlock.cs @@ -0,0 +1,92 @@ +namespace DbMaker +{ + /// + /// item index class + /// + public class IndexBlock + { + private static int LENGTH = 12; + + public IndexBlock(long startIp, long endIp, int dataPtr, int dataLen) + { + StartIp = startIp; + EndIp = endIp; + DataPtr = dataPtr; + DataLen = dataLen; + } + + /** + * start ip address + */ + public long StartIp { get; set; } + + /** + * end ip address + */ + public long EndIp { get; set; } + + /** + * data ptr and data length + */ + public int DataPtr { get; set; } + + /** + * data length + */ + public int DataLen { get; set; } + + public IndexBlock SetStartIp(long startIp) + { + StartIp = startIp; + return this; + } + + public IndexBlock SetEndIp(long endIp) + { + EndIp = endIp; + return this; + } + + public IndexBlock SetDataPtr(int dataPtr) + { + DataPtr = dataPtr; + return this; + } + + public IndexBlock SetDataLen(int dataLen) + { + DataLen = dataLen; + return this; + } + + public static int GetIndexBlockLength() + { + return LENGTH; + } + + /** + * get the bytes for storage + * + * @return byte[] + */ + public byte[] GetBytes() + { + /* + * +------------+-----------+-----------+ + * | 4bytes | 4bytes | 4bytes | + * +------------+-----------+-----------+ + * start ip end ip data ptr + len + */ + var b = new byte[12]; + + Util.writeIntLong(b, 0, StartIp); //start ip + Util.writeIntLong(b, 4, EndIp); //end ip + + //write the data ptr and the length + var mix = DataPtr | DataLen << 24 & 0xFF000000L; + Util.writeIntLong(b, 8, mix); + + return b; + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/Util.cs b/maker/csharp/DbMaker/Util.cs new file mode 100644 index 00000000..60e493ed --- /dev/null +++ b/maker/csharp/DbMaker/Util.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Text; + +namespace DbMaker +{ + /// + /// util class + /// + public static class Util + { + /** + * write specfield bytes to a byte array start from offset + * + * @param b + * @param offset + * @param v + * @param bytes + */ + public static void write(byte[] b, int offset, long v, int bytes) + { + for (int i = 0; i < bytes; i++) + { + b[offset++] = (byte)((v >> (8 * i)) & 0xFF); + } + } + + /** + * write a int to a byte array + * + * @param b + * @param offet + * @param v + */ + public static void writeIntLong(byte[] b, int offset, long v) + { + b[offset++] = (byte)((v >> 0) & 0xFF); + b[offset++] = (byte)((v >> 8) & 0xFF); + b[offset++] = (byte)((v >> 16) & 0xFF); + b[offset] = (byte)((v >> 24) & 0xFF); + } + + /** + * get a int from a byte array start from the specifiled offset + * + * @param b + * @param offset + */ + public static long getIntLong(byte[] b, int offset) + { + return ( + ((b[offset++] & 0x000000FFL)) | + ((b[offset++] << 8) & 0x0000FF00L) | + ((b[offset++] << 16) & 0x00FF0000L) | + ((b[offset] << 24) & 0xFF000000L) + ); + } + + /** + * get a int from a byte array start from the specifield offset + * + * @param b + * @param offset + */ + public static int getInt3(byte[] b, int offset) + { + return ( + (b[offset++] & 0x000000FF) | + (b[offset++] & 0x0000FF00) | + (b[offset] & 0x00FF0000) + ); + } + + public static int getInt2(byte[] b, int offset) + { + return ( + (b[offset++] & 0x000000FF) | + (b[offset] & 0x0000FF00) + ); + } + + public static int getInt1(byte[] b, int offset) + { + return ( + (b[offset] & 0x000000FF) + ); + } + + /** + * string ip to long ip + * + * @param ip + * @return long + */ + public static long ip2long(String ip) + { + IPAddress ipaddress = IPAddress.Parse(ip); + byte[] addbuffer = ipaddress.GetAddressBytes(); + Array.Reverse(addbuffer); + return BitConverter.ToInt64(addbuffer, 0); + } + + /** + * int to ip string + * + * @param ip + * @return string + */ + public static String long2ip(long ip) + { + StringBuilder sb = new StringBuilder(); + + sb + .Append((ip >> 24) & 0xFF).Append('.') + .Append((ip >> 16) & 0xFF).Append('.') + .Append((ip >> 8) & 0xFF).Append('.') + .Append((ip >> 0) & 0xFF); + + return sb.ToString(); + } + + /** + * check the validate of the specifeld ip address + * + * @param ip + * @return boolean + */ + public static Boolean isIpAddress(String ip) + { + String[] p = ip.Split("\\."); + if (p.Length != 4) return false; + + foreach (var pp in p) + { + if (pp.Length > 3) return false; + int val = Convert.ToInt32(pp); //Integer.valueOf(pp); + if (val > 255) return false; + } + return true; + } + } +} From e13f9b4a56543b53d51ce87baaa29d7d0c865507 Mon Sep 17 00:00:00 2001 From: Killer Date: Wed, 10 Jun 2020 18:09:57 +0800 Subject: [PATCH 3/9] add DbSearcher --- maker/csharp/DbMaker/DbSearcher.cs | 462 +++++++++++++++++++++++++++++ maker/csharp/DbMaker/Program.cs | 1 - maker/csharp/DbMaker/Util.cs | 12 +- 3 files changed, 470 insertions(+), 5 deletions(-) create mode 100644 maker/csharp/DbMaker/DbSearcher.cs diff --git a/maker/csharp/DbMaker/DbSearcher.cs b/maker/csharp/DbMaker/DbSearcher.cs new file mode 100644 index 00000000..d60ad760 --- /dev/null +++ b/maker/csharp/DbMaker/DbSearcher.cs @@ -0,0 +1,462 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace DbMaker +{ + /// + /// ip db searcher class (Not thread safe) + /// + public class DbSearcher + { + public const int BTREE_ALGORITHM = 1; + public const int BINARY_ALGORITHM = 2; + public const int MEMORY_ALGORITYM = 3; + + /** + * db config + */ + private DbConfig dbConfig = null; + + /** + * db file access handler + */ + private Stream raf = null; + + /** + * header blocks buffer + */ + private long[] HeaderSip = null; + private int[] HeaderPtr = null; + private int headerLength; + + /** + * super blocks info + */ + private long firstIndexPtr = 0; + private long lastIndexPtr = 0; + private int totalIndexBlocks = 0; + + /** + * for memory mode + * the original db binary string + */ + private byte[] dbBinStr = null; + + /** + * construct class + * + * @param bdConfig + * @param dbFile + * @throws FileNotFoundException + */ + public DbSearcher(DbConfig dbConfig, String dbFile) + { + this.dbConfig = dbConfig; + raf = File.OpenRead(dbFile); //new RandomAccessFile(dbFile, "r"); + } + + /** + * construct method with self-define std ip2region bianry string support + * Thanks to the issue from Wendal at https://gitee.com/lionsoul/ip2region/issues/IILFL + * + * @param dbConfig + * @param dbBinStr + */ + public DbSearcher(DbConfig dbConfig, byte[] dbBinStr) + { + this.dbConfig = dbConfig; + this.dbBinStr = dbBinStr; + + firstIndexPtr = Util.getIntLong(dbBinStr, 0); + lastIndexPtr = Util.getIntLong(dbBinStr, 4); + totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / IndexBlock.GetIndexBlockLength()) + 1; + } + + /** + * get the region with a int ip address with memory binary search algorithm + * + * @param ip + * @throws IOException + */ + public DataBlock memorySearch(long ip) + { + int blen = IndexBlock.GetIndexBlockLength(); + if (dbBinStr == null) + { + dbBinStr = new byte[(int)raf.Length]; + raf.Seek(0L, SeekOrigin.Begin); + //raf.readFully(dbBinStr, 0, dbBinStr.length); + raf.Read(dbBinStr, 0, dbBinStr.Length); + //initialize the global vars + firstIndexPtr = Util.getIntLong(dbBinStr, 0); + lastIndexPtr = Util.getIntLong(dbBinStr, 4); + totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1; + } + + //search the index blocks to define the data + int l = 0, h = totalIndexBlocks; + long sip, eip, dataptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + int p = (int)(firstIndexPtr + m * blen); + + sip = Util.getIntLong(dbBinStr, p); + if (ip < sip) + { + h = m - 1; + } + else + { + eip = Util.getIntLong(dbBinStr, p + 4); + if (ip > eip) + { + l = m + 1; + } + else + { + dataptr = Util.getIntLong(dbBinStr, p + 8); + break; + } + } + } + + //not matched + if (dataptr == 0) return null; + + //get the data + int dataLen = (int)((dataptr >> 24) & 0xFF); + int dataPtr = (int)((dataptr & 0x00FFFFFF)); + int city_id = (int)Util.getIntLong(dbBinStr, dataPtr); + //String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, "UTF-8"); + var region = Encoding.UTF8.GetString(dbBinStr, dataPtr + 4, dataLen - 4); + return new DataBlock(city_id, region, dataPtr); + } + + /** + * get the region throught the ip address with memory binary search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException +*/ + public DataBlock memorySearch(String ip) + { + return memorySearch(Util.ip2long(ip)); + } + + + /** + * get by index ptr + * + * @param indexPtr + * @throws IOException +*/ + public DataBlock getByIndexPtr(long ptr) + { + raf.Seek(ptr, SeekOrigin.Begin); + byte[] + buffer = new byte[12]; + //raf.readFully(buffer, 0, buffer.length); + raf.Read(buffer, 0, buffer.Length); + //long startIp = Util.getIntLong(buffer, 0); + //long endIp = Util.getIntLong(buffer, 4); + long extra = Util.getIntLong(buffer, 8); + + int dataLen = (int)((extra >> 24) & 0xFF); + int dataPtr = (int)((extra & 0x00FFFFFF)); + + raf.Seek(dataPtr, SeekOrigin.Begin); + byte[] data = new byte[dataLen]; + //raf.readFully(data, 0, data.length); + raf.Read(data, 0, data.Length); + int city_id = (int)Util.getIntLong(data, 0); + //String region = new String(data, 4, data.length - 4, "UTF-8"); + String region = Encoding.UTF8.GetString(data, 4, data.Length - 4); + return new DataBlock(city_id, region, dataPtr); + } + + /** + * get the region with a int ip address with b-tree algorithm + * + * @param ip + * @throws IOException +*/ + public DataBlock btreeSearch(long ip) + { + //check and load the header + if (HeaderSip == null) + { + //raf.seek(8L); //pass the super block + raf.Seek(8, SeekOrigin.Begin); + byte[] b = new byte[dbConfig.TotalHeaderSize]; + // byte[] b = new byte[4096]; + //raf.readFully(b, 0, b.length); + raf.Read(b, 0, b.Length); + + //fill the header + int len = b.Length >> 3, idx = 0; //b.lenght / 8 + HeaderSip = new long[len]; + HeaderPtr = new int[len]; + long startIp; + long xDataPtr = 0; + for (int i = 0; i < b.Length; i += 8) + { + startIp = Util.getIntLong(b, i); + xDataPtr = Util.getIntLong(b, i + 4); + if (xDataPtr == 0) break; + + HeaderSip[idx] = startIp; + HeaderPtr[idx] = (int)xDataPtr; + idx++; + } + + headerLength = idx; + } + + //1. define the index block with the binary search + if (ip == HeaderSip[0]) + { + return getByIndexPtr(HeaderPtr[0]); + } + else if (ip == HeaderSip[headerLength - 1]) + { + return getByIndexPtr(HeaderPtr[headerLength - 1]); + } + + int l = 0, h = headerLength, sptr = 0, eptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + + //perfetc matched, just return it + if (ip == HeaderSip[m]) + { + if (m > 0) + { + sptr = HeaderPtr[m - 1]; + eptr = HeaderPtr[m]; + } + else + { + sptr = HeaderPtr[m]; + eptr = HeaderPtr[m + 1]; + } + + break; + } + + //less then the middle value + if (ip < HeaderSip[m]) + { + if (m == 0) + { + sptr = HeaderPtr[m]; + eptr = HeaderPtr[m + 1]; + break; + } + else if (ip > HeaderSip[m - 1]) + { + sptr = HeaderPtr[m - 1]; + eptr = HeaderPtr[m]; + break; + } + h = m - 1; + } + else + { + if (m == headerLength - 1) + { + sptr = HeaderPtr[m - 1]; + eptr = HeaderPtr[m]; + break; + } + else if (ip <= HeaderSip[m + 1]) + { + sptr = HeaderPtr[m]; + eptr = HeaderPtr[m + 1]; + break; + } + l = m + 1; + } + } + + //match nothing just stop it + if (sptr == 0) return null; + + //2. search the index blocks to define the data + int blockLen = eptr - sptr, blen = IndexBlock.GetIndexBlockLength(); + byte[] + iBuffer = new byte[blockLen + blen]; //include the right border block + //raf.seek(sptr); + //raf.readFully(iBuffer, 0, iBuffer.length); + raf.Seek(sptr, SeekOrigin.Begin); + raf.Read(iBuffer, 0, iBuffer.Length); + + l = 0; h = blockLen / blen; + long sip, eip, dataptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + int p = m * blen; + sip = Util.getIntLong(iBuffer, p); + if (ip < sip) + { + h = m - 1; + } + else + { + eip = Util.getIntLong(iBuffer, p + 4); + if (ip > eip) + { + l = m + 1; + } + else + { + dataptr = Util.getIntLong(iBuffer, p + 8); + break; + } + } + } + + //not matched + if (dataptr == 0) return null; + + //3. get the data + int dataLen = (int)((dataptr >> 24) & 0xFF); + int dataPtr = (int)((dataptr & 0x00FFFFFF)); + + //raf.seek(dataPtr); + byte[] data = new byte[dataLen]; + // raf.readFully(data, 0, data.length); + raf.Seek(dataPtr, SeekOrigin.Begin); + raf.Read(data, 0, data.Length); + int city_id = (int)Util.getIntLong(data, 0); + //String region = new String(data, 4, data.length - 4, "UTF-8"); + var region = Encoding.UTF8.GetString(data, 4, data.Length - 4); + return new DataBlock(city_id, region, dataPtr); + } + + /** + * get the region throught the ip address with b-tree search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException + */ + public DataBlock btreeSearch(String ip) + { + return btreeSearch(Util.ip2long(ip)); + } + + /** + * get the region with a int ip address with binary search algorithm + * + * @param ip + * @throws IOException +*/ + public DataBlock binarySearch(long ip) + { + int blen = IndexBlock.GetIndexBlockLength(); + if (totalIndexBlocks == 0) + { + //raf.seek(0L); + byte[] superBytes = new byte[8]; + //raf.readFully(superBytes, 0, superBytes.length); + raf.Seek(0, SeekOrigin.Begin); + raf.Read(superBytes, 0, superBytes.Length); + //initialize the global vars + firstIndexPtr = Util.getIntLong(superBytes, 0); + lastIndexPtr = Util.getIntLong(superBytes, 4); + totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1; + } + + //search the index blocks to define the data + int l = 0, h = totalIndexBlocks; + byte[] + buffer = new byte[blen]; + long sip, eip, dataptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + //raf.seek(firstIndexPtr + m * blen); //set the file pointer + //raf.readFully(buffer, 0, buffer.length); + raf.Seek(firstIndexPtr + m * blen, SeekOrigin.Begin); + raf.Read(buffer, 0, buffer.Length); + sip = Util.getIntLong(buffer, 0); + if (ip < sip) + { + h = m - 1; + } + else + { + eip = Util.getIntLong(buffer, 4); + if (ip > eip) + { + l = m + 1; + } + else + { + dataptr = Util.getIntLong(buffer, 8); + break; + } + } + } + + //not matched + if (dataptr == 0) return null; + + //get the data + int dataLen = (int)((dataptr >> 24) & 0xFF); + int dataPtr = (int)((dataptr & 0x00FFFFFF)); + + //raf.seek(dataPtr); + byte[] data = new byte[dataLen]; + //raf.readFully(data, 0, data.length); + raf.Seek(dataPtr, SeekOrigin.Begin); + raf.Read(data, 0, data.Length); + + int city_id = (int)Util.getIntLong(data, 0); + //String region = new String(data, 4, data.length - 4, "UTF-8"); + var region = Encoding.UTF8.GetString(data, 4, data.Length - 4); + return new DataBlock(city_id, region, dataPtr); + } + + /** + * get the region throught the ip address with binary search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException + */ + public DataBlock binarySearch(String ip) + { + return binarySearch(Util.ip2long(ip)); + } + + /** + * get the db config + * + * @return DbConfig +*/ + public DbConfig getDbConfig() + { + return dbConfig; + } + + /** + * close the db + * + * @throws IOException +*/ + public void close() + { + HeaderSip = null; //let gc do its work + HeaderPtr = null; + dbBinStr = null; + raf.Close(); + } + } +} diff --git a/maker/csharp/DbMaker/Program.cs b/maker/csharp/DbMaker/Program.cs index 97138e2d..e6d0dd7a 100644 --- a/maker/csharp/DbMaker/Program.cs +++ b/maker/csharp/DbMaker/Program.cs @@ -6,7 +6,6 @@ class Program { static void Main(string[] args) { - Console.WriteLine("Hello World!"); } } } diff --git a/maker/csharp/DbMaker/Util.cs b/maker/csharp/DbMaker/Util.cs index 60e493ed..54f6e3aa 100644 --- a/maker/csharp/DbMaker/Util.cs +++ b/maker/csharp/DbMaker/Util.cs @@ -95,10 +95,14 @@ public static int getInt1(byte[] b, int offset) */ public static long ip2long(String ip) { - IPAddress ipaddress = IPAddress.Parse(ip); - byte[] addbuffer = ipaddress.GetAddressBytes(); - Array.Reverse(addbuffer); - return BitConverter.ToInt64(addbuffer, 0); + ip = ip.Trim(); + String[] ips = ip.Split('.'); + long ip1 = Int64.Parse(ips[0]); + long ip2 = Int64.Parse(ips[1]); + long ip3 = Int64.Parse(ips[2]); + long ip4 = Int64.Parse(ips[3]); + long ip2long = 1L * ip1 * 256 * 256 * 256 + ip2 * 256 * 256 + ip3 * 256 + ip4; + return ip2long; } /** From 1d02185af1e65808e169d31411484a4c951c21e6 Mon Sep 17 00:00:00 2001 From: Killer Date: Wed, 10 Jun 2020 18:13:42 +0800 Subject: [PATCH 4/9] clean code --- maker/csharp/DbMaker/DbSearcher.cs | 243 ++++++++++++++++------------- maker/csharp/DbMaker/Program.cs | 7 + 2 files changed, 138 insertions(+), 112 deletions(-) diff --git a/maker/csharp/DbMaker/DbSearcher.cs b/maker/csharp/DbMaker/DbSearcher.cs index d60ad760..f8966941 100644 --- a/maker/csharp/DbMaker/DbSearcher.cs +++ b/maker/csharp/DbMaker/DbSearcher.cs @@ -1,57 +1,54 @@ using System; -using System.Collections.Generic; using System.IO; using System.Text; namespace DbMaker { /// - /// ip db searcher class (Not thread safe) + /// ip db searcher class (Not thread safe) /// - public class DbSearcher + public class DbSearcher : IDisposable { public const int BTREE_ALGORITHM = 1; public const int BINARY_ALGORITHM = 2; public const int MEMORY_ALGORITYM = 3; /** - * db config + * for memory mode + * the original db binary string */ - private DbConfig dbConfig = null; + private byte[] dbBinStr; /** - * db file access handler - */ - private Stream raf = null; + * super blocks info + */ + private long firstIndexPtr; - /** - * header blocks buffer - */ - private long[] HeaderSip = null; - private int[] HeaderPtr = null; private int headerLength; + private int[] HeaderPtr; /** - * super blocks info - */ - private long firstIndexPtr = 0; - private long lastIndexPtr = 0; - private int totalIndexBlocks = 0; + * header blocks buffer + */ + private long[] HeaderSip; + + private long lastIndexPtr; /** - * for memory mode - * the original db binary string + * db file access handler */ - private byte[] dbBinStr = null; + private readonly Stream raf; + + private int totalIndexBlocks; /** * construct class * * @param bdConfig * @param dbFile - * @throws FileNotFoundException - */ - public DbSearcher(DbConfig dbConfig, String dbFile) + * @throws FileNotFoundException + */ + public DbSearcher(DbConfig dbConfig, string dbFile) { this.dbConfig = dbConfig; raf = File.OpenRead(dbFile); //new RandomAccessFile(dbFile, "r"); @@ -71,7 +68,17 @@ public DbSearcher(DbConfig dbConfig, byte[] dbBinStr) firstIndexPtr = Util.getIntLong(dbBinStr, 0); lastIndexPtr = Util.getIntLong(dbBinStr, 4); - totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / IndexBlock.GetIndexBlockLength()) + 1; + totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / IndexBlock.GetIndexBlockLength()) + 1; + } + + /** + * db config + */ + public DbConfig dbConfig { get; set; } + + public void Dispose() + { + Close(); } /** @@ -80,19 +87,19 @@ public DbSearcher(DbConfig dbConfig, byte[] dbBinStr) * @param ip * @throws IOException */ - public DataBlock memorySearch(long ip) + public DataBlock MemorySearch(long ip) { - int blen = IndexBlock.GetIndexBlockLength(); + var blen = IndexBlock.GetIndexBlockLength(); if (dbBinStr == null) { - dbBinStr = new byte[(int)raf.Length]; + dbBinStr = new byte[(int) raf.Length]; raf.Seek(0L, SeekOrigin.Begin); //raf.readFully(dbBinStr, 0, dbBinStr.length); raf.Read(dbBinStr, 0, dbBinStr.Length); //initialize the global vars firstIndexPtr = Util.getIntLong(dbBinStr, 0); lastIndexPtr = Util.getIntLong(dbBinStr, 4); - totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1; + totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1; } //search the index blocks to define the data @@ -100,8 +107,8 @@ public DataBlock memorySearch(long ip) long sip, eip, dataptr = 0; while (l <= h) { - int m = (l + h) >> 1; - int p = (int)(firstIndexPtr + m * blen); + var m = l + h >> 1; + var p = (int) (firstIndexPtr + m * blen); sip = Util.getIntLong(dbBinStr, p); if (ip < sip) @@ -124,12 +131,15 @@ public DataBlock memorySearch(long ip) } //not matched - if (dataptr == 0) return null; + if (dataptr == 0) + { + return null; + } //get the data - int dataLen = (int)((dataptr >> 24) & 0xFF); - int dataPtr = (int)((dataptr & 0x00FFFFFF)); - int city_id = (int)Util.getIntLong(dbBinStr, dataPtr); + var dataLen = (int) (dataptr >> 24 & 0xFF); + var dataPtr = (int) (dataptr & 0x00FFFFFF); + var city_id = (int) Util.getIntLong(dbBinStr, dataPtr); //String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, "UTF-8"); var region = Encoding.UTF8.GetString(dbBinStr, dataPtr + 4, dataLen - 4); return new DataBlock(city_id, region, dataPtr); @@ -140,11 +150,11 @@ public DataBlock memorySearch(long ip) * * @param ip * @return DataBlock - * @throws IOException -*/ - public DataBlock memorySearch(String ip) + * @throws IOException + */ + public DataBlock MemorySearch(string ip) { - return memorySearch(Util.ip2long(ip)); + return MemorySearch(Util.ip2long(ip)); } @@ -152,29 +162,29 @@ public DataBlock memorySearch(String ip) * get by index ptr * * @param indexPtr - * @throws IOException -*/ - public DataBlock getByIndexPtr(long ptr) + * @throws IOException + */ + public DataBlock GetByIndexPtr(long ptr) { raf.Seek(ptr, SeekOrigin.Begin); - byte[] - buffer = new byte[12]; + var + buffer = new byte[12]; //raf.readFully(buffer, 0, buffer.length); raf.Read(buffer, 0, buffer.Length); //long startIp = Util.getIntLong(buffer, 0); //long endIp = Util.getIntLong(buffer, 4); - long extra = Util.getIntLong(buffer, 8); + var extra = Util.getIntLong(buffer, 8); - int dataLen = (int)((extra >> 24) & 0xFF); - int dataPtr = (int)((extra & 0x00FFFFFF)); + var dataLen = (int) (extra >> 24 & 0xFF); + var dataPtr = (int) (extra & 0x00FFFFFF); raf.Seek(dataPtr, SeekOrigin.Begin); - byte[] data = new byte[dataLen]; + var data = new byte[dataLen]; //raf.readFully(data, 0, data.length); raf.Read(data, 0, data.Length); - int city_id = (int)Util.getIntLong(data, 0); + var city_id = (int) Util.getIntLong(data, 0); //String region = new String(data, 4, data.length - 4, "UTF-8"); - String region = Encoding.UTF8.GetString(data, 4, data.Length - 4); + var region = Encoding.UTF8.GetString(data, 4, data.Length - 4); return new DataBlock(city_id, region, dataPtr); } @@ -182,34 +192,37 @@ public DataBlock getByIndexPtr(long ptr) * get the region with a int ip address with b-tree algorithm * * @param ip - * @throws IOException -*/ - public DataBlock btreeSearch(long ip) + * @throws IOException + */ + public DataBlock BTreeSearch(long ip) { //check and load the header if (HeaderSip == null) { //raf.seek(8L); //pass the super block raf.Seek(8, SeekOrigin.Begin); - byte[] b = new byte[dbConfig.TotalHeaderSize]; + var b = new byte[dbConfig.TotalHeaderSize]; // byte[] b = new byte[4096]; //raf.readFully(b, 0, b.length); raf.Read(b, 0, b.Length); //fill the header - int len = b.Length >> 3, idx = 0; //b.lenght / 8 + int len = b.Length >> 3, idx = 0; //b.lenght / 8 HeaderSip = new long[len]; HeaderPtr = new int[len]; long startIp; long xDataPtr = 0; - for (int i = 0; i < b.Length; i += 8) + for (var i = 0; i < b.Length; i += 8) { startIp = Util.getIntLong(b, i); xDataPtr = Util.getIntLong(b, i + 4); - if (xDataPtr == 0) break; + if (xDataPtr == 0) + { + break; + } HeaderSip[idx] = startIp; - HeaderPtr[idx] = (int)xDataPtr; + HeaderPtr[idx] = (int) xDataPtr; idx++; } @@ -219,17 +232,18 @@ public DataBlock btreeSearch(long ip) //1. define the index block with the binary search if (ip == HeaderSip[0]) { - return getByIndexPtr(HeaderPtr[0]); + return GetByIndexPtr(HeaderPtr[0]); } - else if (ip == HeaderSip[headerLength - 1]) + + if (ip == HeaderSip[headerLength - 1]) { - return getByIndexPtr(HeaderPtr[headerLength - 1]); + return GetByIndexPtr(HeaderPtr[headerLength - 1]); } int l = 0, h = headerLength, sptr = 0, eptr = 0; while (l <= h) { - int m = (l + h) >> 1; + var m = l + h >> 1; //perfetc matched, just return it if (ip == HeaderSip[m]) @@ -257,12 +271,14 @@ public DataBlock btreeSearch(long ip) eptr = HeaderPtr[m + 1]; break; } - else if (ip > HeaderSip[m - 1]) + + if (ip > HeaderSip[m - 1]) { sptr = HeaderPtr[m - 1]; eptr = HeaderPtr[m]; break; } + h = m - 1; } else @@ -273,34 +289,40 @@ public DataBlock btreeSearch(long ip) eptr = HeaderPtr[m]; break; } - else if (ip <= HeaderSip[m + 1]) + + if (ip <= HeaderSip[m + 1]) { sptr = HeaderPtr[m]; eptr = HeaderPtr[m + 1]; break; } + l = m + 1; } } //match nothing just stop it - if (sptr == 0) return null; + if (sptr == 0) + { + return null; + } //2. search the index blocks to define the data int blockLen = eptr - sptr, blen = IndexBlock.GetIndexBlockLength(); - byte[] - iBuffer = new byte[blockLen + blen]; //include the right border block + var + iBuffer = new byte[blockLen + blen]; //include the right border block //raf.seek(sptr); //raf.readFully(iBuffer, 0, iBuffer.length); raf.Seek(sptr, SeekOrigin.Begin); raf.Read(iBuffer, 0, iBuffer.Length); - l = 0; h = blockLen / blen; + l = 0; + h = blockLen / blen; long sip, eip, dataptr = 0; while (l <= h) { - int m = (l + h) >> 1; - int p = m * blen; + var m = l + h >> 1; + var p = m * blen; sip = Util.getIntLong(iBuffer, p); if (ip < sip) { @@ -322,18 +344,21 @@ public DataBlock btreeSearch(long ip) } //not matched - if (dataptr == 0) return null; + if (dataptr == 0) + { + return null; + } //3. get the data - int dataLen = (int)((dataptr >> 24) & 0xFF); - int dataPtr = (int)((dataptr & 0x00FFFFFF)); + var dataLen = (int) (dataptr >> 24 & 0xFF); + var dataPtr = (int) (dataptr & 0x00FFFFFF); //raf.seek(dataPtr); - byte[] data = new byte[dataLen]; + var data = new byte[dataLen]; // raf.readFully(data, 0, data.length); raf.Seek(dataPtr, SeekOrigin.Begin); raf.Read(data, 0, data.Length); - int city_id = (int)Util.getIntLong(data, 0); + var city_id = (int) Util.getIntLong(data, 0); //String region = new String(data, 4, data.length - 4, "UTF-8"); var region = Encoding.UTF8.GetString(data, 4, data.Length - 4); return new DataBlock(city_id, region, dataPtr); @@ -344,43 +369,43 @@ public DataBlock btreeSearch(long ip) * * @param ip * @return DataBlock - * @throws IOException - */ - public DataBlock btreeSearch(String ip) + * @throws IOException + */ + public DataBlock BTreeSearch(string ip) { - return btreeSearch(Util.ip2long(ip)); + return BTreeSearch(Util.ip2long(ip)); } /** * get the region with a int ip address with binary search algorithm * * @param ip - * @throws IOException -*/ - public DataBlock binarySearch(long ip) + * @throws IOException + */ + public DataBlock BinarySearch(long ip) { - int blen = IndexBlock.GetIndexBlockLength(); + var blen = IndexBlock.GetIndexBlockLength(); if (totalIndexBlocks == 0) { //raf.seek(0L); - byte[] superBytes = new byte[8]; + var superBytes = new byte[8]; //raf.readFully(superBytes, 0, superBytes.length); raf.Seek(0, SeekOrigin.Begin); raf.Read(superBytes, 0, superBytes.Length); //initialize the global vars firstIndexPtr = Util.getIntLong(superBytes, 0); lastIndexPtr = Util.getIntLong(superBytes, 4); - totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1; + totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1; } //search the index blocks to define the data int l = 0, h = totalIndexBlocks; - byte[] - buffer = new byte[blen]; + var + buffer = new byte[blen]; long sip, eip, dataptr = 0; while (l <= h) { - int m = (l + h) >> 1; + var m = l + h >> 1; //raf.seek(firstIndexPtr + m * blen); //set the file pointer //raf.readFully(buffer, 0, buffer.length); raf.Seek(firstIndexPtr + m * blen, SeekOrigin.Begin); @@ -406,19 +431,22 @@ public DataBlock binarySearch(long ip) } //not matched - if (dataptr == 0) return null; + if (dataptr == 0) + { + return null; + } //get the data - int dataLen = (int)((dataptr >> 24) & 0xFF); - int dataPtr = (int)((dataptr & 0x00FFFFFF)); + var dataLen = (int) (dataptr >> 24 & 0xFF); + var dataPtr = (int) (dataptr & 0x00FFFFFF); //raf.seek(dataPtr); - byte[] data = new byte[dataLen]; + var data = new byte[dataLen]; //raf.readFully(data, 0, data.length); raf.Seek(dataPtr, SeekOrigin.Begin); raf.Read(data, 0, data.Length); - int city_id = (int)Util.getIntLong(data, 0); + var city_id = (int) Util.getIntLong(data, 0); //String region = new String(data, 4, data.length - 4, "UTF-8"); var region = Encoding.UTF8.GetString(data, 4, data.Length - 4); return new DataBlock(city_id, region, dataPtr); @@ -429,34 +457,25 @@ public DataBlock binarySearch(long ip) * * @param ip * @return DataBlock - * @throws IOException - */ - public DataBlock binarySearch(String ip) + * @throws IOException + */ + public DataBlock BinarySearch(string ip) { - return binarySearch(Util.ip2long(ip)); + return BinarySearch(Util.ip2long(ip)); } - /** - * get the db config - * - * @return DbConfig -*/ - public DbConfig getDbConfig() - { - return dbConfig; - } /** * close the db * - * @throws IOException -*/ - public void close() + * @throws IOException + */ + private void Close() { - HeaderSip = null; //let gc do its work + HeaderSip = null; //let gc do its work HeaderPtr = null; dbBinStr = null; - raf.Close(); + raf?.Close(); } } -} +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/Program.cs b/maker/csharp/DbMaker/Program.cs index e6d0dd7a..ebb946c1 100644 --- a/maker/csharp/DbMaker/Program.cs +++ b/maker/csharp/DbMaker/Program.cs @@ -6,6 +6,13 @@ class Program { static void Main(string[] args) { + var fn = @"G:\src\ip2region\data\ip2region.db"; + using (var searcher = new DbSearcher(new DbConfig(), fn)) + { + Console.WriteLine(searcher.BinarySearch("202.102.227.68")); + Console.WriteLine(searcher.BTreeSearch("202.102.227.68")); + Console.ReadLine(); + } } } } From 399f4a5ebcc861b099094f95988b16488719020b Mon Sep 17 00:00:00 2001 From: Soar360 Date: Thu, 11 Jun 2020 22:10:56 +0800 Subject: [PATCH 5/9] rename to Java style. --- maker/csharp/DbMaker/DataBlock.cs | 100 +++-- maker/csharp/DbMaker/DbConfig.cs | 68 +-- maker/csharp/DbMaker/DbMaker.csproj | 8 +- .../csharp/DbMaker/DbMakerConfigException.cs | 8 +- maker/csharp/DbMaker/DbSearcher.cs | 406 +++++++++--------- maker/csharp/DbMaker/HeaderBlock.cs | 69 +-- maker/csharp/DbMaker/IndexBlock.cs | 99 +++-- maker/csharp/DbMaker/Program.cs | 10 +- maker/csharp/DbMaker/RandomAccessFile.cs | 49 +++ maker/csharp/DbMaker/Util.cs | 74 ++-- 10 files changed, 509 insertions(+), 382 deletions(-) create mode 100644 maker/csharp/DbMaker/RandomAccessFile.cs diff --git a/maker/csharp/DbMaker/DataBlock.cs b/maker/csharp/DbMaker/DataBlock.cs index 3c81edb6..d33bf94a 100644 --- a/maker/csharp/DbMaker/DataBlock.cs +++ b/maker/csharp/DbMaker/DataBlock.cs @@ -1,47 +1,89 @@ -using System; +using System; +using System.Text; namespace DbMaker { - /// - /// data block class - /// + + /** + * data block class + * + * @author chenxin + */ public class DataBlock { - /// - /// construct method - /// - /// - /// - /// - public DataBlock(int cityId, string region, int dataPtr) + /** + * city id + */ + private int city_id; + + /** + * region address + */ + private String region; + + /** + * region ptr in the db file + */ + private int dataPtr; + + /** + * construct method + * + * @param city_id + * @param region region string + * @param dataPtr data ptr + */ + public DataBlock(int city_id, String region, int dataPtr) + { + this.city_id = city_id; + this.region = region; + this.dataPtr = dataPtr; + } + + public DataBlock(int city_id, String region) : this(city_id, region, 0) + { + //this(city_id, region, 0); + } + + public int getCityId() { - CityId = cityId; - Region = region; - DataPtr = dataPtr; + return city_id; } - public DataBlock(int cityId, string region) : this(cityId, region, 0) + public DataBlock setCityId(int city_id) { + this.city_id = city_id; + return this; } - /// - /// city id - /// - public int CityId { get; set; } + public String getRegion() + { + return region; + } + + public DataBlock setRegion(String region) + { + this.region = region; + return this; + } - /// - /// region address - /// - public string Region { get; set; } + public int getDataPtr() + { + return dataPtr; + } - /// - /// region ptr in the db file - /// - public int DataPtr { get; set; } + public DataBlock setDataPtr(int dataPtr) + { + this.dataPtr = dataPtr; + return this; + } - public override string ToString() + public override String ToString() { - return String.Join("|", this.CityId, this.Region, this.DataPtr); + StringBuilder sb = new StringBuilder(); + + sb.Append(city_id).Append('|').Append(region).Append('|').Append(dataPtr); + return sb.ToString(); } } } \ No newline at end of file diff --git a/maker/csharp/DbMaker/DbConfig.cs b/maker/csharp/DbMaker/DbConfig.cs index 0f14752a..56f0c58a 100644 --- a/maker/csharp/DbMaker/DbConfig.cs +++ b/maker/csharp/DbMaker/DbConfig.cs @@ -1,49 +1,67 @@ -using System; -using System.Collections.Generic; -using System.Text; - namespace DbMaker { - /// - /// database configuration class - /// + + /** + * database configuration class + * + * @author chenxin + */ public class DbConfig { + /** + * total header data block size + */ + private int totalHeaderSize; + + /** + * max index data block size + * u should always choice the fastest read block size + */ + private int indexBlockSize; + + /** + * construct method + * + * @param totalHeaderSize + * @throws DbMakerConfigException + */ public DbConfig(int totalHeaderSize) { - if (totalHeaderSize % 8 != 0) + if ((totalHeaderSize % 8) != 0) { throw new DbMakerConfigException("totalHeaderSize must be times of 8"); } - TotalHeaderSize = totalHeaderSize; - IndexBlockSize = 8192; //4*2048 + this.totalHeaderSize = totalHeaderSize; + this.indexBlockSize = 8192; //4 * 2048 } public DbConfig() : this(8 * 2048) { + //this(8 * 2048); + } + public int getTotalHeaderSize() + { + return totalHeaderSize; } - /// - /// total header data block size - /// - public int TotalHeaderSize { get; set; } - /// - /// max index data block size - /// u should always choice the fastest read block size - /// - public int IndexBlockSize { get; set; } - - public DbConfig SetTotalHeaderSize(int totalHeaderSize) + public DbConfig setTotalHeaderSize(int totalHeaderSize) { - this.TotalHeaderSize = totalHeaderSize; + this.totalHeaderSize = totalHeaderSize; return this; } - public DbConfig SetIndexBlockSize(int dataBlockSize) + + public int getIndexBlockSize() { - this.IndexBlockSize = dataBlockSize; + return indexBlockSize; + } + + public DbConfig setIndexBlockSize(int dataBlockSize) + { + this.indexBlockSize = dataBlockSize; return this; } } -} + +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/DbMaker.csproj b/maker/csharp/DbMaker/DbMaker.csproj index c73e0d16..2c0dacc0 100644 --- a/maker/csharp/DbMaker/DbMaker.csproj +++ b/maker/csharp/DbMaker/DbMaker.csproj @@ -1,8 +1,8 @@ - - Exe - netcoreapp3.1 - + + Exe + netcoreapp3.1 + diff --git a/maker/csharp/DbMaker/DbMakerConfigException.cs b/maker/csharp/DbMaker/DbMakerConfigException.cs index 5bc49794..7e1399ea 100644 --- a/maker/csharp/DbMaker/DbMakerConfigException.cs +++ b/maker/csharp/DbMaker/DbMakerConfigException.cs @@ -1,14 +1,12 @@ -using System; -using System.Collections.Generic; -using System.Text; +using System; namespace DbMaker { - class DbMakerConfigException : Exception + public class DbMakerConfigException : Exception { public DbMakerConfigException(String message) : base(message) { } } -} +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/DbSearcher.cs b/maker/csharp/DbMaker/DbSearcher.cs index f8966941..e8cb63bf 100644 --- a/maker/csharp/DbMaker/DbSearcher.cs +++ b/maker/csharp/DbMaker/DbSearcher.cs @@ -1,12 +1,14 @@ -using System; -using System.IO; +using System; using System.Text; namespace DbMaker { - /// - /// ip db searcher class (Not thread safe) - /// + + /** + * ip db searcher class (Not thread safe) + * + * @author chenxin +*/ public class DbSearcher : IDisposable { public const int BTREE_ALGORITHM = 1; @@ -14,53 +16,57 @@ public class DbSearcher : IDisposable public const int MEMORY_ALGORITYM = 3; /** - * for memory mode - * the original db binary string - */ - private byte[] dbBinStr; + * db config + */ + private DbConfig dbConfig = null; + + /** + * db file access handler + */ + private RandomAccessFile raf = null; /** - * super blocks info - */ - private long firstIndexPtr; + * header blocks buffer + */ + private long[] HeaderSip = null; + private int[] HeaderPtr = null; private int headerLength; - private int[] HeaderPtr; /** - * header blocks buffer - */ - private long[] HeaderSip; + * super blocks info + */ + private long firstIndexPtr = 0; - private long lastIndexPtr; + private long lastIndexPtr = 0; + private int totalIndexBlocks = 0; /** - * db file access handler - */ - private readonly Stream raf; - - private int totalIndexBlocks; + * for memory mode + * the original db binary string + */ + private byte[] dbBinStr = null; /** - * construct class - * - * @param bdConfig - * @param dbFile - * @throws FileNotFoundException - */ - public DbSearcher(DbConfig dbConfig, string dbFile) + * construct class + * + * @param bdConfig + * @param dbFile + * @throws FileNotFoundException + */ + public DbSearcher(DbConfig dbConfig, String dbFile) { this.dbConfig = dbConfig; - raf = File.OpenRead(dbFile); //new RandomAccessFile(dbFile, "r"); + raf = new RandomAccessFile(dbFile); } /** - * construct method with self-define std ip2region bianry string support - * Thanks to the issue from Wendal at https://gitee.com/lionsoul/ip2region/issues/IILFL - * - * @param dbConfig - * @param dbBinStr - */ + * construct method with self-define std ip2region bianry string support + * Thanks to the issue from Wendal at https://gitee.com/lionsoul/ip2region/issues/IILFL + * + * @param dbConfig + * @param dbBinStr + */ public DbSearcher(DbConfig dbConfig, byte[] dbBinStr) { this.dbConfig = dbConfig; @@ -68,34 +74,24 @@ public DbSearcher(DbConfig dbConfig, byte[] dbBinStr) firstIndexPtr = Util.getIntLong(dbBinStr, 0); lastIndexPtr = Util.getIntLong(dbBinStr, 4); - totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / IndexBlock.GetIndexBlockLength()) + 1; + totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / IndexBlock.getIndexBlockLength()) + 1; } /** - * db config - */ - public DbConfig dbConfig { get; set; } - - public void Dispose() + * get the region with a int ip address with memory binary search algorithm + * + * @param ip + * @throws IOException + */ + public DataBlock memorySearch(long ip) { - Close(); - } - - /** - * get the region with a int ip address with memory binary search algorithm - * - * @param ip - * @throws IOException - */ - public DataBlock MemorySearch(long ip) - { - var blen = IndexBlock.GetIndexBlockLength(); + int blen = IndexBlock.getIndexBlockLength(); if (dbBinStr == null) { - dbBinStr = new byte[(int) raf.Length]; - raf.Seek(0L, SeekOrigin.Begin); - //raf.readFully(dbBinStr, 0, dbBinStr.length); - raf.Read(dbBinStr, 0, dbBinStr.Length); + dbBinStr = new byte[(int) raf.length()]; + raf.seek(0L); + raf.readFully(dbBinStr, 0, dbBinStr.Length); + //initialize the global vars firstIndexPtr = Util.getIntLong(dbBinStr, 0); lastIndexPtr = Util.getIntLong(dbBinStr, 4); @@ -107,8 +103,8 @@ public DataBlock MemorySearch(long ip) long sip, eip, dataptr = 0; while (l <= h) { - var m = l + h >> 1; - var p = (int) (firstIndexPtr + m * blen); + int m = (l + h) >> 1; + int p = (int) (firstIndexPtr + m * blen); sip = Util.getIntLong(dbBinStr, p); if (ip < sip) @@ -131,95 +127,89 @@ public DataBlock MemorySearch(long ip) } //not matched - if (dataptr == 0) - { - return null; - } + if (dataptr == 0) return null; //get the data - var dataLen = (int) (dataptr >> 24 & 0xFF); - var dataPtr = (int) (dataptr & 0x00FFFFFF); - var city_id = (int) Util.getIntLong(dbBinStr, dataPtr); + int dataLen = (int) ((dataptr >> 24) & 0xFF); + int dataPtr = (int) ((dataptr & 0x00FFFFFF)); + int city_id = (int) Util.getIntLong(dbBinStr, dataPtr); //String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, "UTF-8"); - var region = Encoding.UTF8.GetString(dbBinStr, dataPtr + 4, dataLen - 4); + var region = ReadString(dbBinStr, dataPtr + 4, dataLen - 4); return new DataBlock(city_id, region, dataPtr); } + private string ReadString(byte[] bytes, int dataPtr, int dataLen) + { + return Encoding.UTF8.GetString(bytes, dataPtr, dataLen); + } + /** - * get the region throught the ip address with memory binary search algorithm - * - * @param ip - * @return DataBlock - * @throws IOException - */ - public DataBlock MemorySearch(string ip) + * get the region throught the ip address with memory binary search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException + */ + public DataBlock memorySearch(String ip) { - return MemorySearch(Util.ip2long(ip)); + return memorySearch(Util.ip2long(ip)); } /** - * get by index ptr - * - * @param indexPtr - * @throws IOException - */ - public DataBlock GetByIndexPtr(long ptr) + * get by index ptr + * + * @param indexPtr + * @throws IOException + */ + public DataBlock getByIndexPtr(long ptr) { - raf.Seek(ptr, SeekOrigin.Begin); - var - buffer = new byte[12]; - //raf.readFully(buffer, 0, buffer.length); - raf.Read(buffer, 0, buffer.Length); + raf.seek(ptr); + byte[] buffer = new byte[12]; + raf.readFully(buffer, 0, buffer.Length); //long startIp = Util.getIntLong(buffer, 0); //long endIp = Util.getIntLong(buffer, 4); - var extra = Util.getIntLong(buffer, 8); - - var dataLen = (int) (extra >> 24 & 0xFF); - var dataPtr = (int) (extra & 0x00FFFFFF); - - raf.Seek(dataPtr, SeekOrigin.Begin); - var data = new byte[dataLen]; - //raf.readFully(data, 0, data.length); - raf.Read(data, 0, data.Length); - var city_id = (int) Util.getIntLong(data, 0); - //String region = new String(data, 4, data.length - 4, "UTF-8"); - var region = Encoding.UTF8.GetString(data, 4, data.Length - 4); + long extra = Util.getIntLong(buffer, 8); + + int dataLen = (int) ((extra >> 24) & 0xFF); + int dataPtr = (int) ((extra & 0x00FFFFFF)); + + raf.seek(dataPtr); + byte[] data = new byte[dataLen]; + raf.readFully(data, 0, data.Length); + + int city_id = (int) Util.getIntLong(data, 0); + //String region = new String(data, 4, data.Length - 4, "UTF-8"); + var region = ReadString(data, 4, data.Length - 4); return new DataBlock(city_id, region, dataPtr); } /** - * get the region with a int ip address with b-tree algorithm - * - * @param ip - * @throws IOException - */ - public DataBlock BTreeSearch(long ip) + * get the region with a int ip address with b-tree algorithm + * + * @param ip + * @throws IOException + */ + public DataBlock btreeSearch(long ip) { //check and load the header if (HeaderSip == null) { - //raf.seek(8L); //pass the super block - raf.Seek(8, SeekOrigin.Begin); - var b = new byte[dbConfig.TotalHeaderSize]; + raf.seek(8L); //pass the super block + byte[] b = new byte[dbConfig.getTotalHeaderSize()]; // byte[] b = new byte[4096]; - //raf.readFully(b, 0, b.length); - raf.Read(b, 0, b.Length); + raf.readFully(b, 0, b.Length); //fill the header int len = b.Length >> 3, idx = 0; //b.lenght / 8 HeaderSip = new long[len]; - HeaderPtr = new int[len]; - long startIp; - long xDataPtr = 0; - for (var i = 0; i < b.Length; i += 8) + HeaderPtr = new int [len]; + long startIp, xDataPtr; + for (int i = 0; i < b.Length; i += 8) { startIp = Util.getIntLong(b, i); xDataPtr = Util.getIntLong(b, i + 4); - if (xDataPtr == 0) - { - break; - } + if (xDataPtr == 0) break; HeaderSip[idx] = startIp; HeaderPtr[idx] = (int) xDataPtr; @@ -232,18 +222,17 @@ public DataBlock BTreeSearch(long ip) //1. define the index block with the binary search if (ip == HeaderSip[0]) { - return GetByIndexPtr(HeaderPtr[0]); + return getByIndexPtr(HeaderPtr[0]); } - - if (ip == HeaderSip[headerLength - 1]) + else if (ip == HeaderSip[headerLength - 1]) { - return GetByIndexPtr(HeaderPtr[headerLength - 1]); + return getByIndexPtr(HeaderPtr[headerLength - 1]); } int l = 0, h = headerLength, sptr = 0, eptr = 0; while (l <= h) { - var m = l + h >> 1; + int m = (l + h) >> 1; //perfetc matched, just return it if (ip == HeaderSip[m]) @@ -271,8 +260,7 @@ public DataBlock BTreeSearch(long ip) eptr = HeaderPtr[m + 1]; break; } - - if (ip > HeaderSip[m - 1]) + else if (ip > HeaderSip[m - 1]) { sptr = HeaderPtr[m - 1]; eptr = HeaderPtr[m]; @@ -289,8 +277,7 @@ public DataBlock BTreeSearch(long ip) eptr = HeaderPtr[m]; break; } - - if (ip <= HeaderSip[m + 1]) + else if (ip <= HeaderSip[m + 1]) { sptr = HeaderPtr[m]; eptr = HeaderPtr[m + 1]; @@ -302,27 +289,21 @@ public DataBlock BTreeSearch(long ip) } //match nothing just stop it - if (sptr == 0) - { - return null; - } + if (sptr == 0) return null; //2. search the index blocks to define the data - int blockLen = eptr - sptr, blen = IndexBlock.GetIndexBlockLength(); - var - iBuffer = new byte[blockLen + blen]; //include the right border block - //raf.seek(sptr); - //raf.readFully(iBuffer, 0, iBuffer.length); - raf.Seek(sptr, SeekOrigin.Begin); - raf.Read(iBuffer, 0, iBuffer.Length); + int blockLen = eptr - sptr, blen = IndexBlock.getIndexBlockLength(); + byte[] iBuffer = new byte[blockLen + blen]; //include the right border block + raf.seek(sptr); + raf.readFully(iBuffer, 0, iBuffer.Length); l = 0; h = blockLen / blen; long sip, eip, dataptr = 0; while (l <= h) { - var m = l + h >> 1; - var p = m * blen; + int m = (l + h) >> 1; + int p = m * blen; sip = Util.getIntLong(iBuffer, p); if (ip < sip) { @@ -344,54 +325,48 @@ public DataBlock BTreeSearch(long ip) } //not matched - if (dataptr == 0) - { - return null; - } + if (dataptr == 0) return null; //3. get the data - var dataLen = (int) (dataptr >> 24 & 0xFF); - var dataPtr = (int) (dataptr & 0x00FFFFFF); - - //raf.seek(dataPtr); - var data = new byte[dataLen]; - // raf.readFully(data, 0, data.length); - raf.Seek(dataPtr, SeekOrigin.Begin); - raf.Read(data, 0, data.Length); - var city_id = (int) Util.getIntLong(data, 0); - //String region = new String(data, 4, data.length - 4, "UTF-8"); - var region = Encoding.UTF8.GetString(data, 4, data.Length - 4); + int dataLen = (int) ((dataptr >> 24) & 0xFF); + int dataPtr = (int) ((dataptr & 0x00FFFFFF)); + + raf.seek(dataPtr); + byte[] data = new byte[dataLen]; + raf.readFully(data, 0, data.Length); + + int city_id = (int) Util.getIntLong(data, 0); + //String region = new String(data, 4, data.Length - 4, "UTF-8"); + var region = ReadString(data, 4, data.Length - 4); return new DataBlock(city_id, region, dataPtr); } /** - * get the region throught the ip address with b-tree search algorithm - * - * @param ip - * @return DataBlock - * @throws IOException - */ - public DataBlock BTreeSearch(string ip) + * get the region throught the ip address with b-tree search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException + */ + public DataBlock btreeSearch(String ip) { - return BTreeSearch(Util.ip2long(ip)); + return btreeSearch(Util.ip2long(ip)); } /** - * get the region with a int ip address with binary search algorithm - * - * @param ip - * @throws IOException - */ - public DataBlock BinarySearch(long ip) + * get the region with a int ip address with binary search algorithm + * + * @param ip + * @throws IOException + */ + public DataBlock binarySearch(long ip) { - var blen = IndexBlock.GetIndexBlockLength(); + int blen = IndexBlock.getIndexBlockLength(); if (totalIndexBlocks == 0) { - //raf.seek(0L); - var superBytes = new byte[8]; - //raf.readFully(superBytes, 0, superBytes.length); - raf.Seek(0, SeekOrigin.Begin); - raf.Read(superBytes, 0, superBytes.Length); + raf.seek(0L); + byte[] superBytes = new byte[8]; + raf.readFully(superBytes, 0, superBytes.Length); //initialize the global vars firstIndexPtr = Util.getIntLong(superBytes, 0); lastIndexPtr = Util.getIntLong(superBytes, 4); @@ -400,16 +375,13 @@ public DataBlock BinarySearch(long ip) //search the index blocks to define the data int l = 0, h = totalIndexBlocks; - var - buffer = new byte[blen]; + byte[] buffer = new byte[blen]; long sip, eip, dataptr = 0; while (l <= h) { - var m = l + h >> 1; - //raf.seek(firstIndexPtr + m * blen); //set the file pointer - //raf.readFully(buffer, 0, buffer.length); - raf.Seek(firstIndexPtr + m * blen, SeekOrigin.Begin); - raf.Read(buffer, 0, buffer.Length); + int m = (l + h) >> 1; + raf.seek(firstIndexPtr + m * blen); //set the file pointer + raf.readFully(buffer, 0, buffer.Length); sip = Util.getIntLong(buffer, 0); if (ip < sip) { @@ -431,51 +403,61 @@ public DataBlock BinarySearch(long ip) } //not matched - if (dataptr == 0) - { - return null; - } + if (dataptr == 0) return null; //get the data - var dataLen = (int) (dataptr >> 24 & 0xFF); - var dataPtr = (int) (dataptr & 0x00FFFFFF); - - //raf.seek(dataPtr); - var data = new byte[dataLen]; - //raf.readFully(data, 0, data.length); - raf.Seek(dataPtr, SeekOrigin.Begin); - raf.Read(data, 0, data.Length); - - var city_id = (int) Util.getIntLong(data, 0); - //String region = new String(data, 4, data.length - 4, "UTF-8"); - var region = Encoding.UTF8.GetString(data, 4, data.Length - 4); + int dataLen = (int) ((dataptr >> 24) & 0xFF); + int dataPtr = (int) ((dataptr & 0x00FFFFFF)); + + raf.seek(dataPtr); + byte[] data = new byte[dataLen]; + raf.readFully(data, 0, data.Length); + + int city_id = (int) Util.getIntLong(data, 0); + //String region = new String(data, 4, data.Length - 4, "UTF-8"); + var region = ReadString(data, 4, data.Length - 4); return new DataBlock(city_id, region, dataPtr); } /** - * get the region throught the ip address with binary search algorithm - * - * @param ip - * @return DataBlock - * @throws IOException - */ - public DataBlock BinarySearch(string ip) + * get the region throught the ip address with binary search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException + */ + public DataBlock binarySearch(String ip) { - return BinarySearch(Util.ip2long(ip)); + return binarySearch(Util.ip2long(ip)); } + /** + * get the db config + * + * @return DbConfig + */ + public DbConfig getDbConfig() + { + return dbConfig; + } /** - * close the db - * - * @throws IOException - */ - private void Close() + * close the db + * + * @throws IOException + */ + private void close() { HeaderSip = null; //let gc do its work HeaderPtr = null; dbBinStr = null; - raf?.Close(); + raf.close(); + } + + public void Dispose() + { + close(); } } + } \ No newline at end of file diff --git a/maker/csharp/DbMaker/HeaderBlock.cs b/maker/csharp/DbMaker/HeaderBlock.cs index b561e0f3..ac120fc0 100644 --- a/maker/csharp/DbMaker/HeaderBlock.cs +++ b/maker/csharp/DbMaker/HeaderBlock.cs @@ -1,43 +1,57 @@ -namespace DbMaker +namespace DbMaker { - /// - /// header block class - /// + + /** + * header block class + * + * @author chenxin +*/ public class HeaderBlock { + /** + * index block start ip address + */ + private long indexStartIp; + + /** + * ip address + */ + private int indexPtr; + public HeaderBlock(long indexStartIp, int indexPtr) { - IndexStartIp = indexStartIp; - IndexPtr = indexPtr; + this.indexStartIp = indexStartIp; + this.indexPtr = indexPtr; } - /// - /// index block start ip address - /// - public long IndexStartIp { get; set; } - - /// - /// ip address - /// - public int IndexPtr { get; set; } + public long getIndexStartIp() + { + return indexStartIp; + } - public HeaderBlock SetIndexStartIp(long indexStartIp) + public HeaderBlock setIndexStartIp(long indexStartIp) { - IndexStartIp = indexStartIp; + this.indexStartIp = indexStartIp; return this; } - public HeaderBlock SetIndexPtr(int indexPtr) + public int getIndexPtr() { - IndexPtr = indexPtr; + return indexPtr; + } + + public HeaderBlock setIndexPtr(int indexPtr) + { + this.indexPtr = indexPtr; return this; } - /// - /// get the bytes for db storage - /// - /// - public byte[] GetBytes() + /** + * get the bytes for db storage + * + * @return byte[] + */ + public byte[] getBytes() { /* * +------------+-----------+ @@ -45,11 +59,10 @@ public byte[] GetBytes() * +------------+-----------+ * start ip index ptr */ + byte[] b = new byte[8]; - var b = new byte[8]; - - Util.writeIntLong(b, 0, IndexStartIp); - Util.writeIntLong(b, 4, IndexPtr); + Util.writeIntLong(b, 0, indexStartIp); + Util.writeIntLong(b, 4, indexPtr); return b; } diff --git a/maker/csharp/DbMaker/IndexBlock.cs b/maker/csharp/DbMaker/IndexBlock.cs index b456126e..1d2c8435 100644 --- a/maker/csharp/DbMaker/IndexBlock.cs +++ b/maker/csharp/DbMaker/IndexBlock.cs @@ -1,65 +1,88 @@ -namespace DbMaker +namespace DbMaker { - /// - /// item index class - /// + + /** + * item index class + * + * @author chenxin + */ public class IndexBlock { private static int LENGTH = 12; - public IndexBlock(long startIp, long endIp, int dataPtr, int dataLen) - { - StartIp = startIp; - EndIp = endIp; - DataPtr = dataPtr; - DataLen = dataLen; - } - /** - * start ip address - */ - public long StartIp { get; set; } + * start ip address + */ + private long startIp; /** - * end ip address - */ - public long EndIp { get; set; } + * end ip address + */ + private long endIp; /** - * data ptr and data length - */ - public int DataPtr { get; set; } + * data ptr and data length + */ + private int dataPtr; /** - * data length - */ - public int DataLen { get; set; } + * data length + */ + private int dataLen; + + public IndexBlock(long startIp, long endIp, int dataPtr, int dataLen) + { + this.startIp = startIp; + this.endIp = endIp; + this.dataPtr = dataPtr; + this.dataLen = dataLen; + } - public IndexBlock SetStartIp(long startIp) + public long getStartIp() { - StartIp = startIp; + return startIp; + } + + public IndexBlock setStartIp(long startIp) + { + this.startIp = startIp; return this; } - public IndexBlock SetEndIp(long endIp) + public long getEndIp() + { + return endIp; + } + + public IndexBlock setEndIp(long endIp) { - EndIp = endIp; + this.endIp = endIp; return this; } - public IndexBlock SetDataPtr(int dataPtr) + public int getDataPtr() { - DataPtr = dataPtr; + return dataPtr; + } + + public IndexBlock setDataPtr(int dataPtr) + { + this.dataPtr = dataPtr; return this; } - public IndexBlock SetDataLen(int dataLen) + public int getDataLen() + { + return dataLen; + } + + public IndexBlock setDataLen(int dataLen) { - DataLen = dataLen; + this.dataLen = dataLen; return this; } - public static int GetIndexBlockLength() + public static int getIndexBlockLength() { return LENGTH; } @@ -69,7 +92,7 @@ public static int GetIndexBlockLength() * * @return byte[] */ - public byte[] GetBytes() + public byte[] getBytes() { /* * +------------+-----------+-----------+ @@ -77,13 +100,13 @@ public byte[] GetBytes() * +------------+-----------+-----------+ * start ip end ip data ptr + len */ - var b = new byte[12]; + byte[] b = new byte[12]; - Util.writeIntLong(b, 0, StartIp); //start ip - Util.writeIntLong(b, 4, EndIp); //end ip + Util.writeIntLong(b, 0, startIp); //start ip + Util.writeIntLong(b, 4, endIp); //end ip //write the data ptr and the length - var mix = DataPtr | DataLen << 24 & 0xFF000000L; + long mix = dataPtr | ((dataLen << 24) & 0xFF000000L); Util.writeIntLong(b, 8, mix); return b; diff --git a/maker/csharp/DbMaker/Program.cs b/maker/csharp/DbMaker/Program.cs index ebb946c1..c2072e31 100644 --- a/maker/csharp/DbMaker/Program.cs +++ b/maker/csharp/DbMaker/Program.cs @@ -6,13 +6,7 @@ class Program { static void Main(string[] args) { - var fn = @"G:\src\ip2region\data\ip2region.db"; - using (var searcher = new DbSearcher(new DbConfig(), fn)) - { - Console.WriteLine(searcher.BinarySearch("202.102.227.68")); - Console.WriteLine(searcher.BTreeSearch("202.102.227.68")); - Console.ReadLine(); - } + Console.WriteLine("Hello World!"); } } -} +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/RandomAccessFile.cs b/maker/csharp/DbMaker/RandomAccessFile.cs new file mode 100644 index 00000000..71ca6c2c --- /dev/null +++ b/maker/csharp/DbMaker/RandomAccessFile.cs @@ -0,0 +1,49 @@ +using System; +using System.IO; + +namespace DbMaker +{ + public class RandomAccessFile : IDisposable + { + private readonly Stream _stream; + + public RandomAccessFile(String file) : this(File.OpenRead(file)) + { + + } + + public RandomAccessFile(Stream stream) + { + if (stream == null) + { + throw new ArgumentNullException(nameof(stream)); + } + _stream = stream; + } + + public void Dispose() + { + _stream?.Dispose(); + } + + public long length() + { + return _stream.Length; + } + + public void seek(long position) + { + _stream.Seek(position, SeekOrigin.Begin); + } + + public void readFully(byte[] dbBinStr, int offset, int count) + { + _stream.Read(dbBinStr, offset, count); + } + + public void close() + { + _stream.Dispose(); + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/Util.cs b/maker/csharp/DbMaker/Util.cs index 54f6e3aa..8281a6ab 100644 --- a/maker/csharp/DbMaker/Util.cs +++ b/maker/csharp/DbMaker/Util.cs @@ -1,28 +1,31 @@ -using System; -using System.Collections.Generic; -using System.Net; +using System; using System.Text; namespace DbMaker { - /// - /// util class - /// - public static class Util - { - /** - * write specfield bytes to a byte array start from offset + + + /** + * util class * - * @param b - * @param offset - * @param v - * @param bytes + * @author chenxin */ + public class Util + { + /** + * write specfield bytes to a byte array start from offset + * + * @param b + * @param offset + * @param v + * @param bytes + */ public static void write(byte[] b, int offset, long v, int bytes) { for (int i = 0; i < bytes; i++) { - b[offset++] = (byte)((v >> (8 * i)) & 0xFF); + //b[offset++] = (byte)((v >>> (8 * i)) & 0xFF); + b[offset++] = (byte) ((v >> (8 * i)) & 0xFF); } } @@ -35,10 +38,10 @@ public static void write(byte[] b, int offset, long v, int bytes) */ public static void writeIntLong(byte[] b, int offset, long v) { - b[offset++] = (byte)((v >> 0) & 0xFF); - b[offset++] = (byte)((v >> 8) & 0xFF); - b[offset++] = (byte)((v >> 16) & 0xFF); - b[offset] = (byte)((v >> 24) & 0xFF); + b[offset++] = (byte) ((v >> 0) & 0xFF); + b[offset++] = (byte) ((v >> 8) & 0xFF); + b[offset++] = (byte) ((v >> 16) & 0xFF); + b[offset] = (byte) ((v >> 24) & 0xFF); } /** @@ -95,14 +98,16 @@ public static int getInt1(byte[] b, int offset) */ public static long ip2long(String ip) { - ip = ip.Trim(); - String[] ips = ip.Split('.'); - long ip1 = Int64.Parse(ips[0]); - long ip2 = Int64.Parse(ips[1]); - long ip3 = Int64.Parse(ips[2]); - long ip4 = Int64.Parse(ips[3]); - long ip2long = 1L * ip1 * 256 * 256 * 256 + ip2 * 256 * 256 + ip3 * 256 + ip4; - return ip2long; + throw new NotImplementedException(); + //String[] p = ip.Split("\\."); + //if ( p.Length != 4 ) return 0; + + //int p1 = ((Convert.ToInt32(p[0]) << 24) & 0xFF000000); + //int p2 = ((Integer.valueOf(p[1]) << 16) & 0x00FF0000); + //int p3 = ((Integer.valueOf(p[2]) << 8) & 0x0000FF00); + //int p4 = ((Integer.valueOf(p[3]) << 0) & 0x000000FF); + + //return ((p1 | p2 | p3 | p4) & 0xFFFFFFFFL); } /** @@ -116,10 +121,10 @@ public static String long2ip(long ip) StringBuilder sb = new StringBuilder(); sb - .Append((ip >> 24) & 0xFF).Append('.') - .Append((ip >> 16) & 0xFF).Append('.') - .Append((ip >> 8) & 0xFF).Append('.') - .Append((ip >> 0) & 0xFF); + .Append((ip >> 24) & 0xFF).Append('.') + .Append((ip >> 16) & 0xFF).Append('.') + .Append((ip >> 8) & 0xFF).Append('.') + .Append((ip >> 0) & 0xFF); return sb.ToString(); } @@ -135,13 +140,16 @@ public static Boolean isIpAddress(String ip) String[] p = ip.Split("\\."); if (p.Length != 4) return false; + //for ( String pp : p ) { + //for ( String pp : p ) { foreach (var pp in p) { if (pp.Length > 3) return false; - int val = Convert.ToInt32(pp); //Integer.valueOf(pp); + int val = Int32.Parse(pp); //Integer.valueOf(pp); if (val > 255) return false; } + return true; } } -} +} \ No newline at end of file From 06685a3010a4d035cc79f0569d4553397a4a8255 Mon Sep 17 00:00:00 2001 From: Soar360 Date: Thu, 11 Jun 2020 22:12:00 +0800 Subject: [PATCH 6/9] add sln file --- maker/csharp/DbMaker.sln | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 maker/csharp/DbMaker.sln diff --git a/maker/csharp/DbMaker.sln b/maker/csharp/DbMaker.sln new file mode 100644 index 00000000..bd53da23 --- /dev/null +++ b/maker/csharp/DbMaker.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29102.190 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DbMaker", "DbMaker\DbMaker.csproj", "{724424E3-EC3C-4819-A29C-AA7FD7F2357F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {135F4345-E533-41E0-ACA9-7F345351555D} + EndGlobalSection +EndGlobal From cfa29188e397e44b201831ea271d9eb41705f948 Mon Sep 17 00:00:00 2001 From: Soar360 Date: Thu, 11 Jun 2020 23:47:31 +0800 Subject: [PATCH 7/9] add DbMaker.cs --- maker/csharp/DbMaker/DbMaker.cs | 423 +++++++++++++++++++++++ maker/csharp/DbMaker/Extensions.cs | 39 +++ maker/csharp/DbMaker/Program.cs | 12 - maker/csharp/DbMaker/RandomAccessFile.cs | 15 + 4 files changed, 477 insertions(+), 12 deletions(-) create mode 100644 maker/csharp/DbMaker/DbMaker.cs create mode 100644 maker/csharp/DbMaker/Extensions.cs delete mode 100644 maker/csharp/DbMaker/Program.cs diff --git a/maker/csharp/DbMaker/DbMaker.cs b/maker/csharp/DbMaker/DbMaker.cs new file mode 100644 index 00000000..43fefbf9 --- /dev/null +++ b/maker/csharp/DbMaker/DbMaker.cs @@ -0,0 +1,423 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using DbMaker; + +namespace DbMaker +{ + +/** + * fast ip db maker + * + * db struct: + * 1. header part + * 1): super part: + * +------------+-----------+ + * | 4 bytes | 4 bytes | + * +------------+-----------+ + * start index ptr, end index ptr + * + * 2): b-tree index part + * +------------+-----------+-----------+-----------+ + * | 4bytes | 4bytes | 4bytes | 4bytes | ... + * +------------+-----------+-----------+-----------+ + * start ip ptr index ptr + * + * 2. data part: + * +------------+-----------------------+ + * | 2bytes | dynamic length | + * +------------+-----------------------+ + * data length city_id|Country|Province|Area|City|ISP + * + * 3. index part: (ip range) + * +------------+-----------+---------------+ + * | 4bytes | 4bytes | 4bytes | + * +------------+-----------+---------------+ + * start ip end ip 3 byte data ptr & 1 byte data length + * + * @author chenxin +*/ +public class DbMaker +{ + /** + * db config + */ + private DbConfig dbConfig; + + /** + * ip source file path + */ + private FileStream ipSrcFile; + + /** + * buffer + */ + private List indexPool; + private List headerPool; + + /** + * global region Id mapping data + */ + private FileStream globalRegionFile = null; + private Dictionary globalRegionMap = null; + + /** + * region and data ptr mapping data + */ + private Dictionary regionPtrPool = null; + + /** + * construct method + * + * @param config + * @param ipSrcFile tb source ip file + * @param globalRegionFile global_region.csv file offer by lion + * @throws DbMakerConfigException + * @throws IOException + */ + public DbMaker( + DbConfig config, + String ipSrcFile, + String globalRegionFile ) + { + this.dbConfig = config; + this.ipSrcFile = File.OpenRead(ipSrcFile); //new File(ipSrcFile); + this.globalRegionMap = new Dictionary(); //new HashMap(); + this.regionPtrPool = new Dictionary(); //new HashMap(); + if ( globalRegionFile != null ) + { + this.globalRegionFile = File.OpenRead(globalRegionFile); //new File(globalRegionFile); + } + + //if ( this.ipSrcFile.exists() == false ) { + if (!File.Exists(ipSrcFile)) + { + throw new IOException("Error: Invalid file path " + ipSrcFile); + } + } + + /** + * initialize the db file + * + * @param raf + * @throws IOException + */ + private void initDbFile( RandomAccessFile raf ) + { + //1. zero fill the header part + raf.seek(0L); + raf.write(new byte[8]); //super block + raf.write(new byte[dbConfig.getTotalHeaderSize()]); //header block + + headerPool = new List(); //new LinkedList(); + indexPool = new List(); //new LinkedList(); + } + + /** + * make the Db file + * + * @param dbFile target output file path + * @throws IOException + */ + public void make(String dbFile) + { + //check and load the gloabl region + if ( globalRegionFile != null ) { + Console.WriteLine("+-Try to load the global region data ..."); + StreamReader greader = new StreamReader((globalRegionFile)); + String gline = null; + while (!greader.EndOfStream ) { + gline = greader.ReadLine();// != null + if (String.IsNullOrWhiteSpace(gline)) + { + continue; + } + + String[] p = gline.Split(","); + if ( p.Length != 5 ) continue; + + //push the mapping + globalRegionMap.put(p[2], Int32.Parse(p[0])); + } + + greader.Close(); + Console.WriteLine("|--[Ok]"); + } + + //alloc the header size + StreamReader reader = new StreamReader(this.ipSrcFile); + RandomAccessFile raf = new RandomAccessFile(dbFile); + + //init the db file + initDbFile(raf); + Console.WriteLine("+-Db file initialized."); + + //analysis main loop + Console.WriteLine("+-Try to write the data blocks ... "); + String line = null; + while ( !reader.EndOfStream ) { + //line = reader.ReadLine() + line = reader.ReadLine(); + if (String.IsNullOrWhiteSpace(line)) + { + continue; + } + line = line.Trim(); + if ( line.Length == 0 ) continue; + if ( line[0] == '#' ) continue; + + //1. get the start ip + int sIdx = 0, eIdx = 0; + if ( (eIdx = line.IndexOf('|', sIdx + 1)) == -1 ) continue; + String startIp = line.Substring(sIdx, eIdx); + + //2. get the end ip + sIdx = eIdx + 1; + if ( (eIdx = line.IndexOf('|', sIdx + 1)) == -1 ) continue; + String endIp = line.Substring(sIdx, eIdx); + + //3. get the region + sIdx = eIdx + 1; + String region = line.Substring(sIdx); + + Console.WriteLine("+-Try to process item " + line); + addDataBlock(raf, startIp, endIp, region); + Console.WriteLine("|--[Ok]"); + } + Console.WriteLine("|--Data block flushed!"); + Console.WriteLine("|--Data file pointer: "+raf.getFilePointer()+"\n"); + + //write the index bytes + Console.WriteLine("+-Try to write index blocks ... "); + + //record the start block + IndexBlock indexBlock = null; + HeaderBlock hb = null; + indexBlock = indexPool.First(); + long indexStartIp = indexBlock.getStartIp(), + indexStratPtr = raf.getFilePointer(), indexEndPtr; + headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr))); + + int blockLength = IndexBlock.getIndexBlockLength(); + int counter = 0, shotCounter = (dbConfig.getIndexBlockSize()/blockLength) - 1; + //Iterator indexIt = indexPool.iterator(); + //while ( indexIt.hasNext() ) { + foreach(var block in indexPool){ + if ( ++counter >= shotCounter ) { + hb = new HeaderBlock( + block.getStartIp(), + (int)raf.getFilePointer() + ); + + headerPool.add(hb); + counter = 0; + } + + //write the buffer + raf.write(block.getBytes()); + } + + //record the end block + if ( counter > 0 ) { + indexBlock = indexPool.Last(); + hb = new HeaderBlock( + indexBlock.getStartIp(), + ((int)raf.getFilePointer()) - IndexBlock.getIndexBlockLength() + ); + + headerPool.add(hb); + } + + indexEndPtr = raf.getFilePointer(); + Console.WriteLine("|--[Ok]"); + + //write the super blocks + Console.WriteLine("+-Try to write the super blocks ... "); + raf.seek(0L); //reset the file pointer + byte[] superBuffer = new byte[8]; + Util.writeIntLong(superBuffer, 0, indexStratPtr); + Util.writeIntLong(superBuffer, 4, indexEndPtr - blockLength); + raf.write(superBuffer); + Console.WriteLine("|--[Ok]"); + + //write the header blocks + Console.WriteLine("+-Try to write the header blocks ... "); + //Iterator headerIt = headerPool.iterator(); + //while ( headerIt.hasNext() ) { + // HeaderBlock headerBlock = headerIt.next(); + // raf.write(headerBlock.getBytes()); + //} + foreach (var headerBlock in headerPool) + { + raf.write(headerBlock.getBytes()); + } + + //write the copyright and the release timestamp info + Console.WriteLine("+-Try to write the copyright and release date info ... "); + raf.seek(raf.length()); + //Calendar cal = Calendar.getInstance(); + //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd"); + var date = DateTime.Now; + String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd"); +var timespan= new DateTimeOffset(date).ToUnixTimeSeconds(); + raf.write((int)(timespan)); //the unix timestamp + raf.write(copyright.getBytes("UTF-8")); + Console.WriteLine("|--[Ok]"); + + reader.Close(); + raf.close(); + } + + /** + * internal method to add a new data block record + * + * @param raf + * @param startIp + * @param endIp + * @param region data + */ + private void addDataBlock( + RandomAccessFile raf, + String startIp, String endIp, String region) + { + + byte[] data = region.getBytes("UTF-8"); + int dataPtr = 0; + + /*byte[] city = new byte[4]; + int city_id = getCityId(region); + Util.writeIntLong(city, 0, city_id); + dataPtr = (int)raf.getFilePointer(); + raf.write(city); + raf.write(data);*/ + + //check region ptr pool first + if ( regionPtrPool.containsKey(region) ) { + DataBlock dataBlock = regionPtrPool.get(region); + dataPtr = dataBlock.getDataPtr(); + Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region); + } else { + byte[] city = new byte[4]; + int city_id = getCityId(region); + Util.writeIntLong(city, 0, city_id); + dataPtr = (int)raf.getFilePointer(); + raf.write(city); + raf.write(data); + + regionPtrPool.put(region, new DataBlock(city_id, region, dataPtr)); + } + + //add the data index blocks + IndexBlock ib = new IndexBlock( + Util.ip2long(startIp), + Util.ip2long(endIp), + dataPtr, + data.Length + 4 //4 bytes for the city id + ); + indexPool.add(ib); + } + + /** + * get the city id base on the global region data + * + * @param region + * @return int + */ + public int getCityId( String region ) + { + String[] p = region.Split("\\|"); + if ( p.Length != 5 ) return 0; + + String key = null; + int? intv = null; + for ( int i = 3; i >= 0; i-- ) { + if ( p[i].Equals("0") ) continue; + if ( i == 3 + && p[i].IndexOf("省直辖县级") > -1 ) { + key = p[2]+p[3]; + } else { + key = p[i]; + } + + intv = globalRegionMap.get(key); + if ( intv == null ) return 0; + return intv.Value; + } + + return 0; + } + + public DbConfig getDbConfig() + { + return dbConfig; + } + + public DbMaker setDbConfig(DbConfig dbConfig) + { + this.dbConfig = dbConfig; + return this; + } + + public FileStream getIpSrcFile() + { + return ipSrcFile; + } + + public DbMaker setIpSrcFile(FileStream ipSrcFile) + { + this.ipSrcFile = ipSrcFile; + return this; + } + + /** + * make this directly a runnable application + * interface to make the database file + */ + public static void Main(String[] args) + { + String dstDir = "./data/"; + String[] path = new String[]{null, null}; + for ( int i = 0; i < args.Length; i++ ) { + if ( args[i].Equals("-src") ) { + path[0] = args[++i]; + } else if ( args[i].Equals("-region") ) { + path[1] = args[++i]; + } else if ( args[i].Equals("-dst") ) { + dstDir = args[++i]; + } + } + + for ( int i = 0; i < path.Length; i++ ) { + if ( path[i] == null ) { + Console.WriteLine("Usage: java -jar dbMaker.jar " + + "-src [source text file path] " + + "-region [global region file path]"); + Console.WriteLine("eg: java -jar dbMaker.jar " + + "-src ./data/ip.merge.txt -region ./data/origin/global_region.csv"); + //System.exit(0); + Environment.Exit(0); + } + } + + //check and stdlize the destination directory + if ( ! dstDir.EndsWith("/") ) { + dstDir = dstDir + "/"; + } + + try { + DbConfig config = new DbConfig(); + DbMaker dbMaker = new DbMaker(config, path[0], path[1]); + dbMaker.make(dstDir + "ip2region.db"); + } catch (DbMakerConfigException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } +} +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/Extensions.cs b/maker/csharp/DbMaker/Extensions.cs new file mode 100644 index 00000000..19769033 --- /dev/null +++ b/maker/csharp/DbMaker/Extensions.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace DbMaker +{ + public static class Extensions + { + public static byte[] getBytes(this string str, string encode) + { + return Encoding.GetEncoding(encode).GetBytes(str); + } + + public static Boolean containsKey(this IDictionary dic, TKey key) + { + return dic.ContainsKey(key); + } + + public static TValue get(this IDictionary dic, TKey key) + { + return dic[key]; + } + + public static void put(this IDictionary dic, TKey key, TValue value) + { + dic.Add(key,value); + } + + public static void add(this IList list, TItem item) + { + list.Add(item); + } + + public static void printStackTrace(this Exception e) + { + Console.WriteLine(e); + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/Program.cs b/maker/csharp/DbMaker/Program.cs deleted file mode 100644 index c2072e31..00000000 --- a/maker/csharp/DbMaker/Program.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; - -namespace DbMaker -{ - class Program - { - static void Main(string[] args) - { - Console.WriteLine("Hello World!"); - } - } -} \ No newline at end of file diff --git a/maker/csharp/DbMaker/RandomAccessFile.cs b/maker/csharp/DbMaker/RandomAccessFile.cs index 71ca6c2c..d9f50315 100644 --- a/maker/csharp/DbMaker/RandomAccessFile.cs +++ b/maker/csharp/DbMaker/RandomAccessFile.cs @@ -45,5 +45,20 @@ public void close() { _stream.Dispose(); } + + public void write(byte[] bytes) + { + _stream.Write(bytes); + } + + public void write(int i) + { + + } + + public long getFilePointer() + { + return _stream.Position; + } } } \ No newline at end of file From 56a9fe834e541c063f033cb9b58a67a6aa40c7ad Mon Sep 17 00:00:00 2001 From: Soar360 Date: Thu, 11 Jun 2020 23:58:32 +0800 Subject: [PATCH 8/9] Util.cs -> ip2long --- maker/csharp/DbMaker/Util.cs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/maker/csharp/DbMaker/Util.cs b/maker/csharp/DbMaker/Util.cs index 8281a6ab..a92f8af0 100644 --- a/maker/csharp/DbMaker/Util.cs +++ b/maker/csharp/DbMaker/Util.cs @@ -98,16 +98,14 @@ public static int getInt1(byte[] b, int offset) */ public static long ip2long(String ip) { - throw new NotImplementedException(); - //String[] p = ip.Split("\\."); - //if ( p.Length != 4 ) return 0; - - //int p1 = ((Convert.ToInt32(p[0]) << 24) & 0xFF000000); - //int p2 = ((Integer.valueOf(p[1]) << 16) & 0x00FF0000); - //int p3 = ((Integer.valueOf(p[2]) << 8) & 0x0000FF00); - //int p4 = ((Integer.valueOf(p[3]) << 0) & 0x000000FF); - - //return ((p1 | p2 | p3 | p4) & 0xFFFFFFFFL); + ip = ip.Trim(); + String[] ips = ip.Split('.'); + long ip1 = Int64.Parse(ips[0]); + long ip2 = Int64.Parse(ips[1]); + long ip3 = Int64.Parse(ips[2]); + long ip4 = Int64.Parse(ips[3]); + long ip2long = 1L * ip1 * 256 * 256 * 256 + ip2 * 256 * 256 + ip3 * 256 + ip4; + return ip2long; } /** From 9101c218d3a686c809a96fb9918f0a390372c5c2 Mon Sep 17 00:00:00 2001 From: Soar360 Date: Thu, 9 Jul 2020 14:01:07 +0800 Subject: [PATCH 9/9] add CSharp Macker --- maker/csharp/DbMaker/DbMaker.cs | 853 +++++++++--------- maker/csharp/DbMaker/Extensions.cs | 3 +- .../DbMaker/Properties/launchSettings.json | 8 + maker/csharp/DbMaker/RandomAccessFile.cs | 2 +- 4 files changed, 455 insertions(+), 411 deletions(-) create mode 100644 maker/csharp/DbMaker/Properties/launchSettings.json diff --git a/maker/csharp/DbMaker/DbMaker.cs b/maker/csharp/DbMaker/DbMaker.cs index 43fefbf9..5e3dce58 100644 --- a/maker/csharp/DbMaker/DbMaker.cs +++ b/maker/csharp/DbMaker/DbMaker.cs @@ -9,415 +9,450 @@ namespace DbMaker { -/** - * fast ip db maker - * - * db struct: - * 1. header part - * 1): super part: - * +------------+-----------+ - * | 4 bytes | 4 bytes | - * +------------+-----------+ - * start index ptr, end index ptr - * - * 2): b-tree index part - * +------------+-----------+-----------+-----------+ - * | 4bytes | 4bytes | 4bytes | 4bytes | ... - * +------------+-----------+-----------+-----------+ - * start ip ptr index ptr - * - * 2. data part: - * +------------+-----------------------+ - * | 2bytes | dynamic length | - * +------------+-----------------------+ - * data length city_id|Country|Province|Area|City|ISP - * - * 3. index part: (ip range) - * +------------+-----------+---------------+ - * | 4bytes | 4bytes | 4bytes | - * +------------+-----------+---------------+ - * start ip end ip 3 byte data ptr & 1 byte data length - * - * @author chenxin + /** + * fast ip db maker + * + * db struct: + * 1. header part + * 1): super part: + * +------------+-----------+ + * | 4 bytes | 4 bytes | + * +------------+-----------+ + * start index ptr, end index ptr + * + * 2): b-tree index part + * +------------+-----------+-----------+-----------+ + * | 4bytes | 4bytes | 4bytes | 4bytes | ... + * +------------+-----------+-----------+-----------+ + * start ip ptr index ptr + * + * 2. data part: + * +------------+-----------------------+ + * | 2bytes | dynamic length | + * +------------+-----------------------+ + * data length city_id|Country|Province|Area|City|ISP + * + * 3. index part: (ip range) + * +------------+-----------+---------------+ + * | 4bytes | 4bytes | 4bytes | + * +------------+-----------+---------------+ + * start ip end ip 3 byte data ptr & 1 byte data length + * + * @author chenxin */ -public class DbMaker -{ - /** - * db config - */ - private DbConfig dbConfig; - - /** - * ip source file path - */ - private FileStream ipSrcFile; - - /** - * buffer - */ - private List indexPool; - private List headerPool; - - /** - * global region Id mapping data - */ - private FileStream globalRegionFile = null; - private Dictionary globalRegionMap = null; - - /** - * region and data ptr mapping data - */ - private Dictionary regionPtrPool = null; - - /** - * construct method - * - * @param config - * @param ipSrcFile tb source ip file - * @param globalRegionFile global_region.csv file offer by lion - * @throws DbMakerConfigException - * @throws IOException - */ - public DbMaker( - DbConfig config, - String ipSrcFile, - String globalRegionFile ) - { - this.dbConfig = config; - this.ipSrcFile = File.OpenRead(ipSrcFile); //new File(ipSrcFile); - this.globalRegionMap = new Dictionary(); //new HashMap(); - this.regionPtrPool = new Dictionary(); //new HashMap(); - if ( globalRegionFile != null ) - { - this.globalRegionFile = File.OpenRead(globalRegionFile); //new File(globalRegionFile); - } - - //if ( this.ipSrcFile.exists() == false ) { - if (!File.Exists(ipSrcFile)) - { - throw new IOException("Error: Invalid file path " + ipSrcFile); - } - } - - /** - * initialize the db file - * - * @param raf - * @throws IOException - */ - private void initDbFile( RandomAccessFile raf ) - { - //1. zero fill the header part - raf.seek(0L); - raf.write(new byte[8]); //super block - raf.write(new byte[dbConfig.getTotalHeaderSize()]); //header block - - headerPool = new List(); //new LinkedList(); - indexPool = new List(); //new LinkedList(); - } - - /** - * make the Db file - * - * @param dbFile target output file path - * @throws IOException - */ - public void make(String dbFile) - { - //check and load the gloabl region - if ( globalRegionFile != null ) { - Console.WriteLine("+-Try to load the global region data ..."); - StreamReader greader = new StreamReader((globalRegionFile)); - String gline = null; - while (!greader.EndOfStream ) { - gline = greader.ReadLine();// != null - if (String.IsNullOrWhiteSpace(gline)) - { - continue; - } - - String[] p = gline.Split(","); - if ( p.Length != 5 ) continue; - - //push the mapping - globalRegionMap.put(p[2], Int32.Parse(p[0])); - } - - greader.Close(); - Console.WriteLine("|--[Ok]"); - } - - //alloc the header size - StreamReader reader = new StreamReader(this.ipSrcFile); - RandomAccessFile raf = new RandomAccessFile(dbFile); - - //init the db file - initDbFile(raf); - Console.WriteLine("+-Db file initialized."); - - //analysis main loop - Console.WriteLine("+-Try to write the data blocks ... "); - String line = null; - while ( !reader.EndOfStream ) { - //line = reader.ReadLine() - line = reader.ReadLine(); - if (String.IsNullOrWhiteSpace(line)) - { - continue; - } - line = line.Trim(); - if ( line.Length == 0 ) continue; - if ( line[0] == '#' ) continue; - - //1. get the start ip - int sIdx = 0, eIdx = 0; - if ( (eIdx = line.IndexOf('|', sIdx + 1)) == -1 ) continue; - String startIp = line.Substring(sIdx, eIdx); - - //2. get the end ip - sIdx = eIdx + 1; - if ( (eIdx = line.IndexOf('|', sIdx + 1)) == -1 ) continue; - String endIp = line.Substring(sIdx, eIdx); - - //3. get the region - sIdx = eIdx + 1; - String region = line.Substring(sIdx); - - Console.WriteLine("+-Try to process item " + line); - addDataBlock(raf, startIp, endIp, region); - Console.WriteLine("|--[Ok]"); - } - Console.WriteLine("|--Data block flushed!"); - Console.WriteLine("|--Data file pointer: "+raf.getFilePointer()+"\n"); - - //write the index bytes - Console.WriteLine("+-Try to write index blocks ... "); - - //record the start block - IndexBlock indexBlock = null; - HeaderBlock hb = null; - indexBlock = indexPool.First(); - long indexStartIp = indexBlock.getStartIp(), - indexStratPtr = raf.getFilePointer(), indexEndPtr; - headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr))); - - int blockLength = IndexBlock.getIndexBlockLength(); - int counter = 0, shotCounter = (dbConfig.getIndexBlockSize()/blockLength) - 1; - //Iterator indexIt = indexPool.iterator(); - //while ( indexIt.hasNext() ) { - foreach(var block in indexPool){ - if ( ++counter >= shotCounter ) { - hb = new HeaderBlock( - block.getStartIp(), - (int)raf.getFilePointer() - ); - - headerPool.add(hb); - counter = 0; - } - - //write the buffer - raf.write(block.getBytes()); - } - - //record the end block - if ( counter > 0 ) { - indexBlock = indexPool.Last(); - hb = new HeaderBlock( - indexBlock.getStartIp(), - ((int)raf.getFilePointer()) - IndexBlock.getIndexBlockLength() - ); - - headerPool.add(hb); - } - - indexEndPtr = raf.getFilePointer(); - Console.WriteLine("|--[Ok]"); - - //write the super blocks - Console.WriteLine("+-Try to write the super blocks ... "); - raf.seek(0L); //reset the file pointer - byte[] superBuffer = new byte[8]; - Util.writeIntLong(superBuffer, 0, indexStratPtr); - Util.writeIntLong(superBuffer, 4, indexEndPtr - blockLength); - raf.write(superBuffer); - Console.WriteLine("|--[Ok]"); - - //write the header blocks - Console.WriteLine("+-Try to write the header blocks ... "); - //Iterator headerIt = headerPool.iterator(); - //while ( headerIt.hasNext() ) { - // HeaderBlock headerBlock = headerIt.next(); - // raf.write(headerBlock.getBytes()); - //} - foreach (var headerBlock in headerPool) - { - raf.write(headerBlock.getBytes()); - } - - //write the copyright and the release timestamp info - Console.WriteLine("+-Try to write the copyright and release date info ... "); - raf.seek(raf.length()); - //Calendar cal = Calendar.getInstance(); - //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd"); - var date = DateTime.Now; - String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd"); -var timespan= new DateTimeOffset(date).ToUnixTimeSeconds(); - raf.write((int)(timespan)); //the unix timestamp - raf.write(copyright.getBytes("UTF-8")); - Console.WriteLine("|--[Ok]"); - - reader.Close(); - raf.close(); - } - - /** - * internal method to add a new data block record - * - * @param raf - * @param startIp - * @param endIp - * @param region data - */ - private void addDataBlock( - RandomAccessFile raf, - String startIp, String endIp, String region) - { - - byte[] data = region.getBytes("UTF-8"); - int dataPtr = 0; - - /*byte[] city = new byte[4]; - int city_id = getCityId(region); - Util.writeIntLong(city, 0, city_id); - dataPtr = (int)raf.getFilePointer(); - raf.write(city); - raf.write(data);*/ - - //check region ptr pool first - if ( regionPtrPool.containsKey(region) ) { - DataBlock dataBlock = regionPtrPool.get(region); - dataPtr = dataBlock.getDataPtr(); - Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region); - } else { - byte[] city = new byte[4]; - int city_id = getCityId(region); - Util.writeIntLong(city, 0, city_id); - dataPtr = (int)raf.getFilePointer(); - raf.write(city); - raf.write(data); - - regionPtrPool.put(region, new DataBlock(city_id, region, dataPtr)); - } - - //add the data index blocks - IndexBlock ib = new IndexBlock( - Util.ip2long(startIp), - Util.ip2long(endIp), - dataPtr, - data.Length + 4 //4 bytes for the city id - ); - indexPool.add(ib); - } - - /** - * get the city id base on the global region data - * - * @param region - * @return int - */ - public int getCityId( String region ) - { - String[] p = region.Split("\\|"); - if ( p.Length != 5 ) return 0; - - String key = null; - int? intv = null; - for ( int i = 3; i >= 0; i-- ) { - if ( p[i].Equals("0") ) continue; - if ( i == 3 - && p[i].IndexOf("省直辖县级") > -1 ) { - key = p[2]+p[3]; - } else { - key = p[i]; - } - - intv = globalRegionMap.get(key); - if ( intv == null ) return 0; - return intv.Value; - } - - return 0; - } - - public DbConfig getDbConfig() - { - return dbConfig; - } - - public DbMaker setDbConfig(DbConfig dbConfig) - { - this.dbConfig = dbConfig; - return this; - } - - public FileStream getIpSrcFile() - { - return ipSrcFile; - } - - public DbMaker setIpSrcFile(FileStream ipSrcFile) - { - this.ipSrcFile = ipSrcFile; - return this; - } - - /** - * make this directly a runnable application - * interface to make the database file - */ - public static void Main(String[] args) - { - String dstDir = "./data/"; - String[] path = new String[]{null, null}; - for ( int i = 0; i < args.Length; i++ ) { - if ( args[i].Equals("-src") ) { - path[0] = args[++i]; - } else if ( args[i].Equals("-region") ) { - path[1] = args[++i]; - } else if ( args[i].Equals("-dst") ) { - dstDir = args[++i]; - } - } - - for ( int i = 0; i < path.Length; i++ ) { - if ( path[i] == null ) { - Console.WriteLine("Usage: java -jar dbMaker.jar " - + "-src [source text file path] " - + "-region [global region file path]"); - Console.WriteLine("eg: java -jar dbMaker.jar " - + "-src ./data/ip.merge.txt -region ./data/origin/global_region.csv"); - //System.exit(0); - Environment.Exit(0); - } - } - - //check and stdlize the destination directory - if ( ! dstDir.EndsWith("/") ) { - dstDir = dstDir + "/"; - } - - try { - DbConfig config = new DbConfig(); - DbMaker dbMaker = new DbMaker(config, path[0], path[1]); - dbMaker.make(dstDir + "ip2region.db"); - } catch (DbMakerConfigException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + public class DbMaker + { + /** + * db config + */ + private DbConfig dbConfig; + + /** + * ip source file path + */ + private FileStream ipSrcFile; + + /** + * buffer + */ + private List indexPool; + private List headerPool; + + /** + * global region Id mapping data + */ + private FileStream globalRegionFile = null; + private Dictionary globalRegionMap = null; + + /** + * region and data ptr mapping data + */ + private Dictionary regionPtrPool = null; + + /** + * construct method + * + * @param config + * @param ipSrcFile tb source ip file + * @param globalRegionFile global_region.csv file offer by lion + * @throws DbMakerConfigException + * @throws IOException + */ + public DbMaker( + DbConfig config, + String ipSrcFile, + String globalRegionFile) + { + this.dbConfig = config; + this.ipSrcFile = File.OpenRead(ipSrcFile); //new File(ipSrcFile); + this.globalRegionMap = new Dictionary(); //new HashMap(); + this.regionPtrPool = new Dictionary(); //new HashMap(); + if (globalRegionFile != null) + { + this.globalRegionFile = File.OpenRead(globalRegionFile); //new File(globalRegionFile); + } + + //if ( this.ipSrcFile.exists() == false ) { + if (!File.Exists(ipSrcFile)) + { + throw new IOException("Error: Invalid file path " + ipSrcFile); + } + } + + /** + * initialize the db file + * + * @param raf + * @throws IOException + */ + private void initDbFile(RandomAccessFile raf) + { + //1. zero fill the header part + raf.seek(0L); + raf.write(new byte[8]); //super block + raf.write(new byte[dbConfig.getTotalHeaderSize()]); //header block + + headerPool = new List(); //new LinkedList(); + indexPool = new List(); //new LinkedList(); + } + + /** + * make the Db file + * + * @param dbFile target output file path + * @throws IOException + */ + public void make(String dbFile) + { + //check and load the gloabl region + if (globalRegionFile != null) + { + Console.WriteLine("+-Try to load the global region data ..."); + StreamReader greader = new StreamReader((globalRegionFile)); + String gline = null; + while (!greader.EndOfStream) + { + gline = greader.ReadLine();// != null + if (String.IsNullOrWhiteSpace(gline)) + { + continue; + } + + String[] p = gline.Split(","); + if (p.Length != 5) continue; + + //push the mapping + globalRegionMap.put(p[2], Int32.Parse(p[0])); + } + + greader.Close(); + Console.WriteLine("|--[Ok]"); + } + + //alloc the header size + StreamReader reader = new StreamReader(this.ipSrcFile); + RandomAccessFile raf = new RandomAccessFile(dbFile); + + //init the db file + initDbFile(raf); + Console.WriteLine("+-Db file initialized."); + + //analysis main loop + Console.WriteLine("+-Try to write the data blocks ... "); + String line = null; + while (!reader.EndOfStream) + { + //line = reader.ReadLine() + line = reader.ReadLine(); + if (String.IsNullOrWhiteSpace(line)) + { + continue; + } + line = line.Trim(); + if (line.Length == 0) continue; + if (line[0] == '#') continue; + + //1. get the start ip + int sIdx = 0, eIdx = 0; + if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue; + String startIp = line.Substring(sIdx, eIdx); + + //2. get the end ip + sIdx = eIdx + 1; + if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue; + String endIp = line.Substring(sIdx, eIdx - sIdx); + + //3. get the region + sIdx = eIdx + 1; + String region = line.Substring(sIdx); + + Console.WriteLine("+-Try to process item " + line); + addDataBlock(raf, startIp, endIp, region); + Console.WriteLine("|--[Ok]"); + } + Console.WriteLine("|--Data block flushed!"); + Console.WriteLine("|--Data file pointer: " + raf.getFilePointer() + "\n"); + + //write the index bytes + Console.WriteLine("+-Try to write index blocks ... "); + + //record the start block + IndexBlock indexBlock = null; + HeaderBlock hb = null; + indexBlock = indexPool.First(); + long indexStartIp = indexBlock.getStartIp(), + indexStratPtr = raf.getFilePointer(), indexEndPtr; + headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr))); + + int blockLength = IndexBlock.getIndexBlockLength(); + int counter = 0, shotCounter = (dbConfig.getIndexBlockSize() / blockLength) - 1; + //Iterator indexIt = indexPool.iterator(); + //while ( indexIt.hasNext() ) { + foreach (var block in indexPool) + { + if (++counter >= shotCounter) + { + hb = new HeaderBlock( + block.getStartIp(), + (int)raf.getFilePointer() + ); + + headerPool.add(hb); + counter = 0; + } + + //write the buffer + raf.write(block.getBytes()); + } + + //record the end block + if (counter > 0) + { + indexBlock = indexPool.Last(); + hb = new HeaderBlock( + indexBlock.getStartIp(), + ((int)raf.getFilePointer()) - IndexBlock.getIndexBlockLength() + ); + + headerPool.add(hb); + } + + indexEndPtr = raf.getFilePointer(); + Console.WriteLine("|--[Ok]"); + + //write the super blocks + Console.WriteLine("+-Try to write the super blocks ... "); + raf.seek(0L); //reset the file pointer + byte[] superBuffer = new byte[8]; + Util.writeIntLong(superBuffer, 0, indexStratPtr); + Util.writeIntLong(superBuffer, 4, indexEndPtr - blockLength); + raf.write(superBuffer); + Console.WriteLine("|--[Ok]"); + + //write the header blocks + Console.WriteLine("+-Try to write the header blocks ... "); + //Iterator headerIt = headerPool.iterator(); + //while ( headerIt.hasNext() ) { + // HeaderBlock headerBlock = headerIt.next(); + // raf.write(headerBlock.getBytes()); + //} + foreach (var headerBlock in headerPool) + { + raf.write(headerBlock.getBytes()); + } + + //write the copyright and the release timestamp info + Console.WriteLine("+-Try to write the copyright and release date info ... "); + raf.seek(raf.length()); + //Calendar cal = Calendar.getInstance(); + //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd"); + var date = DateTime.Now; + String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd"); + var timespan = new DateTimeOffset(date).ToUnixTimeSeconds(); + raf.write((int)(timespan)); //the unix timestamp + raf.write(copyright.getBytes("UTF-8")); + Console.WriteLine("|--[Ok]"); + + reader.Close(); + raf.close(); + } + + /** + * internal method to add a new data block record + * + * @param raf + * @param startIp + * @param endIp + * @param region data + */ + private void addDataBlock( + RandomAccessFile raf, + String startIp, String endIp, String region) + { + + byte[] data = region.getBytes("UTF-8"); + int dataPtr = 0; + + /*byte[] city = new byte[4]; + int city_id = getCityId(region); + Util.writeIntLong(city, 0, city_id); + dataPtr = (int)raf.getFilePointer(); + raf.write(city); + raf.write(data);*/ + + //check region ptr pool first + if (regionPtrPool.containsKey(region)) + { + DataBlock dataBlock = regionPtrPool.get(region); + dataPtr = dataBlock.getDataPtr(); + Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region); + } + else + { + byte[] city = new byte[4]; + int city_id = getCityId(region); + Util.writeIntLong(city, 0, city_id); + dataPtr = (int)raf.getFilePointer(); + raf.write(city); + raf.write(data); + + regionPtrPool.put(region, new DataBlock(city_id, region, dataPtr)); + } + + //add the data index blocks + IndexBlock ib = new IndexBlock( + Util.ip2long(startIp), + Util.ip2long(endIp), + dataPtr, + data.Length + 4 //4 bytes for the city id + ); + indexPool.add(ib); + } + + /** + * get the city id base on the global region data + * + * @param region + * @return int + */ + public int getCityId(String region) + { + String[] p = region.Split("|"); + if (p.Length != 5) return 0; + + String key = null; + int? intv = null; + for (int i = 3; i >= 0; i--) + { + if (p[i].Equals("0")) continue; + if (i == 3 + && p[i].IndexOf("省直辖县级") > -1) + { + key = p[2] + p[3]; + } + else + { + key = p[i]; + } + + if (!globalRegionMap.ContainsKey(key)) + { + return 0; + } + intv = globalRegionMap.get(key); + //if (intv == null) return 0; + return intv.Value; + } + + return 0; + } + + public DbConfig getDbConfig() + { + return dbConfig; + } + + public DbMaker setDbConfig(DbConfig dbConfig) + { + this.dbConfig = dbConfig; + return this; + } + + public FileStream getIpSrcFile() + { + return ipSrcFile; + } + + public DbMaker setIpSrcFile(FileStream ipSrcFile) + { + this.ipSrcFile = ipSrcFile; + return this; + } + + /** + * make this directly a runnable application + * interface to make the database file + */ + public static void Main(String[] args) + { + String dstDir = "./data/"; + String[] path = new String[] { null, null }; + for (int i = 0; i < args.Length; i++) + { + if (args[i].Equals("-src")) + { + path[0] = args[++i]; + } + else if (args[i].Equals("-region")) + { + path[1] = args[++i]; + } + else if (args[i].Equals("-dst")) + { + dstDir = args[++i]; + } + } + + for (int i = 0; i < path.Length; i++) + { + if (path[i] == null) + { + Console.WriteLine("Usage: DbMaker " + + "-src [source text file path] " + + "-region [global region file path]"); + Console.WriteLine("eg: DbMaker " + + "-src ./data/ip.merge.txt -region ./data/origin/global_region.csv"); + //System.exit(0); + Environment.Exit(0); + } + } + + //check and stdlize the destination directory + if (!dstDir.EndsWith("/")) + { + dstDir = dstDir + "/"; + } + + try + { + if (!Directory.Exists(dstDir)) + { + Directory.CreateDirectory(dstDir); + } + DbConfig config = new DbConfig(); + DbMaker dbMaker = new DbMaker(config, path[0], path[1]); + dbMaker.make(dstDir + "ip2region.db"); + } + catch (DbMakerConfigException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + catch (IOException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } } - } -} + } } \ No newline at end of file diff --git a/maker/csharp/DbMaker/Extensions.cs b/maker/csharp/DbMaker/Extensions.cs index 19769033..092c1f4f 100644 --- a/maker/csharp/DbMaker/Extensions.cs +++ b/maker/csharp/DbMaker/Extensions.cs @@ -23,7 +23,8 @@ public static TValue get(this IDictionary dic, TKey public static void put(this IDictionary dic, TKey key, TValue value) { - dic.Add(key,value); + //dic.Add(key,value); + dic[key] = value; } public static void add(this IList list, TItem item) diff --git a/maker/csharp/DbMaker/Properties/launchSettings.json b/maker/csharp/DbMaker/Properties/launchSettings.json new file mode 100644 index 00000000..73453831 --- /dev/null +++ b/maker/csharp/DbMaker/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "DbMaker": { + "commandName": "Project", + "commandLineArgs": " -src ip.merge.txt -region global_region.csv" + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/RandomAccessFile.cs b/maker/csharp/DbMaker/RandomAccessFile.cs index d9f50315..585821ef 100644 --- a/maker/csharp/DbMaker/RandomAccessFile.cs +++ b/maker/csharp/DbMaker/RandomAccessFile.cs @@ -7,7 +7,7 @@ public class RandomAccessFile : IDisposable { private readonly Stream _stream; - public RandomAccessFile(String file) : this(File.OpenRead(file)) + public RandomAccessFile(String file) : this(File.Open(file, FileMode.OpenOrCreate)) { }