Skip to content

Commit

Permalink
Reduce I/O
Browse files Browse the repository at this point in the history
  • Loading branch information
@ip2location
ip2location committed Nov 21, 2019
1 parent b7c9293 commit 4db3608
Show file tree
Hide file tree
Showing 2 changed files with 116 additions and 27 deletions.
139 changes: 114 additions & 25 deletions IP2ProxyComponent/IP2Proxy.vb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,12 +224,48 @@ Public Class Component
If _MMF Is Nothing Then
Try
_MMF = MemoryMappedFile.OpenExisting(_MapFileName, MemoryMappedFileRights.Read)
Catch Ex As Exception
Catch ex As Exception
Try
_MMF = MemoryMappedFile.CreateFromFile(_DBFilePath, FileMode.Open, _MapFileName, New FileInfo(_DBFilePath).Length, MemoryMappedFileAccess.Read)
Catch Ex2 As Exception
ErrLog(Ex2.Message)
Throw Ex2
Catch ex2 As Exception
Try
Dim len As Long = New FileInfo(_DBFilePath).Length
_MMF = MemoryMappedFile.CreateNew(_MapFileName, len, MemoryMappedFileAccess.ReadWrite)
Using stream As MemoryMappedViewStream = _MMF.CreateViewStream()
Using writer As BinaryWriter = New BinaryWriter(stream)
Using fs As FileStream = New FileStream(_DBFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)
Dim buff(len) As Byte
fs.Read(buff, 0, buff.Length)
writer.Write(buff, 0, buff.Length)
End Using
End Using
End Using
Catch ex3 As Exception
' this part onwards trying Linux specific stuff (no named map)
Try
_MMF = MemoryMappedFile.OpenExisting(Nothing, MemoryMappedFileRights.Read)
Catch ex4 As Exception
Try
_MMF = MemoryMappedFile.CreateFromFile(_DBFilePath, FileMode.Open, Nothing, New FileInfo(_DBFilePath).Length, MemoryMappedFileAccess.Read)
Catch ex5 As Exception
Try
Dim len As Long = New FileInfo(_DBFilePath).Length
_MMF = MemoryMappedFile.CreateNew(Nothing, len, MemoryMappedFileAccess.ReadWrite)
Using stream As MemoryMappedViewStream = _MMF.CreateViewStream()
Using writer As BinaryWriter = New BinaryWriter(stream)
Using fs As FileStream = New FileStream(_DBFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)
Dim buff(len) As Byte
fs.Read(buff, 0, buff.Length)
writer.Write(buff, 0, buff.Length)
End Using
End Using
End Using
Catch ex6 As Exception
ErrLog(ex6.Message & "---" & ex6.StackTrace)
End Try
End Try
End Try
End Try
End Try
End Try
End If
Expand Down Expand Up @@ -309,16 +345,28 @@ Public Class Component
_IPv6ColumnSize = 16 + ((_DBColumn - 1) << 2) ' 4 bytes each column, except IPFrom column which is 16 bytes

' since both IPv4 and IPv6 use 4 bytes for the below columns, can just do it once here
COUNTRY_POSITION_OFFSET = If(COUNTRY_POSITION(_DBType) <> 0, (COUNTRY_POSITION(_DBType) - 1) << 2, 0)
REGION_POSITION_OFFSET = If(REGION_POSITION(_DBType) <> 0, (REGION_POSITION(_DBType) - 1) << 2, 0)
CITY_POSITION_OFFSET = If(CITY_POSITION(_DBType) <> 0, (CITY_POSITION(_DBType) - 1) << 2, 0)
ISP_POSITION_OFFSET = If(ISP_POSITION(_DBType) <> 0, (ISP_POSITION(_DBType) - 1) << 2, 0)
PROXYTYPE_POSITION_OFFSET = If(PROXYTYPE_POSITION(_DBType) <> 0, (PROXYTYPE_POSITION(_DBType) - 1) << 2, 0)
DOMAIN_POSITION_OFFSET = If(DOMAIN_POSITION(_DBType) <> 0, (DOMAIN_POSITION(_DBType) - 1) << 2, 0)
USAGETYPE_POSITION_OFFSET = If(USAGETYPE_POSITION(_DBType) <> 0, (USAGETYPE_POSITION(_DBType) - 1) << 2, 0)
ASN_POSITION_OFFSET = If(ASN_POSITION(_DBType) <> 0, (ASN_POSITION(_DBType) - 1) << 2, 0)
AS_POSITION_OFFSET = If(AS_POSITION(_DBType) <> 0, (AS_POSITION(_DBType) - 1) << 2, 0)
LASTSEEN_POSITION_OFFSET = If(LASTSEEN_POSITION(_DBType) <> 0, (LASTSEEN_POSITION(_DBType) - 1) << 2, 0)
'COUNTRY_POSITION_OFFSET = If(COUNTRY_POSITION(_DBType) <> 0, (COUNTRY_POSITION(_DBType) - 1) << 2, 0)
'REGION_POSITION_OFFSET = If(REGION_POSITION(_DBType) <> 0, (REGION_POSITION(_DBType) - 1) << 2, 0)
'CITY_POSITION_OFFSET = If(CITY_POSITION(_DBType) <> 0, (CITY_POSITION(_DBType) - 1) << 2, 0)
'ISP_POSITION_OFFSET = If(ISP_POSITION(_DBType) <> 0, (ISP_POSITION(_DBType) - 1) << 2, 0)
'PROXYTYPE_POSITION_OFFSET = If(PROXYTYPE_POSITION(_DBType) <> 0, (PROXYTYPE_POSITION(_DBType) - 1) << 2, 0)
'DOMAIN_POSITION_OFFSET = If(DOMAIN_POSITION(_DBType) <> 0, (DOMAIN_POSITION(_DBType) - 1) << 2, 0)
'USAGETYPE_POSITION_OFFSET = If(USAGETYPE_POSITION(_DBType) <> 0, (USAGETYPE_POSITION(_DBType) - 1) << 2, 0)
'ASN_POSITION_OFFSET = If(ASN_POSITION(_DBType) <> 0, (ASN_POSITION(_DBType) - 1) << 2, 0)
'AS_POSITION_OFFSET = If(AS_POSITION(_DBType) <> 0, (AS_POSITION(_DBType) - 1) << 2, 0)
'LASTSEEN_POSITION_OFFSET = If(LASTSEEN_POSITION(_DBType) <> 0, (LASTSEEN_POSITION(_DBType) - 1) << 2, 0)

' slightly different offset for reading by row
COUNTRY_POSITION_OFFSET = If(COUNTRY_POSITION(_DBType) <> 0, (COUNTRY_POSITION(_DBType) - 2) << 2, 0)
REGION_POSITION_OFFSET = If(REGION_POSITION(_DBType) <> 0, (REGION_POSITION(_DBType) - 2) << 2, 0)
CITY_POSITION_OFFSET = If(CITY_POSITION(_DBType) <> 0, (CITY_POSITION(_DBType) - 2) << 2, 0)
ISP_POSITION_OFFSET = If(ISP_POSITION(_DBType) <> 0, (ISP_POSITION(_DBType) - 2) << 2, 0)
PROXYTYPE_POSITION_OFFSET = If(PROXYTYPE_POSITION(_DBType) <> 0, (PROXYTYPE_POSITION(_DBType) - 2) << 2, 0)
DOMAIN_POSITION_OFFSET = If(DOMAIN_POSITION(_DBType) <> 0, (DOMAIN_POSITION(_DBType) - 2) << 2, 0)
USAGETYPE_POSITION_OFFSET = If(USAGETYPE_POSITION(_DBType) <> 0, (USAGETYPE_POSITION(_DBType) - 2) << 2, 0)
ASN_POSITION_OFFSET = If(ASN_POSITION(_DBType) <> 0, (ASN_POSITION(_DBType) - 2) << 2, 0)
AS_POSITION_OFFSET = If(AS_POSITION(_DBType) <> 0, (AS_POSITION(_DBType) - 2) << 2, 0)
LASTSEEN_POSITION_OFFSET = If(LASTSEEN_POSITION(_DBType) <> 0, (LASTSEEN_POSITION(_DBType) - 2) << 2, 0)

COUNTRY_ENABLED = If(COUNTRY_POSITION(_DBType) <> 0, True, False)
REGION_ENABLED = If(REGION_POSITION(_DBType) <> 0, True, False)
Expand Down Expand Up @@ -572,18 +620,25 @@ Public Class Component
Dim [AS] As String = MSG_NOT_SUPPORTED
Dim Last_Seen As String = MSG_NOT_SUPPORTED

Dim FirstCol As Integer = 4 ' for IPv4, IP From is 4 bytes
If IPType = 6 Then ' IPv6
RowOffset = RowOffset + 12 ' coz below is assuming all columns are 4 bytes, so got 12 left to go to make 16 bytes total
FirstCol = 16 ' 16 bytes for IPv6
'RowOffset = RowOffset + 12 ' coz below is assuming all columns are 4 bytes, so got 12 left to go to make 16 bytes total
End If

' read the row here after the IP From column (remaining columns are all 4 bytes)
Dim Row() As Byte = ReadRow(RowOffset + FirstCol, ColumnSize - FirstCol, Accessor, FS)

If PROXYTYPE_ENABLED Then
If Mode = Modes.ALL OrElse Mode = Modes.PROXY_TYPE OrElse Mode = Modes.IS_PROXY Then
Proxy_Type = ReadStr(Read32(RowOffset + PROXYTYPE_POSITION_OFFSET, Accessor, FS), FS)
'Proxy_Type = ReadStr(Read32(RowOffset + PROXYTYPE_POSITION_OFFSET, Accessor, FS), FS)
Proxy_Type = ReadStr(Read32_Row(Row, PROXYTYPE_POSITION_OFFSET), FS)
End If
End If
If COUNTRY_ENABLED Then
If Mode = Modes.ALL OrElse Mode = Modes.COUNTRY_SHORT OrElse Mode = Modes.COUNTRY_LONG OrElse Mode = Modes.IS_PROXY Then
CountryPos = Read32(RowOffset + COUNTRY_POSITION_OFFSET, Accessor, FS)
'CountryPos = Read32(RowOffset + COUNTRY_POSITION_OFFSET, Accessor, FS)
CountryPos = Read32_Row(Row, COUNTRY_POSITION_OFFSET)
End If
If Mode = Modes.ALL OrElse Mode = Modes.COUNTRY_SHORT OrElse Mode = Modes.IS_PROXY Then
Country_Short = ReadStr(CountryPos, FS)
Expand All @@ -594,42 +649,50 @@ Public Class Component
End If
If REGION_ENABLED Then
If Mode = Modes.ALL OrElse Mode = Modes.REGION Then
Region = ReadStr(Read32(RowOffset + REGION_POSITION_OFFSET, Accessor, FS), FS)
'Region = ReadStr(Read32(RowOffset + REGION_POSITION_OFFSET, Accessor, FS), FS)
Region = ReadStr(Read32_Row(Row, REGION_POSITION_OFFSET), FS)
End If
End If
If CITY_ENABLED Then
If Mode = Modes.ALL OrElse Mode = Modes.CITY Then
City = ReadStr(Read32(RowOffset + CITY_POSITION_OFFSET, Accessor, FS), FS)
'City = ReadStr(Read32(RowOffset + CITY_POSITION_OFFSET, Accessor, FS), FS)
City = ReadStr(Read32_Row(Row, CITY_POSITION_OFFSET), FS)
End If
End If
If ISP_ENABLED Then
If Mode = Modes.ALL OrElse Mode = Modes.ISP Then
ISP = ReadStr(Read32(RowOffset + ISP_POSITION_OFFSET, Accessor, FS), FS)
'ISP = ReadStr(Read32(RowOffset + ISP_POSITION_OFFSET, Accessor, FS), FS)
ISP = ReadStr(Read32_Row(Row, ISP_POSITION_OFFSET), FS)
End If
End If
If DOMAIN_ENABLED Then
If Mode = Modes.ALL OrElse Mode = Modes.DOMAIN Then
Domain = ReadStr(Read32(RowOffset + DOMAIN_POSITION_OFFSET, Accessor, FS), FS)
'Domain = ReadStr(Read32(RowOffset + DOMAIN_POSITION_OFFSET, Accessor, FS), FS)
Domain = ReadStr(Read32_Row(Row, DOMAIN_POSITION_OFFSET), FS)
End If
End If
If USAGETYPE_ENABLED Then
If Mode = Modes.ALL OrElse Mode = Modes.USAGE_TYPE Then
Usage_Type = ReadStr(Read32(RowOffset + USAGETYPE_POSITION_OFFSET, Accessor, FS), FS)
'Usage_Type = ReadStr(Read32(RowOffset + USAGETYPE_POSITION_OFFSET, Accessor, FS), FS)
Usage_Type = ReadStr(Read32_Row(Row, USAGETYPE_POSITION_OFFSET), FS)
End If
End If
If ASN_ENABLED Then
If Mode = Modes.ALL OrElse Mode = Modes.ASN Then
ASN = ReadStr(Read32(RowOffset + ASN_POSITION_OFFSET, Accessor, FS), FS)
'ASN = ReadStr(Read32(RowOffset + ASN_POSITION_OFFSET, Accessor, FS), FS)
ASN = ReadStr(Read32_Row(Row, ASN_POSITION_OFFSET), FS)
End If
End If
If AS_ENABLED Then
If Mode = Modes.ALL OrElse Mode = Modes.AS Then
[AS] = ReadStr(Read32(RowOffset + AS_POSITION_OFFSET, Accessor, FS), FS)
'[AS] = ReadStr(Read32(RowOffset + AS_POSITION_OFFSET, Accessor, FS), FS)
[AS] = ReadStr(Read32_Row(Row, AS_POSITION_OFFSET), FS)
End If
End If
If LASTSEEN_ENABLED Then
If Mode = Modes.ALL OrElse Mode = Modes.LAST_SEEN Then
Last_Seen = ReadStr(Read32(RowOffset + LASTSEEN_POSITION_OFFSET, Accessor, FS), FS)
'Last_Seen = ReadStr(Read32(RowOffset + LASTSEEN_POSITION_OFFSET, Accessor, FS), FS)
Last_Seen = ReadStr(Read32_Row(Row, LASTSEEN_POSITION_OFFSET), FS)
End If
End If

Expand Down Expand Up @@ -694,6 +757,19 @@ Public Class Component
End Try
End Function

' Read whole row into array of bytes
Private Function ReadRow(ByVal _Pos As Long, ByVal MyLen As UInt32, ByRef MyAccessor As MemoryMappedViewAccessor, ByRef MyFilestream As FileStream) As Byte()
Dim row(MyLen - 1) As Byte

If _UseMemoryMappedFile Then
MyAccessor.ReadArray(Of Byte)(_Pos, row, 0, MyLen)
Else
MyFilestream.Seek(_Pos - 1, SeekOrigin.Begin)
MyFilestream.Read(row, 0, MyLen)
End If
Return row
End Function

Private Function Read32Or128(ByVal _Pos As Long, ByVal _MyIPType As Integer, ByRef MyAccessor As MemoryMappedViewAccessor, ByRef MyFilestream As FileStream) As BigInteger
If _MyIPType = 4 Then
Return Read32(_Pos, MyAccessor, MyFilestream)
Expand Down Expand Up @@ -729,6 +805,19 @@ Public Class Component
End Try
End Function

' Read 32 bits in byte array
Private Function Read32_Row(ByRef Row() As Byte, ByVal ByteOffset As Integer) As BigInteger
Try
Dim _Byte(3) As Byte ' 4 bytes
Array.Copy(Row, ByteOffset, _Byte, 0, 4)

Return System.BitConverter.ToUInt32(_Byte, 0)
Catch ex As Exception
Throw
'ErrLog("Read32_Row-" & ex.Message)
End Try
End Function

' Read 32 bits in the encrypted database
Private Function Read32(ByVal _Pos As Long, ByRef MyAccessor As MemoryMappedViewAccessor, ByRef MyFilestream As FileStream) As BigInteger
Try
Expand Down
4 changes: 2 additions & 2 deletions IP2ProxyComponent/My Project/AssemblyInfo.vb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,5 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")>

<Assembly: AssemblyVersion("2.1.0.0")>
<Assembly: AssemblyFileVersion("2.1.0.0")>
<Assembly: AssemblyVersion("2.2.0.0")>
<Assembly: AssemblyFileVersion("2.2.0.0")>

0 comments on commit 4db3608

Please sign in to comment.