Skip to content

Commit

Permalink
Fiction and scimag scan
Browse files Browse the repository at this point in the history
  • Loading branch information
@libgenapps
libgenapps committed May 27, 2018
1 parent 3b35b27 commit eb99fac
Show file tree
Hide file tree
Showing 24 changed files with 1,076 additions and 110 deletions.
1 change: 1 addition & 0 deletions LibgenDesktop.Setup/AppFiles.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ static AppFiles()
AddFile(@"Languages\Romanian.lng");
AddFile(@"Languages\Ukrainian.lng");
AddFile(@"Languages\Spanish.lng");
AddFile(@"Languages\French.lng");
AddFile(@"Mirrors\mirrors.config");
AddFile(@"Mirrors\libgen_io_nonfiction.xslt");
AddFile(@"Mirrors\libgen_io_fiction.xslt");
Expand Down
4 changes: 2 additions & 2 deletions LibgenDesktop.Setup/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
{
internal static class Constants
{
public const string CURRENT_VERSION = "1.1.5";
public const string TITLE_VERSION = "1.1.5";
public const string CURRENT_VERSION = "1.1.6";
public const string TITLE_VERSION = "1.1.6";
public const string PRODUCT_TITLE_FORMAT = "Libgen Desktop " + TITLE_VERSION + " ({0}-bit)";
public const string SHORTCUT_TITLE_FORMAT = "Libgen Desktop ({0}-bit)";
public const string PRODUCT_COMPANY = "Libgen Apps";
Expand Down
6 changes: 3 additions & 3 deletions LibgenDesktop/Common/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ namespace LibgenDesktop.Common
{
internal static class Constants
{
public const string CURRENT_VERSION = "1.1.5";
public const string CURRENT_GITHUB_RELEASE_NAME = "1.1.5";
public static readonly DateTime CURRENT_GITHUB_RELEASE_DATE = new DateTime(2018, 5, 24);
public const string CURRENT_VERSION = "1.1.6";
public const string CURRENT_GITHUB_RELEASE_NAME = "1.1.6";
public static readonly DateTime CURRENT_GITHUB_RELEASE_DATE = new DateTime(2018, 5, 27);
public const string CURRENT_DATABASE_VERSION = "1.0";

public const string APP_SETTINGS_FILE_NAME = "libgen.config";
Expand Down
10 changes: 10 additions & 0 deletions LibgenDesktop/LibgenDesktop.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<LangVersion>7.1</LangVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
Expand All @@ -46,6 +47,7 @@
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>true</Prefer32Bit>
<LangVersion>7.1</LangVersion>
</PropertyGroup>
<PropertyGroup>
<ApplicationManifest>app.manifest</ApplicationManifest>
Expand Down Expand Up @@ -209,6 +211,7 @@
<Compile Include="Models\ProgressArgs\ImportSearchTableDefinitionProgress.cs" />
<Compile Include="Models\ProgressArgs\ImportTableDefinitionFoundProgress.cs" />
<Compile Include="Models\ProgressArgs\ExportProgress.cs" />
<Compile Include="Models\ProgressArgs\ScanUnknownProgress.cs" />
<Compile Include="Models\ProgressArgs\ScanCompleteProgress.cs" />
<Compile Include="Models\ProgressArgs\ScanProgress.cs" />
<Compile Include="Models\ProgressArgs\SearchProgress.cs" />
Expand Down Expand Up @@ -237,6 +240,9 @@
<Compile Include="ViewModels\DetailsItems\FictionDetailsItemViewModel.cs" />
<Compile Include="ViewModels\DetailsItems\NonFictionDetailsItemViewModel.cs" />
<Compile Include="ViewModels\DetailsItems\SciMagDetailsItemViewModel.cs" />
<Compile Include="ViewModels\Library\SciMagScanResultItemViewModel.cs" />
<Compile Include="ViewModels\Library\FictionScanResultItemViewModel.cs" />
<Compile Include="ViewModels\Library\NonFictionScanResultItemViewModel.cs" />
<Compile Include="ViewModels\Library\ScanResultItemViewModel.cs" />
<Compile Include="ViewModels\SearchResultItems\SciMagSearchResultItemViewModel.cs" />
<Compile Include="ViewModels\SearchResultItems\FictionSearchResultItemViewModel.cs" />
Expand Down Expand Up @@ -730,6 +736,10 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<TargetPath>Languages\Spanish.lng</TargetPath>
</ContentWithTargetPath>
<ContentWithTargetPath Include="Resources\Languages\French.lng">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<TargetPath>Languages\French.lng</TargetPath>
</ContentWithTargetPath>
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
Expand Down
52 changes: 52 additions & 0 deletions LibgenDesktop/Models/Database/LocalDatabase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,11 @@ public void CreateFictionTables()
ExecuteCommands(SqlScripts.CREATE_FICTION_FTS_TABLE);
}

public void CreateFictionMd5HashIndex()
{
ExecuteCommands(SqlScripts.CREATE_FICTION_MD5HASH_INDEX);
}

public void CreateFictionLastModifiedDateTimeIndex()
{
ExecuteCommands(SqlScripts.CREATE_FICTION_LASTMODIFIEDDATETIME_INDEX);
Expand Down Expand Up @@ -564,6 +569,27 @@ public FictionBook GetFictionBookById(int id)
}
}

public FictionBook GetFictionBookByMd5Hash(string md5Hash)
{
using (SQLiteCommand command = connection.CreateCommand())
{
command.CommandText = SqlScripts.GET_FICTION_BY_MD5HASH;
command.Parameters.AddWithValue("@Md5Hash", md5Hash);
using (SQLiteDataReader dataReader = command.ExecuteReader())
{
if (dataReader.Read())
{
FictionBook book = ReadFictionBook(dataReader);
return book;
}
else
{
return null;
}
}
}
}

public int? GetFictionBookIdByLibgenId(int libgenId)
{
return GetIdByLibgenId(SqlScripts.GET_FICTION_ID_BY_LIBGENID, libgenId);
Expand Down Expand Up @@ -1021,6 +1047,11 @@ public void CreateSciMagTables()
ExecuteCommands(SqlScripts.CREATE_SCIMAG_FTS_TABLE);
}

public void CreateSciMagMd5HashIndex()
{
ExecuteCommands(SqlScripts.CREATE_SCIMAG_MD5HASH_INDEX);
}

public void CreateSciMagAddedDateTimeIndex()
{
ExecuteCommands(SqlScripts.CREATE_SCIMAG_ADDEDDATETIME_INDEX);
Expand Down Expand Up @@ -1057,6 +1088,27 @@ public SciMagArticle GetSciMagArticleById(int id)
}
}

public SciMagArticle GetSciMagArticleByMd5Hash(string md5Hash)
{
using (SQLiteCommand command = connection.CreateCommand())
{
command.CommandText = SqlScripts.GET_SCIMAG_BY_MD5HASH;
command.Parameters.AddWithValue("@Md5Hash", md5Hash);
using (SQLiteDataReader dataReader = command.ExecuteReader())
{
if (dataReader.Read())
{
SciMagArticle article = ReadSciMagArticle(dataReader);
return article;
}
else
{
return null;
}
}
}
}

public int? GetSciMagArticleIdByLibgenId(int libgenId)
{
return GetIdByLibgenId(SqlScripts.GET_SCIMAG_ID_BY_LIBGENID, libgenId);
Expand Down
12 changes: 8 additions & 4 deletions LibgenDesktop/Models/Database/SqlScripts.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ internal static class SqlScripts

public const string GET_NON_FICTION_BY_MD5HASH = "SELECT * FROM non_fiction WHERE Md5Hash = @Md5Hash COLLATE NOCASE";

public const string GET_NON_FICTION_ID_BY_LIBGENID = "SELECT Id FROM non_fiction WHERE LibgenId=@LibgenId LIMIT 1";
public const string GET_NON_FICTION_ID_BY_LIBGENID = "SELECT Id FROM non_fiction WHERE LibgenId = @LibgenId LIMIT 1";

public const string GET_LAST_MODIFIED_NON_FICTION =
"SELECT * FROM non_fiction WHERE LastModifiedDateTime = (SELECT MAX(LastModifiedDateTime) FROM non_fiction) ORDER BY LibgenId DESC LIMIT 1";
Expand Down Expand Up @@ -238,7 +238,9 @@ internal static class SqlScripts

public const string GET_FICTION_BY_ID = "SELECT * FROM fiction WHERE Id = @Id";

public const string GET_FICTION_ID_BY_LIBGENID = "SELECT Id FROM fiction WHERE LibgenId=@LibgenId LIMIT 1";
public const string GET_FICTION_BY_MD5HASH = "SELECT * FROM fiction WHERE Md5Hash = @Md5Hash COLLATE NOCASE";

public const string GET_FICTION_ID_BY_LIBGENID = "SELECT Id FROM fiction WHERE LibgenId = @LibgenId LIMIT 1";

public const string GET_LAST_MODIFIED_FICTION =
"SELECT * FROM fiction WHERE LastModifiedDateTime = (SELECT MAX(LastModifiedDateTime) FROM fiction) ORDER BY LibgenId DESC LIMIT 1";
Expand Down Expand Up @@ -386,7 +388,9 @@ internal static class SqlScripts

public const string GET_SCIMAG_BY_ID = "SELECT * FROM scimag WHERE Id = @Id";

public const string GET_SCIMAG_ID_BY_LIBGENID = "SELECT Id FROM scimag WHERE LibgenId=@LibgenId LIMIT 1";
public const string GET_SCIMAG_BY_MD5HASH = "SELECT * FROM scimag WHERE Md5Hash = @Md5Hash COLLATE NOCASE LIMIT 1";

public const string GET_SCIMAG_ID_BY_LIBGENID = "SELECT Id FROM scimag WHERE LibgenId = @LibgenId LIMIT 1";

public const string GET_LAST_ADDED_SCIMAG =
"SELECT * FROM scimag WHERE AddedDateTime = (SELECT MAX(AddedDateTime) FROM scimag) ORDER BY LibgenId DESC LIMIT 1";
Expand All @@ -406,7 +410,7 @@ internal static class SqlScripts

public const string SCIMAG_INDEX_PREFIX = "IX_scimag_";

public const string CREATE_SCIMAG_MD5HASH_INDEX = "CREATE UNIQUE INDEX " + SCIMAG_INDEX_PREFIX + "Md5Hash ON scimag (Md5Hash COLLATE NOCASE)";
public const string CREATE_SCIMAG_MD5HASH_INDEX = "CREATE INDEX " + SCIMAG_INDEX_PREFIX + "Md5Hash ON scimag (Md5Hash COLLATE NOCASE)";

public const string CREATE_SCIMAG_ADDEDDATETIME_INDEX = "CREATE INDEX " + SCIMAG_INDEX_PREFIX + "AddedDateTime ON scimag (AddedDateTime DESC)";
}
Expand Down
8 changes: 6 additions & 2 deletions LibgenDesktop/Models/Localization/Localizators/LibraryTabLocalizator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ public LibraryTabLocalizator(List<Translation> prioritizedTranslationList, Langu
: base(prioritizedTranslationList, formatter)
{
TabTitle = Format(translation => translation?.TabTitle);
Scan = Format(translation => translation?.Scan);
ScanNonFiction = Format(translation => translation?.ScanNonFiction);
ScanFiction = Format(translation => translation?.ScanFiction);
ScanSciMag = Format(translation => translation?.ScanSciMag);
BrowseDirectoryDialogTitle = Format(translation => translation?.BrowseDirectoryDialogTitle);
CreatingIndexes = Format(translation => translation?.CreatingIndexes);
ScanLog = Format(translation => translation?.ScanLog);
Expand All @@ -20,7 +22,9 @@ public LibraryTabLocalizator(List<Translation> prioritizedTranslationList, Langu
}

public string TabTitle { get; }
public string Scan { get; }
public string ScanNonFiction { get; }
public string ScanFiction { get; }
public string ScanSciMag { get; }
public string BrowseDirectoryDialogTitle { get; }
public string CreatingIndexes { get; }
public string ScanLog { get; }
Expand Down
4 changes: 3 additions & 1 deletion LibgenDesktop/Models/Localization/Translation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,7 +554,9 @@ internal class LibraryColumnsTranslation
internal class LibraryTranslation
{
public string TabTitle { get; set; }
public string Scan { get; set; }
public string ScanNonFiction { get; set; }
public string ScanFiction { get; set; }
public string ScanSciMag { get; set; }
public string BrowseDirectoryDialogTitle { get; set; }
public string ScanStarted { get; set; }
public string CreatingIndexes { get; set; }
Expand Down
84 changes: 53 additions & 31 deletions LibgenDesktop/Models/MainModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -483,27 +483,22 @@ public Task<DownloadFileResult> DownloadFileAsync(string fileUrl, string destina
});
}

public Task ScanAsync(string scanDirectory, IProgress<object> progressHandler)
public Task ScanNonFictionAsync(string scanDirectory, IProgress<object> progressHandler)
{
return Task.Run(() =>
{
int found = 0;
int notFound = 0;
int errors = 0;
if (NonFictionBookCount > 0)
{
Logger.Debug("Retrieving the list of non-fiction table indexes.");
List<string> nonFictionIndexes = localDatabase.GetNonFictionIndexList();
if (!nonFictionIndexes.Contains(SqlScripts.NON_FICTION_INDEX_PREFIX + "Md5Hash"))
{
Logger.Debug("Creating an index on Md5Hash column.");
progressHandler.Report(new GenericProgress(GenericProgress.Event.SCAN_CREATING_INDEXES));
localDatabase.CreateNonFictionMd5HashIndex();
}
ScanDirectory(scanDirectory.ToLower(), scanDirectory, progressHandler, ref found, ref notFound, ref errors);
}
progressHandler.Report(new ScanCompleteProgress(found, notFound, errors));
});
return ScanAsync(scanDirectory, progressHandler, NonFictionBookCount, localDatabase.GetNonFictionIndexList,
SqlScripts.NON_FICTION_INDEX_PREFIX, localDatabase.CreateNonFictionMd5HashIndex, localDatabase.GetNonFictionBookByMd5Hash);
}

public Task ScanFictionAsync(string scanDirectory, IProgress<object> progressHandler)
{
return ScanAsync(scanDirectory, progressHandler, FictionBookCount, localDatabase.GetFictionIndexList,
SqlScripts.FICTION_INDEX_PREFIX, localDatabase.CreateFictionMd5HashIndex, localDatabase.GetFictionBookByMd5Hash);
}

public Task ScanSciMagAsync(string scanDirectory, IProgress<object> progressHandler)
{
return ScanAsync(scanDirectory, progressHandler, SciMagArticleCount, localDatabase.GetSciMagIndexList,
SqlScripts.SCIMAG_INDEX_PREFIX, localDatabase.CreateSciMagMd5HashIndex, localDatabase.GetSciMagArticleByMd5Hash);
}

public Task<DatabaseStats> GetDatabaseStatsAsync()
Expand Down Expand Up @@ -910,8 +905,35 @@ private TableType DetectImportTableType(SqlDumpReader.ParsedTableDefinition pars
return TableType.UNKNOWN;
}

private void ScanDirectory(string rootScanDirectory, string scanDirectory, IProgress<object> progressHandler, ref int found, ref int notFound,
ref int errors)
private Task ScanAsync<T>(string scanDirectory, IProgress<object> progressHandler, int objectsInDatabaseCount,
Func<List<string>> indexRetrievalFunction, string indexPrefix, Action indexCreationAction, Func<string, T> getObjectByMd5HashFunction)
where T: LibgenObject
{
return Task.Run(() =>
{
Logger.Debug($"Scan request in directory = {scanDirectory}, object count = {objectsInDatabaseCount}, object type = {typeof(T).Name}.");
int found = 0;
int notFound = 0;
int errors = 0;
if (objectsInDatabaseCount > 0)
{
Logger.Debug("Retrieving the list of indexes.");
List<string> indexList = indexRetrievalFunction();
if (!indexList.Contains(indexPrefix + "Md5Hash"))
{
Logger.Debug("Creating an index on Md5Hash column.");
progressHandler.Report(new GenericProgress(GenericProgress.Event.SCAN_CREATING_INDEXES));
indexCreationAction();
}
ScanDirectory(scanDirectory.ToLower(), scanDirectory, progressHandler, getObjectByMd5HashFunction, ref found, ref notFound, ref errors);
}
progressHandler.Report(new ScanCompleteProgress(found, notFound, errors));
});
}

private void ScanDirectory<T>(string rootScanDirectory, string scanDirectory, IProgress<object> progressHandler,
Func<string, T> getObjectByMd5HashFunction, ref int found, ref int notFound, ref int errors)
where T : LibgenObject
{
try
{
Expand All @@ -936,47 +958,47 @@ private void ScanDirectory(string rootScanDirectory, string scanDirectory, IProg
{
Logger.Debug($"Couldn't calculate MD5 hash for the file: {filePath}");
Logger.Exception(exception);
progressHandler.Report(new ScanProgress(relativeFilePath, error: true));
progressHandler.Report(new ScanUnknownProgress(relativeFilePath, error: true));
errors++;
continue;
}
try
{
NonFictionBook book = localDatabase.GetNonFictionBookByMd5Hash(md5Hash);
if (book != null)
T libgenObject = getObjectByMd5HashFunction(md5Hash);
if (libgenObject != null)
{
progressHandler.Report(new ScanProgress(relativeFilePath, book.Authors, book.Title, book));
progressHandler.Report(new ScanProgress<T>(relativeFilePath, libgenObject));
found++;
}
else
{
progressHandler.Report(new ScanProgress(relativeFilePath));
progressHandler.Report(new ScanUnknownProgress(relativeFilePath, error: false));
notFound++;
}
}
catch (Exception exception)
{
Logger.Debug($"Couldn't lookup the MD5 hash: {md5Hash} in the database for the file: {filePath}");
Logger.Exception(exception);
progressHandler.Report(new ScanProgress(relativeFilePath, error: true));
progressHandler.Report(new ScanUnknownProgress(relativeFilePath, error: true));
errors++;
continue;
}
}
foreach (string directoryPath in Directory.EnumerateDirectories(scanDirectory))
{
ScanDirectory(rootScanDirectory, directoryPath, progressHandler, ref found, ref notFound, ref errors);
ScanDirectory(rootScanDirectory, directoryPath, progressHandler, getObjectByMd5HashFunction, ref found, ref notFound, ref errors);
}
}
catch (Exception exception)
{
Logger.Exception(exception);
progressHandler.Report(new ScanProgress(scanDirectory, error: true));
progressHandler.Report(new ScanUnknownProgress(scanDirectory, error: true));
errors++;
}
}

private void CheckAndCreateNonFictionIndexes(IProgress<object> progressHandler, CancellationToken cancellationToken)
private void CheckAndCreateNonFictionIndexes(IProgress<object> progressHandler, CancellationToken cancellationToken)
{
Logger.Debug("Retrieving the list of non-fiction table indexes.");
List<string> nonFictionIndexes = localDatabase.GetNonFictionIndexList();
Expand Down
Loading

0 comments on commit eb99fac

Please sign in to comment.