diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..f1c081b --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,42 @@ +name: Build + +on: [pull_request, push] + +jobs: + build-on-ubuntu: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install build dependencies + run: sudo apt install -y build-essential cmake zlib1g-dev + - name: cmake + run: cmake -DUSE_CLANG_TIDY=1 . + - name: make + run: make -j`nproc` + build-on-macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - name: install build dependencies + run: | + brew install llvm + ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy" + brew install cmake zlib + - name: cmake + run: cmake -DUSE_CLANG_TIDY=1 . + - name: make + run: make -j`nproc` + build-on-windows: + runs-on: windows-2019 + steps: + - uses: actions/checkout@v2 + - uses: lukka/get-cmake@latest + - name: Restore artifacts, or setup vcpkg (do not install any package) + uses: lukka/run-vcpkg@v10 + with: + vcpkgGitCommitId: '14e7bb4ae24616ec54ff6b2f6ef4e8659434ea44' + - name: Run CMake consuming CMakePreset.json and vcpkg.json by mean of vcpkg. + uses: lukka/run-cmake@v10 + with: + configurePreset: 'msbuild-vcpkg' + buildPreset: 'msbuild-vcpkg' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..41c83bd --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.idea/ +/cmake-build-debug/ \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..0e1ce42 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "submodules/falltergeist/vfs"] + path = submodules/falltergeist/vfs + url = https://github.com/falltergeist/vfs.git + tag = 0.2.0 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..3c01943 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +Under development +----------------- +- [feature] Initial vfs code extraction (alexeevdv) diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..9dbd4aa --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,64 @@ +cmake_minimum_required(VERSION 3.8) +project(falltergeist_vfs_dat2 VERSION 0.1.0 DESCRIPTION "Virtual File System DAT2 module") + +find_package(Git QUIET) +if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git") + # Update submodules as needed + option(GIT_SUBMODULE "Check submodules during build" ON) + if(GIT_SUBMODULE) + message(STATUS "Submodule update") + execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE GIT_SUBMOD_RESULT) + if(NOT GIT_SUBMOD_RESULT EQUAL "0") + message(FATAL_ERROR "git submodule update --init --recursive failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") + endif() + endif() +endif() + +if(NOT EXISTS "${PROJECT_SOURCE_DIR}/submodules") + message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") +endif() + +add_subdirectory(submodules/falltergeist/vfs) + +find_package(ZLIB REQUIRED) +if(NOT ZLIB_FOUND) + message(FATAL_ERROR "zlib library not found") +endif(NOT ZLIB_FOUND) +include_directories(SYSTEM ${ZLIB_INCLUDE_DIRS}) + +add_library(${PROJECT_NAME} STATIC) +add_library(falltergeist::vfs::dat2 ALIAS ${PROJECT_NAME}) + +target_sources(${PROJECT_NAME} + PRIVATE + src/DatArchiveDriver.cpp + src/DatArchiveFile.cpp + src/DatArchiveStreamWrapper.cpp +) + +set_target_properties(${PROJECT_NAME} PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED YES + CXX_EXTENSIONS NO + VERSION ${PROJECT_VERSION} +) + +include(GNUInstallDirs) +target_include_directories(${PROJECT_NAME} + PRIVATE + # where the library itself will look for its internal headers + ${CMAKE_CURRENT_SOURCE_DIR}/src + PUBLIC + # where top-level project will look for the library's public headers + $ + # where external projects will look for the library's public headers + $ +) + +target_link_libraries( + ${PROJECT_NAME} + ${ZLIB_LIBRARIES} + falltergeist::vfs +) diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000..c61b2e0 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,62 @@ +{ + "version": 3, + "cmakeMinimumRequired": { + "major": 3, + "minor": 8, + "patch": 0 + }, + "configurePresets": [ + { + "name": "ninja", + "displayName": "Ninja Configure Settings", + "description": "Sets build and install directories", + "binaryDir": "${sourceDir}/builds/${presetName}", + "generator": "Ninja" + }, + { + "name": "ninja-toolchain", + "displayName": "Ninja Configure Settings with toolchain", + "description": "Sets build and install directories", + "binaryDir": "${sourceDir}/builds/${presetName}-toolchain", + "generator": "Ninja", + "toolchainFile": "$env{TOOLCHAINFILE}" + }, + { + "name": "msbuild-vcpkg", + "displayName": "MSBuild (vcpkg toolchain) Configure Settings", + "description": "Configure with VS generators and with vcpkg toolchain", + "binaryDir": "${sourceDir}/builds/${presetName}", + "generator": "Visual Studio 16 2019", + "architecture": { + "strategy": "set", + "value": "x64" + }, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": { + "type": "FILEPATH", + "value": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" + } + } + } + ], + "buildPresets": [ + { + "name": "ninja", + "configurePreset": "ninja", + "displayName": "Build with Ninja", + "description": "Build with Ninja" + }, + { + "name": "ninja-toolchain", + "configurePreset": "ninja-toolchain", + "displayName": "Build ninja-toolchain", + "description": "Build ninja with a toolchain" + }, + { + "name": "msbuild-vcpkg", + "configurePreset": "msbuild-vcpkg", + "displayName": "Build MSBuild", + "description": "Build with MSBuild (VS)" + } + ] +} diff --git a/include/falltergeist/vfs/DatArchiveDriver.h b/include/falltergeist/vfs/DatArchiveDriver.h new file mode 100644 index 0000000..3fad001 --- /dev/null +++ b/include/falltergeist/vfs/DatArchiveDriver.h @@ -0,0 +1,31 @@ +#pragma once + +#include "DatArchiveEntry.h" +#include "DatArchiveStreamWrapper.h" +#include "falltergeist/vfs/IDriver.h" +#include +#include + +namespace Falltergeist::VFS { + /** + * DatArchiveDriver provides support for vanilla DAT archives + * It supports only read operations + */ + class DatArchiveDriver final : public IDriver { + public: + DatArchiveDriver(const std::string& path); + + ~DatArchiveDriver() override = default; + + const std::string& name() override; + + bool exists(const std::string& path) override; + + std::shared_ptr open(const std::string& path, IFile::OpenMode mode) override; + + private: + std::string _name; + + DatArchiveStreamWrapper _streamWrapper; + }; +} diff --git a/include/falltergeist/vfs/DatArchiveEntry.h b/include/falltergeist/vfs/DatArchiveEntry.h new file mode 100644 index 0000000..afbe951 --- /dev/null +++ b/include/falltergeist/vfs/DatArchiveEntry.h @@ -0,0 +1,13 @@ +#pragma once + +namespace Falltergeist::VFS { + struct DatArchiveEntry { + unsigned int packedSize = 0; + + unsigned int unpackedSize = 0; + + unsigned int dataOffset = 0; + + bool isCompressed = false; + }; +} diff --git a/include/falltergeist/vfs/DatArchiveFile.h b/include/falltergeist/vfs/DatArchiveFile.h new file mode 100644 index 0000000..ffa455f --- /dev/null +++ b/include/falltergeist/vfs/DatArchiveFile.h @@ -0,0 +1,48 @@ +#pragma once + +#include "DatArchiveEntry.h" +#include "falltergeist/vfs/IFile.h" +#include + +namespace Falltergeist::VFS { + class DatArchiveDriver; + + class DatArchiveFile final : public IFile { + public: + typedef std::function fnReadBytes; + + DatArchiveFile(const DatArchiveEntry& entry, const fnReadBytes& readFunction); + + ~DatArchiveFile() override = default; + + unsigned int size() override; + + bool isOpened() override; + + unsigned int seek(unsigned int position, SeekFrom seekFrom) override; + + unsigned int tell() override; + + unsigned int read(unsigned char* to, unsigned int size) override; + + unsigned int read(char* to, unsigned int size) override; + + unsigned int write(const char* from, unsigned int size) override; + + protected: + friend class DatArchiveDriver; + + void _open(OpenMode mode) override; + + void _close() override; + + private: + bool _isOpened = false; + + unsigned int _seekPosition = 0; + + DatArchiveEntry _entry; + + fnReadBytes _readFunction; + }; +} diff --git a/include/falltergeist/vfs/DatArchiveStreamWrapper.h b/include/falltergeist/vfs/DatArchiveStreamWrapper.h new file mode 100644 index 0000000..9e6c7ed --- /dev/null +++ b/include/falltergeist/vfs/DatArchiveStreamWrapper.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include +#include "DatArchiveEntry.h" + +namespace Falltergeist::VFS { + class DatArchiveStreamWrapper final { + public: + DatArchiveStreamWrapper(const std::string& path); + + DatArchiveStreamWrapper(const DatArchiveStreamWrapper& other) = delete; + + DatArchiveStreamWrapper(DatArchiveStreamWrapper&& other) = delete; + + void seek(unsigned int position); + + unsigned int readBytes(char* destination, unsigned int size); + + const std::map& entries() const; + + private: + uint32_t _actualFileSize = 0; + + uint32_t _filesTreeSize = 0; + + uint32_t _filesCount = 0; + + std::fstream _stream; + + void _readUint8(uint8_t& dest); + + void _readUint32(uint32_t& dest); + + std::map _entries; + }; +} diff --git a/src/DatArchiveDriver.cpp b/src/DatArchiveDriver.cpp new file mode 100644 index 0000000..36aa9ab --- /dev/null +++ b/src/DatArchiveDriver.cpp @@ -0,0 +1,71 @@ +#include "falltergeist/vfs/DatArchiveDriver.h" +#include "falltergeist/vfs/DatArchiveFile.h" +#include "falltergeist/vfs/MemoryFile.h" +#include "zlib.h" +#include + +namespace Falltergeist::VFS { + DatArchiveDriver::DatArchiveDriver(const std::string& path) : _name("DatArchiveDriver"), _streamWrapper(DatArchiveStreamWrapper(path)) { + } + + const std::string& DatArchiveDriver::name() { + return _name; + } + + bool DatArchiveDriver::exists(const std::string& path) { + std::string unixStylePath = path; + std::replace(unixStylePath.begin(), unixStylePath.end(), '\\','/'); + return _streamWrapper.entries().count(path) != 0; + } + + std::shared_ptr DatArchiveDriver::open(const std::string& path, IFile::OpenMode mode) { + if (mode != IFile::OpenMode::Read) { + // Only read operations are supported + return nullptr; + } + + std::string unixStylePath = path; + std::replace(unixStylePath.begin(), unixStylePath.end(), '\\','/'); + + const DatArchiveEntry& entry = _streamWrapper.entries().at(path); + if (entry.isCompressed) { + auto* packedData = new unsigned char[entry.packedSize]; + _streamWrapper.seek(entry.dataOffset); + _streamWrapper.readBytes(reinterpret_cast(packedData), entry.packedSize); + + auto* unpackedData = new unsigned char[entry.unpackedSize]; + + // unpacking + z_stream zStream; + zStream.total_in = entry.packedSize; + zStream.avail_in = entry.packedSize; + zStream.next_in = packedData; + zStream.total_out = zStream.avail_out = static_cast(entry.unpackedSize); + zStream.next_out = unpackedData; + zStream.zalloc = Z_NULL; + zStream.zfree = Z_NULL; + zStream.opaque = Z_NULL; + inflateInit(&zStream); + inflate(&zStream, Z_FINISH); + inflateEnd(&zStream); + + delete[] packedData; + + auto file = std::make_shared(); + file->_open(IFile::OpenMode::ReadWriteTruncate); + file->write(reinterpret_cast(unpackedData), entry.unpackedSize); + + delete[] unpackedData; + + file->_open(mode); + return file; + } + + auto file = std::make_shared(entry, [=](unsigned int seekPosition, unsigned char* to, unsigned char size)-> unsigned int { + _streamWrapper.seek(seekPosition); + return _streamWrapper.readBytes(reinterpret_cast(to), size); + }); + file->_open(mode); + return file; + } +} diff --git a/src/DatArchiveFile.cpp b/src/DatArchiveFile.cpp new file mode 100644 index 0000000..0c04a8c --- /dev/null +++ b/src/DatArchiveFile.cpp @@ -0,0 +1,81 @@ +#include "falltergeist/vfs/DatArchiveFile.h" +#include + +namespace Falltergeist::VFS { + DatArchiveFile::DatArchiveFile(const DatArchiveEntry& entry, const fnReadBytes& readFunction) : _entry(entry), _readFunction(readFunction) { + } + + unsigned int DatArchiveFile::size() { + return _entry.unpackedSize; + } + + void DatArchiveFile::_open(OpenMode mode) { + if (mode != OpenMode::Read) { + return; + } + + _isOpened = true; + } + + bool DatArchiveFile::isOpened() { + return _isOpened; + } + + void DatArchiveFile::_close() { + _isOpened = false; + } + + unsigned int DatArchiveFile::seek(unsigned int position, IFile::SeekFrom seekFrom) { + if (!isOpened()) { + return 0; + } + + if (seekFrom == SeekFrom::Begin) { + _seekPosition = position; + } else if (seekFrom == SeekFrom::End) { + _seekPosition = size() - position; + } else { + _seekPosition += position; + } + + _seekPosition = std::min(_seekPosition, size() - 1); + + return tell(); + } + + unsigned int DatArchiveFile::tell() { + if (!isOpened()) { + return 0; + } + return _seekPosition; + } + + unsigned int DatArchiveFile::read(unsigned char* to, unsigned int size) { + if (!isOpened()) { + return 0; + } + + unsigned int bytesAvailable = std::min(size, this->size() - tell()); + if (bytesAvailable == 0) { + return 0; + } + + _readFunction(_entry.dataOffset + tell(), to, bytesAvailable); + _seekPosition += bytesAvailable; + + return bytesAvailable; + } + + unsigned int DatArchiveFile::read(char* to, unsigned int size) { + return read(reinterpret_cast(to), size); + } + + unsigned int DatArchiveFile::write(const char* from, unsigned int size) { + if (!isOpened()) { + return 0; + } + + // does not support write operations + return 0; + } +} diff --git a/src/DatArchiveStreamWrapper.cpp b/src/DatArchiveStreamWrapper.cpp new file mode 100644 index 0000000..7dbaf84 --- /dev/null +++ b/src/DatArchiveStreamWrapper.cpp @@ -0,0 +1,86 @@ +#include "falltergeist/vfs/DatArchiveStreamWrapper.h" +#include +#include + +namespace Falltergeist::VFS { + DatArchiveStreamWrapper::DatArchiveStreamWrapper(const std::string& path) { + _stream.open(path, std::ios_base::binary | std::ios_base::in); + if (!_stream.is_open()) { + throw std::runtime_error("DatArchiveStreamWrapper - can't open _stream: " + path); + } + + _stream.seekg(0, std::ios_base::end); + _actualFileSize = _stream.tellg(); + _stream.seekg(0, std::ios_base::beg); + + uint32_t fileSizeInDatFile = 0; + // reading data size from dat file + seek(_actualFileSize - 4); + _readUint32(fileSizeInDatFile); + if (fileSizeInDatFile != _actualFileSize) { + throw std::runtime_error("DatArchiveStreamWrapper - wrong file size: " + std::to_string(fileSizeInDatFile) + + " should be: " + std::to_string(_actualFileSize)); + } + + // reading size of files tree + seek(_actualFileSize - 8); + _readUint32(_filesTreeSize); + + // reading total number of items in dat file + seek(_actualFileSize - _filesTreeSize - 8); + _readUint32(_filesCount); + + // reading files data one by one + for (unsigned int i = 0; i != _filesCount; ++i) { + uint32_t filenameSize = 0; + uint8_t compressed = 0; + uint32_t unpackedSize = 0; + uint32_t packedSize = 0; + uint32_t dataOffset = 0; + + _readUint32(filenameSize); + std::string filename; + filename.resize(filenameSize); + readBytes(&filename[0], filename.length()); + std::transform(filename.begin(), filename.end(), filename.begin(), ::tolower); + std::replace(filename.begin(), filename.end(), '\\','/'); + + _readUint8(compressed); + _readUint32(unpackedSize); + _readUint32(packedSize); + _readUint32(dataOffset); + + _entries.insert(std::make_pair(filename, DatArchiveEntry{ + packedSize, + unpackedSize, + dataOffset, + (bool) compressed + })); + } + } + + void DatArchiveStreamWrapper::seek(unsigned int position) { + _stream.seekg(position, std::ios::beg); + } + + unsigned int DatArchiveStreamWrapper::readBytes(char* destination, unsigned int size) { + _stream.read(destination, size); + if (_stream) { + return size; + } + + return _stream.gcount(); + } + + const std::map& DatArchiveStreamWrapper::entries() const { + return _entries; + } + + void DatArchiveStreamWrapper::_readUint8(uint8_t& dest) { + _stream.read(reinterpret_cast(&dest), 1); + } + + void DatArchiveStreamWrapper::_readUint32(uint32_t& dest) { + _stream.read(reinterpret_cast(&dest), 4); + } +} diff --git a/submodules/falltergeist/vfs b/submodules/falltergeist/vfs new file mode 160000 index 0000000..709f239 --- /dev/null +++ b/submodules/falltergeist/vfs @@ -0,0 +1 @@ +Subproject commit 709f2399b933cb64d287f41048fce599e594d02b diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000..5fc3615 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,7 @@ +{ + "name": "falltergeist-vfs-dat2", + "version-string": "0.1.0", + "dependencies": [ + "zlib" + ] +}