Merge pull request #1 from falltergeist/feature/initial-vfs-extraction

Feature: Initial vfs code extraction

.github/workflows/build.yaml

@@ -0,0 +1,42 @@
+name: Build
+
+on: [pull_request, push]
+
+jobs:
+  build-on-ubuntu:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: install build dependencies
+        run: sudo apt install -y build-essential cmake zlib1g-dev
+      - name: cmake
+        run: cmake -DUSE_CLANG_TIDY=1 .
+      - name: make
+        run: make -j`nproc`
+  build-on-macos:
+    runs-on: macos-latest
+    steps:
+       - uses: actions/checkout@v2
+       - name: install build dependencies
+         run: |
+           brew install llvm
+           ln -s "/usr/local/opt/llvm/bin/clang-tidy" "/usr/local/bin/clang-tidy"
+           brew install cmake zlib
+       - name: cmake
+         run: cmake -DUSE_CLANG_TIDY=1 .
+       - name: make
+         run: make -j`nproc`
+  build-on-windows:
+    runs-on: windows-2019
+    steps:
+      - uses: actions/checkout@v2
+      - uses: lukka/get-cmake@latest
+      - name: Restore artifacts, or setup vcpkg (do not install any package)
+        uses: lukka/run-vcpkg@v10
+        with:
+          vcpkgGitCommitId: '14e7bb4ae24616ec54ff6b2f6ef4e8659434ea44'
+      - name: Run CMake consuming CMakePreset.json and vcpkg.json by mean of vcpkg.
+        uses: lukka/run-cmake@v10
+        with:
+          configurePreset: 'msbuild-vcpkg'
+          buildPreset: 'msbuild-vcpkg'

.gitignore

@@ -0,0 +1,2 @@
+/.idea/
+/cmake-build-debug/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "submodules/falltergeist/vfs"]
+    path = submodules/falltergeist/vfs
+    url = https://github.com/falltergeist/vfs.git
+    tag = 0.2.0

CHANGELOG.md

@@ -0,0 +1,3 @@
+Under development
+-----------------
+- [feature] Initial vfs code extraction (alexeevdv)

CMakeLists.txt

@@ -0,0 +1,64 @@
+cmake_minimum_required(VERSION 3.8)
+project(falltergeist_vfs_dat2 VERSION 0.1.0 DESCRIPTION "Virtual File System DAT2 module")
+
+find_package(Git QUIET)
+if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
+    # Update submodules as needed
+    option(GIT_SUBMODULE "Check submodules during build" ON)
+    if(GIT_SUBMODULE)
+        message(STATUS "Submodule update")
+        execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive
+                WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+                RESULT_VARIABLE GIT_SUBMOD_RESULT)
+        if(NOT GIT_SUBMOD_RESULT EQUAL "0")
+            message(FATAL_ERROR "git submodule update --init --recursive failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
+        endif()
+    endif()
+endif()
+
+if(NOT EXISTS "${PROJECT_SOURCE_DIR}/submodules")
+    message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
+endif()
+
+add_subdirectory(submodules/falltergeist/vfs)
+
+find_package(ZLIB REQUIRED)
+if(NOT ZLIB_FOUND)
+    message(FATAL_ERROR "zlib library not found")
+endif(NOT ZLIB_FOUND)
+include_directories(SYSTEM ${ZLIB_INCLUDE_DIRS})
+
+add_library(${PROJECT_NAME} STATIC)
+add_library(falltergeist::vfs::dat2 ALIAS ${PROJECT_NAME})
+
+target_sources(${PROJECT_NAME}
+    PRIVATE
+    src/DatArchiveDriver.cpp
+    src/DatArchiveFile.cpp
+    src/DatArchiveStreamWrapper.cpp
+)
+
+set_target_properties(${PROJECT_NAME} PROPERTIES
+    CXX_STANDARD 17
+    CXX_STANDARD_REQUIRED YES
+    CXX_EXTENSIONS NO
+    VERSION ${PROJECT_VERSION}
+)
+
+include(GNUInstallDirs)
+target_include_directories(${PROJECT_NAME}
+    PRIVATE
+    # where the library itself will look for its internal headers
+    ${CMAKE_CURRENT_SOURCE_DIR}/src
+    PUBLIC
+    # where top-level project will look for the library's public headers
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+    # where external projects will look for the library's public headers
+    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+)
+
+target_link_libraries(
+    ${PROJECT_NAME}
+    ${ZLIB_LIBRARIES}
+    falltergeist::vfs
+)

CMakePresets.json

@@ -0,0 +1,62 @@
+{
+  "version": 3,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 8,
+    "patch": 0
+  },
+  "configurePresets": [
+    {
+      "name": "ninja",
+      "displayName": "Ninja Configure Settings",
+      "description": "Sets build and install directories",
+      "binaryDir": "${sourceDir}/builds/${presetName}",
+      "generator": "Ninja"
+    },
+    {
+      "name": "ninja-toolchain",
+      "displayName": "Ninja Configure Settings with toolchain",
+      "description": "Sets build and install directories",
+      "binaryDir": "${sourceDir}/builds/${presetName}-toolchain",
+      "generator": "Ninja",
+      "toolchainFile": "$env{TOOLCHAINFILE}"
+    },
+    {
+      "name": "msbuild-vcpkg",
+      "displayName": "MSBuild (vcpkg toolchain) Configure Settings",
+      "description": "Configure with VS generators and with vcpkg toolchain",
+      "binaryDir": "${sourceDir}/builds/${presetName}",
+      "generator": "Visual Studio 16 2019",
+      "architecture": {
+        "strategy": "set",
+        "value": "x64"
+      },
+      "cacheVariables": {
+        "CMAKE_TOOLCHAIN_FILE": {
+          "type": "FILEPATH",
+          "value": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
+        }
+      }
+    }
+  ],
+  "buildPresets": [
+    {
+      "name": "ninja",
+      "configurePreset": "ninja",
+      "displayName": "Build with Ninja",
+      "description": "Build with Ninja"
+    },
+    {
+      "name": "ninja-toolchain",
+      "configurePreset": "ninja-toolchain",
+      "displayName": "Build ninja-toolchain",
+      "description": "Build ninja with a toolchain"
+    },
+    {
+      "name": "msbuild-vcpkg",
+      "configurePreset": "msbuild-vcpkg",
+      "displayName": "Build MSBuild",
+      "description": "Build with MSBuild (VS)"
+    }
+  ]
+}

include/falltergeist/vfs/DatArchiveDriver.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "DatArchiveEntry.h"
+#include "DatArchiveStreamWrapper.h"
+#include "falltergeist/vfs/IDriver.h"
+#include <map>
+#include <memory>
+
+namespace Falltergeist::VFS {
+    /**
+     * DatArchiveDriver provides support for vanilla DAT archives
+     * It supports only read operations
+     */
+    class DatArchiveDriver final : public IDriver {
+    public:
+        DatArchiveDriver(const std::string& path);
+
+        ~DatArchiveDriver() override = default;
+
+        const std::string& name() override;
+
+        bool exists(const std::string& path) override;
+
+        std::shared_ptr<IFile> open(const std::string& path, IFile::OpenMode mode) override;
+
+    private:
+        std::string _name;
+
+        DatArchiveStreamWrapper _streamWrapper;
+    };
+}

include/falltergeist/vfs/DatArchiveEntry.h

@@ -0,0 +1,13 @@
+#pragma once
+
+namespace Falltergeist::VFS {
+    struct DatArchiveEntry {
+        unsigned int packedSize = 0;
+
+        unsigned int unpackedSize = 0;
+
+        unsigned int dataOffset = 0;
+
+        bool isCompressed = false;
+    };
+}

include/falltergeist/vfs/DatArchiveFile.h

@@ -0,0 +1,48 @@
+#pragma once
+
+#include "DatArchiveEntry.h"
+#include "falltergeist/vfs/IFile.h"
+#include <functional>
+
+namespace Falltergeist::VFS {
+    class DatArchiveDriver;
+
+    class DatArchiveFile final : public IFile {
+    public:
+        typedef std::function<unsigned int (unsigned int seekPosition, unsigned char* to, unsigned int size)> fnReadBytes;
+
+        DatArchiveFile(const DatArchiveEntry& entry, const fnReadBytes& readFunction);
+
+        ~DatArchiveFile() override = default;
+
+        unsigned int size() override;
+
+        bool isOpened() override;
+
+        unsigned int seek(unsigned int position, SeekFrom seekFrom) override;
+
+        unsigned int tell() override;
+
+        unsigned int read(unsigned char* to, unsigned int size) override;
+
+        unsigned int read(char* to, unsigned int size) override;
+
+        unsigned int write(const char* from, unsigned int size) override;
+
+    protected:
+        friend class DatArchiveDriver;
+
+        void _open(OpenMode mode) override;
+
+        void _close() override;
+
+    private:
+        bool _isOpened = false;
+
+        unsigned int _seekPosition = 0;
+
+        DatArchiveEntry _entry;
+
+        fnReadBytes _readFunction;
+    };
+}

include/falltergeist/vfs/DatArchiveStreamWrapper.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <fstream>
+#include <cstdint>
+#include <map>
+#include "DatArchiveEntry.h"
+
+namespace Falltergeist::VFS {
+    class DatArchiveStreamWrapper final {
+    public:
+        DatArchiveStreamWrapper(const std::string& path);
+
+        DatArchiveStreamWrapper(const DatArchiveStreamWrapper& other) = delete;
+
+        DatArchiveStreamWrapper(DatArchiveStreamWrapper&& other) = delete;
+
+        void seek(unsigned int position);
+
+        unsigned int readBytes(char* destination, unsigned int size);
+
+        const std::map<std::string, DatArchiveEntry>& entries() const;
+
+    private:
+        uint32_t _actualFileSize = 0;
+
+        uint32_t _filesTreeSize = 0;
+
+        uint32_t _filesCount = 0;
+
+        std::fstream _stream;
+
+        void _readUint8(uint8_t& dest);
+
+        void _readUint32(uint32_t& dest);
+
+        std::map<std::string, DatArchiveEntry> _entries;
+    };
+}

src/DatArchiveDriver.cpp

@@ -0,0 +1,71 @@
+#include "falltergeist/vfs/DatArchiveDriver.h"
+#include "falltergeist/vfs/DatArchiveFile.h"
+#include "falltergeist/vfs/MemoryFile.h"
+#include "zlib.h"
+#include <algorithm>
+
+namespace Falltergeist::VFS {
+    DatArchiveDriver::DatArchiveDriver(const std::string& path) : _name("DatArchiveDriver"), _streamWrapper(DatArchiveStreamWrapper(path)) {
+    }
+
+    const std::string& DatArchiveDriver::name() {
+        return _name;
+    }
+
+    bool DatArchiveDriver::exists(const std::string& path) {
+        std::string unixStylePath = path;
+        std::replace(unixStylePath.begin(), unixStylePath.end(), '\\','/');
+        return _streamWrapper.entries().count(path) != 0;
+    }
+
+    std::shared_ptr<IFile> DatArchiveDriver::open(const std::string& path, IFile::OpenMode mode) {
+        if (mode != IFile::OpenMode::Read) {
+            // Only read operations are supported
+            return nullptr;
+        }
+
+        std::string unixStylePath = path;
+        std::replace(unixStylePath.begin(), unixStylePath.end(), '\\','/');
+
+        const DatArchiveEntry& entry = _streamWrapper.entries().at(path);
+        if (entry.isCompressed) {
+            auto* packedData = new unsigned char[entry.packedSize];
+            _streamWrapper.seek(entry.dataOffset);
+            _streamWrapper.readBytes(reinterpret_cast<char*>(packedData), entry.packedSize);
+
+            auto* unpackedData = new unsigned char[entry.unpackedSize];
+
+            // unpacking
+            z_stream zStream;
+            zStream.total_in = entry.packedSize;
+            zStream.avail_in = entry.packedSize;
+            zStream.next_in = packedData;
+            zStream.total_out = zStream.avail_out = static_cast<uint32_t>(entry.unpackedSize);
+            zStream.next_out = unpackedData;
+            zStream.zalloc = Z_NULL;
+            zStream.zfree = Z_NULL;
+            zStream.opaque = Z_NULL;
+            inflateInit(&zStream);
+            inflate(&zStream, Z_FINISH);
+            inflateEnd(&zStream);
+
+            delete[] packedData;
+
+            auto file = std::make_shared<MemoryFile>();
+            file->_open(IFile::OpenMode::ReadWriteTruncate);
+            file->write(reinterpret_cast<const char*>(unpackedData), entry.unpackedSize);
+
+            delete[] unpackedData;
+
+            file->_open(mode);
+            return file;
+        }
+
+        auto file = std::make_shared<DatArchiveFile>(entry, [=](unsigned int seekPosition, unsigned char* to, unsigned char size)-> unsigned int {
+            _streamWrapper.seek(seekPosition);
+            return _streamWrapper.readBytes(reinterpret_cast<char*>(to), size);
+        });
+        file->_open(mode);
+        return file;
+    }
+}