ChunkData.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
#pragma once
 
#include "Memory/MemoryFwd.h"
#include "Misc/SecureHash.h"
#include "Misc/EnumClassFlags.h"
#include "Misc/Guid.h"
#include "BuildPatchFeatureLevel.h"
 
namespace BuildPatchServices
{
    class IFileSystem;
 
    // Constant for the legacy fixed chunk window size, which was 1MiB.
    static const uint32 LegacyFixedChunkWindow = 1024 * 1024;
 
    enum class EChunkStorageFlags : uint8
    {
        None = 0x00,
 
        // Flag for compressed data.
        Compressed = 0x01,
 
        // Flag for encrypted. If also compressed, decrypt first. Encryption will ruin compressibility.
        Encrypted = 0x02,
    };
    ENUM_CLASS_FLAGS(EChunkStorageFlags);
 
    enum class EChunkHashFlags : uint8
    {
        None = 0x00,
 
        // Flag for FRollingHash class used, stored in RollingHash on header.
        RollingPoly64 = 0x01,
 
        // Flag for FSHA1 class used, stored in SHAHash on header.
        Sha1 = 0x02,
    };
    ENUM_CLASS_FLAGS(EChunkHashFlags);
 
    enum class EChunkLoadResult : uint8
    {
        Success = 0,
 
        // Failed to open the file to load the chunk.
        OpenFileFail,
 
        // Could not serialize due to wrong archive type.
        BadArchive,
 
        // The header in the loaded chunk was invalid.
        CorruptHeader,
 
        // The expected file size in the header did not match the size of the file.
        IncorrectFileSize,
 
        // The storage type of the chunk is not one which we support.
        UnsupportedStorage,
 
        // The hash information was missing.
        MissingHashInfo,
 
        // The serialized data was not successfully understood.
        SerializationError,
 
        // The data was saved compressed but decompression failed.
        DecompressFailure,
 
        // The expected data hash in the header did not match the hash of the data.
        HashCheckFailed,
 
        // The size of the file is too big to handle 
        FileSizeTooBig,
 
        // The operation was aborted.
        Aborted
    };
 
    const TCHAR* ToString(const EChunkLoadResult& ChunkLoadResult);
 
    enum class EChunkSaveResult : uint8
    {
        Success = 0,
 
        // Failed to create the file for the chunk.
        FileCreateFail,
 
        // Could not serialize due to wrong archive type.
        BadArchive,
 
        // There was a serialization problem when writing to the chunk file.
        SerializationError
    };
 
    const TCHAR* ToString(const EChunkSaveResult& ChunkSaveResult);
 
    struct FChunkHeader
    {
        FChunkHeader();
        friend FArchive& operator<< (FArchive& Ar, FChunkHeader& Header);
        // The version of this header data.
        uint32 Version;
        // The size of this header.
        uint32 HeaderSize;
        // The GUID for this data.
        FGuid Guid;
        // The size of this data compressed.
        uint32 DataSizeCompressed;
        // The size of this data uncompressed.
        uint32 DataSizeUncompressed;
        // How the chunk data is stored.
        EChunkStorageFlags StoredAs;
        // What type of hash we are using.
        EChunkHashFlags HashType;
        // The FRollingHash hashed value for this chunk data.
        uint64 RollingHash;
        // The FSHA hashed value for this chunk data.
        FSHAHash SHAHash;
    };
 
    // Some helper constants for dealing with chunks that are full of one single byte, usually padding.
    namespace PaddingChunk
    {
        // The A, B, and C components of a chunk Guid indicating that this is a padding chunk. D would be the actual byte padded with.
        static const int32 ChunkIdA = 0x00000001;
        static const int32 ChunkIdB = 0x00000000;
        static const int32 ChunkIdC = 0x00000000;
        // The size of the chunk we use to save out, which would allow a legacy client to actually use one.
        static const uint32 ChunkSize = LegacyFixedChunkWindow;
        FORCEINLINE bool IsPadding(const FGuid& Guid)
        {
            return Guid.A == ChunkIdA && Guid.B == ChunkIdB && Guid.C == ChunkIdC && Guid.D >= 0 && Guid.D <= 255;
        }
        FORCEINLINE uint8 GetPaddingByte(const FGuid& Guid)
        {
            check(IsPadding(Guid));
            return Guid.D;
        }
        FORCEINLINE FGuid MakePaddingGuid(uint8 Byte)
        {
            return FGuid(ChunkIdA, ChunkIdB, ChunkIdC, Byte);
        }
    }
 
    struct FChunkPart
    {
        FChunkPart();
        FChunkPart(const FGuid& Guid, const uint32 Offset, const uint32 Size);
        friend FArchive& operator<<(FArchive& Ar, FChunkPart& ChunkPart);
        bool IsPadding() const
        {
            return PaddingChunk::IsPadding(Guid);
        }
        uint8 GetPaddingByte() const
        {
            return PaddingChunk::GetPaddingByte(Guid);
        }
        // The GUID of the chunk containing this part.
        FGuid Guid;
        // The offset of the first byte into the chunk.
        uint32 Offset;
        // The size of this part.
        uint32 Size;
    };
 
    struct FFileChunkPart
    {
        FFileChunkPart();
        // The file containing this piece
        FString Filename;
        // The offset into the file of this piece
        uint64 FileOffset;
        // The FChunkPart that can be salvaged from this file
        FChunkPart ChunkPart;
    };
 
    struct FChunkInfo
    {
        FChunkInfo();
        // The GUID for this data.
        FGuid Guid;
        // The FRollingHash hashed value for this chunk data.
        uint64 Hash;
        // The FSHA hashed value for this chunk data.
        FSHAHash ShaHash;
        // The group number this chunk divides into.
        uint8 GroupNumber;
        // The window size for this chunk.
        uint32 WindowSize;
        // The file download size for this chunk.
        int64 FileSize;
    };
 
    struct FChunkLocation
    {
        FGuid ChunkId;
        uint64 ByteStart;
        uint32 ByteSize;
    };
 
    struct FChunkDatabaseHeader
    {
        FChunkDatabaseHeader();
        friend FArchive& operator<< (FArchive& Ar, FChunkDatabaseHeader& Header);
        // The version of this header data.
        uint32 Version;
        // The size of this header.
        uint32 HeaderSize;
        // The size of the following data.
        uint64 DataSize;
        // The table of contents.
        TArray<FChunkLocation> Contents;
    };
 
    class IChunkDataAccess
    {
    public:
        virtual ~IChunkDataAccess() {}
 
        virtual void GetDataLock(const uint8** OutChunkData, const FChunkHeader** OutChunkHeader) const = 0;
        virtual void GetDataLock(      uint8** OutChunkData,       FChunkHeader** OutChunkHeader) = 0;
 
        virtual void ReleaseDataLock() const = 0;
    };
 
    class FChunkDataAccessFactory
    {
    public:
        static IChunkDataAccess* Create(uint32 DataSize);
    };
 
    struct FScopeLockedChunkData
    {
    public:
        FScopeLockedChunkData(IChunkDataAccess* ChunkDataAccess);
        ~FScopeLockedChunkData();
 
        FChunkHeader* GetHeader() const;
        
        uint8* GetData() const;
 
    private:
        // The provided IChunkDataAccess which we lock and release.
        IChunkDataAccess* ChunkDataAccess;
        // Pointer to the header data.
        FChunkHeader* ChunkHeader;
        // Pointer to the chunk data.
        uint8* ChunkData;
    };
 
    class IChunkDataSerialization
    {
    public:
        virtual ~IChunkDataSerialization() {}
 
        virtual IChunkDataAccess* LoadFromFile(const FString& Filename, EChunkLoadResult& OutLoadResult) const = 0;
 
        virtual IChunkDataAccess* LoadFromMemory(const TArray<uint8>& Memory, EChunkLoadResult& OutLoadResult) const = 0;
 
        virtual IChunkDataAccess* LoadFromArchive(FArchive& Archive, EChunkLoadResult& OutLoadResult) const = 0;
        
        virtual EChunkSaveResult SaveToFile(const FString& Filename, const IChunkDataAccess* ChunkDataAccess) const = 0;
 
        virtual EChunkSaveResult SaveToMemory(TArray<uint8>& Memory, const IChunkDataAccess* ChunkDataAccess) const = 0;
 
        virtual EChunkSaveResult SaveToArchive(FArchive& Archive, const IChunkDataAccess* ChunkDataAccess) const = 0;
 
        // In performance sensitive situations, can request the serializer to save uncompressed if implemented
        virtual EChunkSaveResult SaveToArchiveUncompressed(FArchive& Archive, const IChunkDataAccess* ChunkDataAccess) const
        {
            return SaveToArchive(Archive, ChunkDataAccess);
        }
 
        // Read the header from the archive, make sure it's sane, and set up to decompress/hash in a separate call. This
        // is to allow the IO to be on a separate thread from decompression/hashing.
        virtual bool ValidateAndRead(FArchive& InArchive, FMutableMemoryView InDestinationBuffer, FChunkHeader& OutHeader, FUniqueBuffer& OutCompressedBuffer) const = 0;
        
        // Take the output of ValidateAndRead and process it in to an actual output chunk.
        virtual bool DecompressValidatedRead(const FChunkHeader& InHeader, FMutableMemoryView InDestionationBuffer, const FUniqueBuffer& InCompressedBuffer) const = 0;
 
        virtual void InjectShaToChunkData(TArray<uint8>& Memory, const FSHAHash& ShaHashData) const = 0;
    };
 
    class FChunkDataSerializationFactory
    {
    public:
        static IChunkDataSerialization* Create(IFileSystem* FileSystem, EFeatureLevel FeatureLevel = EFeatureLevel::Latest);
    };
}