SecureHash.h

Go to the documentation of this file.

// Copyright Epic Games, Inc. All Rights Reserved.
 
#pragma once
 
#include "Containers/Array.h"
#include "Containers/Map.h"
#include "Containers/StringConv.h"
#include "Containers/StringFwd.h"
#include "Containers/StringView.h"
#include "Containers/UnrealString.h"
#include "CoreTypes.h"
#include "HAL/PlatformCrt.h"
#include "HAL/PreprocessorHelpers.h"
#include "HAL/UnrealMemory.h"
#include "Misc/AssertionMacros.h"
#include "Misc/CString.h"
#include "Misc/Guid.h"
#include "Serialization/Archive.h"
#include "Serialization/BufferReader.h"
#include "Serialization/MemoryLayout.h"
#include "Stats/Stats.h"
#include "String/BytesToHex.h"
#include "String/HexToBytes.h"
#include "Templates/UnrealTemplate.h"
 
class FCbFieldView;
class FCbWriter;
class FMemoryImageWriter;
class FMemoryUnfreezeContent;
class FPointerTableBase;
class FSHA1;
struct FMD5Hash;
 
/*-----------------------------------------------------------------------------
    MD5 functions.
-----------------------------------------------------------------------------*/
 
//
// MD5 Context.
//
 
 
//
// MD5 functions.
// FArchive.
//
// CORE_API void appMD5Init( FMD5Context* context );
// CORE_API void appMD5Update( FMD5Context* context, uint8* input, int32 inputLen );
// CORE_API void appMD5Final( uint8* digest, FMD5Context* context );
// CORE_API void appMD5Transform( uint32* state, uint8* block );
// CORE_API void appMD5Encode( uint8* output, uint32* input, int32 len );
// CORE_API void appMD5Decode( uint32* output, uint8* input, int32 len );
 
class FMD5
{
public:
    CORE_API FMD5();
    CORE_API ~FMD5();
 
    CORE_API void Update(const uint8* input, uint64 inputLen);
 
    CORE_API void Final(uint8* digest);
 
    static FString HashAnsiString(const TCHAR* String)
    {
        return HashBytes((unsigned char*)TCHAR_TO_ANSI(String), FCString::Strlen(String));
    }
 
    static FString HashBytes(const uint8* input, uint64 inputLen)
    {
        uint8 Digest[16];
 
        FMD5 Md5Gen;
 
        Md5Gen.Update(input, inputLen);
        Md5Gen.Final(Digest);
 
        FString MD5;
        for (int32 i = 0; i < 16; i++)
        {
            MD5 += FString::Printf(TEXT("%02x"), Digest[i]);
        }
        return MD5;
    }
 
private:
    struct FContext
    {
        uint32 state[4];
        uint32 count[2];
        uint8 buffer[64];
    };
 
    CORE_API void Transform( uint32* state, const uint8* block );
    CORE_API void Encode( uint8* output, const uint32* input, int32 len );
    CORE_API void Decode( uint32* output, const uint8* input, int32 len );
 
    FContext Context;
};
 
struct FMD5Hash;
 
struct FMD5Hash
{
    FMD5Hash() : bIsValid(false) {}
 
    bool IsValid() const { return bIsValid; }
 
    void Set(FMD5& MD5)
    {
        MD5.Final(Bytes);
        bIsValid = true;
    }
 
    friend bool operator==(const FMD5Hash& LHS, const FMD5Hash& RHS)
    {
        return LHS.bIsValid == RHS.bIsValid && (!LHS.bIsValid || FMemory::Memcmp(LHS.Bytes, RHS.Bytes, 16) == 0);
    }
 
    friend bool operator!=(const FMD5Hash& LHS, const FMD5Hash& RHS)
    {
        return LHS.bIsValid != RHS.bIsValid || (LHS.bIsValid && FMemory::Memcmp(LHS.Bytes, RHS.Bytes, 16) != 0);
    }
 
    friend FArchive& operator<<(FArchive& Ar, FMD5Hash& Hash)
    {
        Ar << Hash.bIsValid;
        if (Hash.bIsValid)
        {
            Ar.Serialize(Hash.Bytes, 16);
        }
 
        return Ar;
    }
 
    CORE_API static FMD5Hash HashFile(const TCHAR* InFilename, TArray<uint8>* Buffer = nullptr);
    CORE_API static FMD5Hash HashFileFromArchive(FArchive* Ar, TArray<uint8>* ScratchPad = nullptr);
 
    const uint8* GetBytes() const { return Bytes; }
    const int32 GetSize() const { return sizeof(Bytes); }
 
private:
    bool bIsValid;
 
    uint8 Bytes[16];
 
    friend inline FCbWriter& operator<<(FCbWriter& Writer, const FMD5Hash& Hash) // Hidden friends must be inlined to be performant
    {
        return Hash.WriteCompactBinary(Writer);
    }
    friend bool LoadFromCompactBinary(const FCbFieldView& Field, FMD5Hash& OutHash) { return OutHash.LoadFromCompactBinary(Field); }
    friend FString LexToString(const FMD5Hash& InHash) { return InHash.LexToString(); }
    friend void LexFromString(FMD5Hash& Hash, const TCHAR* InString) { Hash.LexFromString(InString); }
 
    CORE_API FCbWriter& WriteCompactBinary(FCbWriter& Writer) const;
    CORE_API bool LoadFromCompactBinary(const FCbFieldView& Field); // inlining not available because we don't want FCbFieldView defined
    CORE_API FString LexToString() const;
    CORE_API void LexFromString(const TCHAR*);
};
 
inline FGuid MD5HashToGuid(const FMD5Hash& Hash)
{
    FGuid Result;
    FMemory::Memcpy(&Result, Hash.GetBytes(), sizeof(FGuid));
    return Result;
}
 
/*-----------------------------------------------------------------------------
    SHA-1 functions.
-----------------------------------------------------------------------------*/
 
/*
 *  NOTE:
 *  100% free public domain implementation of the SHA-1 algorithm
 *  by Dominik Reichl <dominik.reichl@t-online.de>
 *  Web: http://www.dominik-reichl.de/
 */
 
#define HASHES_SHA_DIVIDER "+++"
 
class FSHAHash
{
public:
    alignas(uint32) uint8 Hash[20];
 
    FSHAHash()
    {
        FMemory::Memset(Hash, 0, sizeof(Hash));
    }
 
    static constexpr int32 GetStringLen() { return UE_ARRAY_COUNT(Hash) * 2; }
 
    inline void AppendString(FString& Out) const
    {
        BytesToHex((const uint8*)Hash, sizeof(Hash), Out);
    }
 
    inline FString ToString() const
    {
        return BytesToHex((const uint8*)Hash, sizeof(Hash));
    }
 
    inline void ToString(TCHAR* Dest, bool bNullTerminate) const
    {
        constexpr auto Count = UE_ARRAY_COUNT(Hash);
        for (int i = 0; i < Count; ++i)
        {
            uint8 Val = Hash[i];
            Dest[i * 2] = NibbleToTChar(Val >> 4);
            Dest[i * 2 + 1] = NibbleToTChar(Val & 15);
        }
 
        if (bNullTerminate)
        {
            Dest[Count * 2] = TEXT('\0');
        }
    }
 
    inline void FromString(const FStringView& Src)
    {
        check(Src.Len() == GetStringLen());
        UE::String::HexToBytes(Src, Hash);
    }
 
    friend bool operator==(const FSHAHash& X, const FSHAHash& Y)
    {
        return FMemory::Memcmp(&X.Hash, &Y.Hash, sizeof(X.Hash)) == 0;
    }
 
    friend bool operator!=(const FSHAHash& X, const FSHAHash& Y)
    {
        return FMemory::Memcmp(&X.Hash, &Y.Hash, sizeof(X.Hash)) != 0;
    }
 
    friend bool operator<(const FSHAHash& X, const FSHAHash& Y)
    {
        return FMemory::Memcmp(&X.Hash, &Y.Hash, sizeof(X.Hash)) < 0;
    }
 
    friend FArchive& operator<<( FArchive& Ar, FSHAHash& G )
    {
        Ar.Serialize(&G.Hash, sizeof(G.Hash));
        return Ar;
    }
    
    friend uint32 GetTypeHash(const FSHAHash& InKey)
    {
        return *reinterpret_cast<const uint32*>(InKey.Hash);
    }
 
    friend FString LexToString(const FSHAHash& InHash) { return InHash.ToString(); }
    friend void LexFromString(FSHAHash& InHash, const TCHAR* InString) { InHash.FromString(InString); }
    friend FStringBuilderBase& operator<<(FStringBuilderBase& Builder, const FSHAHash& InHash) { UE::String::BytesToHex(InHash.Hash, Builder); return Builder; }
    friend FAnsiStringBuilderBase& operator<<(FAnsiStringBuilderBase& Builder, const FSHAHash& InHash) { UE::String::BytesToHex(InHash.Hash, Builder); return Builder; }
    friend FCbWriter& operator<<(FCbWriter& Writer, const FSHAHash& InHash) { return InHash.WriteCompactBinary(Writer); }
    friend bool LoadFromCompactBinary(const FCbFieldView& Field, FSHAHash& OutHash) { return OutHash.LoadFromCompactBinary(Field); }
 
    CORE_API bool LoadFromCompactBinary(const FCbFieldView& Field);
    CORE_API FCbWriter& WriteCompactBinary(FCbWriter& Writer) const;
};
 
namespace Freeze
{
    CORE_API void IntrinsicToString(const FSHAHash& Object, const FTypeLayoutDesc& TypeDesc, const FPlatformTypeLayoutParameters& LayoutParams, FMemoryToStringContext& OutContext);
}
 
DECLARE_INTRINSIC_TYPE_LAYOUT(FSHAHash);
 
class FSHA1
{
public:
 
    enum {DigestSize=20};
    // Constructor and Destructor
    CORE_API FSHA1();
    CORE_API ~FSHA1();
 
    uint32 m_state[5];
    uint64 m_count;
    uint32 __reserved1[1];
    uint8  m_buffer[64];
    uint8  m_digest[20];
    uint32 __reserved2[3];
 
    CORE_API void Reset();
 
    // Update the hash value
    CORE_API void Update(const uint8 *data, uint64 len);
 
    // Update the hash value with string
    CORE_API void UpdateWithString(const TCHAR *data, uint32 len);
 
    // Update with POD data.
    template <typename Type UE_REQUIRES(TIsPODType<Type>::Value)>
    void Update(const Type& InData)
    {
        Update(reinterpret_cast<const uint8*>(&InData), sizeof(Type));
    }
 
    // Finalize hash and report
    CORE_API void Final();
 
    // Finalize hash and return it
    FSHAHash Finalize()
    {
        Final();
        FSHAHash Digest;
        GetHash(reinterpret_cast<uint8*>(&Digest));
        return Digest;
    }
 
    // Report functions: as pre-formatted and raw data
    CORE_API void GetHash(uint8 *puDest) const;
 
    static CORE_API void HashBuffer(const void* Data, uint64 DataSize, uint8* OutHash);
 
    static FSHAHash HashBuffer(const void* Data, uint64 DataSize)
    {
        FSHAHash Hash;
        HashBuffer(Data, DataSize, Hash.Hash);
        return Hash;
    }
 
    static CORE_API void HMACBuffer(const void* Key, uint32 KeySize, const void* Data, uint64 DataSize, uint8* OutHash);
 
    static CORE_API void InitializeFileHashesFromBuffer(uint8* Buffer, uint64 BufferSize, bool bDuplicateKeyMemory=false);
 
    static CORE_API bool GetFileSHAHash(const TCHAR* Pathname, uint8 Hash[20], bool bIsFullPackageHash=true);
 
private:
    // Private SHA-1 transformation
    CORE_API void Transform(const uint8* buffer, uint64 len);
 
    static CORE_API TMap<FString, uint8*> FullFileSHAHashMap;
 
    static CORE_API TMap<FString, uint8*> ScriptSHAHashMap;
};
 
 
class FAsyncSHAVerify
{
protected:
    void* Buffer;
 
    uint64 BufferSize;
 
    uint8 Hash[20];
 
    FString Pathname;
 
    bool bIsUnfoundHashAnError;
 
    bool bShouldDeleteBuffer;
 
public:
 
    FAsyncSHAVerify(
        void* InBuffer, 
        uint64 InBufferSize, 
        bool bInShouldDeleteBuffer, 
        const TCHAR* InPathname, 
        bool bInIsUnfoundHashAnError)
        :   Buffer(InBuffer)
        ,   BufferSize(InBufferSize)
        ,   Pathname(InPathname)
        ,   bIsUnfoundHashAnError(bInIsUnfoundHashAnError)
        ,   bShouldDeleteBuffer(bInShouldDeleteBuffer)
    {
    }
 
    CORE_API void DoWork();
 
    bool CanAbandon()
    {
        return true;
    }
 
    void Abandon()
    {
        if( bShouldDeleteBuffer )
        {
            FMemory::Free( Buffer );
            Buffer = 0;
        }
    }
 
    UE_FORCEINLINE_HINT TStatId GetStatId() const
    {
        RETURN_QUICK_DECLARE_CYCLE_STAT(FAsyncSHAVerify, STATGROUP_ThreadPoolAsyncTasks);
    }
};
 
CORE_API void appOnFailSHAVerification(const TCHAR* FailedPathname, bool bFailedDueToMissingHash);
 
class FBufferReaderWithSHA final : public FBufferReaderBase
{
public:
    FBufferReaderWithSHA( 
        void* Data, 
        int64 Size, 
        bool bInFreeOnClose, 
        const TCHAR* SHASourcePathname, 
        bool bIsPersistent=false, 
        bool bInIsUnfoundHashAnError=false 
        )
    // we force the base class to NOT free buffer on close, as we will let the SHA task do it if needed
    : FBufferReaderBase(Data, Size, bInFreeOnClose, bIsPersistent)
    , SourcePathname(SHASourcePathname)
    , bIsUnfoundHashAnError(bInIsUnfoundHashAnError)
    {
    }
 
    CORE_API virtual ~FBufferReaderWithSHA();
 
    CORE_API bool Close() override;
 
    virtual FString GetArchiveName() const { return TEXT("FBufferReaderWithSHA"); }
 
protected:
    FString SourcePathname;
    bool bIsUnfoundHashAnError;
};