From 4be8c4f88c6a17f99f19cfd79dacc53c963c0654 Mon Sep 17 00:00:00 2001 From: Wanne Date: Fri, 17 Jun 2011 10:56:48 +0000 Subject: [PATCH] - Removed unused old files git-svn-id: https://ja2svn.mooo.com/source/ja2/trunk/GameSource/ja2_v1.13/Build@4503 3b4a5df2-a311-0410-b5c6-a8a6f20db521 --- ext/7z/7zBuf.c | 36 - ext/7z/7zBuf.h | 31 - ext/7z/7zBuf2.c | 45 - ext/7z/7zCrc.c | 35 - ext/7z/7zCrc.h | 24 - ext/7z/7zFile.c | 263 -- ext/7z/7zFile.h | 74 - ext/7z/7zStream.c | 169 -- ext/7z/7zVersion.h | 7 - ext/7z/Alloc.c | 127 - ext/7z/Alloc.h | 32 - ext/7z/Archive/7z/7z.vcproj | 1070 -------- ext/7z/Archive/7z/7zAlloc.c | 77 - ext/7z/Archive/7z/7zAlloc.h | 15 - ext/7z/Archive/7z/7zDecode.c | 254 -- ext/7z/Archive/7z/7zDecode.h | 13 - ext/7z/Archive/7z/7zExtract.c | 93 - ext/7z/Archive/7z/7zExtract.h | 41 - ext/7z/Archive/7z/7zHeader.c | 6 - ext/7z/Archive/7z/7zHeader.h | 57 - ext/7z/Archive/7z/7zIn.c | 1204 --------- ext/7z/Archive/7z/7zIn.h | 41 - ext/7z/Archive/7z/7zItem.c | 127 - ext/7z/Archive/7z/7zItem.h | 84 - ext/7z/Archive/7z/7zMain.c | 262 -- ext/7z/Archive/7z/7z_2005Express.vcproj | 1070 -------- ext/7z/Archive/7z/7z_VS2003.vcproj | 301 --- ext/7z/Bcj2.c | 132 - ext/7z/Bcj2.h | 30 - ext/7z/Bra.c | 133 - ext/7z/Bra.h | 60 - ext/7z/Bra86.c | 85 - ext/7z/BraIA64.c | 67 - ext/7z/CpuArch.h | 69 - ext/7z/LzFind.c | 751 ------ ext/7z/LzFind.h | 107 - ext/7z/LzFindMt.c | 793 ------ ext/7z/LzFindMt.h | 97 - ext/7z/LzHash.h | 54 - ext/7z/LzmaDec.c | 1007 -------- ext/7z/LzmaDec.h | 223 -- ext/7z/LzmaEnc.c | 2281 ----------------- ext/7z/LzmaEnc.h | 72 - ext/7z/LzmaLib.c | 46 - ext/7z/LzmaLib.h | 135 - ext/7z/Threads.c | 109 - ext/7z/Threads.h | 68 - ext/7z/Types.h | 208 -- ext/libpng/projects/visualc71/libpng.vcproj | 606 ----- .../visualc71/libpng_2005Express.vcproj | 605 ----- .../projects/visualc71/libpng_VS2003.vcproj | 355 --- ext/libpng/projects/visualc71/zlib.vcproj | 530 ---- .../visualc71/zlib_2005Express.vcproj | 532 ---- .../projects/visualc71/zlib_VS2003.vcproj | 311 --- ext/utf8/doc/ReleaseNotes | 22 - ext/utf8/doc/utf8cpp.html | 1629 ------------ ext/utf8/source/utf8.h | 34 - ext/utf8/source/utf8/checked.h | 318 --- ext/utf8/source/utf8/core.h | 334 --- ext/utf8/source/utf8/unchecked.h | 228 -- 60 files changed, 17589 deletions(-) delete mode 100644 ext/7z/7zBuf.c delete mode 100644 ext/7z/7zBuf.h delete mode 100644 ext/7z/7zBuf2.c delete mode 100644 ext/7z/7zCrc.c delete mode 100644 ext/7z/7zCrc.h delete mode 100644 ext/7z/7zFile.c delete mode 100644 ext/7z/7zFile.h delete mode 100644 ext/7z/7zStream.c delete mode 100644 ext/7z/7zVersion.h delete mode 100644 ext/7z/Alloc.c delete mode 100644 ext/7z/Alloc.h delete mode 100644 ext/7z/Archive/7z/7z.vcproj delete mode 100644 ext/7z/Archive/7z/7zAlloc.c delete mode 100644 ext/7z/Archive/7z/7zAlloc.h delete mode 100644 ext/7z/Archive/7z/7zDecode.c delete mode 100644 ext/7z/Archive/7z/7zDecode.h delete mode 100644 ext/7z/Archive/7z/7zExtract.c delete mode 100644 ext/7z/Archive/7z/7zExtract.h delete mode 100644 ext/7z/Archive/7z/7zHeader.c delete mode 100644 ext/7z/Archive/7z/7zHeader.h delete mode 100644 ext/7z/Archive/7z/7zIn.c delete mode 100644 ext/7z/Archive/7z/7zIn.h delete mode 100644 ext/7z/Archive/7z/7zItem.c delete mode 100644 ext/7z/Archive/7z/7zItem.h delete mode 100644 ext/7z/Archive/7z/7zMain.c delete mode 100644 ext/7z/Archive/7z/7z_2005Express.vcproj delete mode 100644 ext/7z/Archive/7z/7z_VS2003.vcproj delete mode 100644 ext/7z/Bcj2.c delete mode 100644 ext/7z/Bcj2.h delete mode 100644 ext/7z/Bra.c delete mode 100644 ext/7z/Bra.h delete mode 100644 ext/7z/Bra86.c delete mode 100644 ext/7z/BraIA64.c delete mode 100644 ext/7z/CpuArch.h delete mode 100644 ext/7z/LzFind.c delete mode 100644 ext/7z/LzFind.h delete mode 100644 ext/7z/LzFindMt.c delete mode 100644 ext/7z/LzFindMt.h delete mode 100644 ext/7z/LzHash.h delete mode 100644 ext/7z/LzmaDec.c delete mode 100644 ext/7z/LzmaDec.h delete mode 100644 ext/7z/LzmaEnc.c delete mode 100644 ext/7z/LzmaEnc.h delete mode 100644 ext/7z/LzmaLib.c delete mode 100644 ext/7z/LzmaLib.h delete mode 100644 ext/7z/Threads.c delete mode 100644 ext/7z/Threads.h delete mode 100644 ext/7z/Types.h delete mode 100644 ext/libpng/projects/visualc71/libpng.vcproj delete mode 100644 ext/libpng/projects/visualc71/libpng_2005Express.vcproj delete mode 100644 ext/libpng/projects/visualc71/libpng_VS2003.vcproj delete mode 100644 ext/libpng/projects/visualc71/zlib.vcproj delete mode 100644 ext/libpng/projects/visualc71/zlib_2005Express.vcproj delete mode 100644 ext/libpng/projects/visualc71/zlib_VS2003.vcproj delete mode 100644 ext/utf8/doc/ReleaseNotes delete mode 100644 ext/utf8/doc/utf8cpp.html delete mode 100644 ext/utf8/source/utf8.h delete mode 100644 ext/utf8/source/utf8/checked.h delete mode 100644 ext/utf8/source/utf8/core.h delete mode 100644 ext/utf8/source/utf8/unchecked.h diff --git a/ext/7z/7zBuf.c b/ext/7z/7zBuf.c deleted file mode 100644 index a35fa2f9..00000000 --- a/ext/7z/7zBuf.c +++ /dev/null @@ -1,36 +0,0 @@ -/* 7zBuf.c -- Byte Buffer -2008-03-28 -Igor Pavlov -Public domain */ - -#include "7zBuf.h" - -void Buf_Init(CBuf *p) -{ - p->data = 0; - p->size = 0; -} - -int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc) -{ - p->size = 0; - if (size == 0) - { - p->data = 0; - return 1; - } - p->data = (Byte *)alloc->Alloc(alloc, size); - if (p->data != 0) - { - p->size = size; - return 1; - } - return 0; -} - -void Buf_Free(CBuf *p, ISzAlloc *alloc) -{ - alloc->Free(alloc, p->data); - p->data = 0; - p->size = 0; -} diff --git a/ext/7z/7zBuf.h b/ext/7z/7zBuf.h deleted file mode 100644 index 49b2354b..00000000 --- a/ext/7z/7zBuf.h +++ /dev/null @@ -1,31 +0,0 @@ -/* 7zBuf.h -- Byte Buffer -2008-10-04 : Igor Pavlov : Public domain */ - -#ifndef __7Z_BUF_H -#define __7Z_BUF_H - -#include "Types.h" - -typedef struct -{ - Byte *data; - size_t size; -} CBuf; - -void Buf_Init(CBuf *p); -int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc); -void Buf_Free(CBuf *p, ISzAlloc *alloc); - -typedef struct -{ - Byte *data; - size_t size; - size_t pos; -} CDynBuf; - -void DynBuf_Construct(CDynBuf *p); -void DynBuf_SeekToBeg(CDynBuf *p); -int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAlloc *alloc); -void DynBuf_Free(CDynBuf *p, ISzAlloc *alloc); - -#endif diff --git a/ext/7z/7zBuf2.c b/ext/7z/7zBuf2.c deleted file mode 100644 index 1c8b931a..00000000 --- a/ext/7z/7zBuf2.c +++ /dev/null @@ -1,45 +0,0 @@ -/* 7zBuf2.c -- Byte Buffer -2008-10-04 : Igor Pavlov : Public domain */ - -#include -#include "7zBuf.h" - -void DynBuf_Construct(CDynBuf *p) -{ - p->data = 0; - p->size = 0; - p->pos = 0; -} - -void DynBuf_SeekToBeg(CDynBuf *p) -{ - p->pos = 0; -} - -int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAlloc *alloc) -{ - if (size > p->size - p->pos) - { - size_t newSize = p->pos + size; - Byte *data; - newSize += newSize / 4; - data = (Byte *)alloc->Alloc(alloc, newSize); - if (data == 0) - return 0; - p->size = newSize; - memcpy(data, p->data, p->pos); - alloc->Free(alloc, p->data); - p->data = data; - } - memcpy(p->data + p->pos, buf, size); - p->pos += size; - return 1; -} - -void DynBuf_Free(CDynBuf *p, ISzAlloc *alloc) -{ - alloc->Free(alloc, p->data); - p->data = 0; - p->size = 0; - p->pos = 0; -} diff --git a/ext/7z/7zCrc.c b/ext/7z/7zCrc.c deleted file mode 100644 index 92c47de0..00000000 --- a/ext/7z/7zCrc.c +++ /dev/null @@ -1,35 +0,0 @@ -/* 7zCrc.c -- CRC32 calculation -2008-08-05 -Igor Pavlov -Public domain */ - -#include "7zCrc.h" - -#define kCrcPoly 0xEDB88320 -UInt32 g_CrcTable[256]; - -void MY_FAST_CALL CrcGenerateTable(void) -{ - UInt32 i; - for (i = 0; i < 256; i++) - { - UInt32 r = i; - int j; - for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1)); - g_CrcTable[i] = r; - } -} - -UInt32 MY_FAST_CALL CrcUpdate(UInt32 v, const void *data, size_t size) -{ - const Byte *p = (const Byte *)data; - for (; size > 0 ; size--, p++) - v = CRC_UPDATE_BYTE(v, *p); - return v; -} - -UInt32 MY_FAST_CALL CrcCalc(const void *data, size_t size) -{ - return CrcUpdate(CRC_INIT_VAL, data, size) ^ 0xFFFFFFFF; -} diff --git a/ext/7z/7zCrc.h b/ext/7z/7zCrc.h deleted file mode 100644 index ae230991..00000000 --- a/ext/7z/7zCrc.h +++ /dev/null @@ -1,24 +0,0 @@ -/* 7zCrc.h -- CRC32 calculation -2008-03-13 -Igor Pavlov -Public domain */ - -#ifndef __7Z_CRC_H -#define __7Z_CRC_H - -#include - -#include "Types.h" - -extern UInt32 g_CrcTable[]; - -void MY_FAST_CALL CrcGenerateTable(void); - -#define CRC_INIT_VAL 0xFFFFFFFF -#define CRC_GET_DIGEST(crc) ((crc) ^ 0xFFFFFFFF) -#define CRC_UPDATE_BYTE(crc, b) (g_CrcTable[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) - -UInt32 MY_FAST_CALL CrcUpdate(UInt32 crc, const void *data, size_t size); -UInt32 MY_FAST_CALL CrcCalc(const void *data, size_t size); - -#endif diff --git a/ext/7z/7zFile.c b/ext/7z/7zFile.c deleted file mode 100644 index 2b605a56..00000000 --- a/ext/7z/7zFile.c +++ /dev/null @@ -1,263 +0,0 @@ -/* 7zFile.c -- File IO -2008-11-22 : Igor Pavlov : Public domain */ - -#include "7zFile.h" - -#ifndef USE_WINDOWS_FILE - -#include - -#endif - -#ifdef USE_WINDOWS_FILE - -/* - ReadFile and WriteFile functions in Windows have BUG: - If you Read or Write 64MB or more (probably min_failure_size = 64MB - 32KB + 1) - from/to Network file, it returns ERROR_NO_SYSTEM_RESOURCES - (Insufficient system resources exist to complete the requested service). - Probably in some version of Windows there are problems with other sizes: - for 32 MB (maybe also for 16 MB). - And message can be "Network connection was lost" -*/ - -#define kChunkSizeMax (1 << 22) - -#endif - -void File_Construct(CSzFile *p) -{ - #ifdef USE_WINDOWS_FILE - p->handle = INVALID_HANDLE_VALUE; - #else - p->file = NULL; - #endif -} - -static WRes File_Open(CSzFile *p, const char *name, int writeMode) -{ - #ifdef USE_WINDOWS_FILE - p->handle = CreateFileA(name, - writeMode ? GENERIC_WRITE : GENERIC_READ, - FILE_SHARE_READ, NULL, - writeMode ? CREATE_ALWAYS : OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, NULL); - return (p->handle != INVALID_HANDLE_VALUE) ? 0 : GetLastError(); - #else - p->file = fopen(name, writeMode ? "wb+" : "rb"); - return (p->file != 0) ? 0 : errno; - #endif -} - -WRes InFile_Open(CSzFile *p, const char *name) { return File_Open(p, name, 0); } -WRes OutFile_Open(CSzFile *p, const char *name) { return File_Open(p, name, 1); } - -WRes File_Close(CSzFile *p) -{ - #ifdef USE_WINDOWS_FILE - if (p->handle != INVALID_HANDLE_VALUE) - { - if (!CloseHandle(p->handle)) - return GetLastError(); - p->handle = INVALID_HANDLE_VALUE; - } - #else - if (p->file != NULL) - { - int res = fclose(p->file); - if (res != 0) - return res; - p->file = NULL; - } - #endif - return 0; -} - -WRes File_Read(CSzFile *p, void *data, size_t *size) -{ - size_t originalSize = *size; - if (originalSize == 0) - return 0; - - #ifdef USE_WINDOWS_FILE - - *size = 0; - do - { - DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; - DWORD processed = 0; - BOOL res = ReadFile(p->handle, data, curSize, &processed, NULL); - data = (void *)((Byte *)data + processed); - originalSize -= processed; - *size += processed; - if (!res) - return GetLastError(); - if (processed == 0) - break; - } - while (originalSize > 0); - return 0; - - #else - - *size = fread(data, 1, originalSize, p->file); - if (*size == originalSize) - return 0; - return ferror(p->file); - - #endif -} - -WRes File_Write(CSzFile *p, const void *data, size_t *size) -{ - size_t originalSize = *size; - if (originalSize == 0) - return 0; - - #ifdef USE_WINDOWS_FILE - - *size = 0; - do - { - DWORD curSize = (originalSize > kChunkSizeMax) ? kChunkSizeMax : (DWORD)originalSize; - DWORD processed = 0; - BOOL res = WriteFile(p->handle, data, curSize, &processed, NULL); - data = (void *)((Byte *)data + processed); - originalSize -= processed; - *size += processed; - if (!res) - return GetLastError(); - if (processed == 0) - break; - } - while (originalSize > 0); - return 0; - - #else - - *size = fwrite(data, 1, originalSize, p->file); - if (*size == originalSize) - return 0; - return ferror(p->file); - - #endif -} - -WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin) -{ - #ifdef USE_WINDOWS_FILE - - LARGE_INTEGER value; - DWORD moveMethod; - value.LowPart = (DWORD)*pos; - value.HighPart = (LONG)((UInt64)*pos >> 16 >> 16); /* for case when UInt64 is 32-bit only */ - switch (origin) - { - case SZ_SEEK_SET: moveMethod = FILE_BEGIN; break; - case SZ_SEEK_CUR: moveMethod = FILE_CURRENT; break; - case SZ_SEEK_END: moveMethod = FILE_END; break; - default: return ERROR_INVALID_PARAMETER; - } - value.LowPart = SetFilePointer(p->handle, value.LowPart, &value.HighPart, moveMethod); - if (value.LowPart == 0xFFFFFFFF) - { - WRes res = GetLastError(); - if (res != NO_ERROR) - return res; - } - *pos = ((Int64)value.HighPart << 32) | value.LowPart; - return 0; - - #else - - int moveMethod; - int res; - switch (origin) - { - case SZ_SEEK_SET: moveMethod = SEEK_SET; break; - case SZ_SEEK_CUR: moveMethod = SEEK_CUR; break; - case SZ_SEEK_END: moveMethod = SEEK_END; break; - default: return 1; - } - res = fseek(p->file, (long)*pos, moveMethod); - *pos = ftell(p->file); - return res; - - #endif -} - -WRes File_GetLength(CSzFile *p, UInt64 *length) -{ - #ifdef USE_WINDOWS_FILE - - DWORD sizeHigh; - DWORD sizeLow = GetFileSize(p->handle, &sizeHigh); - if (sizeLow == 0xFFFFFFFF) - { - DWORD res = GetLastError(); - if (res != NO_ERROR) - return res; - } - *length = (((UInt64)sizeHigh) << 32) + sizeLow; - return 0; - - #else - - long pos = ftell(p->file); - int res = fseek(p->file, 0, SEEK_END); - *length = ftell(p->file); - fseek(p->file, pos, SEEK_SET); - return res; - - #endif -} - - -/* ---------- FileSeqInStream ---------- */ - -static SRes FileSeqInStream_Read(void *pp, void *buf, size_t *size) -{ - CFileSeqInStream *p = (CFileSeqInStream *)pp; - return File_Read(&p->file, buf, size) == 0 ? SZ_OK : SZ_ERROR_READ; -} - -void FileSeqInStream_CreateVTable(CFileSeqInStream *p) -{ - p->s.Read = FileSeqInStream_Read; -} - - -/* ---------- FileInStream ---------- */ - -static SRes FileInStream_Read(void *pp, void *buf, size_t *size) -{ - CFileInStream *p = (CFileInStream *)pp; - return (File_Read(&p->file, buf, size) == 0) ? SZ_OK : SZ_ERROR_READ; -} - -static SRes FileInStream_Seek(void *pp, Int64 *pos, ESzSeek origin) -{ - CFileInStream *p = (CFileInStream *)pp; - return File_Seek(&p->file, pos, origin); -} - -void FileInStream_CreateVTable(CFileInStream *p) -{ - p->s.Read = FileInStream_Read; - p->s.Seek = FileInStream_Seek; -} - - -/* ---------- FileOutStream ---------- */ - -static size_t FileOutStream_Write(void *pp, const void *data, size_t size) -{ - CFileOutStream *p = (CFileOutStream *)pp; - File_Write(&p->file, data, &size); - return size; -} - -void FileOutStream_CreateVTable(CFileOutStream *p) -{ - p->s.Write = FileOutStream_Write; -} diff --git a/ext/7z/7zFile.h b/ext/7z/7zFile.h deleted file mode 100644 index 4fd2936c..00000000 --- a/ext/7z/7zFile.h +++ /dev/null @@ -1,74 +0,0 @@ -/* 7zFile.h -- File IO -2008-11-22 : Igor Pavlov : Public domain */ - -#ifndef __7Z_FILE_H -#define __7Z_FILE_H - -#ifdef _WIN32 -#define USE_WINDOWS_FILE -#endif - -#ifdef USE_WINDOWS_FILE -#include -#else -#include -#endif - -#include "Types.h" - - -/* ---------- File ---------- */ - -typedef struct -{ - #ifdef USE_WINDOWS_FILE - HANDLE handle; - #else - FILE *file; - #endif -} CSzFile; - -void File_Construct(CSzFile *p); -WRes InFile_Open(CSzFile *p, const char *name); -WRes OutFile_Open(CSzFile *p, const char *name); -WRes File_Close(CSzFile *p); - -/* reads max(*size, remain file's size) bytes */ -WRes File_Read(CSzFile *p, void *data, size_t *size); - -/* writes *size bytes */ -WRes File_Write(CSzFile *p, const void *data, size_t *size); - -WRes File_Seek(CSzFile *p, Int64 *pos, ESzSeek origin); -WRes File_GetLength(CSzFile *p, UInt64 *length); - - -/* ---------- FileInStream ---------- */ - -typedef struct -{ - ISeqInStream s; - CSzFile file; -} CFileSeqInStream; - -void FileSeqInStream_CreateVTable(CFileSeqInStream *p); - - -typedef struct -{ - ISeekInStream s; - CSzFile file; -} CFileInStream; - -void FileInStream_CreateVTable(CFileInStream *p); - - -typedef struct -{ - ISeqOutStream s; - CSzFile file; -} CFileOutStream; - -void FileOutStream_CreateVTable(CFileOutStream *p); - -#endif diff --git a/ext/7z/7zStream.c b/ext/7z/7zStream.c deleted file mode 100644 index a0a2405d..00000000 --- a/ext/7z/7zStream.c +++ /dev/null @@ -1,169 +0,0 @@ -/* 7zStream.c -- 7z Stream functions -2008-11-23 : Igor Pavlov : Public domain */ - -#include - -#include "Types.h" - -SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType) -{ - while (size != 0) - { - size_t processed = size; - RINOK(stream->Read(stream, buf, &processed)); - if (processed == 0) - return errorType; - buf = (void *)((Byte *)buf + processed); - size -= processed; - } - return SZ_OK; -} - -SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size) -{ - return SeqInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); -} - -SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf) -{ - size_t processed = 1; - RINOK(stream->Read(stream, buf, &processed)); - return (processed == 1) ? SZ_OK : SZ_ERROR_INPUT_EOF; -} - -SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset) -{ - Int64 t = offset; - return stream->Seek(stream, &t, SZ_SEEK_SET); -} - -SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size) -{ - void *lookBuf; - if (*size == 0) - return SZ_OK; - RINOK(stream->Look(stream, &lookBuf, size)); - memcpy(buf, lookBuf, *size); - return stream->Skip(stream, *size); -} - -SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType) -{ - while (size != 0) - { - size_t processed = size; - RINOK(stream->Read(stream, buf, &processed)); - if (processed == 0) - return errorType; - buf = (void *)((Byte *)buf + processed); - size -= processed; - } - return SZ_OK; -} - -SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size) -{ - return LookInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); -} - -static SRes LookToRead_Look_Lookahead(void *pp, void **buf, size_t *size) -{ - SRes res = SZ_OK; - CLookToRead *p = (CLookToRead *)pp; - size_t size2 = p->size - p->pos; - if (size2 == 0 && *size > 0) - { - p->pos = 0; - size2 = LookToRead_BUF_SIZE; - res = p->realStream->Read(p->realStream, p->buf, &size2); - p->size = size2; - } - if (size2 < *size) - *size = size2; - *buf = p->buf + p->pos; - return res; -} - -static SRes LookToRead_Look_Exact(void *pp, void **buf, size_t *size) -{ - SRes res = SZ_OK; - CLookToRead *p = (CLookToRead *)pp; - size_t size2 = p->size - p->pos; - if (size2 == 0 && *size > 0) - { - p->pos = 0; - if (*size > LookToRead_BUF_SIZE) - *size = LookToRead_BUF_SIZE; - res = p->realStream->Read(p->realStream, p->buf, size); - size2 = p->size = *size; - } - if (size2 < *size) - *size = size2; - *buf = p->buf + p->pos; - return res; -} - -static SRes LookToRead_Skip(void *pp, size_t offset) -{ - CLookToRead *p = (CLookToRead *)pp; - p->pos += offset; - return SZ_OK; -} - -static SRes LookToRead_Read(void *pp, void *buf, size_t *size) -{ - CLookToRead *p = (CLookToRead *)pp; - size_t rem = p->size - p->pos; - if (rem == 0) - return p->realStream->Read(p->realStream, buf, size); - if (rem > *size) - rem = *size; - memcpy(buf, p->buf + p->pos, rem); - p->pos += rem; - *size = rem; - return SZ_OK; -} - -static SRes LookToRead_Seek(void *pp, Int64 *pos, ESzSeek origin) -{ - CLookToRead *p = (CLookToRead *)pp; - p->pos = p->size = 0; - return p->realStream->Seek(p->realStream, pos, origin); -} - -void LookToRead_CreateVTable(CLookToRead *p, int lookahead) -{ - p->s.Look = lookahead ? - LookToRead_Look_Lookahead : - LookToRead_Look_Exact; - p->s.Skip = LookToRead_Skip; - p->s.Read = LookToRead_Read; - p->s.Seek = LookToRead_Seek; -} - -void LookToRead_Init(CLookToRead *p) -{ - p->pos = p->size = 0; -} - -static SRes SecToLook_Read(void *pp, void *buf, size_t *size) -{ - CSecToLook *p = (CSecToLook *)pp; - return LookInStream_LookRead(p->realStream, buf, size); -} - -void SecToLook_CreateVTable(CSecToLook *p) -{ - p->s.Read = SecToLook_Read; -} - -static SRes SecToRead_Read(void *pp, void *buf, size_t *size) -{ - CSecToRead *p = (CSecToRead *)pp; - return p->realStream->Read(p->realStream, buf, size); -} - -void SecToRead_CreateVTable(CSecToRead *p) -{ - p->s.Read = SecToRead_Read; -} diff --git a/ext/7z/7zVersion.h b/ext/7z/7zVersion.h deleted file mode 100644 index 5a6bcada..00000000 --- a/ext/7z/7zVersion.h +++ /dev/null @@ -1,7 +0,0 @@ -#define MY_VER_MAJOR 4 -#define MY_VER_MINOR 65 -#define MY_VER_BUILD 0 -#define MY_VERSION "4.65" -#define MY_DATE "2009-02-03" -#define MY_COPYRIGHT ": Igor Pavlov : Public domain" -#define MY_VERSION_COPYRIGHT_DATE MY_VERSION " " MY_COPYRIGHT " : " MY_DATE diff --git a/ext/7z/Alloc.c b/ext/7z/Alloc.c deleted file mode 100644 index bb24a772..00000000 --- a/ext/7z/Alloc.c +++ /dev/null @@ -1,127 +0,0 @@ -/* Alloc.c -- Memory allocation functions -2008-09-24 -Igor Pavlov -Public domain */ - -#ifdef _WIN32 -#include -#endif -#include - -#include "Alloc.h" - -/* #define _SZ_ALLOC_DEBUG */ - -/* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ -#ifdef _SZ_ALLOC_DEBUG -#include -int g_allocCount = 0; -int g_allocCountMid = 0; -int g_allocCountBig = 0; -#endif - -void *MyAlloc(size_t size) -{ - if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - { - void *p = malloc(size); - fprintf(stderr, "\nAlloc %10d bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p); - return p; - } - #else - return malloc(size); - #endif -} - -void MyFree(void *address) -{ - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree; count = %10d, addr = %8X", --g_allocCount, (unsigned)address); - #endif - free(address); -} - -#ifdef _WIN32 - -void *MidAlloc(size_t size) -{ - if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Mid %10d bytes; count = %10d", size, g_allocCountMid++); - #endif - return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); -} - -void MidFree(void *address) -{ - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid); - #endif - if (address == 0) - return; - VirtualFree(address, 0, MEM_RELEASE); -} - -#ifndef MEM_LARGE_PAGES -#undef _7ZIP_LARGE_PAGES -#endif - -#ifdef _7ZIP_LARGE_PAGES -SIZE_T g_LargePageSize = 0; -typedef SIZE_T (WINAPI *GetLargePageMinimumP)(); -#endif - -void SetLargePageSize() -{ - #ifdef _7ZIP_LARGE_PAGES - SIZE_T size = 0; - GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP) - GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum"); - if (largePageMinimum == 0) - return; - size = largePageMinimum(); - if (size == 0 || (size & (size - 1)) != 0) - return; - g_LargePageSize = size; - #endif -} - - -void *BigAlloc(size_t size) -{ - if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Big %10d bytes; count = %10d", size, g_allocCountBig++); - #endif - - #ifdef _7ZIP_LARGE_PAGES - if (g_LargePageSize != 0 && g_LargePageSize <= (1 << 30) && size >= (1 << 18)) - { - void *res = VirtualAlloc(0, (size + g_LargePageSize - 1) & (~(g_LargePageSize - 1)), - MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); - if (res != 0) - return res; - } - #endif - return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); -} - -void BigFree(void *address) -{ - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig); - #endif - - if (address == 0) - return; - VirtualFree(address, 0, MEM_RELEASE); -} - -#endif diff --git a/ext/7z/Alloc.h b/ext/7z/Alloc.h deleted file mode 100644 index a396c6b9..00000000 --- a/ext/7z/Alloc.h +++ /dev/null @@ -1,32 +0,0 @@ -/* Alloc.h -- Memory allocation functions -2008-03-13 -Igor Pavlov -Public domain */ - -#ifndef __COMMON_ALLOC_H -#define __COMMON_ALLOC_H - -#include - -void *MyAlloc(size_t size); -void MyFree(void *address); - -#ifdef _WIN32 - -void SetLargePageSize(); - -void *MidAlloc(size_t size); -void MidFree(void *address); -void *BigAlloc(size_t size); -void BigFree(void *address); - -#else - -#define MidAlloc(size) MyAlloc(size) -#define MidFree(address) MyFree(address) -#define BigAlloc(size) MyAlloc(size) -#define BigFree(address) MyFree(address) - -#endif - -#endif diff --git a/ext/7z/Archive/7z/7z.vcproj b/ext/7z/Archive/7z/7z.vcproj deleted file mode 100644 index 4c85d240..00000000 --- a/ext/7z/Archive/7z/7z.vcproj +++ /dev/null @@ -1,1070 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ext/7z/Archive/7z/7zAlloc.c b/ext/7z/Archive/7z/7zAlloc.c deleted file mode 100644 index b5918d00..00000000 --- a/ext/7z/Archive/7z/7zAlloc.c +++ /dev/null @@ -1,77 +0,0 @@ -/* 7zAlloc.c -- Allocation functions -2008-10-04 : Igor Pavlov : Public domain */ - -#include -#include "7zAlloc.h" - -/* #define _SZ_ALLOC_DEBUG */ -/* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ - -#ifdef _SZ_ALLOC_DEBUG - -#ifdef _WIN32 -#include -#endif - -#include -int g_allocCount = 0; -int g_allocCountTemp = 0; - -#endif - -void *SzAlloc(void *p, size_t size) -{ - p = p; - if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc %10d bytes; count = %10d", size, g_allocCount); - g_allocCount++; - #endif - return malloc(size); -} - -void SzFree(void *p, void *address) -{ - p = p; - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - { - g_allocCount--; - fprintf(stderr, "\nFree; count = %10d", g_allocCount); - } - #endif - free(address); -} - -void *SzAllocTemp(void *p, size_t size) -{ - p = p; - if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_temp %10d bytes; count = %10d", size, g_allocCountTemp); - g_allocCountTemp++; - #ifdef _WIN32 - return HeapAlloc(GetProcessHeap(), 0, size); - #endif - #endif - return malloc(size); -} - -void SzFreeTemp(void *p, void *address) -{ - p = p; - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - { - g_allocCountTemp--; - fprintf(stderr, "\nFree_temp; count = %10d", g_allocCountTemp); - } - #ifdef _WIN32 - HeapFree(GetProcessHeap(), 0, address); - return; - #endif - #endif - free(address); -} diff --git a/ext/7z/Archive/7z/7zAlloc.h b/ext/7z/Archive/7z/7zAlloc.h deleted file mode 100644 index 91505d00..00000000 --- a/ext/7z/Archive/7z/7zAlloc.h +++ /dev/null @@ -1,15 +0,0 @@ -/* 7zAlloc.h -- Allocation functions -2008-10-04 : Igor Pavlov : Public domain */ - -#ifndef __7Z_ALLOC_H -#define __7Z_ALLOC_H - -#include - -void *SzAlloc(void *p, size_t size); -void SzFree(void *p, void *address); - -void *SzAllocTemp(void *p, size_t size); -void SzFreeTemp(void *p, void *address); - -#endif diff --git a/ext/7z/Archive/7z/7zDecode.c b/ext/7z/Archive/7z/7zDecode.c deleted file mode 100644 index 097d2ea3..00000000 --- a/ext/7z/Archive/7z/7zDecode.c +++ /dev/null @@ -1,254 +0,0 @@ -/* 7zDecode.c -- Decoding from 7z folder -2008-11-23 : Igor Pavlov : Public domain */ - -#include - -#include "../../Bcj2.h" -#include "../../Bra.h" -#include "../../LzmaDec.h" -#include "7zDecode.h" - -#define k_Copy 0 -#define k_LZMA 0x30101 -#define k_BCJ 0x03030103 -#define k_BCJ2 0x0303011B - -static SRes SzDecodeLzma(CSzCoderInfo *coder, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain) -{ - CLzmaDec state; - SRes res = SZ_OK; - - LzmaDec_Construct(&state); - RINOK(LzmaDec_AllocateProbs(&state, coder->Props.data, (unsigned)coder->Props.size, allocMain)); - state.dic = outBuffer; - state.dicBufSize = outSize; - LzmaDec_Init(&state); - - for (;;) - { - Byte *inBuf = NULL; - size_t lookahead = (1 << 18); - if (lookahead > inSize) - lookahead = (size_t)inSize; - res = inStream->Look((void *)inStream, (void **)&inBuf, &lookahead); - if (res != SZ_OK) - break; - - { - SizeT inProcessed = (SizeT)lookahead, dicPos = state.dicPos; - ELzmaStatus status; - res = LzmaDec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status); - lookahead -= inProcessed; - inSize -= inProcessed; - if (res != SZ_OK) - break; - if (state.dicPos == state.dicBufSize || (inProcessed == 0 && dicPos == state.dicPos)) - { - if (state.dicBufSize != outSize || lookahead != 0 || - (status != LZMA_STATUS_FINISHED_WITH_MARK && - status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK)) - res = SZ_ERROR_DATA; - break; - } - res = inStream->Skip((void *)inStream, inProcessed); - if (res != SZ_OK) - break; - } - } - - LzmaDec_FreeProbs(&state, allocMain); - return res; -} - -static SRes SzDecodeCopy(UInt64 inSize, ILookInStream *inStream, Byte *outBuffer) -{ - while (inSize > 0) - { - void *inBuf; - size_t curSize = (1 << 18); - if (curSize > inSize) - curSize = (size_t)inSize; - RINOK(inStream->Look((void *)inStream, (void **)&inBuf, &curSize)); - if (curSize == 0) - return SZ_ERROR_INPUT_EOF; - memcpy(outBuffer, inBuf, curSize); - outBuffer += curSize; - inSize -= curSize; - RINOK(inStream->Skip((void *)inStream, curSize)); - } - return SZ_OK; -} - -#define IS_UNSUPPORTED_METHOD(m) ((m) != k_Copy && (m) != k_LZMA) -#define IS_UNSUPPORTED_CODER(c) (IS_UNSUPPORTED_METHOD(c.MethodID) || c.NumInStreams != 1 || c.NumOutStreams != 1) -#define IS_NO_BCJ(c) (c.MethodID != k_BCJ || c.NumInStreams != 1 || c.NumOutStreams != 1) -#define IS_NO_BCJ2(c) (c.MethodID != k_BCJ2 || c.NumInStreams != 4 || c.NumOutStreams != 1) - -SRes CheckSupportedFolder(const CSzFolder *f) -{ - if (f->NumCoders < 1 || f->NumCoders > 4) - return SZ_ERROR_UNSUPPORTED; - if (IS_UNSUPPORTED_CODER(f->Coders[0])) - return SZ_ERROR_UNSUPPORTED; - if (f->NumCoders == 1) - { - if (f->NumPackStreams != 1 || f->PackStreams[0] != 0 || f->NumBindPairs != 0) - return SZ_ERROR_UNSUPPORTED; - return SZ_OK; - } - if (f->NumCoders == 2) - { - if (IS_NO_BCJ(f->Coders[1]) || - f->NumPackStreams != 1 || f->PackStreams[0] != 0 || - f->NumBindPairs != 1 || - f->BindPairs[0].InIndex != 1 || f->BindPairs[0].OutIndex != 0) - return SZ_ERROR_UNSUPPORTED; - return SZ_OK; - } - if (f->NumCoders == 4) - { - if (IS_UNSUPPORTED_CODER(f->Coders[1]) || - IS_UNSUPPORTED_CODER(f->Coders[2]) || - IS_NO_BCJ2(f->Coders[3])) - return SZ_ERROR_UNSUPPORTED; - if (f->NumPackStreams != 4 || - f->PackStreams[0] != 2 || - f->PackStreams[1] != 6 || - f->PackStreams[2] != 1 || - f->PackStreams[3] != 0 || - f->NumBindPairs != 3 || - f->BindPairs[0].InIndex != 5 || f->BindPairs[0].OutIndex != 0 || - f->BindPairs[1].InIndex != 4 || f->BindPairs[1].OutIndex != 1 || - f->BindPairs[2].InIndex != 3 || f->BindPairs[2].OutIndex != 2) - return SZ_ERROR_UNSUPPORTED; - return SZ_OK; - } - return SZ_ERROR_UNSUPPORTED; -} - -UInt64 GetSum(const UInt64 *values, UInt32 index) -{ - UInt64 sum = 0; - UInt32 i; - for (i = 0; i < index; i++) - sum += values[i]; - return sum; -} - -SRes SzDecode2(const UInt64 *packSizes, const CSzFolder *folder, - ILookInStream *inStream, UInt64 startPos, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain, - Byte *tempBuf[]) -{ - UInt32 ci; - SizeT tempSizes[3] = { 0, 0, 0}; - SizeT tempSize3 = 0; - Byte *tempBuf3 = 0; - - RINOK(CheckSupportedFolder(folder)); - - for (ci = 0; ci < folder->NumCoders; ci++) - { - CSzCoderInfo *coder = &folder->Coders[ci]; - - if (coder->MethodID == k_Copy || coder->MethodID == k_LZMA) - { - UInt32 si = 0; - UInt64 offset; - UInt64 inSize; - Byte *outBufCur = outBuffer; - SizeT outSizeCur = outSize; - if (folder->NumCoders == 4) - { - UInt32 indices[] = { 3, 2, 0 }; - UInt64 unpackSize = folder->UnpackSizes[ci]; - si = indices[ci]; - if (ci < 2) - { - Byte *temp; - outSizeCur = (SizeT)unpackSize; - if (outSizeCur != unpackSize) - return SZ_ERROR_MEM; - temp = (Byte *)IAlloc_Alloc(allocMain, outSizeCur); - if (temp == 0 && outSizeCur != 0) - return SZ_ERROR_MEM; - outBufCur = tempBuf[1 - ci] = temp; - tempSizes[1 - ci] = outSizeCur; - } - else if (ci == 2) - { - if (unpackSize > outSize) /* check it */ - return SZ_ERROR_PARAM; - tempBuf3 = outBufCur = outBuffer + (outSize - (size_t)unpackSize); - tempSize3 = outSizeCur = (SizeT)unpackSize; - } - else - return SZ_ERROR_UNSUPPORTED; - } - offset = GetSum(packSizes, si); - inSize = packSizes[si]; - RINOK(LookInStream_SeekTo(inStream, startPos + offset)); - - if (coder->MethodID == k_Copy) - { - if (inSize != outSizeCur) /* check it */ - return SZ_ERROR_DATA; - RINOK(SzDecodeCopy(inSize, inStream, outBufCur)); - } - else - { - RINOK(SzDecodeLzma(coder, inSize, inStream, outBufCur, outSizeCur, allocMain)); - } - } - else if (coder->MethodID == k_BCJ) - { - UInt32 state; - if (ci != 1) - return SZ_ERROR_UNSUPPORTED; - x86_Convert_Init(state); - x86_Convert(outBuffer, outSize, 0, &state, 0); - } - else if (coder->MethodID == k_BCJ2) - { - UInt64 offset = GetSum(packSizes, 1); - UInt64 s3Size = packSizes[1]; - SRes res; - if (ci != 3) - return SZ_ERROR_UNSUPPORTED; - RINOK(LookInStream_SeekTo(inStream, startPos + offset)); - tempSizes[2] = (SizeT)s3Size; - if (tempSizes[2] != s3Size) - return SZ_ERROR_MEM; - tempBuf[2] = (Byte *)IAlloc_Alloc(allocMain, tempSizes[2]); - if (tempBuf[2] == 0 && tempSizes[2] != 0) - return SZ_ERROR_MEM; - res = SzDecodeCopy(s3Size, inStream, tempBuf[2]); - RINOK(res) - - res = Bcj2_Decode( - tempBuf3, tempSize3, - tempBuf[0], tempSizes[0], - tempBuf[1], tempSizes[1], - tempBuf[2], tempSizes[2], - outBuffer, outSize); - RINOK(res) - } - else - return SZ_ERROR_UNSUPPORTED; - } - return SZ_OK; -} - -SRes SzDecode(const UInt64 *packSizes, const CSzFolder *folder, - ILookInStream *inStream, UInt64 startPos, - Byte *outBuffer, size_t outSize, ISzAlloc *allocMain) -{ - Byte *tempBuf[3] = { 0, 0, 0}; - int i; - SRes res = SzDecode2(packSizes, folder, inStream, startPos, - outBuffer, (SizeT)outSize, allocMain, tempBuf); - for (i = 0; i < 3; i++) - IAlloc_Free(allocMain, tempBuf[i]); - return res; -} diff --git a/ext/7z/Archive/7z/7zDecode.h b/ext/7z/Archive/7z/7zDecode.h deleted file mode 100644 index 9607915b..00000000 --- a/ext/7z/Archive/7z/7zDecode.h +++ /dev/null @@ -1,13 +0,0 @@ -/* 7zDecode.h -- Decoding from 7z folder -2008-11-23 : Igor Pavlov : Public domain */ - -#ifndef __7Z_DECODE_H -#define __7Z_DECODE_H - -#include "7zItem.h" - -SRes SzDecode(const UInt64 *packSizes, const CSzFolder *folder, - ILookInStream *stream, UInt64 startPos, - Byte *outBuffer, size_t outSize, ISzAlloc *allocMain); - -#endif diff --git a/ext/7z/Archive/7z/7zExtract.c b/ext/7z/Archive/7z/7zExtract.c deleted file mode 100644 index 0f957add..00000000 --- a/ext/7z/Archive/7z/7zExtract.c +++ /dev/null @@ -1,93 +0,0 @@ -/* 7zExtract.c -- Extracting from 7z archive -2008-11-23 : Igor Pavlov : Public domain */ - -#include "../../7zCrc.h" -#include "7zDecode.h" -#include "7zExtract.h" - -SRes SzAr_Extract( - const CSzArEx *p, - ILookInStream *inStream, - UInt32 fileIndex, - UInt32 *blockIndex, - Byte **outBuffer, - size_t *outBufferSize, - size_t *offset, - size_t *outSizeProcessed, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) -{ - UInt32 folderIndex = p->FileIndexToFolderIndexMap[fileIndex]; - SRes res = SZ_OK; - *offset = 0; - *outSizeProcessed = 0; - if (folderIndex == (UInt32)-1) - { - IAlloc_Free(allocMain, *outBuffer); - *blockIndex = folderIndex; - *outBuffer = 0; - *outBufferSize = 0; - return SZ_OK; - } - - if (*outBuffer == 0 || *blockIndex != folderIndex) - { - CSzFolder *folder = p->db.Folders + folderIndex; - UInt64 unpackSizeSpec = SzFolder_GetUnpackSize(folder); - size_t unpackSize = (size_t)unpackSizeSpec; - UInt64 startOffset = SzArEx_GetFolderStreamPos(p, folderIndex, 0); - - if (unpackSize != unpackSizeSpec) - return SZ_ERROR_MEM; - *blockIndex = folderIndex; - IAlloc_Free(allocMain, *outBuffer); - *outBuffer = 0; - - RINOK(LookInStream_SeekTo(inStream, startOffset)); - - if (res == SZ_OK) - { - *outBufferSize = unpackSize; - if (unpackSize != 0) - { - *outBuffer = (Byte *)IAlloc_Alloc(allocMain, unpackSize); - if (*outBuffer == 0) - res = SZ_ERROR_MEM; - } - if (res == SZ_OK) - { - res = SzDecode(p->db.PackSizes + - p->FolderStartPackStreamIndex[folderIndex], folder, - inStream, startOffset, - *outBuffer, unpackSize, allocTemp); - if (res == SZ_OK) - { - if (folder->UnpackCRCDefined) - { - if (CrcCalc(*outBuffer, unpackSize) != folder->UnpackCRC) - res = SZ_ERROR_CRC; - } - } - } - } - } - if (res == SZ_OK) - { - UInt32 i; - CSzFileItem *fileItem = p->db.Files + fileIndex; - *offset = 0; - for (i = p->FolderStartFileIndex[folderIndex]; i < fileIndex; i++) - *offset += (UInt32)p->db.Files[i].Size; - *outSizeProcessed = (size_t)fileItem->Size; - if (*offset + *outSizeProcessed > *outBufferSize) - return SZ_ERROR_FAIL; - { - if (fileItem->FileCRCDefined) - { - if (CrcCalc(*outBuffer + *offset, *outSizeProcessed) != fileItem->FileCRC) - res = SZ_ERROR_CRC; - } - } - } - return res; -} diff --git a/ext/7z/Archive/7z/7zExtract.h b/ext/7z/Archive/7z/7zExtract.h deleted file mode 100644 index e171f4ab..00000000 --- a/ext/7z/Archive/7z/7zExtract.h +++ /dev/null @@ -1,41 +0,0 @@ -/* 7zExtract.h -- Extracting from 7z archive -2008-11-23 : Igor Pavlov : Public domain */ - -#ifndef __7Z_EXTRACT_H -#define __7Z_EXTRACT_H - -#include "7zIn.h" - -/* - SzExtract extracts file from archive - - *outBuffer must be 0 before first call for each new archive. - - Extracting cache: - If you need to decompress more than one file, you can send - these values from previous call: - *blockIndex, - *outBuffer, - *outBufferSize - You can consider "*outBuffer" as cache of solid block. If your archive is solid, - it will increase decompression speed. - - If you use external function, you can declare these 3 cache variables - (blockIndex, outBuffer, outBufferSize) as static in that external function. - - Free *outBuffer and set *outBuffer to 0, if you want to flush cache. -*/ - -SRes SzAr_Extract( - const CSzArEx *db, - ILookInStream *inStream, - UInt32 fileIndex, /* index of file */ - UInt32 *blockIndex, /* index of solid block */ - Byte **outBuffer, /* pointer to pointer to output buffer (allocated with allocMain) */ - size_t *outBufferSize, /* buffer size for output buffer */ - size_t *offset, /* offset of stream for required file in *outBuffer */ - size_t *outSizeProcessed, /* size of file in *outBuffer */ - ISzAlloc *allocMain, - ISzAlloc *allocTemp); - -#endif diff --git a/ext/7z/Archive/7z/7zHeader.c b/ext/7z/Archive/7z/7zHeader.c deleted file mode 100644 index c44ff4a4..00000000 --- a/ext/7z/Archive/7z/7zHeader.c +++ /dev/null @@ -1,6 +0,0 @@ -/* 7zHeader.c -- 7z Headers -2008-10-04 : Igor Pavlov : Public domain */ - -#include "7zHeader.h" - -Byte k7zSignature[k7zSignatureSize] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C}; diff --git a/ext/7z/Archive/7z/7zHeader.h b/ext/7z/Archive/7z/7zHeader.h deleted file mode 100644 index 1ef8c019..00000000 --- a/ext/7z/Archive/7z/7zHeader.h +++ /dev/null @@ -1,57 +0,0 @@ -/* 7zHeader.h -- 7z Headers -2008-10-04 : Igor Pavlov : Public domain */ - -#ifndef __7Z_HEADER_H -#define __7Z_HEADER_H - -#include "../../Types.h" - -#define k7zSignatureSize 6 -extern Byte k7zSignature[k7zSignatureSize]; - -#define k7zMajorVersion 0 - -#define k7zStartHeaderSize 0x20 - -enum EIdEnum -{ - k7zIdEnd, - - k7zIdHeader, - - k7zIdArchiveProperties, - - k7zIdAdditionalStreamsInfo, - k7zIdMainStreamsInfo, - k7zIdFilesInfo, - - k7zIdPackInfo, - k7zIdUnpackInfo, - k7zIdSubStreamsInfo, - - k7zIdSize, - k7zIdCRC, - - k7zIdFolder, - - k7zIdCodersUnpackSize, - k7zIdNumUnpackStream, - - k7zIdEmptyStream, - k7zIdEmptyFile, - k7zIdAnti, - - k7zIdName, - k7zIdCTime, - k7zIdATime, - k7zIdMTime, - k7zIdWinAttributes, - k7zIdComment, - - k7zIdEncodedHeader, - - k7zIdStartPos, - k7zIdDummy -}; - -#endif diff --git a/ext/7z/Archive/7z/7zIn.c b/ext/7z/Archive/7z/7zIn.c deleted file mode 100644 index 75b0bcd1..00000000 --- a/ext/7z/Archive/7z/7zIn.c +++ /dev/null @@ -1,1204 +0,0 @@ -/* 7zIn.c -- 7z Input functions -2008-12-31 : Igor Pavlov : Public domain */ - -#include "../../7zCrc.h" -#include "../../CpuArch.h" - -#include "7zDecode.h" -#include "7zIn.h" - -#define RINOM(x) { if ((x) == 0) return SZ_ERROR_MEM; } - -#define NUM_FOLDER_CODERS_MAX 32 -#define NUM_CODER_STREAMS_MAX 32 - -void SzArEx_Init(CSzArEx *p) -{ - SzAr_Init(&p->db); - p->FolderStartPackStreamIndex = 0; - p->PackStreamStartPositions = 0; - p->FolderStartFileIndex = 0; - p->FileIndexToFolderIndexMap = 0; -} - -void SzArEx_Free(CSzArEx *p, ISzAlloc *alloc) -{ - IAlloc_Free(alloc, p->FolderStartPackStreamIndex); - IAlloc_Free(alloc, p->PackStreamStartPositions); - IAlloc_Free(alloc, p->FolderStartFileIndex); - IAlloc_Free(alloc, p->FileIndexToFolderIndexMap); - SzAr_Free(&p->db, alloc); - SzArEx_Init(p); -} - -/* -UInt64 GetFolderPackStreamSize(int folderIndex, int streamIndex) const -{ - return PackSizes[FolderStartPackStreamIndex[folderIndex] + streamIndex]; -} - -UInt64 GetFilePackSize(int fileIndex) const -{ - int folderIndex = FileIndexToFolderIndexMap[fileIndex]; - if (folderIndex >= 0) - { - const CSzFolder &folderInfo = Folders[folderIndex]; - if (FolderStartFileIndex[folderIndex] == fileIndex) - return GetFolderFullPackSize(folderIndex); - } - return 0; -} -*/ - -#define MY_ALLOC(T, p, size, alloc) { if ((size) == 0) p = 0; else \ - if ((p = (T *)IAlloc_Alloc(alloc, (size) * sizeof(T))) == 0) return SZ_ERROR_MEM; } - -static SRes SzArEx_Fill(CSzArEx *p, ISzAlloc *alloc) -{ - UInt32 startPos = 0; - UInt64 startPosSize = 0; - UInt32 i; - UInt32 folderIndex = 0; - UInt32 indexInFolder = 0; - MY_ALLOC(UInt32, p->FolderStartPackStreamIndex, p->db.NumFolders, alloc); - for (i = 0; i < p->db.NumFolders; i++) - { - p->FolderStartPackStreamIndex[i] = startPos; - startPos += p->db.Folders[i].NumPackStreams; - } - - MY_ALLOC(UInt64, p->PackStreamStartPositions, p->db.NumPackStreams, alloc); - - for (i = 0; i < p->db.NumPackStreams; i++) - { - p->PackStreamStartPositions[i] = startPosSize; - startPosSize += p->db.PackSizes[i]; - } - - MY_ALLOC(UInt32, p->FolderStartFileIndex, p->db.NumFolders, alloc); - MY_ALLOC(UInt32, p->FileIndexToFolderIndexMap, p->db.NumFiles, alloc); - - for (i = 0; i < p->db.NumFiles; i++) - { - CSzFileItem *file = p->db.Files + i; - int emptyStream = !file->HasStream; - if (emptyStream && indexInFolder == 0) - { - p->FileIndexToFolderIndexMap[i] = (UInt32)-1; - continue; - } - if (indexInFolder == 0) - { - /* - v3.13 incorrectly worked with empty folders - v4.07: Loop for skipping empty folders - */ - for (;;) - { - if (folderIndex >= p->db.NumFolders) - return SZ_ERROR_ARCHIVE; - p->FolderStartFileIndex[folderIndex] = i; - if (p->db.Folders[folderIndex].NumUnpackStreams != 0) - break; - folderIndex++; - } - } - p->FileIndexToFolderIndexMap[i] = folderIndex; - if (emptyStream) - continue; - indexInFolder++; - if (indexInFolder >= p->db.Folders[folderIndex].NumUnpackStreams) - { - folderIndex++; - indexInFolder = 0; - } - } - return SZ_OK; -} - - -UInt64 SzArEx_GetFolderStreamPos(const CSzArEx *p, UInt32 folderIndex, UInt32 indexInFolder) -{ - return p->dataPos + - p->PackStreamStartPositions[p->FolderStartPackStreamIndex[folderIndex] + indexInFolder]; -} - -int SzArEx_GetFolderFullPackSize(const CSzArEx *p, UInt32 folderIndex, UInt64 *resSize) -{ - UInt32 packStreamIndex = p->FolderStartPackStreamIndex[folderIndex]; - CSzFolder *folder = p->db.Folders + folderIndex; - UInt64 size = 0; - UInt32 i; - for (i = 0; i < folder->NumPackStreams; i++) - { - UInt64 t = size + p->db.PackSizes[packStreamIndex + i]; - if (t < size) /* check it */ - return SZ_ERROR_FAIL; - size = t; - } - *resSize = size; - return SZ_OK; -} - - -/* -SRes SzReadTime(const CObjectVector &dataVector, - CObjectVector &files, UInt64 type) -{ - CBoolVector boolVector; - RINOK(ReadBoolVector2(files.Size(), boolVector)) - - CStreamSwitch streamSwitch; - RINOK(streamSwitch.Set(this, &dataVector)); - - for (int i = 0; i < files.Size(); i++) - { - CSzFileItem &file = files[i]; - CArchiveFileTime fileTime; - bool defined = boolVector[i]; - if (defined) - { - UInt32 low, high; - RINOK(SzReadUInt32(low)); - RINOK(SzReadUInt32(high)); - fileTime.dwLowDateTime = low; - fileTime.dwHighDateTime = high; - } - switch(type) - { - case k7zIdCTime: file.IsCTimeDefined = defined; if (defined) file.CTime = fileTime; break; - case k7zIdATime: file.IsATimeDefined = defined; if (defined) file.ATime = fileTime; break; - case k7zIdMTime: file.IsMTimeDefined = defined; if (defined) file.MTime = fileTime; break; - } - } - return SZ_OK; -} -*/ - -static int TestSignatureCandidate(Byte *testBytes) -{ - size_t i; - for (i = 0; i < k7zSignatureSize; i++) - if (testBytes[i] != k7zSignature[i]) - return 0; - return 1; -} - -typedef struct _CSzState -{ - Byte *Data; - size_t Size; -}CSzData; - -static SRes SzReadByte(CSzData *sd, Byte *b) -{ - if (sd->Size == 0) - return SZ_ERROR_ARCHIVE; - sd->Size--; - *b = *sd->Data++; - return SZ_OK; -} - -static SRes SzReadBytes(CSzData *sd, Byte *data, size_t size) -{ - size_t i; - for (i = 0; i < size; i++) - { - RINOK(SzReadByte(sd, data + i)); - } - return SZ_OK; -} - -static SRes SzReadUInt32(CSzData *sd, UInt32 *value) -{ - int i; - *value = 0; - for (i = 0; i < 4; i++) - { - Byte b; - RINOK(SzReadByte(sd, &b)); - *value |= ((UInt32)(b) << (8 * i)); - } - return SZ_OK; -} - -static SRes SzReadNumber(CSzData *sd, UInt64 *value) -{ - Byte firstByte; - Byte mask = 0x80; - int i; - RINOK(SzReadByte(sd, &firstByte)); - *value = 0; - for (i = 0; i < 8; i++) - { - Byte b; - if ((firstByte & mask) == 0) - { - UInt64 highPart = firstByte & (mask - 1); - *value += (highPart << (8 * i)); - return SZ_OK; - } - RINOK(SzReadByte(sd, &b)); - *value |= ((UInt64)b << (8 * i)); - mask >>= 1; - } - return SZ_OK; -} - -static SRes SzReadNumber32(CSzData *sd, UInt32 *value) -{ - UInt64 value64; - RINOK(SzReadNumber(sd, &value64)); - if (value64 >= 0x80000000) - return SZ_ERROR_UNSUPPORTED; - if (value64 >= ((UInt64)(1) << ((sizeof(size_t) - 1) * 8 + 2))) - return SZ_ERROR_UNSUPPORTED; - *value = (UInt32)value64; - return SZ_OK; -} - -static SRes SzReadID(CSzData *sd, UInt64 *value) -{ - return SzReadNumber(sd, value); -} - -static SRes SzSkeepDataSize(CSzData *sd, UInt64 size) -{ - if (size > sd->Size) - return SZ_ERROR_ARCHIVE; - sd->Size -= (size_t)size; - sd->Data += (size_t)size; - return SZ_OK; -} - -static SRes SzSkeepData(CSzData *sd) -{ - UInt64 size; - RINOK(SzReadNumber(sd, &size)); - return SzSkeepDataSize(sd, size); -} - -static SRes SzReadArchiveProperties(CSzData *sd) -{ - for (;;) - { - UInt64 type; - RINOK(SzReadID(sd, &type)); - if (type == k7zIdEnd) - break; - SzSkeepData(sd); - } - return SZ_OK; -} - -static SRes SzWaitAttribute(CSzData *sd, UInt64 attribute) -{ - for (;;) - { - UInt64 type; - RINOK(SzReadID(sd, &type)); - if (type == attribute) - return SZ_OK; - if (type == k7zIdEnd) - return SZ_ERROR_ARCHIVE; - RINOK(SzSkeepData(sd)); - } -} - -static SRes SzReadBoolVector(CSzData *sd, size_t numItems, Byte **v, ISzAlloc *alloc) -{ - Byte b = 0; - Byte mask = 0; - size_t i; - MY_ALLOC(Byte, *v, numItems, alloc); - for (i = 0; i < numItems; i++) - { - if (mask == 0) - { - RINOK(SzReadByte(sd, &b)); - mask = 0x80; - } - (*v)[i] = (Byte)(((b & mask) != 0) ? 1 : 0); - mask >>= 1; - } - return SZ_OK; -} - -static SRes SzReadBoolVector2(CSzData *sd, size_t numItems, Byte **v, ISzAlloc *alloc) -{ - Byte allAreDefined; - size_t i; - RINOK(SzReadByte(sd, &allAreDefined)); - if (allAreDefined == 0) - return SzReadBoolVector(sd, numItems, v, alloc); - MY_ALLOC(Byte, *v, numItems, alloc); - for (i = 0; i < numItems; i++) - (*v)[i] = 1; - return SZ_OK; -} - -static SRes SzReadHashDigests( - CSzData *sd, - size_t numItems, - Byte **digestsDefined, - UInt32 **digests, - ISzAlloc *alloc) -{ - size_t i; - RINOK(SzReadBoolVector2(sd, numItems, digestsDefined, alloc)); - MY_ALLOC(UInt32, *digests, numItems, alloc); - for (i = 0; i < numItems; i++) - if ((*digestsDefined)[i]) - { - RINOK(SzReadUInt32(sd, (*digests) + i)); - } - return SZ_OK; -} - -static SRes SzReadPackInfo( - CSzData *sd, - UInt64 *dataOffset, - UInt32 *numPackStreams, - UInt64 **packSizes, - Byte **packCRCsDefined, - UInt32 **packCRCs, - ISzAlloc *alloc) -{ - UInt32 i; - RINOK(SzReadNumber(sd, dataOffset)); - RINOK(SzReadNumber32(sd, numPackStreams)); - - RINOK(SzWaitAttribute(sd, k7zIdSize)); - - MY_ALLOC(UInt64, *packSizes, (size_t)*numPackStreams, alloc); - - for (i = 0; i < *numPackStreams; i++) - { - RINOK(SzReadNumber(sd, (*packSizes) + i)); - } - - for (;;) - { - UInt64 type; - RINOK(SzReadID(sd, &type)); - if (type == k7zIdEnd) - break; - if (type == k7zIdCRC) - { - RINOK(SzReadHashDigests(sd, (size_t)*numPackStreams, packCRCsDefined, packCRCs, alloc)); - continue; - } - RINOK(SzSkeepData(sd)); - } - if (*packCRCsDefined == 0) - { - MY_ALLOC(Byte, *packCRCsDefined, (size_t)*numPackStreams, alloc); - MY_ALLOC(UInt32, *packCRCs, (size_t)*numPackStreams, alloc); - for (i = 0; i < *numPackStreams; i++) - { - (*packCRCsDefined)[i] = 0; - (*packCRCs)[i] = 0; - } - } - return SZ_OK; -} - -static SRes SzReadSwitch(CSzData *sd) -{ - Byte external; - RINOK(SzReadByte(sd, &external)); - return (external == 0) ? SZ_OK: SZ_ERROR_UNSUPPORTED; -} - -static SRes SzGetNextFolderItem(CSzData *sd, CSzFolder *folder, ISzAlloc *alloc) -{ - UInt32 numCoders, numBindPairs, numPackStreams, i; - UInt32 numInStreams = 0, numOutStreams = 0; - - RINOK(SzReadNumber32(sd, &numCoders)); - if (numCoders > NUM_FOLDER_CODERS_MAX) - return SZ_ERROR_UNSUPPORTED; - folder->NumCoders = numCoders; - - MY_ALLOC(CSzCoderInfo, folder->Coders, (size_t)numCoders, alloc); - - for (i = 0; i < numCoders; i++) - SzCoderInfo_Init(folder->Coders + i); - - for (i = 0; i < numCoders; i++) - { - Byte mainByte; - CSzCoderInfo *coder = folder->Coders + i; - { - unsigned idSize, j; - Byte longID[15]; - RINOK(SzReadByte(sd, &mainByte)); - idSize = (unsigned)(mainByte & 0xF); - RINOK(SzReadBytes(sd, longID, idSize)); - if (idSize > sizeof(coder->MethodID)) - return SZ_ERROR_UNSUPPORTED; - coder->MethodID = 0; - for (j = 0; j < idSize; j++) - coder->MethodID |= (UInt64)longID[idSize - 1 - j] << (8 * j); - - if ((mainByte & 0x10) != 0) - { - RINOK(SzReadNumber32(sd, &coder->NumInStreams)); - RINOK(SzReadNumber32(sd, &coder->NumOutStreams)); - if (coder->NumInStreams > NUM_CODER_STREAMS_MAX || - coder->NumOutStreams > NUM_CODER_STREAMS_MAX) - return SZ_ERROR_UNSUPPORTED; - } - else - { - coder->NumInStreams = 1; - coder->NumOutStreams = 1; - } - if ((mainByte & 0x20) != 0) - { - UInt64 propertiesSize = 0; - RINOK(SzReadNumber(sd, &propertiesSize)); - if (!Buf_Create(&coder->Props, (size_t)propertiesSize, alloc)) - return SZ_ERROR_MEM; - RINOK(SzReadBytes(sd, coder->Props.data, (size_t)propertiesSize)); - } - } - while ((mainByte & 0x80) != 0) - { - RINOK(SzReadByte(sd, &mainByte)); - RINOK(SzSkeepDataSize(sd, (mainByte & 0xF))); - if ((mainByte & 0x10) != 0) - { - UInt32 n; - RINOK(SzReadNumber32(sd, &n)); - RINOK(SzReadNumber32(sd, &n)); - } - if ((mainByte & 0x20) != 0) - { - UInt64 propertiesSize = 0; - RINOK(SzReadNumber(sd, &propertiesSize)); - RINOK(SzSkeepDataSize(sd, propertiesSize)); - } - } - numInStreams += coder->NumInStreams; - numOutStreams += coder->NumOutStreams; - } - - if (numOutStreams == 0) - return SZ_ERROR_UNSUPPORTED; - - folder->NumBindPairs = numBindPairs = numOutStreams - 1; - MY_ALLOC(CBindPair, folder->BindPairs, (size_t)numBindPairs, alloc); - - for (i = 0; i < numBindPairs; i++) - { - CBindPair *bp = folder->BindPairs + i; - RINOK(SzReadNumber32(sd, &bp->InIndex)); - RINOK(SzReadNumber32(sd, &bp->OutIndex)); - } - - if (numInStreams < numBindPairs) - return SZ_ERROR_UNSUPPORTED; - - folder->NumPackStreams = numPackStreams = numInStreams - numBindPairs; - MY_ALLOC(UInt32, folder->PackStreams, (size_t)numPackStreams, alloc); - - if (numPackStreams == 1) - { - for (i = 0; i < numInStreams ; i++) - if (SzFolder_FindBindPairForInStream(folder, i) < 0) - break; - if (i == numInStreams) - return SZ_ERROR_UNSUPPORTED; - folder->PackStreams[0] = i; - } - else - for (i = 0; i < numPackStreams; i++) - { - RINOK(SzReadNumber32(sd, folder->PackStreams + i)); - } - return SZ_OK; -} - -static SRes SzReadUnpackInfo( - CSzData *sd, - UInt32 *numFolders, - CSzFolder **folders, /* for alloc */ - ISzAlloc *alloc, - ISzAlloc *allocTemp) -{ - UInt32 i; - RINOK(SzWaitAttribute(sd, k7zIdFolder)); - RINOK(SzReadNumber32(sd, numFolders)); - { - RINOK(SzReadSwitch(sd)); - - MY_ALLOC(CSzFolder, *folders, (size_t)*numFolders, alloc); - - for (i = 0; i < *numFolders; i++) - SzFolder_Init((*folders) + i); - - for (i = 0; i < *numFolders; i++) - { - RINOK(SzGetNextFolderItem(sd, (*folders) + i, alloc)); - } - } - - RINOK(SzWaitAttribute(sd, k7zIdCodersUnpackSize)); - - for (i = 0; i < *numFolders; i++) - { - UInt32 j; - CSzFolder *folder = (*folders) + i; - UInt32 numOutStreams = SzFolder_GetNumOutStreams(folder); - - MY_ALLOC(UInt64, folder->UnpackSizes, (size_t)numOutStreams, alloc); - - for (j = 0; j < numOutStreams; j++) - { - RINOK(SzReadNumber(sd, folder->UnpackSizes + j)); - } - } - - for (;;) - { - UInt64 type; - RINOK(SzReadID(sd, &type)); - if (type == k7zIdEnd) - return SZ_OK; - if (type == k7zIdCRC) - { - SRes res; - Byte *crcsDefined = 0; - UInt32 *crcs = 0; - res = SzReadHashDigests(sd, *numFolders, &crcsDefined, &crcs, allocTemp); - if (res == SZ_OK) - { - for (i = 0; i < *numFolders; i++) - { - CSzFolder *folder = (*folders) + i; - folder->UnpackCRCDefined = crcsDefined[i]; - folder->UnpackCRC = crcs[i]; - } - } - IAlloc_Free(allocTemp, crcs); - IAlloc_Free(allocTemp, crcsDefined); - RINOK(res); - continue; - } - RINOK(SzSkeepData(sd)); - } -} - -static SRes SzReadSubStreamsInfo( - CSzData *sd, - UInt32 numFolders, - CSzFolder *folders, - UInt32 *numUnpackStreams, - UInt64 **unpackSizes, - Byte **digestsDefined, - UInt32 **digests, - ISzAlloc *allocTemp) -{ - UInt64 type = 0; - UInt32 i; - UInt32 si = 0; - UInt32 numDigests = 0; - - for (i = 0; i < numFolders; i++) - folders[i].NumUnpackStreams = 1; - *numUnpackStreams = numFolders; - - for (;;) - { - RINOK(SzReadID(sd, &type)); - if (type == k7zIdNumUnpackStream) - { - *numUnpackStreams = 0; - for (i = 0; i < numFolders; i++) - { - UInt32 numStreams; - RINOK(SzReadNumber32(sd, &numStreams)); - folders[i].NumUnpackStreams = numStreams; - *numUnpackStreams += numStreams; - } - continue; - } - if (type == k7zIdCRC || type == k7zIdSize) - break; - if (type == k7zIdEnd) - break; - RINOK(SzSkeepData(sd)); - } - - if (*numUnpackStreams == 0) - { - *unpackSizes = 0; - *digestsDefined = 0; - *digests = 0; - } - else - { - *unpackSizes = (UInt64 *)IAlloc_Alloc(allocTemp, (size_t)*numUnpackStreams * sizeof(UInt64)); - RINOM(*unpackSizes); - *digestsDefined = (Byte *)IAlloc_Alloc(allocTemp, (size_t)*numUnpackStreams * sizeof(Byte)); - RINOM(*digestsDefined); - *digests = (UInt32 *)IAlloc_Alloc(allocTemp, (size_t)*numUnpackStreams * sizeof(UInt32)); - RINOM(*digests); - } - - for (i = 0; i < numFolders; i++) - { - /* - v3.13 incorrectly worked with empty folders - v4.07: we check that folder is empty - */ - UInt64 sum = 0; - UInt32 j; - UInt32 numSubstreams = folders[i].NumUnpackStreams; - if (numSubstreams == 0) - continue; - if (type == k7zIdSize) - for (j = 1; j < numSubstreams; j++) - { - UInt64 size; - RINOK(SzReadNumber(sd, &size)); - (*unpackSizes)[si++] = size; - sum += size; - } - (*unpackSizes)[si++] = SzFolder_GetUnpackSize(folders + i) - sum; - } - if (type == k7zIdSize) - { - RINOK(SzReadID(sd, &type)); - } - - for (i = 0; i < *numUnpackStreams; i++) - { - (*digestsDefined)[i] = 0; - (*digests)[i] = 0; - } - - - for (i = 0; i < numFolders; i++) - { - UInt32 numSubstreams = folders[i].NumUnpackStreams; - if (numSubstreams != 1 || !folders[i].UnpackCRCDefined) - numDigests += numSubstreams; - } - - - si = 0; - for (;;) - { - if (type == k7zIdCRC) - { - int digestIndex = 0; - Byte *digestsDefined2 = 0; - UInt32 *digests2 = 0; - SRes res = SzReadHashDigests(sd, numDigests, &digestsDefined2, &digests2, allocTemp); - if (res == SZ_OK) - { - for (i = 0; i < numFolders; i++) - { - CSzFolder *folder = folders + i; - UInt32 numSubstreams = folder->NumUnpackStreams; - if (numSubstreams == 1 && folder->UnpackCRCDefined) - { - (*digestsDefined)[si] = 1; - (*digests)[si] = folder->UnpackCRC; - si++; - } - else - { - UInt32 j; - for (j = 0; j < numSubstreams; j++, digestIndex++) - { - (*digestsDefined)[si] = digestsDefined2[digestIndex]; - (*digests)[si] = digests2[digestIndex]; - si++; - } - } - } - } - IAlloc_Free(allocTemp, digestsDefined2); - IAlloc_Free(allocTemp, digests2); - RINOK(res); - } - else if (type == k7zIdEnd) - return SZ_OK; - else - { - RINOK(SzSkeepData(sd)); - } - RINOK(SzReadID(sd, &type)); - } -} - - -static SRes SzReadStreamsInfo( - CSzData *sd, - UInt64 *dataOffset, - CSzAr *p, - UInt32 *numUnpackStreams, - UInt64 **unpackSizes, /* allocTemp */ - Byte **digestsDefined, /* allocTemp */ - UInt32 **digests, /* allocTemp */ - ISzAlloc *alloc, - ISzAlloc *allocTemp) -{ - for (;;) - { - UInt64 type; - RINOK(SzReadID(sd, &type)); - if ((UInt64)(int)type != type) - return SZ_ERROR_UNSUPPORTED; - switch((int)type) - { - case k7zIdEnd: - return SZ_OK; - case k7zIdPackInfo: - { - RINOK(SzReadPackInfo(sd, dataOffset, &p->NumPackStreams, - &p->PackSizes, &p->PackCRCsDefined, &p->PackCRCs, alloc)); - break; - } - case k7zIdUnpackInfo: - { - RINOK(SzReadUnpackInfo(sd, &p->NumFolders, &p->Folders, alloc, allocTemp)); - break; - } - case k7zIdSubStreamsInfo: - { - RINOK(SzReadSubStreamsInfo(sd, p->NumFolders, p->Folders, - numUnpackStreams, unpackSizes, digestsDefined, digests, allocTemp)); - break; - } - default: - return SZ_ERROR_UNSUPPORTED; - } - } -} - -Byte kUtf8Limits[5] = { 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; - -static SRes SzReadFileNames(CSzData *sd, UInt32 numFiles, CSzFileItem *files, ISzAlloc *alloc) -{ - UInt32 i; - for (i = 0; i < numFiles; i++) - { - UInt32 len = 0; - UInt32 pos = 0; - CSzFileItem *file = files + i; - while (pos + 2 <= sd->Size) - { - int numAdds; - UInt32 value = (UInt32)(sd->Data[pos] | (((UInt32)sd->Data[pos + 1]) << 8)); - pos += 2; - len++; - if (value == 0) - break; - if (value < 0x80) - continue; - if (value >= 0xD800 && value < 0xE000) - { - UInt32 c2; - if (value >= 0xDC00) - return SZ_ERROR_ARCHIVE; - if (pos + 2 > sd->Size) - return SZ_ERROR_ARCHIVE; - c2 = (UInt32)(sd->Data[pos] | (((UInt32)sd->Data[pos + 1]) << 8)); - pos += 2; - if (c2 < 0xDC00 || c2 >= 0xE000) - return SZ_ERROR_ARCHIVE; - value = ((value - 0xD800) << 10) | (c2 - 0xDC00); - } - for (numAdds = 1; numAdds < 5; numAdds++) - if (value < (((UInt32)1) << (numAdds * 5 + 6))) - break; - len += numAdds; - } - - MY_ALLOC(char, file->Name, (size_t)len, alloc); - - len = 0; - while (2 <= sd->Size) - { - int numAdds; - UInt32 value = (UInt32)(sd->Data[0] | (((UInt32)sd->Data[1]) << 8)); - SzSkeepDataSize(sd, 2); - if (value < 0x80) - { - file->Name[len++] = (char)value; - if (value == 0) - break; - continue; - } - if (value >= 0xD800 && value < 0xE000) - { - UInt32 c2 = (UInt32)(sd->Data[0] | (((UInt32)sd->Data[1]) << 8)); - SzSkeepDataSize(sd, 2); - value = ((value - 0xD800) << 10) | (c2 - 0xDC00); - } - for (numAdds = 1; numAdds < 5; numAdds++) - if (value < (((UInt32)1) << (numAdds * 5 + 6))) - break; - file->Name[len++] = (char)(kUtf8Limits[numAdds - 1] + (value >> (6 * numAdds))); - do - { - numAdds--; - file->Name[len++] = (char)(0x80 + ((value >> (6 * numAdds)) & 0x3F)); - } - while (numAdds > 0); - - len += numAdds; - } - } - return SZ_OK; -} - -static SRes SzReadHeader2( - CSzArEx *p, /* allocMain */ - CSzData *sd, - UInt64 **unpackSizes, /* allocTemp */ - Byte **digestsDefined, /* allocTemp */ - UInt32 **digests, /* allocTemp */ - Byte **emptyStreamVector, /* allocTemp */ - Byte **emptyFileVector, /* allocTemp */ - Byte **lwtVector, /* allocTemp */ - ISzAlloc *allocMain, - ISzAlloc *allocTemp) -{ - UInt64 type; - UInt32 numUnpackStreams = 0; - UInt32 numFiles = 0; - CSzFileItem *files = 0; - UInt32 numEmptyStreams = 0; - UInt32 i; - - RINOK(SzReadID(sd, &type)); - - if (type == k7zIdArchiveProperties) - { - RINOK(SzReadArchiveProperties(sd)); - RINOK(SzReadID(sd, &type)); - } - - - if (type == k7zIdMainStreamsInfo) - { - RINOK(SzReadStreamsInfo(sd, - &p->dataPos, - &p->db, - &numUnpackStreams, - unpackSizes, - digestsDefined, - digests, allocMain, allocTemp)); - p->dataPos += p->startPosAfterHeader; - RINOK(SzReadID(sd, &type)); - } - - if (type == k7zIdEnd) - return SZ_OK; - if (type != k7zIdFilesInfo) - return SZ_ERROR_ARCHIVE; - - RINOK(SzReadNumber32(sd, &numFiles)); - p->db.NumFiles = numFiles; - - MY_ALLOC(CSzFileItem, files, (size_t)numFiles, allocMain); - - p->db.Files = files; - for (i = 0; i < numFiles; i++) - SzFile_Init(files + i); - - for (;;) - { - UInt64 type; - UInt64 size; - RINOK(SzReadID(sd, &type)); - if (type == k7zIdEnd) - break; - RINOK(SzReadNumber(sd, &size)); - - if ((UInt64)(int)type != type) - { - RINOK(SzSkeepDataSize(sd, size)); - } - else - switch((int)type) - { - case k7zIdName: - { - RINOK(SzReadSwitch(sd)); - RINOK(SzReadFileNames(sd, numFiles, files, allocMain)) - break; - } - case k7zIdEmptyStream: - { - RINOK(SzReadBoolVector(sd, numFiles, emptyStreamVector, allocTemp)); - numEmptyStreams = 0; - for (i = 0; i < numFiles; i++) - if ((*emptyStreamVector)[i]) - numEmptyStreams++; - break; - } - case k7zIdEmptyFile: - { - RINOK(SzReadBoolVector(sd, numEmptyStreams, emptyFileVector, allocTemp)); - break; - } - case k7zIdMTime: - { - RINOK(SzReadBoolVector2(sd, numFiles, lwtVector, allocTemp)); - RINOK(SzReadSwitch(sd)); - for (i = 0; i < numFiles; i++) - { - CSzFileItem *f = &files[i]; - Byte defined = (*lwtVector)[i]; - f->MTimeDefined = defined; - f->MTime.Low = f->MTime.High = 0; - if (defined) - { - RINOK(SzReadUInt32(sd, &f->MTime.Low)); - RINOK(SzReadUInt32(sd, &f->MTime.High)); - } - } - break; - } - default: - { - RINOK(SzSkeepDataSize(sd, size)); - } - } - } - - { - UInt32 emptyFileIndex = 0; - UInt32 sizeIndex = 0; - for (i = 0; i < numFiles; i++) - { - CSzFileItem *file = files + i; - file->IsAnti = 0; - if (*emptyStreamVector == 0) - file->HasStream = 1; - else - file->HasStream = (Byte)((*emptyStreamVector)[i] ? 0 : 1); - if (file->HasStream) - { - file->IsDir = 0; - file->Size = (*unpackSizes)[sizeIndex]; - file->FileCRC = (*digests)[sizeIndex]; - file->FileCRCDefined = (Byte)(*digestsDefined)[sizeIndex]; - sizeIndex++; - } - else - { - if (*emptyFileVector == 0) - file->IsDir = 1; - else - file->IsDir = (Byte)((*emptyFileVector)[emptyFileIndex] ? 0 : 1); - emptyFileIndex++; - file->Size = 0; - file->FileCRCDefined = 0; - } - } - } - return SzArEx_Fill(p, allocMain); -} - -static SRes SzReadHeader( - CSzArEx *p, - CSzData *sd, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) -{ - UInt64 *unpackSizes = 0; - Byte *digestsDefined = 0; - UInt32 *digests = 0; - Byte *emptyStreamVector = 0; - Byte *emptyFileVector = 0; - Byte *lwtVector = 0; - SRes res = SzReadHeader2(p, sd, - &unpackSizes, &digestsDefined, &digests, - &emptyStreamVector, &emptyFileVector, &lwtVector, - allocMain, allocTemp); - IAlloc_Free(allocTemp, unpackSizes); - IAlloc_Free(allocTemp, digestsDefined); - IAlloc_Free(allocTemp, digests); - IAlloc_Free(allocTemp, emptyStreamVector); - IAlloc_Free(allocTemp, emptyFileVector); - IAlloc_Free(allocTemp, lwtVector); - return res; -} - -static SRes SzReadAndDecodePackedStreams2( - ILookInStream *inStream, - CSzData *sd, - CBuf *outBuffer, - UInt64 baseOffset, - CSzAr *p, - UInt64 **unpackSizes, - Byte **digestsDefined, - UInt32 **digests, - ISzAlloc *allocTemp) -{ - - UInt32 numUnpackStreams = 0; - UInt64 dataStartPos; - CSzFolder *folder; - UInt64 unpackSize; - SRes res; - - RINOK(SzReadStreamsInfo(sd, &dataStartPos, p, - &numUnpackStreams, unpackSizes, digestsDefined, digests, - allocTemp, allocTemp)); - - dataStartPos += baseOffset; - if (p->NumFolders != 1) - return SZ_ERROR_ARCHIVE; - - folder = p->Folders; - unpackSize = SzFolder_GetUnpackSize(folder); - - RINOK(LookInStream_SeekTo(inStream, dataStartPos)); - - if (!Buf_Create(outBuffer, (size_t)unpackSize, allocTemp)) - return SZ_ERROR_MEM; - - res = SzDecode(p->PackSizes, folder, - inStream, dataStartPos, - outBuffer->data, (size_t)unpackSize, allocTemp); - RINOK(res); - if (folder->UnpackCRCDefined) - if (CrcCalc(outBuffer->data, (size_t)unpackSize) != folder->UnpackCRC) - return SZ_ERROR_CRC; - return SZ_OK; -} - -static SRes SzReadAndDecodePackedStreams( - ILookInStream *inStream, - CSzData *sd, - CBuf *outBuffer, - UInt64 baseOffset, - ISzAlloc *allocTemp) -{ - CSzAr p; - UInt64 *unpackSizes = 0; - Byte *digestsDefined = 0; - UInt32 *digests = 0; - SRes res; - SzAr_Init(&p); - res = SzReadAndDecodePackedStreams2(inStream, sd, outBuffer, baseOffset, - &p, &unpackSizes, &digestsDefined, &digests, - allocTemp); - SzAr_Free(&p, allocTemp); - IAlloc_Free(allocTemp, unpackSizes); - IAlloc_Free(allocTemp, digestsDefined); - IAlloc_Free(allocTemp, digests); - return res; -} - -static SRes SzArEx_Open2( - CSzArEx *p, - ILookInStream *inStream, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) -{ - Byte header[k7zStartHeaderSize]; - UInt64 nextHeaderOffset, nextHeaderSize; - size_t nextHeaderSizeT; - UInt32 nextHeaderCRC; - CBuf buffer; - SRes res; - - RINOK(LookInStream_Read2(inStream, header, k7zStartHeaderSize, SZ_ERROR_NO_ARCHIVE)); - - if (!TestSignatureCandidate(header)) - return SZ_ERROR_NO_ARCHIVE; - if (header[6] != k7zMajorVersion) - return SZ_ERROR_UNSUPPORTED; - - nextHeaderOffset = GetUi64(header + 12); - nextHeaderSize = GetUi64(header + 20); - nextHeaderCRC = GetUi32(header + 28); - - p->startPosAfterHeader = k7zStartHeaderSize; - - if (CrcCalc(header + 12, 20) != GetUi32(header + 8)) - return SZ_ERROR_CRC; - - nextHeaderSizeT = (size_t)nextHeaderSize; - if (nextHeaderSizeT != nextHeaderSize) - return SZ_ERROR_MEM; - if (nextHeaderSizeT == 0) - return SZ_OK; - if (nextHeaderOffset > nextHeaderOffset + nextHeaderSize || - nextHeaderOffset > nextHeaderOffset + nextHeaderSize + k7zStartHeaderSize) - return SZ_ERROR_NO_ARCHIVE; - - { - Int64 pos = 0; - RINOK(inStream->Seek(inStream, &pos, SZ_SEEK_END)); - if ((UInt64)pos < nextHeaderOffset || - (UInt64)pos < k7zStartHeaderSize + nextHeaderOffset || - (UInt64)pos < k7zStartHeaderSize + nextHeaderOffset + nextHeaderSize) - return SZ_ERROR_INPUT_EOF; - } - - RINOK(LookInStream_SeekTo(inStream, k7zStartHeaderSize + nextHeaderOffset)); - - if (!Buf_Create(&buffer, nextHeaderSizeT, allocTemp)) - return SZ_ERROR_MEM; - - res = LookInStream_Read(inStream, buffer.data, nextHeaderSizeT); - if (res == SZ_OK) - { - res = SZ_ERROR_ARCHIVE; - if (CrcCalc(buffer.data, nextHeaderSizeT) == nextHeaderCRC) - { - CSzData sd; - UInt64 type; - sd.Data = buffer.data; - sd.Size = buffer.size; - res = SzReadID(&sd, &type); - if (res == SZ_OK) - { - if (type == k7zIdEncodedHeader) - { - CBuf outBuffer; - Buf_Init(&outBuffer); - res = SzReadAndDecodePackedStreams(inStream, &sd, &outBuffer, p->startPosAfterHeader, allocTemp); - if (res != SZ_OK) - Buf_Free(&outBuffer, allocTemp); - else - { - Buf_Free(&buffer, allocTemp); - buffer.data = outBuffer.data; - buffer.size = outBuffer.size; - sd.Data = buffer.data; - sd.Size = buffer.size; - res = SzReadID(&sd, &type); - } - } - } - if (res == SZ_OK) - { - if (type == k7zIdHeader) - res = SzReadHeader(p, &sd, allocMain, allocTemp); - else - res = SZ_ERROR_UNSUPPORTED; - } - } - } - Buf_Free(&buffer, allocTemp); - return res; -} - -SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, ISzAlloc *allocMain, ISzAlloc *allocTemp) -{ - SRes res = SzArEx_Open2(p, inStream, allocMain, allocTemp); - if (res != SZ_OK) - SzArEx_Free(p, allocMain); - return res; -} diff --git a/ext/7z/Archive/7z/7zIn.h b/ext/7z/Archive/7z/7zIn.h deleted file mode 100644 index 717fe52c..00000000 --- a/ext/7z/Archive/7z/7zIn.h +++ /dev/null @@ -1,41 +0,0 @@ -/* 7zIn.h -- 7z Input functions -2008-11-23 : Igor Pavlov : Public domain */ - -#ifndef __7Z_IN_H -#define __7Z_IN_H - -#include "7zHeader.h" -#include "7zItem.h" - -typedef struct -{ - CSzAr db; - - UInt64 startPosAfterHeader; - UInt64 dataPos; - - UInt32 *FolderStartPackStreamIndex; - UInt64 *PackStreamStartPositions; - UInt32 *FolderStartFileIndex; - UInt32 *FileIndexToFolderIndexMap; -} CSzArEx; - -void SzArEx_Init(CSzArEx *p); -void SzArEx_Free(CSzArEx *p, ISzAlloc *alloc); -UInt64 SzArEx_GetFolderStreamPos(const CSzArEx *p, UInt32 folderIndex, UInt32 indexInFolder); -int SzArEx_GetFolderFullPackSize(const CSzArEx *p, UInt32 folderIndex, UInt64 *resSize); - -/* -Errors: -SZ_ERROR_NO_ARCHIVE -SZ_ERROR_ARCHIVE -SZ_ERROR_UNSUPPORTED -SZ_ERROR_MEM -SZ_ERROR_CRC -SZ_ERROR_INPUT_EOF -SZ_ERROR_FAIL -*/ - -SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, ISzAlloc *allocMain, ISzAlloc *allocTemp); - -#endif diff --git a/ext/7z/Archive/7z/7zItem.c b/ext/7z/Archive/7z/7zItem.c deleted file mode 100644 index a11d5008..00000000 --- a/ext/7z/Archive/7z/7zItem.c +++ /dev/null @@ -1,127 +0,0 @@ -/* 7zItem.c -- 7z Items -2008-10-04 : Igor Pavlov : Public domain */ - -#include "7zItem.h" - -void SzCoderInfo_Init(CSzCoderInfo *p) -{ - Buf_Init(&p->Props); -} - -void SzCoderInfo_Free(CSzCoderInfo *p, ISzAlloc *alloc) -{ - Buf_Free(&p->Props, alloc); - SzCoderInfo_Init(p); -} - -void SzFolder_Init(CSzFolder *p) -{ - p->Coders = 0; - p->BindPairs = 0; - p->PackStreams = 0; - p->UnpackSizes = 0; - p->NumCoders = 0; - p->NumBindPairs = 0; - p->NumPackStreams = 0; - p->UnpackCRCDefined = 0; - p->UnpackCRC = 0; - p->NumUnpackStreams = 0; -} - -void SzFolder_Free(CSzFolder *p, ISzAlloc *alloc) -{ - UInt32 i; - if (p->Coders) - for (i = 0; i < p->NumCoders; i++) - SzCoderInfo_Free(&p->Coders[i], alloc); - IAlloc_Free(alloc, p->Coders); - IAlloc_Free(alloc, p->BindPairs); - IAlloc_Free(alloc, p->PackStreams); - IAlloc_Free(alloc, p->UnpackSizes); - SzFolder_Init(p); -} - -UInt32 SzFolder_GetNumOutStreams(CSzFolder *p) -{ - UInt32 result = 0; - UInt32 i; - for (i = 0; i < p->NumCoders; i++) - result += p->Coders[i].NumOutStreams; - return result; -} - -int SzFolder_FindBindPairForInStream(CSzFolder *p, UInt32 inStreamIndex) -{ - UInt32 i; - for (i = 0; i < p->NumBindPairs; i++) - if (p->BindPairs[i].InIndex == inStreamIndex) - return i; - return -1; -} - - -int SzFolder_FindBindPairForOutStream(CSzFolder *p, UInt32 outStreamIndex) -{ - UInt32 i; - for (i = 0; i < p->NumBindPairs; i++) - if (p->BindPairs[i].OutIndex == outStreamIndex) - return i; - return -1; -} - -UInt64 SzFolder_GetUnpackSize(CSzFolder *p) -{ - int i = (int)SzFolder_GetNumOutStreams(p); - if (i == 0) - return 0; - for (i--; i >= 0; i--) - if (SzFolder_FindBindPairForOutStream(p, i) < 0) - return p->UnpackSizes[i]; - /* throw 1; */ - return 0; -} - -void SzFile_Init(CSzFileItem *p) -{ - p->HasStream = 1; - p->IsDir = 0; - p->IsAnti = 0; - p->FileCRCDefined = 0; - p->MTimeDefined = 0; - p->Name = 0; -} - -static void SzFile_Free(CSzFileItem *p, ISzAlloc *alloc) -{ - IAlloc_Free(alloc, p->Name); - SzFile_Init(p); -} - -void SzAr_Init(CSzAr *p) -{ - p->PackSizes = 0; - p->PackCRCsDefined = 0; - p->PackCRCs = 0; - p->Folders = 0; - p->Files = 0; - p->NumPackStreams = 0; - p->NumFolders = 0; - p->NumFiles = 0; -} - -void SzAr_Free(CSzAr *p, ISzAlloc *alloc) -{ - UInt32 i; - if (p->Folders) - for (i = 0; i < p->NumFolders; i++) - SzFolder_Free(&p->Folders[i], alloc); - if (p->Files) - for (i = 0; i < p->NumFiles; i++) - SzFile_Free(&p->Files[i], alloc); - IAlloc_Free(alloc, p->PackSizes); - IAlloc_Free(alloc, p->PackCRCsDefined); - IAlloc_Free(alloc, p->PackCRCs); - IAlloc_Free(alloc, p->Folders); - IAlloc_Free(alloc, p->Files); - SzAr_Init(p); -} diff --git a/ext/7z/Archive/7z/7zItem.h b/ext/7z/Archive/7z/7zItem.h deleted file mode 100644 index 11599454..00000000 --- a/ext/7z/Archive/7z/7zItem.h +++ /dev/null @@ -1,84 +0,0 @@ -/* 7zItem.h -- 7z Items -2008-10-04 : Igor Pavlov : Public domain */ - -#ifndef __7Z_ITEM_H -#define __7Z_ITEM_H - -#include "../../7zBuf.h" - -typedef struct -{ - UInt32 NumInStreams; - UInt32 NumOutStreams; - UInt64 MethodID; - CBuf Props; -} CSzCoderInfo; - -void SzCoderInfo_Init(CSzCoderInfo *p); -void SzCoderInfo_Free(CSzCoderInfo *p, ISzAlloc *alloc); - -typedef struct -{ - UInt32 InIndex; - UInt32 OutIndex; -} CBindPair; - -typedef struct -{ - CSzCoderInfo *Coders; - CBindPair *BindPairs; - UInt32 *PackStreams; - UInt64 *UnpackSizes; - UInt32 NumCoders; - UInt32 NumBindPairs; - UInt32 NumPackStreams; - int UnpackCRCDefined; - UInt32 UnpackCRC; - - UInt32 NumUnpackStreams; -} CSzFolder; - -void SzFolder_Init(CSzFolder *p); -UInt64 SzFolder_GetUnpackSize(CSzFolder *p); -int SzFolder_FindBindPairForInStream(CSzFolder *p, UInt32 inStreamIndex); -UInt32 SzFolder_GetNumOutStreams(CSzFolder *p); -UInt64 SzFolder_GetUnpackSize(CSzFolder *p); - -typedef struct -{ - UInt32 Low; - UInt32 High; -} CNtfsFileTime; - -typedef struct -{ - CNtfsFileTime MTime; - UInt64 Size; - char *Name; - UInt32 FileCRC; - - Byte HasStream; - Byte IsDir; - Byte IsAnti; - Byte FileCRCDefined; - Byte MTimeDefined; -} CSzFileItem; - -void SzFile_Init(CSzFileItem *p); - -typedef struct -{ - UInt64 *PackSizes; - Byte *PackCRCsDefined; - UInt32 *PackCRCs; - CSzFolder *Folders; - CSzFileItem *Files; - UInt32 NumPackStreams; - UInt32 NumFolders; - UInt32 NumFiles; -} CSzAr; - -void SzAr_Init(CSzAr *p); -void SzAr_Free(CSzAr *p, ISzAlloc *alloc); - -#endif diff --git a/ext/7z/Archive/7z/7zMain.c b/ext/7z/Archive/7z/7zMain.c deleted file mode 100644 index 4d0b1b38..00000000 --- a/ext/7z/Archive/7z/7zMain.c +++ /dev/null @@ -1,262 +0,0 @@ -/* 7zMain.c - Test application for 7z Decoder -2008-11-23 : Igor Pavlov : Public domain */ - -#include -#include -#include - -#include "../../7zCrc.h" -#include "../../7zFile.h" -#include "../../7zVersion.h" - -#include "7zAlloc.h" -#include "7zExtract.h" -#include "7zIn.h" - -static void ConvertNumberToString(UInt64 value, char *s) -{ - char temp[32]; - int pos = 0; - do - { - temp[pos++] = (char)('0' + (int)(value % 10)); - value /= 10; - } - while (value != 0); - do - *s++ = temp[--pos]; - while (pos > 0); - *s = '\0'; -} - -#define PERIOD_4 (4 * 365 + 1) -#define PERIOD_100 (PERIOD_4 * 25 - 1) -#define PERIOD_400 (PERIOD_100 * 4 + 1) - -static void ConvertFileTimeToString(CNtfsFileTime *ft, char *s) -{ - unsigned year, mon, day, hour, min, sec; - UInt64 v64 = ft->Low | ((UInt64)ft->High << 32); - Byte ms[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - unsigned temp; - UInt32 v; - v64 /= 10000000; - sec = (unsigned)(v64 % 60); - v64 /= 60; - min = (unsigned)(v64 % 60); - v64 /= 60; - hour = (unsigned)(v64 % 24); - v64 /= 24; - - v = (UInt32)v64; - - year = (unsigned)(1601 + v / PERIOD_400 * 400); - v %= PERIOD_400; - - temp = (unsigned)(v / PERIOD_100); - if (temp == 4) - temp = 3; - year += temp * 100; - v -= temp * PERIOD_100; - - temp = v / PERIOD_4; - if (temp == 25) - temp = 24; - year += temp * 4; - v -= temp * PERIOD_4; - - temp = v / 365; - if (temp == 4) - temp = 3; - year += temp; - v -= temp * 365; - - if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) - ms[1] = 29; - for (mon = 1; mon <= 12; mon++) - { - unsigned s = ms[mon - 1]; - if (v < s) - break; - v -= s; - } - day = (unsigned)v + 1; - sprintf(s, "%04d-%02d-%02d %02d:%02d:%02d", year, mon, day, hour, min, sec); -} - -void PrintError(char *sz) -{ - printf("\nERROR: %s\n", sz); -} - -int MY_CDECL main(int numargs, char *args[]) -{ - CFileInStream archiveStream; - CLookToRead lookStream; - CSzArEx db; - SRes res; - ISzAlloc allocImp; - ISzAlloc allocTempImp; - - printf("\n7z ANSI-C Decoder " MY_VERSION_COPYRIGHT_DATE "\n"); - if (numargs == 1) - { - printf( - "\nUsage: 7zDec \n\n" - "\n" - " e: Extract files from archive\n" - " l: List contents of archive\n" - " t: Test integrity of archive\n"); - return 0; - } - if (numargs < 3) - { - PrintError("incorrect command"); - return 1; - } - - if (InFile_Open(&archiveStream.file, args[2])) - { - PrintError("can not open input file"); - return 1; - } - - - FileInStream_CreateVTable(&archiveStream); - LookToRead_CreateVTable(&lookStream, False); - - lookStream.realStream = &archiveStream.s; - LookToRead_Init(&lookStream); - - allocImp.Alloc = SzAlloc; - allocImp.Free = SzFree; - - allocTempImp.Alloc = SzAllocTemp; - allocTempImp.Free = SzFreeTemp; - - CrcGenerateTable(); - - SzArEx_Init(&db); - res = SzArEx_Open(&db, &lookStream.s, &allocImp, &allocTempImp); - if (res == SZ_OK) - { - char *command = args[1]; - int listCommand = 0, testCommand = 0, extractCommand = 0; - if (strcmp(command, "l") == 0) listCommand = 1; - else if (strcmp(command, "t") == 0) testCommand = 1; - else if (strcmp(command, "e") == 0) extractCommand = 1; - - if (listCommand) - { - UInt32 i; - for (i = 0; i < db.db.NumFiles; i++) - { - CSzFileItem *f = db.db.Files + i; - char s[32], t[32]; - ConvertNumberToString(f->Size, s); - if (f->MTimeDefined) - ConvertFileTimeToString(&f->MTime, t); - else - strcpy(t, " "); - - printf("%s %10s %s\n", t, s, f->Name); - } - } - else if (testCommand || extractCommand) - { - UInt32 i; - - /* - if you need cache, use these 3 variables. - if you use external function, you can make these variable as static. - */ - UInt32 blockIndex = 0xFFFFFFFF; /* it can have any value before first call (if outBuffer = 0) */ - Byte *outBuffer = 0; /* it must be 0 before first call for each new archive. */ - size_t outBufferSize = 0; /* it can have any value before first call (if outBuffer = 0) */ - - printf("\n"); - for (i = 0; i < db.db.NumFiles; i++) - { - size_t offset; - size_t outSizeProcessed; - CSzFileItem *f = db.db.Files + i; - if (f->IsDir) - printf("Directory "); - else - printf(testCommand ? - "Testing ": - "Extracting"); - printf(" %s", f->Name); - if (f->IsDir) - { - printf("\n"); - continue; - } - res = SzAr_Extract(&db, &lookStream.s, i, - &blockIndex, &outBuffer, &outBufferSize, - &offset, &outSizeProcessed, - &allocImp, &allocTempImp); - if (res != SZ_OK) - break; - if (!testCommand) - { - CSzFile outFile; - size_t processedSize; - char *fileName = f->Name; - size_t nameLen = strlen(f->Name); - for (; nameLen > 0; nameLen--) - if (f->Name[nameLen - 1] == '/') - { - fileName = f->Name + nameLen; - break; - } - - if (OutFile_Open(&outFile, fileName)) - { - PrintError("can not open output file"); - res = SZ_ERROR_FAIL; - break; - } - processedSize = outSizeProcessed; - if (File_Write(&outFile, outBuffer + offset, &processedSize) != 0 || - processedSize != outSizeProcessed) - { - PrintError("can not write output file"); - res = SZ_ERROR_FAIL; - break; - } - if (File_Close(&outFile)) - { - PrintError("can not close output file"); - res = SZ_ERROR_FAIL; - break; - } - } - printf("\n"); - } - IAlloc_Free(&allocImp, outBuffer); - } - else - { - PrintError("incorrect command"); - res = SZ_ERROR_FAIL; - } - } - SzArEx_Free(&db, &allocImp); - - File_Close(&archiveStream.file); - if (res == SZ_OK) - { - printf("\nEverything is Ok\n"); - return 0; - } - if (res == SZ_ERROR_UNSUPPORTED) - PrintError("decoder doesn't support this archive"); - else if (res == SZ_ERROR_MEM) - PrintError("can not allocate memory"); - else if (res == SZ_ERROR_CRC) - PrintError("CRC error"); - else - printf("\nERROR #%d\n", res); - return 1; -} diff --git a/ext/7z/Archive/7z/7z_2005Express.vcproj b/ext/7z/Archive/7z/7z_2005Express.vcproj deleted file mode 100644 index 3f95adc9..00000000 --- a/ext/7z/Archive/7z/7z_2005Express.vcproj +++ /dev/null @@ -1,1070 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ext/7z/Archive/7z/7z_VS2003.vcproj b/ext/7z/Archive/7z/7z_VS2003.vcproj deleted file mode 100644 index 2244293c..00000000 --- a/ext/7z/Archive/7z/7z_VS2003.vcproj +++ /dev/null @@ -1,301 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ext/7z/Bcj2.c b/ext/7z/Bcj2.c deleted file mode 100644 index 474bdd45..00000000 --- a/ext/7z/Bcj2.c +++ /dev/null @@ -1,132 +0,0 @@ -/* Bcj2.c -- Converter for x86 code (BCJ2) -2008-10-04 : Igor Pavlov : Public domain */ - -#include "Bcj2.h" - -#ifdef _LZMA_PROB32 -#define CProb UInt32 -#else -#define CProb UInt16 -#endif - -#define IsJcc(b0, b1) ((b0) == 0x0F && ((b1) & 0xF0) == 0x80) -#define IsJ(b0, b1) ((b1 & 0xFE) == 0xE8 || IsJcc(b0, b1)) - -#define kNumTopBits 24 -#define kTopValue ((UInt32)1 << kNumTopBits) - -#define kNumBitModelTotalBits 11 -#define kBitModelTotal (1 << kNumBitModelTotalBits) -#define kNumMoveBits 5 - -#define RC_READ_BYTE (*buffer++) -#define RC_TEST { if (buffer == bufferLim) return SZ_ERROR_DATA; } -#define RC_INIT2 code = 0; range = 0xFFFFFFFF; \ - { int i; for (i = 0; i < 5; i++) { RC_TEST; code = (code << 8) | RC_READ_BYTE; }} - -#define NORMALIZE if (range < kTopValue) { RC_TEST; range <<= 8; code = (code << 8) | RC_READ_BYTE; } - -#define IF_BIT_0(p) ttt = *(p); bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) -#define UPDATE_0(p) range = bound; *(p) = (CProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); NORMALIZE; -#define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CProb)(ttt - (ttt >> kNumMoveBits)); NORMALIZE; - -int Bcj2_Decode( - const Byte *buf0, SizeT size0, - const Byte *buf1, SizeT size1, - const Byte *buf2, SizeT size2, - const Byte *buf3, SizeT size3, - Byte *outBuf, SizeT outSize) -{ - CProb p[256 + 2]; - SizeT inPos = 0, outPos = 0; - - const Byte *buffer, *bufferLim; - UInt32 range, code; - Byte prevByte = 0; - - unsigned int i; - for (i = 0; i < sizeof(p) / sizeof(p[0]); i++) - p[i] = kBitModelTotal >> 1; - - buffer = buf3; - bufferLim = buffer + size3; - RC_INIT2 - - if (outSize == 0) - return SZ_OK; - - for (;;) - { - Byte b; - CProb *prob; - UInt32 bound; - UInt32 ttt; - - SizeT limit = size0 - inPos; - if (outSize - outPos < limit) - limit = outSize - outPos; - while (limit != 0) - { - Byte b = buf0[inPos]; - outBuf[outPos++] = b; - if (IsJ(prevByte, b)) - break; - inPos++; - prevByte = b; - limit--; - } - - if (limit == 0 || outPos == outSize) - break; - - b = buf0[inPos++]; - - if (b == 0xE8) - prob = p + prevByte; - else if (b == 0xE9) - prob = p + 256; - else - prob = p + 257; - - IF_BIT_0(prob) - { - UPDATE_0(prob) - prevByte = b; - } - else - { - UInt32 dest; - const Byte *v; - UPDATE_1(prob) - if (b == 0xE8) - { - v = buf1; - if (size1 < 4) - return SZ_ERROR_DATA; - buf1 += 4; - size1 -= 4; - } - else - { - v = buf2; - if (size2 < 4) - return SZ_ERROR_DATA; - buf2 += 4; - size2 -= 4; - } - dest = (((UInt32)v[0] << 24) | ((UInt32)v[1] << 16) | - ((UInt32)v[2] << 8) | ((UInt32)v[3])) - ((UInt32)outPos + 4); - outBuf[outPos++] = (Byte)dest; - if (outPos == outSize) - break; - outBuf[outPos++] = (Byte)(dest >> 8); - if (outPos == outSize) - break; - outBuf[outPos++] = (Byte)(dest >> 16); - if (outPos == outSize) - break; - outBuf[outPos++] = prevByte = (Byte)(dest >> 24); - } - } - return (outPos == outSize) ? SZ_OK : SZ_ERROR_DATA; -} diff --git a/ext/7z/Bcj2.h b/ext/7z/Bcj2.h deleted file mode 100644 index f58e3bcb..00000000 --- a/ext/7z/Bcj2.h +++ /dev/null @@ -1,30 +0,0 @@ -/* Bcj2.h -- Converter for x86 code (BCJ2) -2008-10-04 : Igor Pavlov : Public domain */ - -#ifndef __BCJ2_H -#define __BCJ2_H - -#include "Types.h" - -/* -Conditions: - outSize <= FullOutputSize, - where FullOutputSize is full size of output stream of x86_2 filter. - -If buf0 overlaps outBuf, there are two required conditions: - 1) (buf0 >= outBuf) - 2) (buf0 + size0 >= outBuf + FullOutputSize). - -Returns: - SZ_OK - SZ_ERROR_DATA - Data error -*/ - -int Bcj2_Decode( - const Byte *buf0, SizeT size0, - const Byte *buf1, SizeT size1, - const Byte *buf2, SizeT size2, - const Byte *buf3, SizeT size3, - Byte *outBuf, SizeT outSize); - -#endif diff --git a/ext/7z/Bra.c b/ext/7z/Bra.c deleted file mode 100644 index 3b75f0cc..00000000 --- a/ext/7z/Bra.c +++ /dev/null @@ -1,133 +0,0 @@ -/* Bra.c -- Converters for RISC code -2008-10-04 : Igor Pavlov : Public domain */ - -#include "Bra.h" - -SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) -{ - SizeT i; - if (size < 4) - return 0; - size -= 4; - ip += 8; - for (i = 0; i <= size; i += 4) - { - if (data[i + 3] == 0xEB) - { - UInt32 dest; - UInt32 src = ((UInt32)data[i + 2] << 16) | ((UInt32)data[i + 1] << 8) | (data[i + 0]); - src <<= 2; - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - dest >>= 2; - data[i + 2] = (Byte)(dest >> 16); - data[i + 1] = (Byte)(dest >> 8); - data[i + 0] = (Byte)dest; - } - } - return i; -} - -SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) -{ - SizeT i; - if (size < 4) - return 0; - size -= 4; - ip += 4; - for (i = 0; i <= size; i += 2) - { - if ((data[i + 1] & 0xF8) == 0xF0 && - (data[i + 3] & 0xF8) == 0xF8) - { - UInt32 dest; - UInt32 src = - (((UInt32)data[i + 1] & 0x7) << 19) | - ((UInt32)data[i + 0] << 11) | - (((UInt32)data[i + 3] & 0x7) << 8) | - (data[i + 2]); - - src <<= 1; - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - dest >>= 1; - - data[i + 1] = (Byte)(0xF0 | ((dest >> 19) & 0x7)); - data[i + 0] = (Byte)(dest >> 11); - data[i + 3] = (Byte)(0xF8 | ((dest >> 8) & 0x7)); - data[i + 2] = (Byte)dest; - i += 2; - } - } - return i; -} - -SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) -{ - SizeT i; - if (size < 4) - return 0; - size -= 4; - for (i = 0; i <= size; i += 4) - { - if ((data[i] >> 2) == 0x12 && (data[i + 3] & 3) == 1) - { - UInt32 src = ((UInt32)(data[i + 0] & 3) << 24) | - ((UInt32)data[i + 1] << 16) | - ((UInt32)data[i + 2] << 8) | - ((UInt32)data[i + 3] & (~3)); - - UInt32 dest; - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - data[i + 0] = (Byte)(0x48 | ((dest >> 24) & 0x3)); - data[i + 1] = (Byte)(dest >> 16); - data[i + 2] = (Byte)(dest >> 8); - data[i + 3] &= 0x3; - data[i + 3] |= dest; - } - } - return i; -} - -SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) -{ - UInt32 i; - if (size < 4) - return 0; - size -= 4; - for (i = 0; i <= size; i += 4) - { - if (data[i] == 0x40 && (data[i + 1] & 0xC0) == 0x00 || - data[i] == 0x7F && (data[i + 1] & 0xC0) == 0xC0) - { - UInt32 src = - ((UInt32)data[i + 0] << 24) | - ((UInt32)data[i + 1] << 16) | - ((UInt32)data[i + 2] << 8) | - ((UInt32)data[i + 3]); - UInt32 dest; - - src <<= 2; - if (encoding) - dest = ip + i + src; - else - dest = src - (ip + i); - dest >>= 2; - - dest = (((0 - ((dest >> 22) & 1)) << 22) & 0x3FFFFFFF) | (dest & 0x3FFFFF) | 0x40000000; - - data[i + 0] = (Byte)(dest >> 24); - data[i + 1] = (Byte)(dest >> 16); - data[i + 2] = (Byte)(dest >> 8); - data[i + 3] = (Byte)dest; - } - } - return i; -} diff --git a/ext/7z/Bra.h b/ext/7z/Bra.h deleted file mode 100644 index b9018eb9..00000000 --- a/ext/7z/Bra.h +++ /dev/null @@ -1,60 +0,0 @@ -/* Bra.h -- Branch converters for executables -2008-10-04 : Igor Pavlov : Public domain */ - -#ifndef __BRA_H -#define __BRA_H - -#include "Types.h" - -/* -These functions convert relative addresses to absolute addresses -in CALL instructions to increase the compression ratio. - - In: - data - data buffer - size - size of data - ip - current virtual Instruction Pinter (IP) value - state - state variable for x86 converter - encoding - 0 (for decoding), 1 (for encoding) - - Out: - state - state variable for x86 converter - - Returns: - The number of processed bytes. If you call these functions with multiple calls, - you must start next call with first byte after block of processed bytes. - - Type Endian Alignment LookAhead - - x86 little 1 4 - ARMT little 2 2 - ARM little 4 0 - PPC big 4 0 - SPARC big 4 0 - IA64 little 16 0 - - size must be >= Alignment + LookAhead, if it's not last block. - If (size < Alignment + LookAhead), converter returns 0. - - Example: - - UInt32 ip = 0; - for () - { - ; size must be >= Alignment + LookAhead, if it's not last block - SizeT processed = Convert(data, size, ip, 1); - data += processed; - size -= processed; - ip += processed; - } -*/ - -#define x86_Convert_Init(state) { state = 0; } -SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding); -SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); -SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding); - -#endif diff --git a/ext/7z/Bra86.c b/ext/7z/Bra86.c deleted file mode 100644 index 93566cb2..00000000 --- a/ext/7z/Bra86.c +++ /dev/null @@ -1,85 +0,0 @@ -/* Bra86.c -- Converter for x86 code (BCJ) -2008-10-04 : Igor Pavlov : Public domain */ - -#include "Bra.h" - -#define Test86MSByte(b) ((b) == 0 || (b) == 0xFF) - -const Byte kMaskToAllowedStatus[8] = {1, 1, 1, 0, 1, 0, 0, 0}; -const Byte kMaskToBitNumber[8] = {0, 1, 2, 2, 3, 3, 3, 3}; - -SizeT x86_Convert(Byte *data, SizeT size, UInt32 ip, UInt32 *state, int encoding) -{ - SizeT bufferPos = 0, prevPosT; - UInt32 prevMask = *state & 0x7; - if (size < 5) - return 0; - ip += 5; - prevPosT = (SizeT)0 - 1; - - for (;;) - { - Byte *p = data + bufferPos; - Byte *limit = data + size - 4; - for (; p < limit; p++) - if ((*p & 0xFE) == 0xE8) - break; - bufferPos = (SizeT)(p - data); - if (p >= limit) - break; - prevPosT = bufferPos - prevPosT; - if (prevPosT > 3) - prevMask = 0; - else - { - prevMask = (prevMask << ((int)prevPosT - 1)) & 0x7; - if (prevMask != 0) - { - Byte b = p[4 - kMaskToBitNumber[prevMask]]; - if (!kMaskToAllowedStatus[prevMask] || Test86MSByte(b)) - { - prevPosT = bufferPos; - prevMask = ((prevMask << 1) & 0x7) | 1; - bufferPos++; - continue; - } - } - } - prevPosT = bufferPos; - - if (Test86MSByte(p[4])) - { - UInt32 src = ((UInt32)p[4] << 24) | ((UInt32)p[3] << 16) | ((UInt32)p[2] << 8) | ((UInt32)p[1]); - UInt32 dest; - for (;;) - { - Byte b; - int index; - if (encoding) - dest = (ip + (UInt32)bufferPos) + src; - else - dest = src - (ip + (UInt32)bufferPos); - if (prevMask == 0) - break; - index = kMaskToBitNumber[prevMask] * 8; - b = (Byte)(dest >> (24 - index)); - if (!Test86MSByte(b)) - break; - src = dest ^ ((1 << (32 - index)) - 1); - } - p[4] = (Byte)(~(((dest >> 24) & 1) - 1)); - p[3] = (Byte)(dest >> 16); - p[2] = (Byte)(dest >> 8); - p[1] = (Byte)dest; - bufferPos += 5; - } - else - { - prevMask = ((prevMask << 1) & 0x7) | 1; - bufferPos++; - } - } - prevPosT = bufferPos - prevPosT; - *state = ((prevPosT > 3) ? 0 : ((prevMask << ((int)prevPosT - 1)) & 0x7)); - return bufferPos; -} diff --git a/ext/7z/BraIA64.c b/ext/7z/BraIA64.c deleted file mode 100644 index f359f16a..00000000 --- a/ext/7z/BraIA64.c +++ /dev/null @@ -1,67 +0,0 @@ -/* BraIA64.c -- Converter for IA-64 code -2008-10-04 : Igor Pavlov : Public domain */ - -#include "Bra.h" - -static const Byte kBranchTable[32] = -{ - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 4, 4, 6, 6, 0, 0, 7, 7, - 4, 4, 0, 0, 4, 4, 0, 0 -}; - -SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) -{ - SizeT i; - if (size < 16) - return 0; - size -= 16; - for (i = 0; i <= size; i += 16) - { - UInt32 instrTemplate = data[i] & 0x1F; - UInt32 mask = kBranchTable[instrTemplate]; - UInt32 bitPos = 5; - int slot; - for (slot = 0; slot < 3; slot++, bitPos += 41) - { - UInt32 bytePos, bitRes; - UInt64 instruction, instNorm; - int j; - if (((mask >> slot) & 1) == 0) - continue; - bytePos = (bitPos >> 3); - bitRes = bitPos & 0x7; - instruction = 0; - for (j = 0; j < 6; j++) - instruction += (UInt64)data[i + j + bytePos] << (8 * j); - - instNorm = instruction >> bitRes; - if (((instNorm >> 37) & 0xF) == 0x5 && ((instNorm >> 9) & 0x7) == 0) - { - UInt32 src = (UInt32)((instNorm >> 13) & 0xFFFFF); - UInt32 dest; - src |= ((UInt32)(instNorm >> 36) & 1) << 20; - - src <<= 4; - - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - - dest >>= 4; - - instNorm &= ~((UInt64)(0x8FFFFF) << 13); - instNorm |= ((UInt64)(dest & 0xFFFFF) << 13); - instNorm |= ((UInt64)(dest & 0x100000) << (36 - 20)); - - instruction &= (1 << bitRes) - 1; - instruction |= (instNorm << bitRes); - for (j = 0; j < 6; j++) - data[i + j + bytePos] = (Byte)(instruction >> (8 * j)); - } - } - } - return i; -} diff --git a/ext/7z/CpuArch.h b/ext/7z/CpuArch.h deleted file mode 100644 index 006361f2..00000000 --- a/ext/7z/CpuArch.h +++ /dev/null @@ -1,69 +0,0 @@ -/* CpuArch.h -2008-08-05 -Igor Pavlov -Public domain */ - -#ifndef __CPUARCH_H -#define __CPUARCH_H - -/* -LITTLE_ENDIAN_UNALIGN means: - 1) CPU is LITTLE_ENDIAN - 2) it's allowed to make unaligned memory accesses -if LITTLE_ENDIAN_UNALIGN is not defined, it means that we don't know -about these properties of platform. -*/ - -#if defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) || defined(__i386__) || defined(__x86_64__) -#define LITTLE_ENDIAN_UNALIGN -#endif - -#ifdef LITTLE_ENDIAN_UNALIGN - -#define GetUi16(p) (*(const UInt16 *)(p)) -#define GetUi32(p) (*(const UInt32 *)(p)) -#define GetUi64(p) (*(const UInt64 *)(p)) -#define SetUi32(p, d) *(UInt32 *)(p) = (d); - -#else - -#define GetUi16(p) (((const Byte *)(p))[0] | ((UInt16)((const Byte *)(p))[1] << 8)) - -#define GetUi32(p) ( \ - ((const Byte *)(p))[0] | \ - ((UInt32)((const Byte *)(p))[1] << 8) | \ - ((UInt32)((const Byte *)(p))[2] << 16) | \ - ((UInt32)((const Byte *)(p))[3] << 24)) - -#define GetUi64(p) (GetUi32(p) | ((UInt64)GetUi32(((const Byte *)(p)) + 4) << 32)) - -#define SetUi32(p, d) { UInt32 _x_ = (d); \ - ((Byte *)(p))[0] = (Byte)_x_; \ - ((Byte *)(p))[1] = (Byte)(_x_ >> 8); \ - ((Byte *)(p))[2] = (Byte)(_x_ >> 16); \ - ((Byte *)(p))[3] = (Byte)(_x_ >> 24); } - -#endif - -#if defined(LITTLE_ENDIAN_UNALIGN) && defined(_WIN64) && (_MSC_VER >= 1300) - -#pragma intrinsic(_byteswap_ulong) -#pragma intrinsic(_byteswap_uint64) -#define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p)) -#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p)) - -#else - -#define GetBe32(p) ( \ - ((UInt32)((const Byte *)(p))[0] << 24) | \ - ((UInt32)((const Byte *)(p))[1] << 16) | \ - ((UInt32)((const Byte *)(p))[2] << 8) | \ - ((const Byte *)(p))[3] ) - -#define GetBe64(p) (((UInt64)GetBe32(p) << 32) | GetBe32(((const Byte *)(p)) + 4)) - -#endif - -#define GetBe16(p) (((UInt16)((const Byte *)(p))[0] << 8) | ((const Byte *)(p))[1]) - -#endif diff --git a/ext/7z/LzFind.c b/ext/7z/LzFind.c deleted file mode 100644 index e0ebe623..00000000 --- a/ext/7z/LzFind.c +++ /dev/null @@ -1,751 +0,0 @@ -/* LzFind.c -- Match finder for LZ algorithms -2008-10-04 : Igor Pavlov : Public domain */ - -#include - -#include "LzFind.h" -#include "LzHash.h" - -#define kEmptyHashValue 0 -#define kMaxValForNormalize ((UInt32)0xFFFFFFFF) -#define kNormalizeStepMin (1 << 10) /* it must be power of 2 */ -#define kNormalizeMask (~(kNormalizeStepMin - 1)) -#define kMaxHistorySize ((UInt32)3 << 30) - -#define kStartMaxLen 3 - -static void LzInWindow_Free(CMatchFinder *p, ISzAlloc *alloc) -{ - if (!p->directInput) - { - alloc->Free(alloc, p->bufferBase); - p->bufferBase = 0; - } -} - -/* keepSizeBefore + keepSizeAfter + keepSizeReserv must be < 4G) */ - -static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAlloc *alloc) -{ - UInt32 blockSize = p->keepSizeBefore + p->keepSizeAfter + keepSizeReserv; - if (p->directInput) - { - p->blockSize = blockSize; - return 1; - } - if (p->bufferBase == 0 || p->blockSize != blockSize) - { - LzInWindow_Free(p, alloc); - p->blockSize = blockSize; - p->bufferBase = (Byte *)alloc->Alloc(alloc, (size_t)blockSize); - } - return (p->bufferBase != 0); -} - -Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p) { return p->buffer; } -Byte MatchFinder_GetIndexByte(CMatchFinder *p, Int32 index) { return p->buffer[index]; } - -UInt32 MatchFinder_GetNumAvailableBytes(CMatchFinder *p) { return p->streamPos - p->pos; } - -void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue) -{ - p->posLimit -= subValue; - p->pos -= subValue; - p->streamPos -= subValue; -} - -static void MatchFinder_ReadBlock(CMatchFinder *p) -{ - if (p->streamEndWasReached || p->result != SZ_OK) - return; - for (;;) - { - Byte *dest = p->buffer + (p->streamPos - p->pos); - size_t size = (p->bufferBase + p->blockSize - dest); - if (size == 0) - return; - p->result = p->stream->Read(p->stream, dest, &size); - if (p->result != SZ_OK) - return; - if (size == 0) - { - p->streamEndWasReached = 1; - return; - } - p->streamPos += (UInt32)size; - if (p->streamPos - p->pos > p->keepSizeAfter) - return; - } -} - -void MatchFinder_MoveBlock(CMatchFinder *p) -{ - memmove(p->bufferBase, - p->buffer - p->keepSizeBefore, - (size_t)(p->streamPos - p->pos + p->keepSizeBefore)); - p->buffer = p->bufferBase + p->keepSizeBefore; -} - -int MatchFinder_NeedMove(CMatchFinder *p) -{ - /* if (p->streamEndWasReached) return 0; */ - return ((size_t)(p->bufferBase + p->blockSize - p->buffer) <= p->keepSizeAfter); -} - -void MatchFinder_ReadIfRequired(CMatchFinder *p) -{ - if (p->streamEndWasReached) - return; - if (p->keepSizeAfter >= p->streamPos - p->pos) - MatchFinder_ReadBlock(p); -} - -static void MatchFinder_CheckAndMoveAndRead(CMatchFinder *p) -{ - if (MatchFinder_NeedMove(p)) - MatchFinder_MoveBlock(p); - MatchFinder_ReadBlock(p); -} - -static void MatchFinder_SetDefaultSettings(CMatchFinder *p) -{ - p->cutValue = 32; - p->btMode = 1; - p->numHashBytes = 4; - /* p->skipModeBits = 0; */ - p->directInput = 0; - p->bigHash = 0; -} - -#define kCrcPoly 0xEDB88320 - -void MatchFinder_Construct(CMatchFinder *p) -{ - UInt32 i; - p->bufferBase = 0; - p->directInput = 0; - p->hash = 0; - MatchFinder_SetDefaultSettings(p); - - for (i = 0; i < 256; i++) - { - UInt32 r = i; - int j; - for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1)); - p->crc[i] = r; - } -} - -static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAlloc *alloc) -{ - alloc->Free(alloc, p->hash); - p->hash = 0; -} - -void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc) -{ - MatchFinder_FreeThisClassMemory(p, alloc); - LzInWindow_Free(p, alloc); -} - -static CLzRef* AllocRefs(UInt32 num, ISzAlloc *alloc) -{ - size_t sizeInBytes = (size_t)num * sizeof(CLzRef); - if (sizeInBytes / sizeof(CLzRef) != num) - return 0; - return (CLzRef *)alloc->Alloc(alloc, sizeInBytes); -} - -int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, - UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAlloc *alloc) -{ - UInt32 sizeReserv; - if (historySize > kMaxHistorySize) - { - MatchFinder_Free(p, alloc); - return 0; - } - sizeReserv = historySize >> 1; - if (historySize > ((UInt32)2 << 30)) - sizeReserv = historySize >> 2; - sizeReserv += (keepAddBufferBefore + matchMaxLen + keepAddBufferAfter) / 2 + (1 << 19); - - p->keepSizeBefore = historySize + keepAddBufferBefore + 1; - p->keepSizeAfter = matchMaxLen + keepAddBufferAfter; - /* we need one additional byte, since we use MoveBlock after pos++ and before dictionary using */ - if (LzInWindow_Create(p, sizeReserv, alloc)) - { - UInt32 newCyclicBufferSize = (historySize /* >> p->skipModeBits */) + 1; - UInt32 hs; - p->matchMaxLen = matchMaxLen; - { - p->fixedHashSize = 0; - if (p->numHashBytes == 2) - hs = (1 << 16) - 1; - else - { - hs = historySize - 1; - hs |= (hs >> 1); - hs |= (hs >> 2); - hs |= (hs >> 4); - hs |= (hs >> 8); - hs >>= 1; - /* hs >>= p->skipModeBits; */ - hs |= 0xFFFF; /* don't change it! It's required for Deflate */ - if (hs > (1 << 24)) - { - if (p->numHashBytes == 3) - hs = (1 << 24) - 1; - else - hs >>= 1; - } - } - p->hashMask = hs; - hs++; - if (p->numHashBytes > 2) p->fixedHashSize += kHash2Size; - if (p->numHashBytes > 3) p->fixedHashSize += kHash3Size; - if (p->numHashBytes > 4) p->fixedHashSize += kHash4Size; - hs += p->fixedHashSize; - } - - { - UInt32 prevSize = p->hashSizeSum + p->numSons; - UInt32 newSize; - p->historySize = historySize; - p->hashSizeSum = hs; - p->cyclicBufferSize = newCyclicBufferSize; - p->numSons = (p->btMode ? newCyclicBufferSize * 2 : newCyclicBufferSize); - newSize = p->hashSizeSum + p->numSons; - if (p->hash != 0 && prevSize == newSize) - return 1; - MatchFinder_FreeThisClassMemory(p, alloc); - p->hash = AllocRefs(newSize, alloc); - if (p->hash != 0) - { - p->son = p->hash + p->hashSizeSum; - return 1; - } - } - } - MatchFinder_Free(p, alloc); - return 0; -} - -static void MatchFinder_SetLimits(CMatchFinder *p) -{ - UInt32 limit = kMaxValForNormalize - p->pos; - UInt32 limit2 = p->cyclicBufferSize - p->cyclicBufferPos; - if (limit2 < limit) - limit = limit2; - limit2 = p->streamPos - p->pos; - if (limit2 <= p->keepSizeAfter) - { - if (limit2 > 0) - limit2 = 1; - } - else - limit2 -= p->keepSizeAfter; - if (limit2 < limit) - limit = limit2; - { - UInt32 lenLimit = p->streamPos - p->pos; - if (lenLimit > p->matchMaxLen) - lenLimit = p->matchMaxLen; - p->lenLimit = lenLimit; - } - p->posLimit = p->pos + limit; -} - -void MatchFinder_Init(CMatchFinder *p) -{ - UInt32 i; - for (i = 0; i < p->hashSizeSum; i++) - p->hash[i] = kEmptyHashValue; - p->cyclicBufferPos = 0; - p->buffer = p->bufferBase; - p->pos = p->streamPos = p->cyclicBufferSize; - p->result = SZ_OK; - p->streamEndWasReached = 0; - MatchFinder_ReadBlock(p); - MatchFinder_SetLimits(p); -} - -static UInt32 MatchFinder_GetSubValue(CMatchFinder *p) -{ - return (p->pos - p->historySize - 1) & kNormalizeMask; -} - -void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, UInt32 numItems) -{ - UInt32 i; - for (i = 0; i < numItems; i++) - { - UInt32 value = items[i]; - if (value <= subValue) - value = kEmptyHashValue; - else - value -= subValue; - items[i] = value; - } -} - -static void MatchFinder_Normalize(CMatchFinder *p) -{ - UInt32 subValue = MatchFinder_GetSubValue(p); - MatchFinder_Normalize3(subValue, p->hash, p->hashSizeSum + p->numSons); - MatchFinder_ReduceOffsets(p, subValue); -} - -static void MatchFinder_CheckLimits(CMatchFinder *p) -{ - if (p->pos == kMaxValForNormalize) - MatchFinder_Normalize(p); - if (!p->streamEndWasReached && p->keepSizeAfter == p->streamPos - p->pos) - MatchFinder_CheckAndMoveAndRead(p); - if (p->cyclicBufferPos == p->cyclicBufferSize) - p->cyclicBufferPos = 0; - MatchFinder_SetLimits(p); -} - -static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) -{ - son[_cyclicBufferPos] = curMatch; - for (;;) - { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) - return distances; - { - const Byte *pb = cur - delta; - curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; - if (pb[maxLen] == cur[maxLen] && *pb == *cur) - { - UInt32 len = 0; - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - if (maxLen < len) - { - *distances++ = maxLen = len; - *distances++ = delta - 1; - if (len == lenLimit) - return distances; - } - } - } - } -} - -UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) -{ - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; - for (;;) - { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) - { - *ptr0 = *ptr1 = kEmptyHashValue; - return distances; - } - { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); - const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); - if (pb[len] == cur[len]) - { - if (++len != lenLimit && pb[len] == cur[len]) - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - if (maxLen < len) - { - *distances++ = maxLen = len; - *distances++ = delta - 1; - if (len == lenLimit) - { - *ptr1 = pair[0]; - *ptr0 = pair[1]; - return distances; - } - } - } - if (pb[len] < cur[len]) - { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; - } - } - } -} - -static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) -{ - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; - for (;;) - { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) - { - *ptr0 = *ptr1 = kEmptyHashValue; - return; - } - { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); - const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); - if (pb[len] == cur[len]) - { - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - { - if (len == lenLimit) - { - *ptr1 = pair[0]; - *ptr0 = pair[1]; - return; - } - } - } - if (pb[len] < cur[len]) - { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; - } - } - } -} - -#define MOVE_POS \ - ++p->cyclicBufferPos; \ - p->buffer++; \ - if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p); - -#define MOVE_POS_RET MOVE_POS return offset; - -static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } - -#define GET_MATCHES_HEADER2(minLen, ret_op) \ - UInt32 lenLimit; UInt32 hashValue; const Byte *cur; UInt32 curMatch; \ - lenLimit = p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ - cur = p->buffer; - -#define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0) -#define SKIP_HEADER(minLen) GET_MATCHES_HEADER2(minLen, continue) - -#define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue - -#define GET_MATCHES_FOOTER(offset, maxLen) \ - offset = (UInt32)(GetMatchesSpec1(lenLimit, curMatch, MF_PARAMS(p), \ - distances + offset, maxLen) - distances); MOVE_POS_RET; - -#define SKIP_FOOTER \ - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; - -static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - UInt32 offset; - GET_MATCHES_HEADER(2) - HASH2_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - offset = 0; - GET_MATCHES_FOOTER(offset, 1) -} - -UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - UInt32 offset; - GET_MATCHES_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - offset = 0; - GET_MATCHES_FOOTER(offset, 2) -} - -static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - UInt32 hash2Value, delta2, maxLen, offset; - GET_MATCHES_HEADER(3) - - HASH3_CALC; - - delta2 = p->pos - p->hash[hash2Value]; - curMatch = p->hash[kFix3HashSize + hashValue]; - - p->hash[hash2Value] = - p->hash[kFix3HashSize + hashValue] = p->pos; - - - maxLen = 2; - offset = 0; - if (delta2 < p->cyclicBufferSize && *(cur - delta2) == *cur) - { - for (; maxLen != lenLimit; maxLen++) - if (cur[(ptrdiff_t)maxLen - delta2] != cur[maxLen]) - break; - distances[0] = maxLen; - distances[1] = delta2 - 1; - offset = 2; - if (maxLen == lenLimit) - { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); - MOVE_POS_RET; - } - } - GET_MATCHES_FOOTER(offset, maxLen) -} - -static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - UInt32 hash2Value, hash3Value, delta2, delta3, maxLen, offset; - GET_MATCHES_HEADER(4) - - HASH4_CALC; - - delta2 = p->pos - p->hash[ hash2Value]; - delta3 = p->pos - p->hash[kFix3HashSize + hash3Value]; - curMatch = p->hash[kFix4HashSize + hashValue]; - - p->hash[ hash2Value] = - p->hash[kFix3HashSize + hash3Value] = - p->hash[kFix4HashSize + hashValue] = p->pos; - - maxLen = 1; - offset = 0; - if (delta2 < p->cyclicBufferSize && *(cur - delta2) == *cur) - { - distances[0] = maxLen = 2; - distances[1] = delta2 - 1; - offset = 2; - } - if (delta2 != delta3 && delta3 < p->cyclicBufferSize && *(cur - delta3) == *cur) - { - maxLen = 3; - distances[offset + 1] = delta3 - 1; - offset += 2; - delta2 = delta3; - } - if (offset != 0) - { - for (; maxLen != lenLimit; maxLen++) - if (cur[(ptrdiff_t)maxLen - delta2] != cur[maxLen]) - break; - distances[offset - 2] = maxLen; - if (maxLen == lenLimit) - { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); - MOVE_POS_RET; - } - } - if (maxLen < 3) - maxLen = 3; - GET_MATCHES_FOOTER(offset, maxLen) -} - -static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - UInt32 hash2Value, hash3Value, delta2, delta3, maxLen, offset; - GET_MATCHES_HEADER(4) - - HASH4_CALC; - - delta2 = p->pos - p->hash[ hash2Value]; - delta3 = p->pos - p->hash[kFix3HashSize + hash3Value]; - curMatch = p->hash[kFix4HashSize + hashValue]; - - p->hash[ hash2Value] = - p->hash[kFix3HashSize + hash3Value] = - p->hash[kFix4HashSize + hashValue] = p->pos; - - maxLen = 1; - offset = 0; - if (delta2 < p->cyclicBufferSize && *(cur - delta2) == *cur) - { - distances[0] = maxLen = 2; - distances[1] = delta2 - 1; - offset = 2; - } - if (delta2 != delta3 && delta3 < p->cyclicBufferSize && *(cur - delta3) == *cur) - { - maxLen = 3; - distances[offset + 1] = delta3 - 1; - offset += 2; - delta2 = delta3; - } - if (offset != 0) - { - for (; maxLen != lenLimit; maxLen++) - if (cur[(ptrdiff_t)maxLen - delta2] != cur[maxLen]) - break; - distances[offset - 2] = maxLen; - if (maxLen == lenLimit) - { - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS_RET; - } - } - if (maxLen < 3) - maxLen = 3; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances + offset, maxLen) - (distances)); - MOVE_POS_RET -} - -UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) -{ - UInt32 offset; - GET_MATCHES_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), - distances, 2) - (distances)); - MOVE_POS_RET -} - -static void Bt2_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - SKIP_HEADER(2) - HASH2_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - SKIP_FOOTER - } - while (--num != 0); -} - -void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - SKIP_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - SKIP_FOOTER - } - while (--num != 0); -} - -static void Bt3_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - UInt32 hash2Value; - SKIP_HEADER(3) - HASH3_CALC; - curMatch = p->hash[kFix3HashSize + hashValue]; - p->hash[hash2Value] = - p->hash[kFix3HashSize + hashValue] = p->pos; - SKIP_FOOTER - } - while (--num != 0); -} - -static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - UInt32 hash2Value, hash3Value; - SKIP_HEADER(4) - HASH4_CALC; - curMatch = p->hash[kFix4HashSize + hashValue]; - p->hash[ hash2Value] = - p->hash[kFix3HashSize + hash3Value] = p->pos; - p->hash[kFix4HashSize + hashValue] = p->pos; - SKIP_FOOTER - } - while (--num != 0); -} - -static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - UInt32 hash2Value, hash3Value; - SKIP_HEADER(4) - HASH4_CALC; - curMatch = p->hash[kFix4HashSize + hashValue]; - p->hash[ hash2Value] = - p->hash[kFix3HashSize + hash3Value] = - p->hash[kFix4HashSize + hashValue] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } - while (--num != 0); -} - -void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num) -{ - do - { - SKIP_HEADER(3) - HASH_ZIP_CALC; - curMatch = p->hash[hashValue]; - p->hash[hashValue] = p->pos; - p->son[p->cyclicBufferPos] = curMatch; - MOVE_POS - } - while (--num != 0); -} - -void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable) -{ - vTable->Init = (Mf_Init_Func)MatchFinder_Init; - vTable->GetIndexByte = (Mf_GetIndexByte_Func)MatchFinder_GetIndexByte; - vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinder_GetNumAvailableBytes; - vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinder_GetPointerToCurrentPos; - if (!p->btMode) - { - vTable->GetMatches = (Mf_GetMatches_Func)Hc4_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Hc4_MatchFinder_Skip; - } - else if (p->numHashBytes == 2) - { - vTable->GetMatches = (Mf_GetMatches_Func)Bt2_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt2_MatchFinder_Skip; - } - else if (p->numHashBytes == 3) - { - vTable->GetMatches = (Mf_GetMatches_Func)Bt3_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt3_MatchFinder_Skip; - } - else - { - vTable->GetMatches = (Mf_GetMatches_Func)Bt4_MatchFinder_GetMatches; - vTable->Skip = (Mf_Skip_Func)Bt4_MatchFinder_Skip; - } -} diff --git a/ext/7z/LzFind.h b/ext/7z/LzFind.h deleted file mode 100644 index 423d67e0..00000000 --- a/ext/7z/LzFind.h +++ /dev/null @@ -1,107 +0,0 @@ -/* LzFind.h -- Match finder for LZ algorithms -2008-10-04 : Igor Pavlov : Public domain */ - -#ifndef __LZFIND_H -#define __LZFIND_H - -#include "Types.h" - -typedef UInt32 CLzRef; - -typedef struct _CMatchFinder -{ - Byte *buffer; - UInt32 pos; - UInt32 posLimit; - UInt32 streamPos; - UInt32 lenLimit; - - UInt32 cyclicBufferPos; - UInt32 cyclicBufferSize; /* it must be = (historySize + 1) */ - - UInt32 matchMaxLen; - CLzRef *hash; - CLzRef *son; - UInt32 hashMask; - UInt32 cutValue; - - Byte *bufferBase; - ISeqInStream *stream; - int streamEndWasReached; - - UInt32 blockSize; - UInt32 keepSizeBefore; - UInt32 keepSizeAfter; - - UInt32 numHashBytes; - int directInput; - int btMode; - /* int skipModeBits; */ - int bigHash; - UInt32 historySize; - UInt32 fixedHashSize; - UInt32 hashSizeSum; - UInt32 numSons; - SRes result; - UInt32 crc[256]; -} CMatchFinder; - -#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer) -#define Inline_MatchFinder_GetIndexByte(p, index) ((p)->buffer[(Int32)(index)]) - -#define Inline_MatchFinder_GetNumAvailableBytes(p) ((p)->streamPos - (p)->pos) - -int MatchFinder_NeedMove(CMatchFinder *p); -Byte *MatchFinder_GetPointerToCurrentPos(CMatchFinder *p); -void MatchFinder_MoveBlock(CMatchFinder *p); -void MatchFinder_ReadIfRequired(CMatchFinder *p); - -void MatchFinder_Construct(CMatchFinder *p); - -/* Conditions: - historySize <= 3 GB - keepAddBufferBefore + matchMaxLen + keepAddBufferAfter < 511MB -*/ -int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, - UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAlloc *alloc); -void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc); -void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, UInt32 numItems); -void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue); - -UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *buffer, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, - UInt32 *distances, UInt32 maxLen); - -/* -Conditions: - Mf_GetNumAvailableBytes_Func must be called before each Mf_GetMatchLen_Func. - Mf_GetPointerToCurrentPos_Func's result must be used only before any other function -*/ - -typedef void (*Mf_Init_Func)(void *object); -typedef Byte (*Mf_GetIndexByte_Func)(void *object, Int32 index); -typedef UInt32 (*Mf_GetNumAvailableBytes_Func)(void *object); -typedef const Byte * (*Mf_GetPointerToCurrentPos_Func)(void *object); -typedef UInt32 (*Mf_GetMatches_Func)(void *object, UInt32 *distances); -typedef void (*Mf_Skip_Func)(void *object, UInt32); - -typedef struct _IMatchFinder -{ - Mf_Init_Func Init; - Mf_GetIndexByte_Func GetIndexByte; - Mf_GetNumAvailableBytes_Func GetNumAvailableBytes; - Mf_GetPointerToCurrentPos_Func GetPointerToCurrentPos; - Mf_GetMatches_Func GetMatches; - Mf_Skip_Func Skip; -} IMatchFinder; - -void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable); - -void MatchFinder_Init(CMatchFinder *p); -UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); -UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances); -void Bt3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); -void Hc3Zip_MatchFinder_Skip(CMatchFinder *p, UInt32 num); - -#endif diff --git a/ext/7z/LzFindMt.c b/ext/7z/LzFindMt.c deleted file mode 100644 index 0ef13473..00000000 --- a/ext/7z/LzFindMt.c +++ /dev/null @@ -1,793 +0,0 @@ -/* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2008-10-04 : Igor Pavlov : Public domain */ - -#include "LzHash.h" - -#include "LzFindMt.h" - -void MtSync_Construct(CMtSync *p) -{ - p->wasCreated = False; - p->csWasInitialized = False; - p->csWasEntered = False; - Thread_Construct(&p->thread); - Event_Construct(&p->canStart); - Event_Construct(&p->wasStarted); - Event_Construct(&p->wasStopped); - Semaphore_Construct(&p->freeSemaphore); - Semaphore_Construct(&p->filledSemaphore); -} - -void MtSync_GetNextBlock(CMtSync *p) -{ - if (p->needStart) - { - p->numProcessedBlocks = 1; - p->needStart = False; - p->stopWriting = False; - p->exit = False; - Event_Reset(&p->wasStarted); - Event_Reset(&p->wasStopped); - - Event_Set(&p->canStart); - Event_Wait(&p->wasStarted); - } - else - { - CriticalSection_Leave(&p->cs); - p->csWasEntered = False; - p->numProcessedBlocks++; - Semaphore_Release1(&p->freeSemaphore); - } - Semaphore_Wait(&p->filledSemaphore); - CriticalSection_Enter(&p->cs); - p->csWasEntered = True; -} - -/* MtSync_StopWriting must be called if Writing was started */ - -void MtSync_StopWriting(CMtSync *p) -{ - UInt32 myNumBlocks = p->numProcessedBlocks; - if (!Thread_WasCreated(&p->thread) || p->needStart) - return; - p->stopWriting = True; - if (p->csWasEntered) - { - CriticalSection_Leave(&p->cs); - p->csWasEntered = False; - } - Semaphore_Release1(&p->freeSemaphore); - - Event_Wait(&p->wasStopped); - - while (myNumBlocks++ != p->numProcessedBlocks) - { - Semaphore_Wait(&p->filledSemaphore); - Semaphore_Release1(&p->freeSemaphore); - } - p->needStart = True; -} - -void MtSync_Destruct(CMtSync *p) -{ - if (Thread_WasCreated(&p->thread)) - { - MtSync_StopWriting(p); - p->exit = True; - if (p->needStart) - Event_Set(&p->canStart); - Thread_Wait(&p->thread); - Thread_Close(&p->thread); - } - if (p->csWasInitialized) - { - CriticalSection_Delete(&p->cs); - p->csWasInitialized = False; - } - - Event_Close(&p->canStart); - Event_Close(&p->wasStarted); - Event_Close(&p->wasStopped); - Semaphore_Close(&p->freeSemaphore); - Semaphore_Close(&p->filledSemaphore); - - p->wasCreated = False; -} - -#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } - -static SRes MtSync_Create2(CMtSync *p, unsigned (MY_STD_CALL *startAddress)(void *), void *obj, UInt32 numBlocks) -{ - if (p->wasCreated) - return SZ_OK; - - RINOK_THREAD(CriticalSection_Init(&p->cs)); - p->csWasInitialized = True; - - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canStart)); - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStarted)); - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->wasStopped)); - - RINOK_THREAD(Semaphore_Create(&p->freeSemaphore, numBlocks, numBlocks)); - RINOK_THREAD(Semaphore_Create(&p->filledSemaphore, 0, numBlocks)); - - p->needStart = True; - - RINOK_THREAD(Thread_Create(&p->thread, startAddress, obj)); - p->wasCreated = True; - return SZ_OK; -} - -static SRes MtSync_Create(CMtSync *p, unsigned (MY_STD_CALL *startAddress)(void *), void *obj, UInt32 numBlocks) -{ - SRes res = MtSync_Create2(p, startAddress, obj, numBlocks); - if (res != SZ_OK) - MtSync_Destruct(p); - return res; -} - -void MtSync_Init(CMtSync *p) { p->needStart = True; } - -#define kMtMaxValForNormalize 0xFFFFFFFF - -#define DEF_GetHeads2(name, v, action) \ -static void GetHeads ## name(const Byte *p, UInt32 pos, \ -UInt32 *hash, UInt32 hashMask, UInt32 *heads, UInt32 numHeads, const UInt32 *crc) \ -{ action; for (; numHeads != 0; numHeads--) { \ -const UInt32 value = (v); p++; *heads++ = pos - hash[value]; hash[value] = pos++; } } - -#define DEF_GetHeads(name, v) DEF_GetHeads2(name, v, ;) - -DEF_GetHeads2(2, (p[0] | ((UInt32)p[1] << 8)), hashMask = hashMask; crc = crc; ) -DEF_GetHeads(3, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8)) & hashMask) -DEF_GetHeads(4, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ (crc[p[3]] << 5)) & hashMask) -DEF_GetHeads(4b, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ ((UInt32)p[3] << 16)) & hashMask) -DEF_GetHeads(5, (crc[p[0]] ^ p[1] ^ ((UInt32)p[2] << 8) ^ (crc[p[3]] << 5) ^ (crc[p[4]] << 3)) & hashMask) - -void HashThreadFunc(CMatchFinderMt *mt) -{ - CMtSync *p = &mt->hashSync; - for (;;) - { - UInt32 numProcessedBlocks = 0; - Event_Wait(&p->canStart); - Event_Set(&p->wasStarted); - for (;;) - { - if (p->exit) - return; - if (p->stopWriting) - { - p->numProcessedBlocks = numProcessedBlocks; - Event_Set(&p->wasStopped); - break; - } - - { - CMatchFinder *mf = mt->MatchFinder; - if (MatchFinder_NeedMove(mf)) - { - CriticalSection_Enter(&mt->btSync.cs); - CriticalSection_Enter(&mt->hashSync.cs); - { - const Byte *beforePtr = MatchFinder_GetPointerToCurrentPos(mf); - const Byte *afterPtr; - MatchFinder_MoveBlock(mf); - afterPtr = MatchFinder_GetPointerToCurrentPos(mf); - mt->pointerToCurPos -= beforePtr - afterPtr; - mt->buffer -= beforePtr - afterPtr; - } - CriticalSection_Leave(&mt->btSync.cs); - CriticalSection_Leave(&mt->hashSync.cs); - continue; - } - - Semaphore_Wait(&p->freeSemaphore); - - MatchFinder_ReadIfRequired(mf); - if (mf->pos > (kMtMaxValForNormalize - kMtHashBlockSize)) - { - UInt32 subValue = (mf->pos - mf->historySize - 1); - MatchFinder_ReduceOffsets(mf, subValue); - MatchFinder_Normalize3(subValue, mf->hash + mf->fixedHashSize, mf->hashMask + 1); - } - { - UInt32 *heads = mt->hashBuf + ((numProcessedBlocks++) & kMtHashNumBlocksMask) * kMtHashBlockSize; - UInt32 num = mf->streamPos - mf->pos; - heads[0] = 2; - heads[1] = num; - if (num >= mf->numHashBytes) - { - num = num - mf->numHashBytes + 1; - if (num > kMtHashBlockSize - 2) - num = kMtHashBlockSize - 2; - mt->GetHeadsFunc(mf->buffer, mf->pos, mf->hash + mf->fixedHashSize, mf->hashMask, heads + 2, num, mf->crc); - heads[0] += num; - } - mf->pos += num; - mf->buffer += num; - } - } - - Semaphore_Release1(&p->filledSemaphore); - } - } -} - -void MatchFinderMt_GetNextBlock_Hash(CMatchFinderMt *p) -{ - MtSync_GetNextBlock(&p->hashSync); - p->hashBufPosLimit = p->hashBufPos = ((p->hashSync.numProcessedBlocks - 1) & kMtHashNumBlocksMask) * kMtHashBlockSize; - p->hashBufPosLimit += p->hashBuf[p->hashBufPos++]; - p->hashNumAvail = p->hashBuf[p->hashBufPos++]; -} - -#define kEmptyHashValue 0 - -/* #define MFMT_GM_INLINE */ - -#ifdef MFMT_GM_INLINE - -#define NO_INLINE MY_FAST_CALL - -Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, - UInt32 *_distances, UInt32 _maxLen, const UInt32 *hash, Int32 limit, UInt32 size, UInt32 *posRes) -{ - do - { - UInt32 *distances = _distances + 1; - UInt32 curMatch = pos - *hash++; - - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; - UInt32 cutValue = _cutValue; - UInt32 maxLen = _maxLen; - for (;;) - { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) - { - *ptr0 = *ptr1 = kEmptyHashValue; - break; - } - { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); - const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); - if (pb[len] == cur[len]) - { - if (++len != lenLimit && pb[len] == cur[len]) - while (++len != lenLimit) - if (pb[len] != cur[len]) - break; - if (maxLen < len) - { - *distances++ = maxLen = len; - *distances++ = delta - 1; - if (len == lenLimit) - { - *ptr1 = pair[0]; - *ptr0 = pair[1]; - break; - } - } - } - if (pb[len] < cur[len]) - { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; - } - } - } - pos++; - _cyclicBufferPos++; - cur++; - { - UInt32 num = (UInt32)(distances - _distances); - *_distances = num - 1; - _distances += num; - limit -= num; - } - } - while (limit > 0 && --size != 0); - *posRes = pos; - return limit; -} - -#endif - -void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) -{ - UInt32 numProcessed = 0; - UInt32 curPos = 2; - UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); - distances[1] = p->hashNumAvail; - while (curPos < limit) - { - if (p->hashBufPos == p->hashBufPosLimit) - { - MatchFinderMt_GetNextBlock_Hash(p); - distances[1] = numProcessed + p->hashNumAvail; - if (p->hashNumAvail >= p->numHashBytes) - continue; - for (; p->hashNumAvail != 0; p->hashNumAvail--) - distances[curPos++] = 0; - break; - } - { - UInt32 size = p->hashBufPosLimit - p->hashBufPos; - UInt32 lenLimit = p->matchMaxLen; - UInt32 pos = p->pos; - UInt32 cyclicBufferPos = p->cyclicBufferPos; - if (lenLimit >= p->hashNumAvail) - lenLimit = p->hashNumAvail; - { - UInt32 size2 = p->hashNumAvail - lenLimit + 1; - if (size2 < size) - size = size2; - size2 = p->cyclicBufferSize - cyclicBufferPos; - if (size2 < size) - size = size2; - } - #ifndef MFMT_GM_INLINE - while (curPos < limit && size-- != 0) - { - UInt32 *startDistances = distances + curPos; - UInt32 num = (UInt32)(GetMatchesSpec1(lenLimit, pos - p->hashBuf[p->hashBufPos++], - pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, - startDistances + 1, p->numHashBytes - 1) - startDistances); - *startDistances = num - 1; - curPos += num; - cyclicBufferPos++; - pos++; - p->buffer++; - } - #else - { - UInt32 posRes; - curPos = limit - GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, - distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, (Int32)(limit - curPos) , size, &posRes); - p->hashBufPos += posRes - pos; - cyclicBufferPos += posRes - pos; - p->buffer += posRes - pos; - pos = posRes; - } - #endif - - numProcessed += pos - p->pos; - p->hashNumAvail -= pos - p->pos; - p->pos = pos; - if (cyclicBufferPos == p->cyclicBufferSize) - cyclicBufferPos = 0; - p->cyclicBufferPos = cyclicBufferPos; - } - } - distances[0] = curPos; -} - -void BtFillBlock(CMatchFinderMt *p, UInt32 globalBlockIndex) -{ - CMtSync *sync = &p->hashSync; - if (!sync->needStart) - { - CriticalSection_Enter(&sync->cs); - sync->csWasEntered = True; - } - - BtGetMatches(p, p->btBuf + (globalBlockIndex & kMtBtNumBlocksMask) * kMtBtBlockSize); - - if (p->pos > kMtMaxValForNormalize - kMtBtBlockSize) - { - UInt32 subValue = p->pos - p->cyclicBufferSize; - MatchFinder_Normalize3(subValue, p->son, p->cyclicBufferSize * 2); - p->pos -= subValue; - } - - if (!sync->needStart) - { - CriticalSection_Leave(&sync->cs); - sync->csWasEntered = False; - } -} - -void BtThreadFunc(CMatchFinderMt *mt) -{ - CMtSync *p = &mt->btSync; - for (;;) - { - UInt32 blockIndex = 0; - Event_Wait(&p->canStart); - Event_Set(&p->wasStarted); - for (;;) - { - if (p->exit) - return; - if (p->stopWriting) - { - p->numProcessedBlocks = blockIndex; - MtSync_StopWriting(&mt->hashSync); - Event_Set(&p->wasStopped); - break; - } - Semaphore_Wait(&p->freeSemaphore); - BtFillBlock(mt, blockIndex++); - Semaphore_Release1(&p->filledSemaphore); - } - } -} - -void MatchFinderMt_Construct(CMatchFinderMt *p) -{ - p->hashBuf = 0; - MtSync_Construct(&p->hashSync); - MtSync_Construct(&p->btSync); -} - -void MatchFinderMt_FreeMem(CMatchFinderMt *p, ISzAlloc *alloc) -{ - alloc->Free(alloc, p->hashBuf); - p->hashBuf = 0; -} - -void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAlloc *alloc) -{ - MtSync_Destruct(&p->hashSync); - MtSync_Destruct(&p->btSync); - MatchFinderMt_FreeMem(p, alloc); -} - -#define kHashBufferSize (kMtHashBlockSize * kMtHashNumBlocks) -#define kBtBufferSize (kMtBtBlockSize * kMtBtNumBlocks) - -static unsigned MY_STD_CALL HashThreadFunc2(void *p) { HashThreadFunc((CMatchFinderMt *)p); return 0; } -static unsigned MY_STD_CALL BtThreadFunc2(void *p) -{ - Byte allocaDummy[0x180]; - int i = 0; - for (i = 0; i < 16; i++) - allocaDummy[i] = (Byte)i; - BtThreadFunc((CMatchFinderMt *)p); - return 0; -} - -SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore, - UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAlloc *alloc) -{ - CMatchFinder *mf = p->MatchFinder; - p->historySize = historySize; - if (kMtBtBlockSize <= matchMaxLen * 4) - return SZ_ERROR_PARAM; - if (p->hashBuf == 0) - { - p->hashBuf = (UInt32 *)alloc->Alloc(alloc, (kHashBufferSize + kBtBufferSize) * sizeof(UInt32)); - if (p->hashBuf == 0) - return SZ_ERROR_MEM; - p->btBuf = p->hashBuf + kHashBufferSize; - } - keepAddBufferBefore += (kHashBufferSize + kBtBufferSize); - keepAddBufferAfter += kMtHashBlockSize; - if (!MatchFinder_Create(mf, historySize, keepAddBufferBefore, matchMaxLen, keepAddBufferAfter, alloc)) - return SZ_ERROR_MEM; - - RINOK(MtSync_Create(&p->hashSync, HashThreadFunc2, p, kMtHashNumBlocks)); - RINOK(MtSync_Create(&p->btSync, BtThreadFunc2, p, kMtBtNumBlocks)); - return SZ_OK; -} - -/* Call it after ReleaseStream / SetStream */ -void MatchFinderMt_Init(CMatchFinderMt *p) -{ - CMatchFinder *mf = p->MatchFinder; - p->btBufPos = p->btBufPosLimit = 0; - p->hashBufPos = p->hashBufPosLimit = 0; - MatchFinder_Init(mf); - p->pointerToCurPos = MatchFinder_GetPointerToCurrentPos(mf); - p->btNumAvailBytes = 0; - p->lzPos = p->historySize + 1; - - p->hash = mf->hash; - p->fixedHashSize = mf->fixedHashSize; - p->crc = mf->crc; - - p->son = mf->son; - p->matchMaxLen = mf->matchMaxLen; - p->numHashBytes = mf->numHashBytes; - p->pos = mf->pos; - p->buffer = mf->buffer; - p->cyclicBufferPos = mf->cyclicBufferPos; - p->cyclicBufferSize = mf->cyclicBufferSize; - p->cutValue = mf->cutValue; -} - -/* ReleaseStream is required to finish multithreading */ -void MatchFinderMt_ReleaseStream(CMatchFinderMt *p) -{ - MtSync_StopWriting(&p->btSync); - /* p->MatchFinder->ReleaseStream(); */ -} - -void MatchFinderMt_Normalize(CMatchFinderMt *p) -{ - MatchFinder_Normalize3(p->lzPos - p->historySize - 1, p->hash, p->fixedHashSize); - p->lzPos = p->historySize + 1; -} - -void MatchFinderMt_GetNextBlock_Bt(CMatchFinderMt *p) -{ - UInt32 blockIndex; - MtSync_GetNextBlock(&p->btSync); - blockIndex = ((p->btSync.numProcessedBlocks - 1) & kMtBtNumBlocksMask); - p->btBufPosLimit = p->btBufPos = blockIndex * kMtBtBlockSize; - p->btBufPosLimit += p->btBuf[p->btBufPos++]; - p->btNumAvailBytes = p->btBuf[p->btBufPos++]; - if (p->lzPos >= kMtMaxValForNormalize - kMtBtBlockSize) - MatchFinderMt_Normalize(p); -} - -const Byte * MatchFinderMt_GetPointerToCurrentPos(CMatchFinderMt *p) -{ - return p->pointerToCurPos; -} - -#define GET_NEXT_BLOCK_IF_REQUIRED if (p->btBufPos == p->btBufPosLimit) MatchFinderMt_GetNextBlock_Bt(p); - -UInt32 MatchFinderMt_GetNumAvailableBytes(CMatchFinderMt *p) -{ - GET_NEXT_BLOCK_IF_REQUIRED; - return p->btNumAvailBytes; -} - -Byte MatchFinderMt_GetIndexByte(CMatchFinderMt *p, Int32 index) -{ - return p->pointerToCurPos[index]; -} - -UInt32 * MixMatches2(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) -{ - UInt32 hash2Value, curMatch2; - UInt32 *hash = p->hash; - const Byte *cur = p->pointerToCurPos; - UInt32 lzPos = p->lzPos; - MT_HASH2_CALC - - curMatch2 = hash[hash2Value]; - hash[hash2Value] = lzPos; - - if (curMatch2 >= matchMinPos) - if (cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) - { - *distances++ = 2; - *distances++ = lzPos - curMatch2 - 1; - } - return distances; -} - -UInt32 * MixMatches3(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) -{ - UInt32 hash2Value, hash3Value, curMatch2, curMatch3; - UInt32 *hash = p->hash; - const Byte *cur = p->pointerToCurPos; - UInt32 lzPos = p->lzPos; - MT_HASH3_CALC - - curMatch2 = hash[ hash2Value]; - curMatch3 = hash[kFix3HashSize + hash3Value]; - - hash[ hash2Value] = - hash[kFix3HashSize + hash3Value] = - lzPos; - - if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) - { - distances[1] = lzPos - curMatch2 - 1; - if (cur[(ptrdiff_t)curMatch2 - lzPos + 2] == cur[2]) - { - distances[0] = 3; - return distances + 2; - } - distances[0] = 2; - distances += 2; - } - if (curMatch3 >= matchMinPos && cur[(ptrdiff_t)curMatch3 - lzPos] == cur[0]) - { - *distances++ = 3; - *distances++ = lzPos - curMatch3 - 1; - } - return distances; -} - -/* -UInt32 *MixMatches4(CMatchFinderMt *p, UInt32 matchMinPos, UInt32 *distances) -{ - UInt32 hash2Value, hash3Value, hash4Value, curMatch2, curMatch3, curMatch4; - UInt32 *hash = p->hash; - const Byte *cur = p->pointerToCurPos; - UInt32 lzPos = p->lzPos; - MT_HASH4_CALC - - curMatch2 = hash[ hash2Value]; - curMatch3 = hash[kFix3HashSize + hash3Value]; - curMatch4 = hash[kFix4HashSize + hash4Value]; - - hash[ hash2Value] = - hash[kFix3HashSize + hash3Value] = - hash[kFix4HashSize + hash4Value] = - lzPos; - - if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) - { - distances[1] = lzPos - curMatch2 - 1; - if (cur[(ptrdiff_t)curMatch2 - lzPos + 2] == cur[2]) - { - distances[0] = (cur[(ptrdiff_t)curMatch2 - lzPos + 3] == cur[3]) ? 4 : 3; - return distances + 2; - } - distances[0] = 2; - distances += 2; - } - if (curMatch3 >= matchMinPos && cur[(ptrdiff_t)curMatch3 - lzPos] == cur[0]) - { - distances[1] = lzPos - curMatch3 - 1; - if (cur[(ptrdiff_t)curMatch3 - lzPos + 3] == cur[3]) - { - distances[0] = 4; - return distances + 2; - } - distances[0] = 3; - distances += 2; - } - - if (curMatch4 >= matchMinPos) - if ( - cur[(ptrdiff_t)curMatch4 - lzPos] == cur[0] && - cur[(ptrdiff_t)curMatch4 - lzPos + 3] == cur[3] - ) - { - *distances++ = 4; - *distances++ = lzPos - curMatch4 - 1; - } - return distances; -} -*/ - -#define INCREASE_LZ_POS p->lzPos++; p->pointerToCurPos++; - -UInt32 MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *distances) -{ - const UInt32 *btBuf = p->btBuf + p->btBufPos; - UInt32 len = *btBuf++; - p->btBufPos += 1 + len; - p->btNumAvailBytes--; - { - UInt32 i; - for (i = 0; i < len; i += 2) - { - *distances++ = *btBuf++; - *distances++ = *btBuf++; - } - } - INCREASE_LZ_POS - return len; -} - -UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *distances) -{ - const UInt32 *btBuf = p->btBuf + p->btBufPos; - UInt32 len = *btBuf++; - p->btBufPos += 1 + len; - - if (len == 0) - { - if (p->btNumAvailBytes-- >= 4) - len = (UInt32)(p->MixMatchesFunc(p, p->lzPos - p->historySize, distances) - (distances)); - } - else - { - /* Condition: there are matches in btBuf with length < p->numHashBytes */ - UInt32 *distances2; - p->btNumAvailBytes--; - distances2 = p->MixMatchesFunc(p, p->lzPos - btBuf[1], distances); - do - { - *distances2++ = *btBuf++; - *distances2++ = *btBuf++; - } - while ((len -= 2) != 0); - len = (UInt32)(distances2 - (distances)); - } - INCREASE_LZ_POS - return len; -} - -#define SKIP_HEADER2 do { GET_NEXT_BLOCK_IF_REQUIRED -#define SKIP_HEADER(n) SKIP_HEADER2 if (p->btNumAvailBytes-- >= (n)) { const Byte *cur = p->pointerToCurPos; UInt32 *hash = p->hash; -#define SKIP_FOOTER } INCREASE_LZ_POS p->btBufPos += p->btBuf[p->btBufPos] + 1; } while (--num != 0); - -void MatchFinderMt0_Skip(CMatchFinderMt *p, UInt32 num) -{ - SKIP_HEADER2 { p->btNumAvailBytes--; - SKIP_FOOTER -} - -void MatchFinderMt2_Skip(CMatchFinderMt *p, UInt32 num) -{ - SKIP_HEADER(2) - UInt32 hash2Value; - MT_HASH2_CALC - hash[hash2Value] = p->lzPos; - SKIP_FOOTER -} - -void MatchFinderMt3_Skip(CMatchFinderMt *p, UInt32 num) -{ - SKIP_HEADER(3) - UInt32 hash2Value, hash3Value; - MT_HASH3_CALC - hash[kFix3HashSize + hash3Value] = - hash[ hash2Value] = - p->lzPos; - SKIP_FOOTER -} - -/* -void MatchFinderMt4_Skip(CMatchFinderMt *p, UInt32 num) -{ - SKIP_HEADER(4) - UInt32 hash2Value, hash3Value, hash4Value; - MT_HASH4_CALC - hash[kFix4HashSize + hash4Value] = - hash[kFix3HashSize + hash3Value] = - hash[ hash2Value] = - p->lzPos; - SKIP_FOOTER -} -*/ - -void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable) -{ - vTable->Init = (Mf_Init_Func)MatchFinderMt_Init; - vTable->GetIndexByte = (Mf_GetIndexByte_Func)MatchFinderMt_GetIndexByte; - vTable->GetNumAvailableBytes = (Mf_GetNumAvailableBytes_Func)MatchFinderMt_GetNumAvailableBytes; - vTable->GetPointerToCurrentPos = (Mf_GetPointerToCurrentPos_Func)MatchFinderMt_GetPointerToCurrentPos; - vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt_GetMatches; - switch(p->MatchFinder->numHashBytes) - { - case 2: - p->GetHeadsFunc = GetHeads2; - p->MixMatchesFunc = (Mf_Mix_Matches)0; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt0_Skip; - vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches; - break; - case 3: - p->GetHeadsFunc = GetHeads3; - p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches2; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt2_Skip; - break; - default: - /* case 4: */ - p->GetHeadsFunc = p->MatchFinder->bigHash ? GetHeads4b : GetHeads4; - /* p->GetHeadsFunc = GetHeads4; */ - p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches3; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt3_Skip; - break; - /* - default: - p->GetHeadsFunc = GetHeads5; - p->MixMatchesFunc = (Mf_Mix_Matches)MixMatches4; - vTable->Skip = (Mf_Skip_Func)MatchFinderMt4_Skip; - break; - */ - } -} diff --git a/ext/7z/LzFindMt.h b/ext/7z/LzFindMt.h deleted file mode 100644 index b7ead2d4..00000000 --- a/ext/7z/LzFindMt.h +++ /dev/null @@ -1,97 +0,0 @@ -/* LzFindMt.h -- multithreaded Match finder for LZ algorithms -2008-10-04 : Igor Pavlov : Public domain */ - -#ifndef __LZFINDMT_H -#define __LZFINDMT_H - -#include "Threads.h" -#include "LzFind.h" - -#define kMtHashBlockSize (1 << 13) -#define kMtHashNumBlocks (1 << 3) -#define kMtHashNumBlocksMask (kMtHashNumBlocks - 1) - -#define kMtBtBlockSize (1 << 14) -#define kMtBtNumBlocks (1 << 6) -#define kMtBtNumBlocksMask (kMtBtNumBlocks - 1) - -typedef struct _CMtSync -{ - Bool wasCreated; - Bool needStart; - Bool exit; - Bool stopWriting; - - CThread thread; - CAutoResetEvent canStart; - CAutoResetEvent wasStarted; - CAutoResetEvent wasStopped; - CSemaphore freeSemaphore; - CSemaphore filledSemaphore; - Bool csWasInitialized; - Bool csWasEntered; - CCriticalSection cs; - UInt32 numProcessedBlocks; -} CMtSync; - -typedef UInt32 * (*Mf_Mix_Matches)(void *p, UInt32 matchMinPos, UInt32 *distances); - -/* kMtCacheLineDummy must be >= size_of_CPU_cache_line */ -#define kMtCacheLineDummy 128 - -typedef void (*Mf_GetHeads)(const Byte *buffer, UInt32 pos, - UInt32 *hash, UInt32 hashMask, UInt32 *heads, UInt32 numHeads, const UInt32 *crc); - -typedef struct _CMatchFinderMt -{ - /* LZ */ - const Byte *pointerToCurPos; - UInt32 *btBuf; - UInt32 btBufPos; - UInt32 btBufPosLimit; - UInt32 lzPos; - UInt32 btNumAvailBytes; - - UInt32 *hash; - UInt32 fixedHashSize; - UInt32 historySize; - const UInt32 *crc; - - Mf_Mix_Matches MixMatchesFunc; - - /* LZ + BT */ - CMtSync btSync; - Byte btDummy[kMtCacheLineDummy]; - - /* BT */ - UInt32 *hashBuf; - UInt32 hashBufPos; - UInt32 hashBufPosLimit; - UInt32 hashNumAvail; - - CLzRef *son; - UInt32 matchMaxLen; - UInt32 numHashBytes; - UInt32 pos; - Byte *buffer; - UInt32 cyclicBufferPos; - UInt32 cyclicBufferSize; /* it must be historySize + 1 */ - UInt32 cutValue; - - /* BT + Hash */ - CMtSync hashSync; - /* Byte hashDummy[kMtCacheLineDummy]; */ - - /* Hash */ - Mf_GetHeads GetHeadsFunc; - CMatchFinder *MatchFinder; -} CMatchFinderMt; - -void MatchFinderMt_Construct(CMatchFinderMt *p); -void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAlloc *alloc); -SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore, - UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAlloc *alloc); -void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable); -void MatchFinderMt_ReleaseStream(CMatchFinderMt *p); - -#endif diff --git a/ext/7z/LzHash.h b/ext/7z/LzHash.h deleted file mode 100644 index c9234175..00000000 --- a/ext/7z/LzHash.h +++ /dev/null @@ -1,54 +0,0 @@ -/* LzHash.h -- HASH functions for LZ algorithms -2008-10-04 : Igor Pavlov : Public domain */ - -#ifndef __LZHASH_H -#define __LZHASH_H - -#define kHash2Size (1 << 10) -#define kHash3Size (1 << 16) -#define kHash4Size (1 << 20) - -#define kFix3HashSize (kHash2Size) -#define kFix4HashSize (kHash2Size + kHash3Size) -#define kFix5HashSize (kHash2Size + kHash3Size + kHash4Size) - -#define HASH2_CALC hashValue = cur[0] | ((UInt32)cur[1] << 8); - -#define HASH3_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - hash2Value = temp & (kHash2Size - 1); \ - hashValue = (temp ^ ((UInt32)cur[2] << 8)) & p->hashMask; } - -#define HASH4_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - hash2Value = temp & (kHash2Size - 1); \ - hash3Value = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); \ - hashValue = (temp ^ ((UInt32)cur[2] << 8) ^ (p->crc[cur[3]] << 5)) & p->hashMask; } - -#define HASH5_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - hash2Value = temp & (kHash2Size - 1); \ - hash3Value = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); \ - hash4Value = (temp ^ ((UInt32)cur[2] << 8) ^ (p->crc[cur[3]] << 5)); \ - hashValue = (hash4Value ^ (p->crc[cur[4]] << 3)) & p->hashMask; \ - hash4Value &= (kHash4Size - 1); } - -/* #define HASH_ZIP_CALC hashValue = ((cur[0] | ((UInt32)cur[1] << 8)) ^ p->crc[cur[2]]) & 0xFFFF; */ -#define HASH_ZIP_CALC hashValue = ((cur[2] | ((UInt32)cur[0] << 8)) ^ p->crc[cur[1]]) & 0xFFFF; - - -#define MT_HASH2_CALC \ - hash2Value = (p->crc[cur[0]] ^ cur[1]) & (kHash2Size - 1); - -#define MT_HASH3_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - hash2Value = temp & (kHash2Size - 1); \ - hash3Value = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); } - -#define MT_HASH4_CALC { \ - UInt32 temp = p->crc[cur[0]] ^ cur[1]; \ - hash2Value = temp & (kHash2Size - 1); \ - hash3Value = (temp ^ ((UInt32)cur[2] << 8)) & (kHash3Size - 1); \ - hash4Value = (temp ^ ((UInt32)cur[2] << 8) ^ (p->crc[cur[3]] << 5)) & (kHash4Size - 1); } - -#endif diff --git a/ext/7z/LzmaDec.c b/ext/7z/LzmaDec.c deleted file mode 100644 index eb525017..00000000 --- a/ext/7z/LzmaDec.c +++ /dev/null @@ -1,1007 +0,0 @@ -/* LzmaDec.c -- LZMA Decoder -2008-11-06 : Igor Pavlov : Public domain */ - -#include "LzmaDec.h" - -#include - -#define kNumTopBits 24 -#define kTopValue ((UInt32)1 << kNumTopBits) - -#define kNumBitModelTotalBits 11 -#define kBitModelTotal (1 << kNumBitModelTotalBits) -#define kNumMoveBits 5 - -#define RC_INIT_SIZE 5 - -#define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } - -#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) -#define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); -#define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); -#define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ - { UPDATE_0(p); i = (i + i); A0; } else \ - { UPDATE_1(p); i = (i + i) + 1; A1; } -#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;) - -#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); } -#define TREE_DECODE(probs, limit, i) \ - { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } - -/* #define _LZMA_SIZE_OPT */ - -#ifdef _LZMA_SIZE_OPT -#define TREE_6_DECODE(probs, i) TREE_DECODE(probs, (1 << 6), i) -#else -#define TREE_6_DECODE(probs, i) \ - { i = 1; \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - TREE_GET_BIT(probs, i); \ - i -= 0x40; } -#endif - -#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } - -#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) -#define UPDATE_0_CHECK range = bound; -#define UPDATE_1_CHECK range -= bound; code -= bound; -#define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \ - { UPDATE_0_CHECK; i = (i + i); A0; } else \ - { UPDATE_1_CHECK; i = (i + i) + 1; A1; } -#define GET_BIT_CHECK(p, i) GET_BIT2_CHECK(p, i, ; , ;) -#define TREE_DECODE_CHECK(probs, limit, i) \ - { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } - - -#define kNumPosBitsMax 4 -#define kNumPosStatesMax (1 << kNumPosBitsMax) - -#define kLenNumLowBits 3 -#define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) -#define kLenNumHighBits 8 -#define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define LenChoice 0 -#define LenChoice2 (LenChoice + 1) -#define LenLow (LenChoice2 + 1) -#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) -#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) -#define kNumLenProbs (LenHigh + kLenNumHighSymbols) - - -#define kNumStates 12 -#define kNumLitStates 7 - -#define kStartPosModelIndex 4 -#define kEndPosModelIndex 14 -#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) - -#define kNumPosSlotBits 6 -#define kNumLenToPosStates 4 - -#define kNumAlignBits 4 -#define kAlignTableSize (1 << kNumAlignBits) - -#define kMatchMinLen 2 -#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) - -#define IsMatch 0 -#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) -#define IsRepG0 (IsRep + kNumStates) -#define IsRepG1 (IsRepG0 + kNumStates) -#define IsRepG2 (IsRepG1 + kNumStates) -#define IsRep0Long (IsRepG2 + kNumStates) -#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) -#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) -#define LenCoder (Align + kAlignTableSize) -#define RepLenCoder (LenCoder + kNumLenProbs) -#define Literal (RepLenCoder + kNumLenProbs) - -#define LZMA_BASE_SIZE 1846 -#define LZMA_LIT_SIZE 768 - -#define LzmaProps_GetNumProbs(p) ((UInt32)LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) - -#if Literal != LZMA_BASE_SIZE -StopCompilingDueBUG -#endif - -static const Byte kLiteralNextStates[kNumStates * 2] = -{ - 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5, - 7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10 -}; - -#define LZMA_DIC_MIN (1 << 12) - -/* First LZMA-symbol is always decoded. -And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization -Out: - Result: - SZ_OK - OK - SZ_ERROR_DATA - Error - p->remainLen: - < kMatchSpecLenStart : normal remain - = kMatchSpecLenStart : finished - = kMatchSpecLenStart + 1 : Flush marker - = kMatchSpecLenStart + 2 : State Init Marker -*/ - -static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte *bufLimit) -{ - CLzmaProb *probs = p->probs; - - unsigned state = p->state; - UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; - unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; - unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1; - unsigned lc = p->prop.lc; - - Byte *dic = p->dic; - SizeT dicBufSize = p->dicBufSize; - SizeT dicPos = p->dicPos; - - UInt32 processedPos = p->processedPos; - UInt32 checkDicSize = p->checkDicSize; - unsigned len = 0; - - const Byte *buf = p->buf; - UInt32 range = p->range; - UInt32 code = p->code; - - do - { - CLzmaProb *prob; - UInt32 bound; - unsigned ttt; - unsigned posState = processedPos & pbMask; - - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; - IF_BIT_0(prob) - { - unsigned symbol; - UPDATE_0(prob); - prob = probs + Literal; - if (checkDicSize != 0 || processedPos != 0) - prob += (LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) + - (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc)))); - - if (state < kNumLitStates) - { - symbol = 1; - do { GET_BIT(prob + symbol, symbol) } while (symbol < 0x100); - } - else - { - unsigned matchByte = p->dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)]; - unsigned offs = 0x100; - symbol = 1; - do - { - unsigned bit; - CLzmaProb *probLit; - matchByte <<= 1; - bit = (matchByte & offs); - probLit = prob + offs + bit + symbol; - GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit) - } - while (symbol < 0x100); - } - dic[dicPos++] = (Byte)symbol; - processedPos++; - - state = kLiteralNextStates[state]; - /* if (state < 4) state = 0; else if (state < 10) state -= 3; else state -= 6; */ - continue; - } - else - { - UPDATE_1(prob); - prob = probs + IsRep + state; - IF_BIT_0(prob) - { - UPDATE_0(prob); - state += kNumStates; - prob = probs + LenCoder; - } - else - { - UPDATE_1(prob); - if (checkDicSize == 0 && processedPos == 0) - return SZ_ERROR_DATA; - prob = probs + IsRepG0 + state; - IF_BIT_0(prob) - { - UPDATE_0(prob); - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; - IF_BIT_0(prob) - { - UPDATE_0(prob); - dic[dicPos] = dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)]; - dicPos++; - processedPos++; - state = state < kNumLitStates ? 9 : 11; - continue; - } - UPDATE_1(prob); - } - else - { - UInt32 distance; - UPDATE_1(prob); - prob = probs + IsRepG1 + state; - IF_BIT_0(prob) - { - UPDATE_0(prob); - distance = rep1; - } - else - { - UPDATE_1(prob); - prob = probs + IsRepG2 + state; - IF_BIT_0(prob) - { - UPDATE_0(prob); - distance = rep2; - } - else - { - UPDATE_1(prob); - distance = rep3; - rep3 = rep2; - } - rep2 = rep1; - } - rep1 = rep0; - rep0 = distance; - } - state = state < kNumLitStates ? 8 : 11; - prob = probs + RepLenCoder; - } - { - unsigned limit, offset; - CLzmaProb *probLen = prob + LenChoice; - IF_BIT_0(probLen) - { - UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); - offset = 0; - limit = (1 << kLenNumLowBits); - } - else - { - UPDATE_1(probLen); - probLen = prob + LenChoice2; - IF_BIT_0(probLen) - { - UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); - offset = kLenNumLowSymbols; - limit = (1 << kLenNumMidBits); - } - else - { - UPDATE_1(probLen); - probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; - limit = (1 << kLenNumHighBits); - } - } - TREE_DECODE(probLen, limit, len); - len += offset; - } - - if (state >= kNumStates) - { - UInt32 distance; - prob = probs + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); - TREE_6_DECODE(prob, distance); - if (distance >= kStartPosModelIndex) - { - unsigned posSlot = (unsigned)distance; - int numDirectBits = (int)(((distance >> 1) - 1)); - distance = (2 | (distance & 1)); - if (posSlot < kEndPosModelIndex) - { - distance <<= numDirectBits; - prob = probs + SpecPos + distance - posSlot - 1; - { - UInt32 mask = 1; - unsigned i = 1; - do - { - GET_BIT2(prob + i, i, ; , distance |= mask); - mask <<= 1; - } - while (--numDirectBits != 0); - } - } - else - { - numDirectBits -= kNumAlignBits; - do - { - NORMALIZE - range >>= 1; - - { - UInt32 t; - code -= range; - t = (0 - ((UInt32)code >> 31)); /* (UInt32)((Int32)code >> 31) */ - distance = (distance << 1) + (t + 1); - code += range & t; - } - /* - distance <<= 1; - if (code >= range) - { - code -= range; - distance |= 1; - } - */ - } - while (--numDirectBits != 0); - prob = probs + Align; - distance <<= kNumAlignBits; - { - unsigned i = 1; - GET_BIT2(prob + i, i, ; , distance |= 1); - GET_BIT2(prob + i, i, ; , distance |= 2); - GET_BIT2(prob + i, i, ; , distance |= 4); - GET_BIT2(prob + i, i, ; , distance |= 8); - } - if (distance == (UInt32)0xFFFFFFFF) - { - len += kMatchSpecLenStart; - state -= kNumStates; - break; - } - } - } - rep3 = rep2; - rep2 = rep1; - rep1 = rep0; - rep0 = distance + 1; - if (checkDicSize == 0) - { - if (distance >= processedPos) - return SZ_ERROR_DATA; - } - else if (distance >= checkDicSize) - return SZ_ERROR_DATA; - state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; - /* state = kLiteralNextStates[state]; */ - } - - len += kMatchMinLen; - - if (limit == dicPos) - return SZ_ERROR_DATA; - { - SizeT rem = limit - dicPos; - unsigned curLen = ((rem < len) ? (unsigned)rem : len); - SizeT pos = (dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0); - - processedPos += curLen; - - len -= curLen; - if (pos + curLen <= dicBufSize) - { - Byte *dest = dic + dicPos; - ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; - const Byte *lim = dest + curLen; - dicPos += curLen; - do - *(dest) = (Byte)*(dest + src); - while (++dest != lim); - } - else - { - do - { - dic[dicPos++] = dic[pos]; - if (++pos == dicBufSize) - pos = 0; - } - while (--curLen != 0); - } - } - } - } - while (dicPos < limit && buf < bufLimit); - NORMALIZE; - p->buf = buf; - p->range = range; - p->code = code; - p->remainLen = len; - p->dicPos = dicPos; - p->processedPos = processedPos; - p->reps[0] = rep0; - p->reps[1] = rep1; - p->reps[2] = rep2; - p->reps[3] = rep3; - p->state = state; - - return SZ_OK; -} - -static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) -{ - if (p->remainLen != 0 && p->remainLen < kMatchSpecLenStart) - { - Byte *dic = p->dic; - SizeT dicPos = p->dicPos; - SizeT dicBufSize = p->dicBufSize; - unsigned len = p->remainLen; - UInt32 rep0 = p->reps[0]; - if (limit - dicPos < len) - len = (unsigned)(limit - dicPos); - - if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) - p->checkDicSize = p->prop.dicSize; - - p->processedPos += len; - p->remainLen -= len; - while (len-- != 0) - { - dic[dicPos] = dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)]; - dicPos++; - } - p->dicPos = dicPos; - } -} - -static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) -{ - do - { - SizeT limit2 = limit; - if (p->checkDicSize == 0) - { - UInt32 rem = p->prop.dicSize - p->processedPos; - if (limit - p->dicPos > rem) - limit2 = p->dicPos + rem; - } - RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit)); - if (p->processedPos >= p->prop.dicSize) - p->checkDicSize = p->prop.dicSize; - LzmaDec_WriteRem(p, limit); - } - while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); - - if (p->remainLen > kMatchSpecLenStart) - { - p->remainLen = kMatchSpecLenStart; - } - return 0; -} - -typedef enum -{ - DUMMY_ERROR, /* unexpected end of input stream */ - DUMMY_LIT, - DUMMY_MATCH, - DUMMY_REP -} ELzmaDummy; - -static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inSize) -{ - UInt32 range = p->range; - UInt32 code = p->code; - const Byte *bufLimit = buf + inSize; - CLzmaProb *probs = p->probs; - unsigned state = p->state; - ELzmaDummy res; - - { - CLzmaProb *prob; - UInt32 bound; - unsigned ttt; - unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1); - - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK - - /* if (bufLimit - buf >= 7) return DUMMY_LIT; */ - - prob = probs + Literal; - if (p->checkDicSize != 0 || p->processedPos != 0) - prob += (LZMA_LIT_SIZE * - ((((p->processedPos) & ((1 << (p->prop.lp)) - 1)) << p->prop.lc) + - (p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc)))); - - if (state < kNumLitStates) - { - unsigned symbol = 1; - do { GET_BIT_CHECK(prob + symbol, symbol) } while (symbol < 0x100); - } - else - { - unsigned matchByte = p->dic[p->dicPos - p->reps[0] + - ((p->dicPos < p->reps[0]) ? p->dicBufSize : 0)]; - unsigned offs = 0x100; - unsigned symbol = 1; - do - { - unsigned bit; - CLzmaProb *probLit; - matchByte <<= 1; - bit = (matchByte & offs); - probLit = prob + offs + bit + symbol; - GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit) - } - while (symbol < 0x100); - } - res = DUMMY_LIT; - } - else - { - unsigned len; - UPDATE_1_CHECK; - - prob = probs + IsRep + state; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - state = 0; - prob = probs + LenCoder; - res = DUMMY_MATCH; - } - else - { - UPDATE_1_CHECK; - res = DUMMY_REP; - prob = probs + IsRepG0 + state; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - NORMALIZE_CHECK; - return DUMMY_REP; - } - else - { - UPDATE_1_CHECK; - } - } - else - { - UPDATE_1_CHECK; - prob = probs + IsRepG1 + state; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - } - else - { - UPDATE_1_CHECK; - prob = probs + IsRepG2 + state; - IF_BIT_0_CHECK(prob) - { - UPDATE_0_CHECK; - } - else - { - UPDATE_1_CHECK; - } - } - } - state = kNumStates; - prob = probs + RepLenCoder; - } - { - unsigned limit, offset; - CLzmaProb *probLen = prob + LenChoice; - IF_BIT_0_CHECK(probLen) - { - UPDATE_0_CHECK; - probLen = prob + LenLow + (posState << kLenNumLowBits); - offset = 0; - limit = 1 << kLenNumLowBits; - } - else - { - UPDATE_1_CHECK; - probLen = prob + LenChoice2; - IF_BIT_0_CHECK(probLen) - { - UPDATE_0_CHECK; - probLen = prob + LenMid + (posState << kLenNumMidBits); - offset = kLenNumLowSymbols; - limit = 1 << kLenNumMidBits; - } - else - { - UPDATE_1_CHECK; - probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; - limit = 1 << kLenNumHighBits; - } - } - TREE_DECODE_CHECK(probLen, limit, len); - len += offset; - } - - if (state < 4) - { - unsigned posSlot; - prob = probs + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << - kNumPosSlotBits); - TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); - if (posSlot >= kStartPosModelIndex) - { - int numDirectBits = ((posSlot >> 1) - 1); - - /* if (bufLimit - buf >= 8) return DUMMY_MATCH; */ - - if (posSlot < kEndPosModelIndex) - { - prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1; - } - else - { - numDirectBits -= kNumAlignBits; - do - { - NORMALIZE_CHECK - range >>= 1; - code -= range & (((code - range) >> 31) - 1); - /* if (code >= range) code -= range; */ - } - while (--numDirectBits != 0); - prob = probs + Align; - numDirectBits = kNumAlignBits; - } - { - unsigned i = 1; - do - { - GET_BIT_CHECK(prob + i, i); - } - while (--numDirectBits != 0); - } - } - } - } - } - NORMALIZE_CHECK; - return res; -} - - -static void LzmaDec_InitRc(CLzmaDec *p, const Byte *data) -{ - p->code = ((UInt32)data[1] << 24) | ((UInt32)data[2] << 16) | ((UInt32)data[3] << 8) | ((UInt32)data[4]); - p->range = 0xFFFFFFFF; - p->needFlush = 0; -} - -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState) -{ - p->needFlush = 1; - p->remainLen = 0; - p->tempBufSize = 0; - - if (initDic) - { - p->processedPos = 0; - p->checkDicSize = 0; - p->needInitState = 1; - } - if (initState) - p->needInitState = 1; -} - -void LzmaDec_Init(CLzmaDec *p) -{ - p->dicPos = 0; - LzmaDec_InitDicAndState(p, True, True); -} - -static void LzmaDec_InitStateReal(CLzmaDec *p) -{ - UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (p->prop.lc + p->prop.lp)); - UInt32 i; - CLzmaProb *probs = p->probs; - for (i = 0; i < numProbs; i++) - probs[i] = kBitModelTotal >> 1; - p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; - p->state = 0; - p->needInitState = 0; -} - -SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, - ELzmaFinishMode finishMode, ELzmaStatus *status) -{ - SizeT inSize = *srcLen; - (*srcLen) = 0; - LzmaDec_WriteRem(p, dicLimit); - - *status = LZMA_STATUS_NOT_SPECIFIED; - - while (p->remainLen != kMatchSpecLenStart) - { - int checkEndMarkNow; - - if (p->needFlush != 0) - { - for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) - p->tempBuf[p->tempBufSize++] = *src++; - if (p->tempBufSize < RC_INIT_SIZE) - { - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (p->tempBuf[0] != 0) - return SZ_ERROR_DATA; - - LzmaDec_InitRc(p, p->tempBuf); - p->tempBufSize = 0; - } - - checkEndMarkNow = 0; - if (p->dicPos >= dicLimit) - { - if (p->remainLen == 0 && p->code == 0) - { - *status = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK; - return SZ_OK; - } - if (finishMode == LZMA_FINISH_ANY) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_OK; - } - if (p->remainLen != 0) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; - } - checkEndMarkNow = 1; - } - - if (p->needInitState) - LzmaDec_InitStateReal(p); - - if (p->tempBufSize == 0) - { - SizeT processed; - const Byte *bufLimit; - if (inSize < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) - { - int dummyRes = LzmaDec_TryDummy(p, src, inSize); - if (dummyRes == DUMMY_ERROR) - { - memcpy(p->tempBuf, src, inSize); - p->tempBufSize = (unsigned)inSize; - (*srcLen) += inSize; - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (checkEndMarkNow && dummyRes != DUMMY_MATCH) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; - } - bufLimit = src; - } - else - bufLimit = src + inSize - LZMA_REQUIRED_INPUT_MAX; - p->buf = src; - if (LzmaDec_DecodeReal2(p, dicLimit, bufLimit) != 0) - return SZ_ERROR_DATA; - processed = (SizeT)(p->buf - src); - (*srcLen) += processed; - src += processed; - inSize -= processed; - } - else - { - unsigned rem = p->tempBufSize, lookAhead = 0; - while (rem < LZMA_REQUIRED_INPUT_MAX && lookAhead < inSize) - p->tempBuf[rem++] = src[lookAhead++]; - p->tempBufSize = rem; - if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) - { - int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, rem); - if (dummyRes == DUMMY_ERROR) - { - (*srcLen) += lookAhead; - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (checkEndMarkNow && dummyRes != DUMMY_MATCH) - { - *status = LZMA_STATUS_NOT_FINISHED; - return SZ_ERROR_DATA; - } - } - p->buf = p->tempBuf; - if (LzmaDec_DecodeReal2(p, dicLimit, p->buf) != 0) - return SZ_ERROR_DATA; - lookAhead -= (rem - (unsigned)(p->buf - p->tempBuf)); - (*srcLen) += lookAhead; - src += lookAhead; - inSize -= lookAhead; - p->tempBufSize = 0; - } - } - if (p->code == 0) - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA; -} - -SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) -{ - SizeT outSize = *destLen; - SizeT inSize = *srcLen; - *srcLen = *destLen = 0; - for (;;) - { - SizeT inSizeCur = inSize, outSizeCur, dicPos; - ELzmaFinishMode curFinishMode; - SRes res; - if (p->dicPos == p->dicBufSize) - p->dicPos = 0; - dicPos = p->dicPos; - if (outSize > p->dicBufSize - dicPos) - { - outSizeCur = p->dicBufSize; - curFinishMode = LZMA_FINISH_ANY; - } - else - { - outSizeCur = dicPos + outSize; - curFinishMode = finishMode; - } - - res = LzmaDec_DecodeToDic(p, outSizeCur, src, &inSizeCur, curFinishMode, status); - src += inSizeCur; - inSize -= inSizeCur; - *srcLen += inSizeCur; - outSizeCur = p->dicPos - dicPos; - memcpy(dest, p->dic + dicPos, outSizeCur); - dest += outSizeCur; - outSize -= outSizeCur; - *destLen += outSizeCur; - if (res != 0) - return res; - if (outSizeCur == 0 || outSize == 0) - return SZ_OK; - } -} - -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc) -{ - alloc->Free(alloc, p->probs); - p->probs = 0; -} - -static void LzmaDec_FreeDict(CLzmaDec *p, ISzAlloc *alloc) -{ - alloc->Free(alloc, p->dic); - p->dic = 0; -} - -void LzmaDec_Free(CLzmaDec *p, ISzAlloc *alloc) -{ - LzmaDec_FreeProbs(p, alloc); - LzmaDec_FreeDict(p, alloc); -} - -SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) -{ - UInt32 dicSize; - Byte d; - - if (size < LZMA_PROPS_SIZE) - return SZ_ERROR_UNSUPPORTED; - else - dicSize = data[1] | ((UInt32)data[2] << 8) | ((UInt32)data[3] << 16) | ((UInt32)data[4] << 24); - - if (dicSize < LZMA_DIC_MIN) - dicSize = LZMA_DIC_MIN; - p->dicSize = dicSize; - - d = data[0]; - if (d >= (9 * 5 * 5)) - return SZ_ERROR_UNSUPPORTED; - - p->lc = d % 9; - d /= 9; - p->pb = d / 5; - p->lp = d % 5; - - return SZ_OK; -} - -static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAlloc *alloc) -{ - UInt32 numProbs = LzmaProps_GetNumProbs(propNew); - if (p->probs == 0 || numProbs != p->numProbs) - { - LzmaDec_FreeProbs(p, alloc); - p->probs = (CLzmaProb *)alloc->Alloc(alloc, numProbs * sizeof(CLzmaProb)); - p->numProbs = numProbs; - if (p->probs == 0) - return SZ_ERROR_MEM; - } - return SZ_OK; -} - -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) -{ - CLzmaProps propNew; - RINOK(LzmaProps_Decode(&propNew, props, propsSize)); - RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); - p->prop = propNew; - return SZ_OK; -} - -SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) -{ - CLzmaProps propNew; - SizeT dicBufSize; - RINOK(LzmaProps_Decode(&propNew, props, propsSize)); - RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); - dicBufSize = propNew.dicSize; - if (p->dic == 0 || dicBufSize != p->dicBufSize) - { - LzmaDec_FreeDict(p, alloc); - p->dic = (Byte *)alloc->Alloc(alloc, dicBufSize); - if (p->dic == 0) - { - LzmaDec_FreeProbs(p, alloc); - return SZ_ERROR_MEM; - } - } - p->dicBufSize = dicBufSize; - p->prop = propNew; - return SZ_OK; -} - -SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAlloc *alloc) -{ - CLzmaDec p; - SRes res; - SizeT inSize = *srcLen; - SizeT outSize = *destLen; - *srcLen = *destLen = 0; - if (inSize < RC_INIT_SIZE) - return SZ_ERROR_INPUT_EOF; - - LzmaDec_Construct(&p); - res = LzmaDec_AllocateProbs(&p, propData, propSize, alloc); - if (res != 0) - return res; - p.dic = dest; - p.dicBufSize = outSize; - - LzmaDec_Init(&p); - - *srcLen = inSize; - res = LzmaDec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status); - - if (res == SZ_OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT) - res = SZ_ERROR_INPUT_EOF; - - (*destLen) = p.dicPos; - LzmaDec_FreeProbs(&p, alloc); - return res; -} diff --git a/ext/7z/LzmaDec.h b/ext/7z/LzmaDec.h deleted file mode 100644 index ad7d7057..00000000 --- a/ext/7z/LzmaDec.h +++ /dev/null @@ -1,223 +0,0 @@ -/* LzmaDec.h -- LZMA Decoder -2008-10-04 : Igor Pavlov : Public domain */ - -#ifndef __LZMADEC_H -#define __LZMADEC_H - -#include "Types.h" - -/* #define _LZMA_PROB32 */ -/* _LZMA_PROB32 can increase the speed on some CPUs, - but memory usage for CLzmaDec::probs will be doubled in that case */ - -#ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 -#else -#define CLzmaProb UInt16 -#endif - - -/* ---------- LZMA Properties ---------- */ - -#define LZMA_PROPS_SIZE 5 - -typedef struct _CLzmaProps -{ - unsigned lc, lp, pb; - UInt32 dicSize; -} CLzmaProps; - -/* LzmaProps_Decode - decodes properties -Returns: - SZ_OK - SZ_ERROR_UNSUPPORTED - Unsupported properties -*/ - -SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size); - - -/* ---------- LZMA Decoder state ---------- */ - -/* LZMA_REQUIRED_INPUT_MAX = number of required input bytes for worst case. - Num bits = log2((2^11 / 31) ^ 22) + 26 < 134 + 26 = 160; */ - -#define LZMA_REQUIRED_INPUT_MAX 20 - -typedef struct -{ - CLzmaProps prop; - CLzmaProb *probs; - Byte *dic; - const Byte *buf; - UInt32 range, code; - SizeT dicPos; - SizeT dicBufSize; - UInt32 processedPos; - UInt32 checkDicSize; - unsigned state; - UInt32 reps[4]; - unsigned remainLen; - int needFlush; - int needInitState; - UInt32 numProbs; - unsigned tempBufSize; - Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; -} CLzmaDec; - -#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; } - -void LzmaDec_Init(CLzmaDec *p); - -/* There are two types of LZMA streams: - 0) Stream with end mark. That end mark adds about 6 bytes to compressed size. - 1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */ - -typedef enum -{ - LZMA_FINISH_ANY, /* finish at any point */ - LZMA_FINISH_END /* block must be finished at the end */ -} ELzmaFinishMode; - -/* ELzmaFinishMode has meaning only if the decoding reaches output limit !!! - - You must use LZMA_FINISH_END, when you know that current output buffer - covers last bytes of block. In other cases you must use LZMA_FINISH_ANY. - - If LZMA decoder sees end marker before reaching output limit, it returns SZ_OK, - and output value of destLen will be less than output buffer size limit. - You can check status result also. - - You can use multiple checks to test data integrity after full decompression: - 1) Check Result and "status" variable. - 2) Check that output(destLen) = uncompressedSize, if you know real uncompressedSize. - 3) Check that output(srcLen) = compressedSize, if you know real compressedSize. - You must use correct finish mode in that case. */ - -typedef enum -{ - LZMA_STATUS_NOT_SPECIFIED, /* use main error code instead */ - LZMA_STATUS_FINISHED_WITH_MARK, /* stream was finished with end mark. */ - LZMA_STATUS_NOT_FINISHED, /* stream was not finished */ - LZMA_STATUS_NEEDS_MORE_INPUT, /* you must provide more input bytes */ - LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK /* there is probability that stream was finished without end mark */ -} ELzmaStatus; - -/* ELzmaStatus is used only as output value for function call */ - - -/* ---------- Interfaces ---------- */ - -/* There are 3 levels of interfaces: - 1) Dictionary Interface - 2) Buffer Interface - 3) One Call Interface - You can select any of these interfaces, but don't mix functions from different - groups for same object. */ - - -/* There are two variants to allocate state for Dictionary Interface: - 1) LzmaDec_Allocate / LzmaDec_Free - 2) LzmaDec_AllocateProbs / LzmaDec_FreeProbs - You can use variant 2, if you set dictionary buffer manually. - For Buffer Interface you must always use variant 1. - -LzmaDec_Allocate* can return: - SZ_OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_UNSUPPORTED - Unsupported properties -*/ - -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc); -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc); - -SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAlloc *alloc); -void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc); - -/* ---------- Dictionary Interface ---------- */ - -/* You can use it, if you want to eliminate the overhead for data copying from - dictionary to some other external buffer. - You must work with CLzmaDec variables directly in this interface. - - STEPS: - LzmaDec_Constr() - LzmaDec_Allocate() - for (each new stream) - { - LzmaDec_Init() - while (it needs more decompression) - { - LzmaDec_DecodeToDic() - use data from CLzmaDec::dic and update CLzmaDec::dicPos - } - } - LzmaDec_Free() -*/ - -/* LzmaDec_DecodeToDic - - The decoding to internal dictionary buffer (CLzmaDec::dic). - You must manually update CLzmaDec::dicPos, if it reaches CLzmaDec::dicBufSize !!! - -finishMode: - It has meaning only if the decoding reaches output limit (dicLimit). - LZMA_FINISH_ANY - Decode just dicLimit bytes. - LZMA_FINISH_END - Stream must be finished after dicLimit. - -Returns: - SZ_OK - status: - LZMA_STATUS_FINISHED_WITH_MARK - LZMA_STATUS_NOT_FINISHED - LZMA_STATUS_NEEDS_MORE_INPUT - LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK - SZ_ERROR_DATA - Data error -*/ - -SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, - const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); - - -/* ---------- Buffer Interface ---------- */ - -/* It's zlib-like interface. - See LzmaDec_DecodeToDic description for information about STEPS and return results, - but you must use LzmaDec_DecodeToBuf instead of LzmaDec_DecodeToDic and you don't need - to work with CLzmaDec variables manually. - -finishMode: - It has meaning only if the decoding reaches output limit (*destLen). - LZMA_FINISH_ANY - Decode just destLen bytes. - LZMA_FINISH_END - Stream must be finished after (*destLen). -*/ - -SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); - - -/* ---------- One Call Interface ---------- */ - -/* LzmaDecode - -finishMode: - It has meaning only if the decoding reaches output limit (*destLen). - LZMA_FINISH_ANY - Decode just destLen bytes. - LZMA_FINISH_END - Stream must be finished after (*destLen). - -Returns: - SZ_OK - status: - LZMA_STATUS_FINISHED_WITH_MARK - LZMA_STATUS_NOT_FINISHED - LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK - SZ_ERROR_DATA - Data error - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_UNSUPPORTED - Unsupported properties - SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). -*/ - -SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAlloc *alloc); - -#endif diff --git a/ext/7z/LzmaEnc.c b/ext/7z/LzmaEnc.c deleted file mode 100644 index 529fd98f..00000000 --- a/ext/7z/LzmaEnc.c +++ /dev/null @@ -1,2281 +0,0 @@ -/* LzmaEnc.c -- LZMA Encoder -2009-02-02 : Igor Pavlov : Public domain */ - -#include - -/* #define SHOW_STAT */ -/* #define SHOW_STAT2 */ - -#if defined(SHOW_STAT) || defined(SHOW_STAT2) -#include -#endif - -#include "LzmaEnc.h" - -#include "LzFind.h" -#ifdef COMPRESS_MF_MT -#include "LzFindMt.h" -#endif - -#ifdef SHOW_STAT -static int ttt = 0; -#endif - -#define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1) - -#define kBlockSize (9 << 10) -#define kUnpackBlockSize (1 << 18) -#define kMatchArraySize (1 << 21) -#define kMatchRecordMaxSize ((LZMA_MATCH_LEN_MAX * 2 + 3) * LZMA_MATCH_LEN_MAX) - -#define kNumMaxDirectBits (31) - -#define kNumTopBits 24 -#define kTopValue ((UInt32)1 << kNumTopBits) - -#define kNumBitModelTotalBits 11 -#define kBitModelTotal (1 << kNumBitModelTotalBits) -#define kNumMoveBits 5 -#define kProbInitValue (kBitModelTotal >> 1) - -#define kNumMoveReducingBits 4 -#define kNumBitPriceShiftBits 4 -#define kBitPrice (1 << kNumBitPriceShiftBits) - -void LzmaEncProps_Init(CLzmaEncProps *p) -{ - p->level = 5; - p->dictSize = p->mc = 0; - p->lc = p->lp = p->pb = p->algo = p->fb = p->btMode = p->numHashBytes = p->numThreads = -1; - p->writeEndMark = 0; -} - -void LzmaEncProps_Normalize(CLzmaEncProps *p) -{ - int level = p->level; - if (level < 0) level = 5; - p->level = level; - if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26))); - if (p->lc < 0) p->lc = 3; - if (p->lp < 0) p->lp = 0; - if (p->pb < 0) p->pb = 2; - if (p->algo < 0) p->algo = (level < 5 ? 0 : 1); - if (p->fb < 0) p->fb = (level < 7 ? 32 : 64); - if (p->btMode < 0) p->btMode = (p->algo == 0 ? 0 : 1); - if (p->numHashBytes < 0) p->numHashBytes = 4; - if (p->mc == 0) p->mc = (16 + (p->fb >> 1)) >> (p->btMode ? 0 : 1); - if (p->numThreads < 0) - p->numThreads = - #ifdef COMPRESS_MF_MT - ((p->btMode && p->algo) ? 2 : 1); - #else - 1; - #endif -} - -UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2) -{ - CLzmaEncProps props = *props2; - LzmaEncProps_Normalize(&props); - return props.dictSize; -} - -/* #define LZMA_LOG_BSR */ -/* Define it for Intel's CPU */ - - -#ifdef LZMA_LOG_BSR - -#define kDicLogSizeMaxCompress 30 - -#define BSR2_RET(pos, res) { unsigned long i; _BitScanReverse(&i, (pos)); res = (i + i) + ((pos >> (i - 1)) & 1); } - -UInt32 GetPosSlot1(UInt32 pos) -{ - UInt32 res; - BSR2_RET(pos, res); - return res; -} -#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < 2) res = pos; else BSR2_RET(pos, res); } - -#else - -#define kNumLogBits (9 + (int)sizeof(size_t) / 2) -#define kDicLogSizeMaxCompress ((kNumLogBits - 1) * 2 + 7) - -void LzmaEnc_FastPosInit(Byte *g_FastPos) -{ - int c = 2, slotFast; - g_FastPos[0] = 0; - g_FastPos[1] = 1; - - for (slotFast = 2; slotFast < kNumLogBits * 2; slotFast++) - { - UInt32 k = (1 << ((slotFast >> 1) - 1)); - UInt32 j; - for (j = 0; j < k; j++, c++) - g_FastPos[c] = (Byte)slotFast; - } -} - -#define BSR2_RET(pos, res) { UInt32 i = 6 + ((kNumLogBits - 1) & \ - (0 - (((((UInt32)1 << (kNumLogBits + 6)) - 1) - pos) >> 31))); \ - res = p->g_FastPos[pos >> i] + (i * 2); } -/* -#define BSR2_RET(pos, res) { res = (pos < (1 << (kNumLogBits + 6))) ? \ - p->g_FastPos[pos >> 6] + 12 : \ - p->g_FastPos[pos >> (6 + kNumLogBits - 1)] + (6 + (kNumLogBits - 1)) * 2; } -*/ - -#define GetPosSlot1(pos) p->g_FastPos[pos] -#define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos]; else BSR2_RET(pos, res); } - -#endif - - -#define LZMA_NUM_REPS 4 - -typedef unsigned CState; - -typedef struct _COptimal -{ - UInt32 price; - - CState state; - int prev1IsChar; - int prev2; - - UInt32 posPrev2; - UInt32 backPrev2; - - UInt32 posPrev; - UInt32 backPrev; - UInt32 backs[LZMA_NUM_REPS]; -} COptimal; - -#define kNumOpts (1 << 12) - -#define kNumLenToPosStates 4 -#define kNumPosSlotBits 6 -#define kDicLogSizeMin 0 -#define kDicLogSizeMax 32 -#define kDistTableSizeMax (kDicLogSizeMax * 2) - - -#define kNumAlignBits 4 -#define kAlignTableSize (1 << kNumAlignBits) -#define kAlignMask (kAlignTableSize - 1) - -#define kStartPosModelIndex 4 -#define kEndPosModelIndex 14 -#define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex) - -#define kNumFullDistances (1 << (kEndPosModelIndex / 2)) - -#ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 -#else -#define CLzmaProb UInt16 -#endif - -#define LZMA_PB_MAX 4 -#define LZMA_LC_MAX 8 -#define LZMA_LP_MAX 4 - -#define LZMA_NUM_PB_STATES_MAX (1 << LZMA_PB_MAX) - - -#define kLenNumLowBits 3 -#define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) -#define kLenNumHighBits 8 -#define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define kLenNumSymbolsTotal (kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) - -#define LZMA_MATCH_LEN_MIN 2 -#define LZMA_MATCH_LEN_MAX (LZMA_MATCH_LEN_MIN + kLenNumSymbolsTotal - 1) - -#define kNumStates 12 - -typedef struct -{ - CLzmaProb choice; - CLzmaProb choice2; - CLzmaProb low[LZMA_NUM_PB_STATES_MAX << kLenNumLowBits]; - CLzmaProb mid[LZMA_NUM_PB_STATES_MAX << kLenNumMidBits]; - CLzmaProb high[kLenNumHighSymbols]; -} CLenEnc; - -typedef struct -{ - CLenEnc p; - UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; - UInt32 tableSize; - UInt32 counters[LZMA_NUM_PB_STATES_MAX]; -} CLenPriceEnc; - -typedef struct _CRangeEnc -{ - UInt32 range; - Byte cache; - UInt64 low; - UInt64 cacheSize; - Byte *buf; - Byte *bufLim; - Byte *bufBase; - ISeqOutStream *outStream; - UInt64 processed; - SRes res; -} CRangeEnc; - -typedef struct _CSeqInStreamBuf -{ - ISeqInStream funcTable; - const Byte *data; - SizeT rem; -} CSeqInStreamBuf; - -static SRes MyRead(void *pp, void *data, size_t *size) -{ - size_t curSize = *size; - CSeqInStreamBuf *p = (CSeqInStreamBuf *)pp; - if (p->rem < curSize) - curSize = p->rem; - memcpy(data, p->data, curSize); - p->rem -= curSize; - p->data += curSize; - *size = curSize; - return SZ_OK; -} - -typedef struct -{ - CLzmaProb *litProbs; - - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb isRep[kNumStates]; - CLzmaProb isRepG0[kNumStates]; - CLzmaProb isRepG1[kNumStates]; - CLzmaProb isRepG2[kNumStates]; - CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; - - CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; - - CLenPriceEnc lenEnc; - CLenPriceEnc repLenEnc; - - UInt32 reps[LZMA_NUM_REPS]; - UInt32 state; -} CSaveState; - -typedef struct _CLzmaEnc -{ - IMatchFinder matchFinder; - void *matchFinderObj; - - #ifdef COMPRESS_MF_MT - Bool mtMode; - CMatchFinderMt matchFinderMt; - #endif - - CMatchFinder matchFinderBase; - - #ifdef COMPRESS_MF_MT - Byte pad[128]; - #endif - - UInt32 optimumEndIndex; - UInt32 optimumCurrentIndex; - - UInt32 longestMatchLength; - UInt32 numPairs; - UInt32 numAvail; - COptimal opt[kNumOpts]; - - #ifndef LZMA_LOG_BSR - Byte g_FastPos[1 << kNumLogBits]; - #endif - - UInt32 ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; - UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1]; - UInt32 numFastBytes; - UInt32 additionalOffset; - UInt32 reps[LZMA_NUM_REPS]; - UInt32 state; - - UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; - UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; - UInt32 alignPrices[kAlignTableSize]; - UInt32 alignPriceCount; - - UInt32 distTableSize; - - unsigned lc, lp, pb; - unsigned lpMask, pbMask; - - CLzmaProb *litProbs; - - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb isRep[kNumStates]; - CLzmaProb isRepG0[kNumStates]; - CLzmaProb isRepG1[kNumStates]; - CLzmaProb isRepG2[kNumStates]; - CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; - - CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; - - CLenPriceEnc lenEnc; - CLenPriceEnc repLenEnc; - - unsigned lclp; - - Bool fastMode; - - CRangeEnc rc; - - Bool writeEndMark; - UInt64 nowPos64; - UInt32 matchPriceCount; - Bool finished; - Bool multiThread; - - SRes result; - UInt32 dictSize; - UInt32 matchFinderCycles; - - ISeqInStream *inStream; - CSeqInStreamBuf seqBufInStream; - - CSaveState saveState; -} CLzmaEnc; - -void LzmaEnc_SaveState(CLzmaEncHandle pp) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - CSaveState *dest = &p->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; - dest->state = p->state; - - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); - memcpy(dest->litProbs, p->litProbs, (0x300 << p->lclp) * sizeof(CLzmaProb)); -} - -void LzmaEnc_RestoreState(CLzmaEncHandle pp) -{ - CLzmaEnc *dest = (CLzmaEnc *)pp; - const CSaveState *p = &dest->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; - dest->state = p->state; - - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); - memcpy(dest->litProbs, p->litProbs, (0x300 << dest->lclp) * sizeof(CLzmaProb)); -} - -SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - CLzmaEncProps props = *props2; - LzmaEncProps_Normalize(&props); - - if (props.lc > LZMA_LC_MAX || props.lp > LZMA_LP_MAX || props.pb > LZMA_PB_MAX || - props.dictSize > (1 << kDicLogSizeMaxCompress) || props.dictSize > (1 << 30)) - return SZ_ERROR_PARAM; - p->dictSize = props.dictSize; - p->matchFinderCycles = props.mc; - { - unsigned fb = props.fb; - if (fb < 5) - fb = 5; - if (fb > LZMA_MATCH_LEN_MAX) - fb = LZMA_MATCH_LEN_MAX; - p->numFastBytes = fb; - } - p->lc = props.lc; - p->lp = props.lp; - p->pb = props.pb; - p->fastMode = (props.algo == 0); - p->matchFinderBase.btMode = props.btMode; - { - UInt32 numHashBytes = 4; - if (props.btMode) - { - if (props.numHashBytes < 2) - numHashBytes = 2; - else if (props.numHashBytes < 4) - numHashBytes = props.numHashBytes; - } - p->matchFinderBase.numHashBytes = numHashBytes; - } - - p->matchFinderBase.cutValue = props.mc; - - p->writeEndMark = props.writeEndMark; - - #ifdef COMPRESS_MF_MT - /* - if (newMultiThread != _multiThread) - { - ReleaseMatchFinder(); - _multiThread = newMultiThread; - } - */ - p->multiThread = (props.numThreads > 1); - #endif - - return SZ_OK; -} - -static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; -static const int kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; -static const int kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; -static const int kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; - -#define IsCharState(s) ((s) < 7) - -#define GetLenToPosState(len) (((len) < kNumLenToPosStates + 1) ? (len) - 2 : kNumLenToPosStates - 1) - -#define kInfinityPrice (1 << 30) - -static void RangeEnc_Construct(CRangeEnc *p) -{ - p->outStream = 0; - p->bufBase = 0; -} - -#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) - -#define RC_BUF_SIZE (1 << 16) -static int RangeEnc_Alloc(CRangeEnc *p, ISzAlloc *alloc) -{ - if (p->bufBase == 0) - { - p->bufBase = (Byte *)alloc->Alloc(alloc, RC_BUF_SIZE); - if (p->bufBase == 0) - return 0; - p->bufLim = p->bufBase + RC_BUF_SIZE; - } - return 1; -} - -static void RangeEnc_Free(CRangeEnc *p, ISzAlloc *alloc) -{ - alloc->Free(alloc, p->bufBase); - p->bufBase = 0; -} - -static void RangeEnc_Init(CRangeEnc *p) -{ - /* Stream.Init(); */ - p->low = 0; - p->range = 0xFFFFFFFF; - p->cacheSize = 1; - p->cache = 0; - - p->buf = p->bufBase; - - p->processed = 0; - p->res = SZ_OK; -} - -static void RangeEnc_FlushStream(CRangeEnc *p) -{ - size_t num; - if (p->res != SZ_OK) - return; - num = p->buf - p->bufBase; - if (num != p->outStream->Write(p->outStream, p->bufBase, num)) - p->res = SZ_ERROR_WRITE; - p->processed += num; - p->buf = p->bufBase; -} - -static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) -{ - if ((UInt32)p->low < (UInt32)0xFF000000 || (int)(p->low >> 32) != 0) - { - Byte temp = p->cache; - do - { - Byte *buf = p->buf; - *buf++ = (Byte)(temp + (Byte)(p->low >> 32)); - p->buf = buf; - if (buf == p->bufLim) - RangeEnc_FlushStream(p); - temp = 0xFF; - } - while (--p->cacheSize != 0); - p->cache = (Byte)((UInt32)p->low >> 24); - } - p->cacheSize++; - p->low = (UInt32)p->low << 8; -} - -static void RangeEnc_FlushData(CRangeEnc *p) -{ - int i; - for (i = 0; i < 5; i++) - RangeEnc_ShiftLow(p); -} - -static void RangeEnc_EncodeDirectBits(CRangeEnc *p, UInt32 value, int numBits) -{ - do - { - p->range >>= 1; - p->low += p->range & (0 - ((value >> --numBits) & 1)); - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); - } - } - while (numBits != 0); -} - -static void RangeEnc_EncodeBit(CRangeEnc *p, CLzmaProb *prob, UInt32 symbol) -{ - UInt32 ttt = *prob; - UInt32 newBound = (p->range >> kNumBitModelTotalBits) * ttt; - if (symbol == 0) - { - p->range = newBound; - ttt += (kBitModelTotal - ttt) >> kNumMoveBits; - } - else - { - p->low += newBound; - p->range -= newBound; - ttt -= ttt >> kNumMoveBits; - } - *prob = (CLzmaProb)ttt; - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); - } -} - -static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol) -{ - symbol |= 0x100; - do - { - RangeEnc_EncodeBit(p, probs + (symbol >> 8), (symbol >> 7) & 1); - symbol <<= 1; - } - while (symbol < 0x10000); -} - -static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol, UInt32 matchByte) -{ - UInt32 offs = 0x100; - symbol |= 0x100; - do - { - matchByte <<= 1; - RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (symbol >> 8)), (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); - } - while (symbol < 0x10000); -} - -void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) -{ - UInt32 i; - for (i = (1 << kNumMoveReducingBits) / 2; i < kBitModelTotal; i += (1 << kNumMoveReducingBits)) - { - const int kCyclesBits = kNumBitPriceShiftBits; - UInt32 w = i; - UInt32 bitCount = 0; - int j; - for (j = 0; j < kCyclesBits; j++) - { - w = w * w; - bitCount <<= 1; - while (w >= ((UInt32)1 << 16)) - { - w >>= 1; - bitCount++; - } - } - ProbPrices[i >> kNumMoveReducingBits] = ((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); - } -} - - -#define GET_PRICE(prob, symbol) \ - p->ProbPrices[((prob) ^ (((-(int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; - -#define GET_PRICEa(prob, symbol) \ - ProbPrices[((prob) ^ ((-((int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; - -#define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] -#define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] - -#define GET_PRICE_0a(prob) ProbPrices[(prob) >> kNumMoveReducingBits] -#define GET_PRICE_1a(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] - -static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 symbol, UInt32 *ProbPrices) -{ - UInt32 price = 0; - symbol |= 0x100; - do - { - price += GET_PRICEa(probs[symbol >> 8], (symbol >> 7) & 1); - symbol <<= 1; - } - while (symbol < 0x10000); - return price; -} - -static UInt32 LitEnc_GetPriceMatched(const CLzmaProb *probs, UInt32 symbol, UInt32 matchByte, UInt32 *ProbPrices) -{ - UInt32 price = 0; - UInt32 offs = 0x100; - symbol |= 0x100; - do - { - matchByte <<= 1; - price += GET_PRICEa(probs[offs + (matchByte & offs) + (symbol >> 8)], (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); - } - while (symbol < 0x10000); - return price; -} - - -static void RcTree_Encode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) -{ - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0;) - { - UInt32 bit; - i--; - bit = (symbol >> i) & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); - m = (m << 1) | bit; - } -} - -static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) -{ - UInt32 m = 1; - int i; - for (i = 0; i < numBitLevels; i++) - { - UInt32 bit = symbol & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); - m = (m << 1) | bit; - symbol >>= 1; - } -} - -static UInt32 RcTree_GetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, UInt32 *ProbPrices) -{ - UInt32 price = 0; - symbol |= (1 << numBitLevels); - while (symbol != 1) - { - price += GET_PRICEa(probs[symbol >> 1], symbol & 1); - symbol >>= 1; - } - return price; -} - -static UInt32 RcTree_ReverseGetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, UInt32 *ProbPrices) -{ - UInt32 price = 0; - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0; i--) - { - UInt32 bit = symbol & 1; - symbol >>= 1; - price += GET_PRICEa(probs[m], bit); - m = (m << 1) | bit; - } - return price; -} - - -static void LenEnc_Init(CLenEnc *p) -{ - unsigned i; - p->choice = p->choice2 = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumLowBits); i++) - p->low[i] = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumMidBits); i++) - p->mid[i] = kProbInitValue; - for (i = 0; i < kLenNumHighSymbols; i++) - p->high[i] = kProbInitValue; -} - -static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState) -{ - if (symbol < kLenNumLowSymbols) - { - RangeEnc_EncodeBit(rc, &p->choice, 0); - RcTree_Encode(rc, p->low + (posState << kLenNumLowBits), kLenNumLowBits, symbol); - } - else - { - RangeEnc_EncodeBit(rc, &p->choice, 1); - if (symbol < kLenNumLowSymbols + kLenNumMidSymbols) - { - RangeEnc_EncodeBit(rc, &p->choice2, 0); - RcTree_Encode(rc, p->mid + (posState << kLenNumMidBits), kLenNumMidBits, symbol - kLenNumLowSymbols); - } - else - { - RangeEnc_EncodeBit(rc, &p->choice2, 1); - RcTree_Encode(rc, p->high, kLenNumHighBits, symbol - kLenNumLowSymbols - kLenNumMidSymbols); - } - } -} - -static void LenEnc_SetPrices(CLenEnc *p, UInt32 posState, UInt32 numSymbols, UInt32 *prices, UInt32 *ProbPrices) -{ - UInt32 a0 = GET_PRICE_0a(p->choice); - UInt32 a1 = GET_PRICE_1a(p->choice); - UInt32 b0 = a1 + GET_PRICE_0a(p->choice2); - UInt32 b1 = a1 + GET_PRICE_1a(p->choice2); - UInt32 i = 0; - for (i = 0; i < kLenNumLowSymbols; i++) - { - if (i >= numSymbols) - return; - prices[i] = a0 + RcTree_GetPrice(p->low + (posState << kLenNumLowBits), kLenNumLowBits, i, ProbPrices); - } - for (; i < kLenNumLowSymbols + kLenNumMidSymbols; i++) - { - if (i >= numSymbols) - return; - prices[i] = b0 + RcTree_GetPrice(p->mid + (posState << kLenNumMidBits), kLenNumMidBits, i - kLenNumLowSymbols, ProbPrices); - } - for (; i < numSymbols; i++) - prices[i] = b1 + RcTree_GetPrice(p->high, kLenNumHighBits, i - kLenNumLowSymbols - kLenNumMidSymbols, ProbPrices); -} - -static void MY_FAST_CALL LenPriceEnc_UpdateTable(CLenPriceEnc *p, UInt32 posState, UInt32 *ProbPrices) -{ - LenEnc_SetPrices(&p->p, posState, p->tableSize, p->prices[posState], ProbPrices); - p->counters[posState] = p->tableSize; -} - -static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, UInt32 numPosStates, UInt32 *ProbPrices) -{ - UInt32 posState; - for (posState = 0; posState < numPosStates; posState++) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} - -static void LenEnc_Encode2(CLenPriceEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState, Bool updatePrice, UInt32 *ProbPrices) -{ - LenEnc_Encode(&p->p, rc, symbol, posState); - if (updatePrice) - if (--p->counters[posState] == 0) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} - - - - -static void MovePos(CLzmaEnc *p, UInt32 num) -{ - #ifdef SHOW_STAT - ttt += num; - printf("\n MovePos %d", num); - #endif - if (num != 0) - { - p->additionalOffset += num; - p->matchFinder.Skip(p->matchFinderObj, num); - } -} - -static UInt32 ReadMatchDistances(CLzmaEnc *p, UInt32 *numDistancePairsRes) -{ - UInt32 lenRes = 0, numPairs; - p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); - numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); - #ifdef SHOW_STAT - printf("\n i = %d numPairs = %d ", ttt, numPairs / 2); - ttt++; - { - UInt32 i; - for (i = 0; i < numPairs; i += 2) - printf("%2d %6d | ", p->matches[i], p->matches[i + 1]); - } - #endif - if (numPairs > 0) - { - lenRes = p->matches[numPairs - 2]; - if (lenRes == p->numFastBytes) - { - const Byte *pby = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - UInt32 distance = p->matches[numPairs - 1] + 1; - UInt32 numAvail = p->numAvail; - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - { - const Byte *pby2 = pby - distance; - for (; lenRes < numAvail && pby[lenRes] == pby2[lenRes]; lenRes++); - } - } - } - p->additionalOffset++; - *numDistancePairsRes = numPairs; - return lenRes; -} - - -#define MakeAsChar(p) (p)->backPrev = (UInt32)(-1); (p)->prev1IsChar = False; -#define MakeAsShortRep(p) (p)->backPrev = 0; (p)->prev1IsChar = False; -#define IsShortRep(p) ((p)->backPrev == 0) - -static UInt32 GetRepLen1Price(CLzmaEnc *p, UInt32 state, UInt32 posState) -{ - return - GET_PRICE_0(p->isRepG0[state]) + - GET_PRICE_0(p->isRep0Long[state][posState]); -} - -static UInt32 GetPureRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 state, UInt32 posState) -{ - UInt32 price; - if (repIndex == 0) - { - price = GET_PRICE_0(p->isRepG0[state]); - price += GET_PRICE_1(p->isRep0Long[state][posState]); - } - else - { - price = GET_PRICE_1(p->isRepG0[state]); - if (repIndex == 1) - price += GET_PRICE_0(p->isRepG1[state]); - else - { - price += GET_PRICE_1(p->isRepG1[state]); - price += GET_PRICE(p->isRepG2[state], repIndex - 2); - } - } - return price; -} - -static UInt32 GetRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 len, UInt32 state, UInt32 posState) -{ - return p->repLenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN] + - GetPureRepPrice(p, repIndex, state, posState); -} - -static UInt32 Backward(CLzmaEnc *p, UInt32 *backRes, UInt32 cur) -{ - UInt32 posMem = p->opt[cur].posPrev; - UInt32 backMem = p->opt[cur].backPrev; - p->optimumEndIndex = cur; - do - { - if (p->opt[cur].prev1IsChar) - { - MakeAsChar(&p->opt[posMem]) - p->opt[posMem].posPrev = posMem - 1; - if (p->opt[cur].prev2) - { - p->opt[posMem - 1].prev1IsChar = False; - p->opt[posMem - 1].posPrev = p->opt[cur].posPrev2; - p->opt[posMem - 1].backPrev = p->opt[cur].backPrev2; - } - } - { - UInt32 posPrev = posMem; - UInt32 backCur = backMem; - - backMem = p->opt[posPrev].backPrev; - posMem = p->opt[posPrev].posPrev; - - p->opt[posPrev].backPrev = backCur; - p->opt[posPrev].posPrev = cur; - cur = posPrev; - } - } - while (cur != 0); - *backRes = p->opt[0].backPrev; - p->optimumCurrentIndex = p->opt[0].posPrev; - return p->optimumCurrentIndex; -} - -#define LIT_PROBS(pos, prevByte) (p->litProbs + ((((pos) & p->lpMask) << p->lc) + ((prevByte) >> (8 - p->lc))) * 0x300) - -static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) -{ - UInt32 numAvail, mainLen, numPairs, repMaxIndex, i, posState, lenEnd, len, cur; - UInt32 matchPrice, repMatchPrice, normalMatchPrice; - UInt32 reps[LZMA_NUM_REPS], repLens[LZMA_NUM_REPS]; - UInt32 *matches; - const Byte *data; - Byte curByte, matchByte; - if (p->optimumEndIndex != p->optimumCurrentIndex) - { - const COptimal *opt = &p->opt[p->optimumCurrentIndex]; - UInt32 lenRes = opt->posPrev - p->optimumCurrentIndex; - *backRes = opt->backPrev; - p->optimumCurrentIndex = opt->posPrev; - return lenRes; - } - p->optimumCurrentIndex = p->optimumEndIndex = 0; - - if (p->additionalOffset == 0) - mainLen = ReadMatchDistances(p, &numPairs); - else - { - mainLen = p->longestMatchLength; - numPairs = p->numPairs; - } - - numAvail = p->numAvail; - if (numAvail < 2) - { - *backRes = (UInt32)(-1); - return 1; - } - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repMaxIndex = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 lenTest; - const Byte *data2; - reps[i] = p->reps[i]; - data2 = data - (reps[i] + 1); - if (data[0] != data2[0] || data[1] != data2[1]) - { - repLens[i] = 0; - continue; - } - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - repLens[i] = lenTest; - if (lenTest > repLens[repMaxIndex]) - repMaxIndex = i; - } - if (repLens[repMaxIndex] >= p->numFastBytes) - { - UInt32 lenRes; - *backRes = repMaxIndex; - lenRes = repLens[repMaxIndex]; - MovePos(p, lenRes - 1); - return lenRes; - } - - matches = p->matches; - if (mainLen >= p->numFastBytes) - { - *backRes = matches[numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); - return mainLen; - } - curByte = *data; - matchByte = *(data - (reps[0] + 1)); - - if (mainLen < 2 && curByte != matchByte && repLens[repMaxIndex] < 2) - { - *backRes = (UInt32)-1; - return 1; - } - - p->opt[0].state = (CState)p->state; - - posState = (position & p->pbMask); - - { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + - (!IsCharState(p->state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - } - - MakeAsChar(&p->opt[1]); - - matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); - repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); - - if (matchByte == curByte) - { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, p->state, posState); - if (shortRepPrice < p->opt[1].price) - { - p->opt[1].price = shortRepPrice; - MakeAsShortRep(&p->opt[1]); - } - } - lenEnd = ((mainLen >= repLens[repMaxIndex]) ? mainLen : repLens[repMaxIndex]); - - if (lenEnd < 2) - { - *backRes = p->opt[1].backPrev; - return 1; - } - - p->opt[1].posPrev = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - p->opt[0].backs[i] = reps[i]; - - len = lenEnd; - do - p->opt[len--].price = kInfinityPrice; - while (len >= 2); - - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 repLen = repLens[i]; - UInt32 price; - if (repLen < 2) - continue; - price = repMatchPrice + GetPureRepPrice(p, i, p->state, posState); - do - { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][repLen - 2]; - COptimal *opt = &p->opt[repLen]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = i; - opt->prev1IsChar = False; - } - } - while (--repLen >= 2); - } - - normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); - - len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); - if (len <= mainLen) - { - UInt32 offs = 0; - while (len > matches[offs]) - offs += 2; - for (; ; len++) - { - COptimal *opt; - UInt32 distance = matches[offs + 1]; - - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN]; - UInt32 lenToPosState = GetLenToPosState(len); - if (distance < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][distance]; - else - { - UInt32 slot; - GetPosSlot2(distance, slot); - curAndLenPrice += p->alignPrices[distance & kAlignMask] + p->posSlotPrices[lenToPosState][slot]; - } - opt = &p->opt[len]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = distance + LZMA_NUM_REPS; - opt->prev1IsChar = False; - } - if (len == matches[offs]) - { - offs += 2; - if (offs == numPairs) - break; - } - } - } - - cur = 0; - - #ifdef SHOW_STAT2 - if (position >= 0) - { - unsigned i; - printf("\n pos = %4X", position); - for (i = cur; i <= lenEnd; i++) - printf("\nprice[%4X] = %d", position - cur + i, p->opt[i].price); - } - #endif - - for (;;) - { - UInt32 numAvailFull, newLen, numPairs, posPrev, state, posState, startLen; - UInt32 curPrice, curAnd1Price, matchPrice, repMatchPrice; - Bool nextIsChar; - Byte curByte, matchByte; - const Byte *data; - COptimal *curOpt; - COptimal *nextOpt; - - cur++; - if (cur == lenEnd) - return Backward(p, backRes, cur); - - newLen = ReadMatchDistances(p, &numPairs); - if (newLen >= p->numFastBytes) - { - p->numPairs = numPairs; - p->longestMatchLength = newLen; - return Backward(p, backRes, cur); - } - position++; - curOpt = &p->opt[cur]; - posPrev = curOpt->posPrev; - if (curOpt->prev1IsChar) - { - posPrev--; - if (curOpt->prev2) - { - state = p->opt[curOpt->posPrev2].state; - if (curOpt->backPrev2 < LZMA_NUM_REPS) - state = kRepNextStates[state]; - else - state = kMatchNextStates[state]; - } - else - state = p->opt[posPrev].state; - state = kLiteralNextStates[state]; - } - else - state = p->opt[posPrev].state; - if (posPrev == cur - 1) - { - if (IsShortRep(curOpt)) - state = kShortRepNextStates[state]; - else - state = kLiteralNextStates[state]; - } - else - { - UInt32 pos; - const COptimal *prevOpt; - if (curOpt->prev1IsChar && curOpt->prev2) - { - posPrev = curOpt->posPrev2; - pos = curOpt->backPrev2; - state = kRepNextStates[state]; - } - else - { - pos = curOpt->backPrev; - if (pos < LZMA_NUM_REPS) - state = kRepNextStates[state]; - else - state = kMatchNextStates[state]; - } - prevOpt = &p->opt[posPrev]; - if (pos < LZMA_NUM_REPS) - { - UInt32 i; - reps[0] = prevOpt->backs[pos]; - for (i = 1; i <= pos; i++) - reps[i] = prevOpt->backs[i - 1]; - for (; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i]; - } - else - { - UInt32 i; - reps[0] = (pos - LZMA_NUM_REPS); - for (i = 1; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i - 1]; - } - } - curOpt->state = (CState)state; - - curOpt->backs[0] = reps[0]; - curOpt->backs[1] = reps[1]; - curOpt->backs[2] = reps[2]; - curOpt->backs[3] = reps[3]; - - curPrice = curOpt->price; - nextIsChar = False; - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - curByte = *data; - matchByte = *(data - (reps[0] + 1)); - - posState = (position & p->pbMask); - - curAnd1Price = curPrice + GET_PRICE_0(p->isMatch[state][posState]); - { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - curAnd1Price += - (!IsCharState(state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - } - - nextOpt = &p->opt[cur + 1]; - - if (curAnd1Price < nextOpt->price) - { - nextOpt->price = curAnd1Price; - nextOpt->posPrev = cur; - MakeAsChar(nextOpt); - nextIsChar = True; - } - - matchPrice = curPrice + GET_PRICE_1(p->isMatch[state][posState]); - repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); - - if (matchByte == curByte && !(nextOpt->posPrev < cur && nextOpt->backPrev == 0)) - { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, state, posState); - if (shortRepPrice <= nextOpt->price) - { - nextOpt->price = shortRepPrice; - nextOpt->posPrev = cur; - MakeAsShortRep(nextOpt); - nextIsChar = True; - } - } - numAvailFull = p->numAvail; - { - UInt32 temp = kNumOpts - 1 - cur; - if (temp < numAvailFull) - numAvailFull = temp; - } - - if (numAvailFull < 2) - continue; - numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); - - if (!nextIsChar && matchByte != curByte) /* speed optimization */ - { - /* try Literal + rep0 */ - UInt32 temp; - UInt32 lenTest2; - const Byte *data2 = data - (reps[0] + 1); - UInt32 limit = p->numFastBytes + 1; - if (limit > numAvailFull) - limit = numAvailFull; - - for (temp = 1; temp < limit && data[temp] == data2[temp]; temp++); - lenTest2 = temp - 1; - if (lenTest2 >= 2) - { - UInt32 state2 = kLiteralNextStates[state]; - UInt32 posStateNext = (position + 1) & p->pbMask; - UInt32 nextRepMatchPrice = curAnd1Price + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - /* for (; lenTest2 >= 2; lenTest2--) */ - { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = False; - } - } - } - } - - startLen = 2; /* speed optimization */ - { - UInt32 repIndex; - for (repIndex = 0; repIndex < LZMA_NUM_REPS; repIndex++) - { - UInt32 lenTest; - UInt32 lenTestTemp; - UInt32 price; - const Byte *data2 = data - (reps[repIndex] + 1); - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - while (lenEnd < cur + lenTest) - p->opt[++lenEnd].price = kInfinityPrice; - lenTestTemp = lenTest; - price = repMatchPrice + GetPureRepPrice(p, repIndex, state, posState); - do - { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][lenTest - 2]; - COptimal *opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = repIndex; - opt->prev1IsChar = False; - } - } - while (--lenTest >= 2); - lenTest = lenTestTemp; - - if (repIndex == 0) - startLen = lenTest + 1; - - /* if (_maxMode) */ - { - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; - UInt32 nextRepMatchPrice; - if (limit > numAvailFull) - limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) - { - UInt32 state2 = kRepNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = - price + p->repLenEnc.prices[posState][lenTest - 2] + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (position + lenTest + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - - /* for (; lenTest2 >= 2; lenTest2--) */ - { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + lenTest + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = repIndex; - } - } - } - } - } - } - /* for (UInt32 lenTest = 2; lenTest <= newLen; lenTest++) */ - if (newLen > numAvail) - { - newLen = numAvail; - for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2); - matches[numPairs] = newLen; - numPairs += 2; - } - if (newLen >= startLen) - { - UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); - UInt32 offs, curBack, posSlot; - UInt32 lenTest; - while (lenEnd < cur + newLen) - p->opt[++lenEnd].price = kInfinityPrice; - - offs = 0; - while (startLen > matches[offs]) - offs += 2; - curBack = matches[offs + 1]; - GetPosSlot2(curBack, posSlot); - for (lenTest = /*2*/ startLen; ; lenTest++) - { - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][lenTest - LZMA_MATCH_LEN_MIN]; - UInt32 lenToPosState = GetLenToPosState(lenTest); - COptimal *opt; - if (curBack < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][curBack]; - else - curAndLenPrice += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[curBack & kAlignMask]; - - opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = curBack + LZMA_NUM_REPS; - opt->prev1IsChar = False; - } - - if (/*_maxMode && */lenTest == matches[offs]) - { - /* Try Match + Literal + Rep0 */ - const Byte *data2 = data - (curBack + 1); - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; - UInt32 nextRepMatchPrice; - if (limit > numAvailFull) - limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) - { - UInt32 state2 = kMatchNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = curAndLenPrice + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (posStateNext + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - - /* for (; lenTest2 >= 2; lenTest2--) */ - { - UInt32 offset = cur + lenTest + 1 + lenTest2; - UInt32 curAndLenPrice; - COptimal *opt; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) - { - opt->price = curAndLenPrice; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = curBack + LZMA_NUM_REPS; - } - } - } - offs += 2; - if (offs == numPairs) - break; - curBack = matches[offs + 1]; - if (curBack >= kNumFullDistances) - GetPosSlot2(curBack, posSlot); - } - } - } - } -} - -#define ChangePair(smallDist, bigDist) (((bigDist) >> 7) > (smallDist)) - -static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) -{ - UInt32 numAvail, mainLen, mainDist, numPairs, repIndex, repLen, i; - const Byte *data; - const UInt32 *matches; - - if (p->additionalOffset == 0) - mainLen = ReadMatchDistances(p, &numPairs); - else - { - mainLen = p->longestMatchLength; - numPairs = p->numPairs; - } - - numAvail = p->numAvail; - *backRes = (UInt32)-1; - if (numAvail < 2) - return 1; - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - - repLen = repIndex = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 len; - const Byte *data2 = data - (p->reps[i] + 1); - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - for (len = 2; len < numAvail && data[len] == data2[len]; len++); - if (len >= p->numFastBytes) - { - *backRes = i; - MovePos(p, len - 1); - return len; - } - if (len > repLen) - { - repIndex = i; - repLen = len; - } - } - - matches = p->matches; - if (mainLen >= p->numFastBytes) - { - *backRes = matches[numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); - return mainLen; - } - - mainDist = 0; /* for GCC */ - if (mainLen >= 2) - { - mainDist = matches[numPairs - 1]; - while (numPairs > 2 && mainLen == matches[numPairs - 4] + 1) - { - if (!ChangePair(matches[numPairs - 3], mainDist)) - break; - numPairs -= 2; - mainLen = matches[numPairs - 2]; - mainDist = matches[numPairs - 1]; - } - if (mainLen == 2 && mainDist >= 0x80) - mainLen = 1; - } - - if (repLen >= 2 && ( - (repLen + 1 >= mainLen) || - (repLen + 2 >= mainLen && mainDist >= (1 << 9)) || - (repLen + 3 >= mainLen && mainDist >= (1 << 15)))) - { - *backRes = repIndex; - MovePos(p, repLen - 1); - return repLen; - } - - if (mainLen < 2 || numAvail <= 2) - return 1; - - p->longestMatchLength = ReadMatchDistances(p, &p->numPairs); - if (p->longestMatchLength >= 2) - { - UInt32 newDistance = matches[p->numPairs - 1]; - if ((p->longestMatchLength >= mainLen && newDistance < mainDist) || - (p->longestMatchLength == mainLen + 1 && !ChangePair(mainDist, newDistance)) || - (p->longestMatchLength > mainLen + 1) || - (p->longestMatchLength + 1 >= mainLen && mainLen >= 3 && ChangePair(newDistance, mainDist))) - return 1; - } - - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 len, limit; - const Byte *data2 = data - (p->reps[i] + 1); - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - limit = mainLen - 1; - for (len = 2; len < limit && data[len] == data2[len]; len++); - if (len >= limit) - return 1; - } - *backRes = mainDist + LZMA_NUM_REPS; - MovePos(p, mainLen - 2); - return mainLen; -} - -static void WriteEndMarker(CLzmaEnc *p, UInt32 posState) -{ - UInt32 len; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); - p->state = kMatchNextStates[p->state]; - len = LZMA_MATCH_LEN_MIN; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, (1 << kNumPosSlotBits) - 1); - RangeEnc_EncodeDirectBits(&p->rc, (((UInt32)1 << 30) - 1) >> kNumAlignBits, 30 - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); -} - -static SRes CheckErrors(CLzmaEnc *p) -{ - if (p->result != SZ_OK) - return p->result; - if (p->rc.res != SZ_OK) - p->result = SZ_ERROR_WRITE; - if (p->matchFinderBase.result != SZ_OK) - p->result = SZ_ERROR_READ; - if (p->result != SZ_OK) - p->finished = True; - return p->result; -} - -static SRes Flush(CLzmaEnc *p, UInt32 nowPos) -{ - /* ReleaseMFStream(); */ - p->finished = True; - if (p->writeEndMark) - WriteEndMarker(p, nowPos & p->pbMask); - RangeEnc_FlushData(&p->rc); - RangeEnc_FlushStream(&p->rc); - return CheckErrors(p); -} - -static void FillAlignPrices(CLzmaEnc *p) -{ - UInt32 i; - for (i = 0; i < kAlignTableSize; i++) - p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); - p->alignPriceCount = 0; -} - -static void FillDistancesPrices(CLzmaEnc *p) -{ - UInt32 tempPrices[kNumFullDistances]; - UInt32 i, lenToPosState; - for (i = kStartPosModelIndex; i < kNumFullDistances; i++) - { - UInt32 posSlot = GetPosSlot1(i); - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base - posSlot - 1, footerBits, i - base, p->ProbPrices); - } - - for (lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) - { - UInt32 posSlot; - const CLzmaProb *encoder = p->posSlotEncoder[lenToPosState]; - UInt32 *posSlotPrices = p->posSlotPrices[lenToPosState]; - for (posSlot = 0; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] = RcTree_GetPrice(encoder, kNumPosSlotBits, posSlot, p->ProbPrices); - for (posSlot = kEndPosModelIndex; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] += ((((posSlot >> 1) - 1) - kNumAlignBits) << kNumBitPriceShiftBits); - - { - UInt32 *distancesPrices = p->distancesPrices[lenToPosState]; - UInt32 i; - for (i = 0; i < kStartPosModelIndex; i++) - distancesPrices[i] = posSlotPrices[i]; - for (; i < kNumFullDistances; i++) - distancesPrices[i] = posSlotPrices[GetPosSlot1(i)] + tempPrices[i]; - } - } - p->matchPriceCount = 0; -} - -void LzmaEnc_Construct(CLzmaEnc *p) -{ - RangeEnc_Construct(&p->rc); - MatchFinder_Construct(&p->matchFinderBase); - #ifdef COMPRESS_MF_MT - MatchFinderMt_Construct(&p->matchFinderMt); - p->matchFinderMt.MatchFinder = &p->matchFinderBase; - #endif - - { - CLzmaEncProps props; - LzmaEncProps_Init(&props); - LzmaEnc_SetProps(p, &props); - } - - #ifndef LZMA_LOG_BSR - LzmaEnc_FastPosInit(p->g_FastPos); - #endif - - LzmaEnc_InitPriceTables(p->ProbPrices); - p->litProbs = 0; - p->saveState.litProbs = 0; -} - -CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc) -{ - void *p; - p = alloc->Alloc(alloc, sizeof(CLzmaEnc)); - if (p != 0) - LzmaEnc_Construct((CLzmaEnc *)p); - return p; -} - -void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAlloc *alloc) -{ - alloc->Free(alloc, p->litProbs); - alloc->Free(alloc, p->saveState.litProbs); - p->litProbs = 0; - p->saveState.litProbs = 0; -} - -void LzmaEnc_Destruct(CLzmaEnc *p, ISzAlloc *alloc, ISzAlloc *allocBig) -{ - #ifdef COMPRESS_MF_MT - MatchFinderMt_Destruct(&p->matchFinderMt, allocBig); - #endif - MatchFinder_Free(&p->matchFinderBase, allocBig); - LzmaEnc_FreeLits(p, alloc); - RangeEnc_Free(&p->rc, alloc); -} - -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig) -{ - LzmaEnc_Destruct((CLzmaEnc *)p, alloc, allocBig); - alloc->Free(alloc, p); -} - -static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize, UInt32 maxUnpackSize) -{ - UInt32 nowPos32, startPos32; - if (p->inStream != 0) - { - p->matchFinderBase.stream = p->inStream; - p->matchFinder.Init(p->matchFinderObj); - p->inStream = 0; - } - - if (p->finished) - return p->result; - RINOK(CheckErrors(p)); - - nowPos32 = (UInt32)p->nowPos64; - startPos32 = nowPos32; - - if (p->nowPos64 == 0) - { - UInt32 numPairs; - Byte curByte; - if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) - return Flush(p, nowPos32); - ReadMatchDistances(p, &numPairs); - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][0], 0); - p->state = kLiteralNextStates[p->state]; - curByte = p->matchFinder.GetIndexByte(p->matchFinderObj, 0 - p->additionalOffset); - LitEnc_Encode(&p->rc, p->litProbs, curByte); - p->additionalOffset--; - nowPos32++; - } - - if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0) - for (;;) - { - UInt32 pos, len, posState; - - if (p->fastMode) - len = GetOptimumFast(p, &pos); - else - len = GetOptimum(p, nowPos32, &pos); - - #ifdef SHOW_STAT2 - printf("\n pos = %4X, len = %d pos = %d", nowPos32, len, pos); - #endif - - posState = nowPos32 & p->pbMask; - if (len == 1 && pos == (UInt32)-1) - { - Byte curByte; - CLzmaProb *probs; - const Byte *data; - - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 0); - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; - curByte = *data; - probs = LIT_PROBS(nowPos32, *(data - 1)); - if (IsCharState(p->state)) - LitEnc_Encode(&p->rc, probs, curByte); - else - LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0] - 1)); - p->state = kLiteralNextStates[p->state]; - } - else - { - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - if (pos < LZMA_NUM_REPS) - { - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 1); - if (pos == 0) - { - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 0); - RangeEnc_EncodeBit(&p->rc, &p->isRep0Long[p->state][posState], ((len == 1) ? 0 : 1)); - } - else - { - UInt32 distance = p->reps[pos]; - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 1); - if (pos == 1) - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 0); - else - { - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRepG2[p->state], pos - 2); - if (pos == 3) - p->reps[3] = p->reps[2]; - p->reps[2] = p->reps[1]; - } - p->reps[1] = p->reps[0]; - p->reps[0] = distance; - } - if (len == 1) - p->state = kShortRepNextStates[p->state]; - else - { - LenEnc_Encode2(&p->repLenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - p->state = kRepNextStates[p->state]; - } - } - else - { - UInt32 posSlot; - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); - p->state = kMatchNextStates[p->state]; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - pos -= LZMA_NUM_REPS; - GetPosSlot(pos, posSlot); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, posSlot); - - if (posSlot >= kStartPosModelIndex) - { - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - UInt32 posReduced = pos - base; - - if (posSlot < kEndPosModelIndex) - RcTree_ReverseEncode(&p->rc, p->posEncoders + base - posSlot - 1, footerBits, posReduced); - else - { - RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); - p->alignPriceCount++; - } - } - p->reps[3] = p->reps[2]; - p->reps[2] = p->reps[1]; - p->reps[1] = p->reps[0]; - p->reps[0] = pos; - p->matchPriceCount++; - } - } - p->additionalOffset -= len; - nowPos32 += len; - if (p->additionalOffset == 0) - { - UInt32 processed; - if (!p->fastMode) - { - if (p->matchPriceCount >= (1 << 7)) - FillDistancesPrices(p); - if (p->alignPriceCount >= kAlignTableSize) - FillAlignPrices(p); - } - if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) - break; - processed = nowPos32 - startPos32; - if (useLimits) - { - if (processed + kNumOpts + 300 >= maxUnpackSize || - RangeEnc_GetProcessed(&p->rc) + kNumOpts * 2 >= maxPackSize) - break; - } - else if (processed >= (1 << 15)) - { - p->nowPos64 += nowPos32 - startPos32; - return CheckErrors(p); - } - } - } - p->nowPos64 += nowPos32 - startPos32; - return Flush(p, nowPos32); -} - -#define kBigHashDicLimit ((UInt32)1 << 24) - -static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) -{ - UInt32 beforeSize = kNumOpts; - Bool btMode; - if (!RangeEnc_Alloc(&p->rc, alloc)) - return SZ_ERROR_MEM; - btMode = (p->matchFinderBase.btMode != 0); - #ifdef COMPRESS_MF_MT - p->mtMode = (p->multiThread && !p->fastMode && btMode); - #endif - - { - unsigned lclp = p->lc + p->lp; - if (p->litProbs == 0 || p->saveState.litProbs == 0 || p->lclp != lclp) - { - LzmaEnc_FreeLits(p, alloc); - p->litProbs = (CLzmaProb *)alloc->Alloc(alloc, (0x300 << lclp) * sizeof(CLzmaProb)); - p->saveState.litProbs = (CLzmaProb *)alloc->Alloc(alloc, (0x300 << lclp) * sizeof(CLzmaProb)); - if (p->litProbs == 0 || p->saveState.litProbs == 0) - { - LzmaEnc_FreeLits(p, alloc); - return SZ_ERROR_MEM; - } - p->lclp = lclp; - } - } - - p->matchFinderBase.bigHash = (p->dictSize > kBigHashDicLimit); - - if (beforeSize + p->dictSize < keepWindowSize) - beforeSize = keepWindowSize - p->dictSize; - - #ifdef COMPRESS_MF_MT - if (p->mtMode) - { - RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)); - p->matchFinderObj = &p->matchFinderMt; - MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); - } - else - #endif - { - if (!MatchFinder_Create(&p->matchFinderBase, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)) - return SZ_ERROR_MEM; - p->matchFinderObj = &p->matchFinderBase; - MatchFinder_CreateVTable(&p->matchFinderBase, &p->matchFinder); - } - return SZ_OK; -} - -void LzmaEnc_Init(CLzmaEnc *p) -{ - UInt32 i; - p->state = 0; - for (i = 0 ; i < LZMA_NUM_REPS; i++) - p->reps[i] = 0; - - RangeEnc_Init(&p->rc); - - - for (i = 0; i < kNumStates; i++) - { - UInt32 j; - for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) - { - p->isMatch[i][j] = kProbInitValue; - p->isRep0Long[i][j] = kProbInitValue; - } - p->isRep[i] = kProbInitValue; - p->isRepG0[i] = kProbInitValue; - p->isRepG1[i] = kProbInitValue; - p->isRepG2[i] = kProbInitValue; - } - - { - UInt32 num = 0x300 << (p->lp + p->lc); - for (i = 0; i < num; i++) - p->litProbs[i] = kProbInitValue; - } - - { - for (i = 0; i < kNumLenToPosStates; i++) - { - CLzmaProb *probs = p->posSlotEncoder[i]; - UInt32 j; - for (j = 0; j < (1 << kNumPosSlotBits); j++) - probs[j] = kProbInitValue; - } - } - { - for (i = 0; i < kNumFullDistances - kEndPosModelIndex; i++) - p->posEncoders[i] = kProbInitValue; - } - - LenEnc_Init(&p->lenEnc.p); - LenEnc_Init(&p->repLenEnc.p); - - for (i = 0; i < (1 << kNumAlignBits); i++) - p->posAlignEncoder[i] = kProbInitValue; - - p->optimumEndIndex = 0; - p->optimumCurrentIndex = 0; - p->additionalOffset = 0; - - p->pbMask = (1 << p->pb) - 1; - p->lpMask = (1 << p->lp) - 1; -} - -void LzmaEnc_InitPrices(CLzmaEnc *p) -{ - if (!p->fastMode) - { - FillDistancesPrices(p); - FillAlignPrices(p); - } - - p->lenEnc.tableSize = - p->repLenEnc.tableSize = - p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, p->ProbPrices); - LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, p->ProbPrices); -} - -static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) -{ - UInt32 i; - for (i = 0; i < (UInt32)kDicLogSizeMaxCompress; i++) - if (p->dictSize <= ((UInt32)1 << i)) - break; - p->distTableSize = i * 2; - - p->finished = False; - p->result = SZ_OK; - RINOK(LzmaEnc_Alloc(p, keepWindowSize, alloc, allocBig)); - LzmaEnc_Init(p); - LzmaEnc_InitPrices(p); - p->nowPos64 = 0; - return SZ_OK; -} - -static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqInStream *inStream, ISeqOutStream *outStream, - ISzAlloc *alloc, ISzAlloc *allocBig) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - p->inStream = inStream; - p->rc.outStream = outStream; - return LzmaEnc_AllocAndInit(p, 0, alloc, allocBig); -} - -SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, - ISeqInStream *inStream, UInt32 keepWindowSize, - ISzAlloc *alloc, ISzAlloc *allocBig) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - p->inStream = inStream; - return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); -} - -static void LzmaEnc_SetInputBuf(CLzmaEnc *p, const Byte *src, SizeT srcLen) -{ - p->seqBufInStream.funcTable.Read = MyRead; - p->seqBufInStream.data = src; - p->seqBufInStream.rem = srcLen; -} - -SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, - UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - LzmaEnc_SetInputBuf(p, src, srcLen); - p->inStream = &p->seqBufInStream.funcTable; - return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); -} - -void LzmaEnc_Finish(CLzmaEncHandle pp) -{ - #ifdef COMPRESS_MF_MT - CLzmaEnc *p = (CLzmaEnc *)pp; - if (p->mtMode) - MatchFinderMt_ReleaseStream(&p->matchFinderMt); - #else - pp = pp; - #endif -} - -typedef struct _CSeqOutStreamBuf -{ - ISeqOutStream funcTable; - Byte *data; - SizeT rem; - Bool overflow; -} CSeqOutStreamBuf; - -static size_t MyWrite(void *pp, const void *data, size_t size) -{ - CSeqOutStreamBuf *p = (CSeqOutStreamBuf *)pp; - if (p->rem < size) - { - size = p->rem; - p->overflow = True; - } - memcpy(p->data, data, size); - p->rem -= size; - p->data += size; - return size; -} - - -UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp) -{ - const CLzmaEnc *p = (CLzmaEnc *)pp; - return p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); -} - -const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) -{ - const CLzmaEnc *p = (CLzmaEnc *)pp; - return p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; -} - -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, - Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - UInt64 nowPos64; - SRes res; - CSeqOutStreamBuf outStream; - - outStream.funcTable.Write = MyWrite; - outStream.data = dest; - outStream.rem = *destLen; - outStream.overflow = False; - - p->writeEndMark = False; - p->finished = False; - p->result = SZ_OK; - - if (reInit) - LzmaEnc_Init(p); - LzmaEnc_InitPrices(p); - nowPos64 = p->nowPos64; - RangeEnc_Init(&p->rc); - p->rc.outStream = &outStream.funcTable; - - res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize); - - *unpackSize = (UInt32)(p->nowPos64 - nowPos64); - *destLen -= outStream.rem; - if (outStream.overflow) - return SZ_ERROR_OUTPUT_EOF; - - return res; -} - -SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress, - ISzAlloc *alloc, ISzAlloc *allocBig) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - SRes res = SZ_OK; - - #ifdef COMPRESS_MF_MT - Byte allocaDummy[0x300]; - int i = 0; - for (i = 0; i < 16; i++) - allocaDummy[i] = (Byte)i; - #endif - - RINOK(LzmaEnc_Prepare(pp, inStream, outStream, alloc, allocBig)); - - for (;;) - { - res = LzmaEnc_CodeOneBlock(p, False, 0, 0); - if (res != SZ_OK || p->finished != 0) - break; - if (progress != 0) - { - res = progress->Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); - if (res != SZ_OK) - { - res = SZ_ERROR_PROGRESS; - break; - } - } - } - LzmaEnc_Finish(pp); - return res; -} - -SRes LzmaEnc_WriteProperties(CLzmaEncHandle pp, Byte *props, SizeT *size) -{ - CLzmaEnc *p = (CLzmaEnc *)pp; - int i; - UInt32 dictSize = p->dictSize; - if (*size < LZMA_PROPS_SIZE) - return SZ_ERROR_PARAM; - *size = LZMA_PROPS_SIZE; - props[0] = (Byte)((p->pb * 5 + p->lp) * 9 + p->lc); - - for (i = 11; i <= 30; i++) - { - if (dictSize <= ((UInt32)2 << i)) - { - dictSize = (2 << i); - break; - } - if (dictSize <= ((UInt32)3 << i)) - { - dictSize = (3 << i); - break; - } - } - - for (i = 0; i < 4; i++) - props[1 + i] = (Byte)(dictSize >> (8 * i)); - return SZ_OK; -} - -SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig) -{ - SRes res; - CLzmaEnc *p = (CLzmaEnc *)pp; - - CSeqOutStreamBuf outStream; - - LzmaEnc_SetInputBuf(p, src, srcLen); - - outStream.funcTable.Write = MyWrite; - outStream.data = dest; - outStream.rem = *destLen; - outStream.overflow = False; - - p->writeEndMark = writeEndMark; - res = LzmaEnc_Encode(pp, &outStream.funcTable, &p->seqBufInStream.funcTable, - progress, alloc, allocBig); - - *destLen -= outStream.rem; - if (outStream.overflow) - return SZ_ERROR_OUTPUT_EOF; - return res; -} - -SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig) -{ - CLzmaEnc *p = (CLzmaEnc *)LzmaEnc_Create(alloc); - SRes res; - if (p == 0) - return SZ_ERROR_MEM; - - res = LzmaEnc_SetProps(p, props); - if (res == SZ_OK) - { - res = LzmaEnc_WriteProperties(p, propsEncoded, propsSize); - if (res == SZ_OK) - res = LzmaEnc_MemEncode(p, dest, destLen, src, srcLen, - writeEndMark, progress, alloc, allocBig); - } - - LzmaEnc_Destroy(p, alloc, allocBig); - return res; -} diff --git a/ext/7z/LzmaEnc.h b/ext/7z/LzmaEnc.h deleted file mode 100644 index e3d84fa3..00000000 --- a/ext/7z/LzmaEnc.h +++ /dev/null @@ -1,72 +0,0 @@ -/* LzmaEnc.h -- LZMA Encoder -2008-10-04 : Igor Pavlov : Public domain */ - -#ifndef __LZMAENC_H -#define __LZMAENC_H - -#include "Types.h" - -#define LZMA_PROPS_SIZE 5 - -typedef struct _CLzmaEncProps -{ - int level; /* 0 <= level <= 9 */ - UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version - (1 << 12) <= dictSize <= (1 << 30) for 64-bit version - default = (1 << 24) */ - int lc; /* 0 <= lc <= 8, default = 3 */ - int lp; /* 0 <= lp <= 4, default = 0 */ - int pb; /* 0 <= pb <= 4, default = 2 */ - int algo; /* 0 - fast, 1 - normal, default = 1 */ - int fb; /* 5 <= fb <= 273, default = 32 */ - int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */ - int numHashBytes; /* 2, 3 or 4, default = 4 */ - UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ - unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */ - int numThreads; /* 1 or 2, default = 2 */ -} CLzmaEncProps; - -void LzmaEncProps_Init(CLzmaEncProps *p); -void LzmaEncProps_Normalize(CLzmaEncProps *p); -UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2); - - -/* ---------- CLzmaEncHandle Interface ---------- */ - -/* LzmaEnc_* functions can return the following exit codes: -Returns: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - Write callback error. - SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ - -typedef void * CLzmaEncHandle; - -CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc); -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig); -SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props); -SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size); -SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); -SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); - -/* ---------- One Call Interface ---------- */ - -/* LzmaEncode -Return code: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ - -SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); - -#endif diff --git a/ext/7z/LzmaLib.c b/ext/7z/LzmaLib.c deleted file mode 100644 index 3e3cf40a..00000000 --- a/ext/7z/LzmaLib.c +++ /dev/null @@ -1,46 +0,0 @@ -/* LzmaLib.c -- LZMA library wrapper -2008-08-05 -Igor Pavlov -Public domain */ - -#include "LzmaEnc.h" -#include "LzmaDec.h" -#include "Alloc.h" -#include "LzmaLib.h" - -static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); } -static void SzFree(void *p, void *address) { p = p; MyFree(address); } -static ISzAlloc g_Alloc = { SzAlloc, SzFree }; - -MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t srcLen, - unsigned char *outProps, size_t *outPropsSize, - int level, /* 0 <= level <= 9, default = 5 */ - unsigned dictSize, /* use (1 << N) or (3 << N). 4 KB < dictSize <= 128 MB */ - int lc, /* 0 <= lc <= 8, default = 3 */ - int lp, /* 0 <= lp <= 4, default = 0 */ - int pb, /* 0 <= pb <= 4, default = 2 */ - int fb, /* 5 <= fb <= 273, default = 32 */ - int numThreads /* 1 or 2, default = 2 */ -) -{ - CLzmaEncProps props; - LzmaEncProps_Init(&props); - props.level = level; - props.dictSize = dictSize; - props.lc = lc; - props.lp = lp; - props.pb = pb; - props.fb = fb; - props.numThreads = numThreads; - - return LzmaEncode(dest, destLen, src, srcLen, &props, outProps, outPropsSize, 0, - NULL, &g_Alloc, &g_Alloc); -} - - -MY_STDAPI LzmaUncompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t *srcLen, - const unsigned char *props, size_t propsSize) -{ - ELzmaStatus status; - return LzmaDecode(dest, destLen, src, srcLen, props, (unsigned)propsSize, LZMA_FINISH_ANY, &status, &g_Alloc); -} diff --git a/ext/7z/LzmaLib.h b/ext/7z/LzmaLib.h deleted file mode 100644 index 849d347e..00000000 --- a/ext/7z/LzmaLib.h +++ /dev/null @@ -1,135 +0,0 @@ -/* LzmaLib.h -- LZMA library interface -2008-08-05 -Igor Pavlov -Public domain */ - -#ifndef __LZMALIB_H -#define __LZMALIB_H - -#include "Types.h" - -#ifdef __cplusplus - #define MY_EXTERN_C extern "C" -#else - #define MY_EXTERN_C extern -#endif - -#define MY_STDAPI MY_EXTERN_C int MY_STD_CALL - -#define LZMA_PROPS_SIZE 5 - -/* -RAM requirements for LZMA: - for compression: (dictSize * 11.5 + 6 MB) + state_size - for decompression: dictSize + state_size - state_size = (4 + (1.5 << (lc + lp))) KB - by default (lc=3, lp=0), state_size = 16 KB. - -LZMA properties (5 bytes) format - Offset Size Description - 0 1 lc, lp and pb in encoded form. - 1 4 dictSize (little endian). -*/ - -/* -LzmaCompress ------------- - -outPropsSize - - In: the pointer to the size of outProps buffer; *outPropsSize = LZMA_PROPS_SIZE = 5. - Out: the pointer to the size of written properties in outProps buffer; *outPropsSize = LZMA_PROPS_SIZE = 5. - - LZMA Encoder will use defult values for any parameter, if it is - -1 for any from: level, loc, lp, pb, fb, numThreads - 0 for dictSize - -level - compression level: 0 <= level <= 9; - - level dictSize algo fb - 0: 16 KB 0 32 - 1: 64 KB 0 32 - 2: 256 KB 0 32 - 3: 1 MB 0 32 - 4: 4 MB 0 32 - 5: 16 MB 1 32 - 6: 32 MB 1 32 - 7+: 64 MB 1 64 - - The default value for "level" is 5. - - algo = 0 means fast method - algo = 1 means normal method - -dictSize - The dictionary size in bytes. The maximum value is - 128 MB = (1 << 27) bytes for 32-bit version - 1 GB = (1 << 30) bytes for 64-bit version - The default value is 16 MB = (1 << 24) bytes. - It's recommended to use the dictionary that is larger than 4 KB and - that can be calculated as (1 << N) or (3 << N) sizes. - -lc - The number of literal context bits (high bits of previous literal). - It can be in the range from 0 to 8. The default value is 3. - Sometimes lc=4 gives the gain for big files. - -lp - The number of literal pos bits (low bits of current position for literals). - It can be in the range from 0 to 4. The default value is 0. - The lp switch is intended for periodical data when the period is equal to 2^lp. - For example, for 32-bit (4 bytes) periodical data you can use lp=2. Often it's - better to set lc=0, if you change lp switch. - -pb - The number of pos bits (low bits of current position). - It can be in the range from 0 to 4. The default value is 2. - The pb switch is intended for periodical data when the period is equal 2^pb. - -fb - Word size (the number of fast bytes). - It can be in the range from 5 to 273. The default value is 32. - Usually, a big number gives a little bit better compression ratio and - slower compression process. - -numThreads - The number of thereads. 1 or 2. The default value is 2. - Fast mode (algo = 0) can use only 1 thread. - -Out: - destLen - processed output size -Returns: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ - -MY_STDAPI LzmaCompress(unsigned char *dest, size_t *destLen, const unsigned char *src, size_t srcLen, - unsigned char *outProps, size_t *outPropsSize, /* *outPropsSize must be = 5 */ - int level, /* 0 <= level <= 9, default = 5 */ - unsigned dictSize, /* default = (1 << 24) */ - int lc, /* 0 <= lc <= 8, default = 3 */ - int lp, /* 0 <= lp <= 4, default = 0 */ - int pb, /* 0 <= pb <= 4, default = 2 */ - int fb, /* 5 <= fb <= 273, default = 32 */ - int numThreads /* 1 or 2, default = 2 */ - ); - -/* -LzmaUncompress --------------- -In: - dest - output data - destLen - output data size - src - input data - srcLen - input data size -Out: - destLen - processed output size - srcLen - processed input size -Returns: - SZ_OK - OK - SZ_ERROR_DATA - Data error - SZ_ERROR_MEM - Memory allocation arror - SZ_ERROR_UNSUPPORTED - Unsupported properties - SZ_ERROR_INPUT_EOF - it needs more bytes in input buffer (src) -*/ - -MY_STDAPI LzmaUncompress(unsigned char *dest, size_t *destLen, const unsigned char *src, SizeT *srcLen, - const unsigned char *props, size_t propsSize); - -#endif diff --git a/ext/7z/Threads.c b/ext/7z/Threads.c deleted file mode 100644 index c7d8b0a8..00000000 --- a/ext/7z/Threads.c +++ /dev/null @@ -1,109 +0,0 @@ -/* Threads.c -- multithreading library -2008-08-05 -Igor Pavlov -Public domain */ - -#include "Threads.h" -#include - -static WRes GetError() -{ - DWORD res = GetLastError(); - return (res) ? (WRes)(res) : 1; -} - -WRes HandleToWRes(HANDLE h) { return (h != 0) ? 0 : GetError(); } -WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); } - -static WRes MyCloseHandle(HANDLE *h) -{ - if (*h != NULL) - if (!CloseHandle(*h)) - return GetError(); - *h = NULL; - return 0; -} - -WRes Thread_Create(CThread *thread, THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE *startAddress)(void *), LPVOID parameter) -{ - unsigned threadId; /* Windows Me/98/95: threadId parameter may not be NULL in _beginthreadex/CreateThread functions */ - thread->handle = - /* CreateThread(0, 0, startAddress, parameter, 0, &threadId); */ - (HANDLE)_beginthreadex(NULL, 0, startAddress, parameter, 0, &threadId); - /* maybe we must use errno here, but probably GetLastError() is also OK. */ - return HandleToWRes(thread->handle); -} - -WRes WaitObject(HANDLE h) -{ - return (WRes)WaitForSingleObject(h, INFINITE); -} - -WRes Thread_Wait(CThread *thread) -{ - if (thread->handle == NULL) - return 1; - return WaitObject(thread->handle); -} - -WRes Thread_Close(CThread *thread) -{ - return MyCloseHandle(&thread->handle); -} - -WRes Event_Create(CEvent *p, BOOL manualReset, int initialSignaled) -{ - p->handle = CreateEvent(NULL, manualReset, (initialSignaled ? TRUE : FALSE), NULL); - return HandleToWRes(p->handle); -} - -WRes ManualResetEvent_Create(CManualResetEvent *p, int initialSignaled) - { return Event_Create(p, TRUE, initialSignaled); } -WRes ManualResetEvent_CreateNotSignaled(CManualResetEvent *p) - { return ManualResetEvent_Create(p, 0); } - -WRes AutoResetEvent_Create(CAutoResetEvent *p, int initialSignaled) - { return Event_Create(p, FALSE, initialSignaled); } -WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p) - { return AutoResetEvent_Create(p, 0); } - -WRes Event_Set(CEvent *p) { return BOOLToWRes(SetEvent(p->handle)); } -WRes Event_Reset(CEvent *p) { return BOOLToWRes(ResetEvent(p->handle)); } -WRes Event_Wait(CEvent *p) { return WaitObject(p->handle); } -WRes Event_Close(CEvent *p) { return MyCloseHandle(&p->handle); } - - -WRes Semaphore_Create(CSemaphore *p, UInt32 initiallyCount, UInt32 maxCount) -{ - p->handle = CreateSemaphore(NULL, (LONG)initiallyCount, (LONG)maxCount, NULL); - return HandleToWRes(p->handle); -} - -WRes Semaphore_Release(CSemaphore *p, LONG releaseCount, LONG *previousCount) -{ - return BOOLToWRes(ReleaseSemaphore(p->handle, releaseCount, previousCount)); -} -WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 releaseCount) -{ - return Semaphore_Release(p, (LONG)releaseCount, NULL); -} -WRes Semaphore_Release1(CSemaphore *p) -{ - return Semaphore_ReleaseN(p, 1); -} - -WRes Semaphore_Wait(CSemaphore *p) { return WaitObject(p->handle); } -WRes Semaphore_Close(CSemaphore *p) { return MyCloseHandle(&p->handle); } - -WRes CriticalSection_Init(CCriticalSection *p) -{ - /* InitializeCriticalSection can raise only STATUS_NO_MEMORY exception */ - __try - { - InitializeCriticalSection(p); - /* InitializeCriticalSectionAndSpinCount(p, 0); */ - } - __except (EXCEPTION_EXECUTE_HANDLER) { return 1; } - return 0; -} - diff --git a/ext/7z/Threads.h b/ext/7z/Threads.h deleted file mode 100644 index 3d907294..00000000 --- a/ext/7z/Threads.h +++ /dev/null @@ -1,68 +0,0 @@ -/* Threads.h -- multithreading library -2008-11-22 : Igor Pavlov : Public domain */ - -#ifndef __7Z_THRESDS_H -#define __7Z_THRESDS_H - -#include "Types.h" - -typedef struct _CThread -{ - HANDLE handle; -} CThread; - -#define Thread_Construct(thread) (thread)->handle = NULL -#define Thread_WasCreated(thread) ((thread)->handle != NULL) - -typedef unsigned THREAD_FUNC_RET_TYPE; -#define THREAD_FUNC_CALL_TYPE MY_STD_CALL -#define THREAD_FUNC_DECL THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE - -WRes Thread_Create(CThread *thread, THREAD_FUNC_RET_TYPE (THREAD_FUNC_CALL_TYPE *startAddress)(void *), LPVOID parameter); -WRes Thread_Wait(CThread *thread); -WRes Thread_Close(CThread *thread); - -typedef struct _CEvent -{ - HANDLE handle; -} CEvent; - -typedef CEvent CAutoResetEvent; -typedef CEvent CManualResetEvent; - -#define Event_Construct(event) (event)->handle = NULL -#define Event_IsCreated(event) ((event)->handle != NULL) - -WRes ManualResetEvent_Create(CManualResetEvent *event, int initialSignaled); -WRes ManualResetEvent_CreateNotSignaled(CManualResetEvent *event); -WRes AutoResetEvent_Create(CAutoResetEvent *event, int initialSignaled); -WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *event); -WRes Event_Set(CEvent *event); -WRes Event_Reset(CEvent *event); -WRes Event_Wait(CEvent *event); -WRes Event_Close(CEvent *event); - - -typedef struct _CSemaphore -{ - HANDLE handle; -} CSemaphore; - -#define Semaphore_Construct(p) (p)->handle = NULL - -WRes Semaphore_Create(CSemaphore *p, UInt32 initiallyCount, UInt32 maxCount); -WRes Semaphore_ReleaseN(CSemaphore *p, UInt32 num); -WRes Semaphore_Release1(CSemaphore *p); -WRes Semaphore_Wait(CSemaphore *p); -WRes Semaphore_Close(CSemaphore *p); - - -typedef CRITICAL_SECTION CCriticalSection; - -WRes CriticalSection_Init(CCriticalSection *p); -#define CriticalSection_Delete(p) DeleteCriticalSection(p) -#define CriticalSection_Enter(p) EnterCriticalSection(p) -#define CriticalSection_Leave(p) LeaveCriticalSection(p) - -#endif - diff --git a/ext/7z/Types.h b/ext/7z/Types.h deleted file mode 100644 index 2638196a..00000000 --- a/ext/7z/Types.h +++ /dev/null @@ -1,208 +0,0 @@ -/* Types.h -- Basic types -2008-11-23 : Igor Pavlov : Public domain */ - -#ifndef __7Z_TYPES_H -#define __7Z_TYPES_H - -#include - -#ifdef _WIN32 -#include -#endif - -#define SZ_OK 0 - -#define SZ_ERROR_DATA 1 -#define SZ_ERROR_MEM 2 -#define SZ_ERROR_CRC 3 -#define SZ_ERROR_UNSUPPORTED 4 -#define SZ_ERROR_PARAM 5 -#define SZ_ERROR_INPUT_EOF 6 -#define SZ_ERROR_OUTPUT_EOF 7 -#define SZ_ERROR_READ 8 -#define SZ_ERROR_WRITE 9 -#define SZ_ERROR_PROGRESS 10 -#define SZ_ERROR_FAIL 11 -#define SZ_ERROR_THREAD 12 - -#define SZ_ERROR_ARCHIVE 16 -#define SZ_ERROR_NO_ARCHIVE 17 - -typedef int SRes; - -#ifdef _WIN32 -typedef DWORD WRes; -#else -typedef int WRes; -#endif - -#ifndef RINOK -#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; } -#endif - -typedef unsigned char Byte; -typedef short Int16; -typedef unsigned short UInt16; - -#ifdef _LZMA_UINT32_IS_ULONG -typedef long Int32; -typedef unsigned long UInt32; -#else -typedef int Int32; -typedef unsigned int UInt32; -#endif - -#ifdef _SZ_NO_INT_64 - -/* define _SZ_NO_INT_64, if your compiler doesn't support 64-bit integers. - NOTES: Some code will work incorrectly in that case! */ - -typedef long Int64; -typedef unsigned long UInt64; - -#else - -#if defined(_MSC_VER) || defined(__BORLANDC__) -typedef __int64 Int64; -typedef unsigned __int64 UInt64; -#else -typedef long long int Int64; -typedef unsigned long long int UInt64; -#endif - -#endif - -#ifdef _LZMA_NO_SYSTEM_SIZE_T -typedef UInt32 SizeT; -#else -typedef size_t SizeT; -#endif - -typedef int Bool; -#define True 1 -#define False 0 - - -#ifdef _MSC_VER - -#if _MSC_VER >= 1300 -#define MY_NO_INLINE __declspec(noinline) -#else -#define MY_NO_INLINE -#endif - -#define MY_CDECL __cdecl -#define MY_STD_CALL __stdcall -#define MY_FAST_CALL MY_NO_INLINE __fastcall - -#else - -#define MY_CDECL -#define MY_STD_CALL -#define MY_FAST_CALL - -#endif - - -/* The following interfaces use first parameter as pointer to structure */ - -typedef struct -{ - SRes (*Read)(void *p, void *buf, size_t *size); - /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. - (output(*size) < input(*size)) is allowed */ -} ISeqInStream; - -/* it can return SZ_ERROR_INPUT_EOF */ -SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size); -SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType); -SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf); - -typedef struct -{ - size_t (*Write)(void *p, const void *buf, size_t size); - /* Returns: result - the number of actually written bytes. - (result < size) means error */ -} ISeqOutStream; - -typedef enum -{ - SZ_SEEK_SET = 0, - SZ_SEEK_CUR = 1, - SZ_SEEK_END = 2 -} ESzSeek; - -typedef struct -{ - SRes (*Read)(void *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ - SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); -} ISeekInStream; - -typedef struct -{ - SRes (*Look)(void *p, void **buf, size_t *size); - /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. - (output(*size) > input(*size)) is not allowed - (output(*size) < input(*size)) is allowed */ - SRes (*Skip)(void *p, size_t offset); - /* offset must be <= output(*size) of Look */ - - SRes (*Read)(void *p, void *buf, size_t *size); - /* reads directly (without buffer). It's same as ISeqInStream::Read */ - SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); -} ILookInStream; - -SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size); -SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset); - -/* reads via ILookInStream::Read */ -SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType); -SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size); - -#define LookToRead_BUF_SIZE (1 << 14) - -typedef struct -{ - ILookInStream s; - ISeekInStream *realStream; - size_t pos; - size_t size; - Byte buf[LookToRead_BUF_SIZE]; -} CLookToRead; - -void LookToRead_CreateVTable(CLookToRead *p, int lookahead); -void LookToRead_Init(CLookToRead *p); - -typedef struct -{ - ISeqInStream s; - ILookInStream *realStream; -} CSecToLook; - -void SecToLook_CreateVTable(CSecToLook *p); - -typedef struct -{ - ISeqInStream s; - ILookInStream *realStream; -} CSecToRead; - -void SecToRead_CreateVTable(CSecToRead *p); - -typedef struct -{ - SRes (*Progress)(void *p, UInt64 inSize, UInt64 outSize); - /* Returns: result. (result != SZ_OK) means break. - Value (UInt64)(Int64)-1 for size means unknown value. */ -} ICompressProgress; - -typedef struct -{ - void *(*Alloc)(void *p, size_t size); - void (*Free)(void *p, void *address); /* address can be 0 */ -} ISzAlloc; - -#define IAlloc_Alloc(p, size) (p)->Alloc((p), size) -#define IAlloc_Free(p, a) (p)->Free((p), a) - -#endif diff --git a/ext/libpng/projects/visualc71/libpng.vcproj b/ext/libpng/projects/visualc71/libpng.vcproj deleted file mode 100644 index d3942d5d..00000000 --- a/ext/libpng/projects/visualc71/libpng.vcproj +++ /dev/null @@ -1,606 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ext/libpng/projects/visualc71/libpng_2005Express.vcproj b/ext/libpng/projects/visualc71/libpng_2005Express.vcproj deleted file mode 100644 index d999082f..00000000 --- a/ext/libpng/projects/visualc71/libpng_2005Express.vcproj +++ /dev/null @@ -1,605 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ext/libpng/projects/visualc71/libpng_VS2003.vcproj b/ext/libpng/projects/visualc71/libpng_VS2003.vcproj deleted file mode 100644 index d86c4a44..00000000 --- a/ext/libpng/projects/visualc71/libpng_VS2003.vcproj +++ /dev/null @@ -1,355 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ext/libpng/projects/visualc71/zlib.vcproj b/ext/libpng/projects/visualc71/zlib.vcproj deleted file mode 100644 index 78309e0a..00000000 --- a/ext/libpng/projects/visualc71/zlib.vcproj +++ /dev/null @@ -1,530 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ext/libpng/projects/visualc71/zlib_2005Express.vcproj b/ext/libpng/projects/visualc71/zlib_2005Express.vcproj deleted file mode 100644 index af792c26..00000000 --- a/ext/libpng/projects/visualc71/zlib_2005Express.vcproj +++ /dev/null @@ -1,532 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ext/libpng/projects/visualc71/zlib_VS2003.vcproj b/ext/libpng/projects/visualc71/zlib_VS2003.vcproj deleted file mode 100644 index 527abe8c..00000000 --- a/ext/libpng/projects/visualc71/zlib_VS2003.vcproj +++ /dev/null @@ -1,311 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ext/utf8/doc/ReleaseNotes b/ext/utf8/doc/ReleaseNotes deleted file mode 100644 index 83adf8a0..00000000 --- a/ext/utf8/doc/ReleaseNotes +++ /dev/null @@ -1,22 +0,0 @@ -utf8 cpp library -<<<<<<< .mine -Release 2.3 -======= -Release 2.2.1 ->>>>>>> .r98 - -<<<<<<< .mine -This is a bug fix release. -======= -This is a minor bug fix release aimed at supporting GCC 4.3. ->>>>>>> .r98 - -<<<<<<< .mine -Changes from version 2.2 -- Bug fix: ID: 2830326 : multiple definition of `utf8::internal::is_overlong_sequence -======= -Changes from version 2.2 -- Bug fix [ ID: 2823847 ] "warning from GCC 4.3" ->>>>>>> .r98 - -Files included in the release: utf8.h, core.h, checked.h, unchecked.h, utf8cpp.html, ReleaseNotes diff --git a/ext/utf8/doc/utf8cpp.html b/ext/utf8/doc/utf8cpp.html deleted file mode 100644 index 069c2be5..00000000 --- a/ext/utf8/doc/utf8cpp.html +++ /dev/null @@ -1,1629 +0,0 @@ - - - - - - - - - UTF8-CPP: UTF-8 with C++ in a Portable Way - - - - -

- UTF8-CPP: UTF-8 with C++ in a Portable Way -

-

- The Sourceforge project page -

- -

- Introduction -

-

- Many C++ developers miss an easy and portable way of handling Unicode encoded - strings. The original C++ Standard (known as C++98 or C++03) is Unicode agnostic, - and while some work is being done to introduce Unicode to the next incarnation - called C++0x, for the moment nothing of the sort is available. In the meantime, - developers use third party libraries like ICU, OS specific capabilities, or simply - roll out their own solutions. -

-

- In order to easily handle UTF-8 encoded Unicode strings, I came up with a small - generic library. For anybody used to work with STL algorithms and iterators, it should be - easy and natural to use. The code is freely available for any purpose - check out - the license at the beginning of the utf8.h file. If you run into - bugs or performance issues, please let me know and I'll do my best to address them. -

-

- The purpose of this article is not to offer an introduction to Unicode in general, - and UTF-8 in particular. If you are not familiar with Unicode, be sure to check out - Unicode Home Page or some other source of - information for Unicode. Also, it is not my aim to advocate the use of UTF-8 - encoded strings in C++ programs; if you want to handle UTF-8 encoded strings from - C++, I am sure you have good reasons for it. -

-

- Examples of use -

-

- Introductionary Sample -

-

- To illustrate the use of the library, let's start with a small but complete program - that opens a file containing UTF-8 encoded text, reads it line by line, checks each line - for invalid UTF-8 byte sequences, and converts it to UTF-16 encoding and back to UTF-8: -

-
-#include <fstream>
-#include <iostream>
-#include <string>
-#include <vector>
-#include "utf8.h"
-using namespace std;
-int main(int argc, char** argv)
-{
-    if (argc != 2) {
-        cout << "\nUsage: docsample filename\n";
-        return 0;
-    }
-
-    const char* test_file_path = argv[1];
-    // Open the test file (contains UTF-8 encoded text)
-    ifstream fs8(test_file_path);
-    if (!fs8.is_open()) {
-    cout << "Could not open " << test_file_path << endl;
-    return 0;
-    }
-
-    unsigned line_count = 1;
-    string line;
-    // Play with all the lines in the file
-    while (getline(fs8, line)) {
-       // check for invalid utf-8 (for a simple yes/no check, there is also utf8::is_valid function)
-        string::iterator end_it = utf8::find_invalid(line.begin(), line.end());
-        if (end_it != line.end()) {
-            cout << "Invalid UTF-8 encoding detected at line " << line_count << "\n";
-            cout << "This part is fine: " << string(line.begin(), end_it) << "\n";
-        }
-
-        // Get the line length (at least for the valid part)
-        int length = utf8::distance(line.begin(), end_it);
-        cout << "Length of line " << line_count << " is " << length <<  "\n";
-
-        // Convert it to utf-16
-        vector<unsigned short> utf16line;
-        utf8::utf8to16(line.begin(), end_it, back_inserter(utf16line));
-
-        // And back to utf-8
-        string utf8line; 
-        utf8::utf16to8(utf16line.begin(), utf16line.end(), back_inserter(utf8line));
-
-        // Confirm that the conversion went OK:
-        if (utf8line != string(line.begin(), end_it))
-            cout << "Error in UTF-16 conversion at line: " << line_count << "\n";        
-
-        line_count++;
-    }
-    return 0;
-}
-
-

- In the previous code sample, for each line we performed - a detection of invalid UTF-8 sequences with find_invalid; the number - of characters (more precisely - the number of Unicode code points, including the end - of line and even BOM if there is one) in each line was - determined with a use of utf8::distance; finally, we have converted - each line to UTF-16 encoding with utf8to16 and back to UTF-8 with - utf16to8. -

-

Checking if a file contains valid UTF-8 text

-

-Here is a function that checks whether the content of a file is valid UTF-8 encoded text without -reading the content into the memory: -

-
    
-bool valid_utf8_file(iconst char* file_name)
-{
-    ifstream ifs(file_name);
-    if (!ifs)
-        return false; // even better, throw here
-
-    istreambuf_iterator<char> it(ifs.rdbuf());
-    istreambuf_iterator<char> eos;
-
-    return utf8::is_valid(it, eos);
-}
-
-

-Because the function utf8::is_valid() works with input iterators, we were able -to pass an istreambuf_iterator to it and read the content of the file directly -without loading it to the memory first.

-

-Note that other functions that take input iterator arguments can be used in a similar way. For -instance, to read the content of a UTF-8 encoded text file and convert the text to UTF-16, just -do something like: -

-
-    utf8::utf8to16(it, eos, back_inserter(u16string));
-
-

Ensure that a string contains valid UTF-8 text

-

-If we have some text that "probably" contains UTF-8 encoded text and we want to -replace any invalid UTF-8 sequence with a replacement character, something like -the following function may be used: -

-
-void fix_utf8_string(std::string& str)
-{
-    std::string temp;
-    utf8::replace_invalid(str.begin(), str.end(), back_inserter(temp));
-    str = temp;
-}
-
-

The function will replace any invalid UTF-8 sequence with a Unicode replacement character. -There is an overloaded function that enables the caller to supply their own replacement character. -

-

- Reference -

-

- Functions From utf8 Namespace -

-

- utf8::append -

-

- Available in version 1.0 and later. -

-

- Encodes a 32 bit code point as a UTF-8 sequence of octets and appends the sequence - to a UTF-8 string. -

-
-template <typename octet_iterator>
-octet_iterator append(uint32_t cp, octet_iterator result);
-   
-
-

- cp: A 32 bit integer representing a code point to append to the - sequence.
- result: An output iterator to the place in the sequence where to - append the code point.
- Return value: An iterator pointing to the place - after the newly appended sequence. -

-

- Example of use: -

-
-unsigned char u[5] = {0,0,0,0,0};
-unsigned char* end = append(0x0448, u);
-assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
-
-

- Note that append does not allocate any memory - it is the burden of - the caller to make sure there is enough memory allocated for the operation. To make - things more interesting, append can add anywhere between 1 and 4 - octets to the sequence. In practice, you would most often want to use - std::back_inserter to ensure that the necessary memory is allocated. -

-

- In case of an invalid code point, a utf8::invalid_code_point exception - is thrown. -

-

- utf8::next -

-

- Available in version 1.0 and later. -

-

- Given the iterator to the beginning of the UTF-8 sequence, it returns the code - point and moves the iterator to the next position. -

-
-template <typename octet_iterator> 
-uint32_t next(octet_iterator& it, octet_iterator end);
-   
-
-

- it: a reference to an iterator pointing to the beginning of an UTF-8 - encoded code point. After the function returns, it is incremented to point to the - beginning of the next code point.
- end: end of the UTF-8 sequence to be processed. If it - gets equal to end during the extraction of a code point, an - utf8::not_enough_room exception is thrown.
- Return value: the 32 bit representation of the - processed UTF-8 code point. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-char* w = twochars;
-int cp = next(w, twochars + 6);
-assert (cp == 0x65e5);
-assert (w == twochars + 3);
-
-

- This function is typically used to iterate through a UTF-8 encoded string. -

-

- In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is - thrown. -

-

- utf8::peek_next -

-

- Available in version 2.1 and later. -

-

- Given the iterator to the beginning of the UTF-8 sequence, it returns the code - point for the following sequence without changing the value of the iterator. -

-
-template <typename octet_iterator> 
-uint32_t peek_next(octet_iterator it, octet_iterator end);
-   
-
-

- it: an iterator pointing to the beginning of an UTF-8 - encoded code point.
- end: end of the UTF-8 sequence to be processed. If it - gets equal to end during the extraction of a code point, an - utf8::not_enough_room exception is thrown.
- Return value: the 32 bit representation of the - processed UTF-8 code point. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-char* w = twochars;
-int cp = peek_next(w, twochars + 6);
-assert (cp == 0x65e5);
-assert (w == twochars);
-
-

- In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is - thrown. -

-

- utf8::prior -

-

- Available in version 1.02 and later. -

-

- Given a reference to an iterator pointing to an octet in a UTF-8 seqence, it - decreases the iterator until it hits the beginning of the previous UTF-8 encoded - code point and returns the 32 bits representation of the code point. -

-
-template <typename octet_iterator> 
-uint32_t prior(octet_iterator& it, octet_iterator start);
-   
-
-

- it: a reference pointing to an octet within a UTF-8 encoded string. - After the function returns, it is decremented to point to the beginning of the - previous code point.
- start: an iterator to the beginning of the sequence where the search - for the beginning of a code point is performed. It is a - safety measure to prevent passing the beginning of the string in the search for a - UTF-8 lead octet.
- Return value: the 32 bit representation of the - previous code point. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-unsigned char* w = twochars + 3;
-int cp = prior (w, twochars);
-assert (cp == 0x65e5);
-assert (w == twochars);
-
-

- This function has two purposes: one is two iterate backwards through a UTF-8 - encoded string. Note that it is usually a better idea to iterate forward instead, - since utf8::next is faster. The second purpose is to find a beginning - of a UTF-8 sequence if we have a random position within a string. -

-

- it will typically point to the beginning of - a code point, and start will point to the - beginning of the string to ensure we don't go backwards too far. it is - decreased until it points to a lead UTF-8 octet, and then the UTF-8 sequence - beginning with that octet is decoded to a 32 bit representation and returned. -

-

- In case pass_end is reached before a UTF-8 lead octet is hit, or if an - invalid UTF-8 sequence is started by the lead octet, an invalid_utf8 - exception is thrown. -

-

- utf8::previous -

-

- Deprecated in version 1.02 and later. -

-

- Given a reference to an iterator pointing to an octet in a UTF-8 seqence, it - decreases the iterator until it hits the beginning of the previous UTF-8 encoded - code point and returns the 32 bits representation of the code point. -

-
-template <typename octet_iterator> 
-uint32_t previous(octet_iterator& it, octet_iterator pass_start);
-   
-
-

- it: a reference pointing to an octet within a UTF-8 encoded string. - After the function returns, it is decremented to point to the beginning of the - previous code point.
- pass_start: an iterator to the point in the sequence where the search - for the beginning of a code point is aborted if no result was reached. It is a - safety measure to prevent passing the beginning of the string in the search for a - UTF-8 lead octet.
- Return value: the 32 bit representation of the - previous code point. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-unsigned char* w = twochars + 3;
-int cp = previous (w, twochars - 1);
-assert (cp == 0x65e5);
-assert (w == twochars);
-
-

- utf8::previous is deprecated, and utf8::prior should - be used instead, although the existing code can continue using this function. - The problem is the parameter pass_start that points to the position - just before the beginning of the sequence. Standard containers don't have the - concept of "pass start" and the function can not be used with their iterators. -

-

- it will typically point to the beginning of - a code point, and pass_start will point to the octet just before the - beginning of the string to ensure we don't go backwards too far. it is - decreased until it points to a lead UTF-8 octet, and then the UTF-8 sequence - beginning with that octet is decoded to a 32 bit representation and returned. -

-

- In case pass_end is reached before a UTF-8 lead octet is hit, or if an - invalid UTF-8 sequence is started by the lead octet, an invalid_utf8 - exception is thrown -

-

- utf8::advance -

-

- Available in version 1.0 and later. -

-

- Advances an iterator by the specified number of code points within an UTF-8 - sequence. -

-
-template <typename octet_iterator, typename distance_type> 
-void advance (octet_iterator& it, distance_type n, octet_iterator end);
-   
-
-

- it: a reference to an iterator pointing to the beginning of an UTF-8 - encoded code point. After the function returns, it is incremented to point to the - nth following code point.
- n: a positive integer that shows how many code points we want to - advance.
- end: end of the UTF-8 sequence to be processed. If it - gets equal to end during the extraction of a code point, an - utf8::not_enough_room exception is thrown.
-

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-unsigned char* w = twochars;
-advance (w, 2, twochars + 6);
-assert (w == twochars + 5);
-
-

- This function works only "forward". In case of a negative n, there is - no effect. -

-

- In case of an invalid code point, a utf8::invalid_code_point exception - is thrown. -

-

- utf8::distance -

-

- Available in version 1.0 and later. -

-

- Given the iterators to two UTF-8 encoded code points in a seqence, returns the - number of code points between them. -

-
-template <typename octet_iterator> 
-typename std::iterator_traits<octet_iterator>::difference_type distance (octet_iterator first, octet_iterator last);
-   
-
-

- first: an iterator to a beginning of a UTF-8 encoded code point.
- last: an iterator to a "post-end" of the last UTF-8 encoded code - point in the sequence we are trying to determine the length. It can be the - beginning of a new code point, or not.
- Return value the distance between the iterators, - in code points. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-size_t dist = utf8::distance(twochars, twochars + 5);
-assert (dist == 2);
-
-

- This function is used to find the length (in code points) of a UTF-8 encoded - string. The reason it is called distance, rather than, say, - length is mainly because developers are used that length is an - O(1) function. Computing the length of an UTF-8 string is a linear operation, and - it looked better to model it after std::distance algorithm. -

-

- In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is - thrown. If last does not point to the past-of-end of a UTF-8 seqence, - a utf8::not_enough_room exception is thrown. -

-

- utf8::utf16to8 -

-

- Available in version 1.0 and later. -

-

- Converts a UTF-16 encoded string to UTF-8. -

-
-template <typename u16bit_iterator, typename octet_iterator>
-octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result);
-   
-
-

- start: an iterator pointing to the beginning of the UTF-16 encoded - string to convert.
- end: an iterator pointing to pass-the-end of the UTF-16 encoded - string to convert.
- result: an output iterator to the place in the UTF-8 string where to - append the result of conversion.
- Return value: An iterator pointing to the place - after the appended UTF-8 string. -

-

- Example of use: -

-
-unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
-vector<unsigned char> utf8result;
-utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
-assert (utf8result.size() == 10);    
-
-

- In case of invalid UTF-16 sequence, a utf8::invalid_utf16 exception is - thrown. -

-

- utf8::utf8to16 -

-

- Available in version 1.0 and later. -

-

- Converts an UTF-8 encoded string to UTF-16 -

-
-template <typename u16bit_iterator, typename octet_iterator>
-u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result);
-   
-
-

- start: an iterator pointing to the beginning of the UTF-8 encoded - string to convert. < br /> end: an iterator pointing to - pass-the-end of the UTF-8 encoded string to convert.
- result: an output iterator to the place in the UTF-16 string where to - append the result of conversion.
- Return value: An iterator pointing to the place - after the appended UTF-16 string. -

-

- Example of use: -

-
-char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
-vector <unsigned short> utf16result;
-utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
-assert (utf16result.size() == 4);
-assert (utf16result[2] == 0xd834);
-assert (utf16result[3] == 0xdd1e);
-
-

- In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is - thrown. If end does not point to the past-of-end of a UTF-8 seqence, a - utf8::not_enough_room exception is thrown. -

-

- utf8::utf32to8 -

-

- Available in version 1.0 and later. -

-

- Converts a UTF-32 encoded string to UTF-8. -

-
-template <typename octet_iterator, typename u32bit_iterator>
-octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result);
-   
-
-

- start: an iterator pointing to the beginning of the UTF-32 encoded - string to convert.
- end: an iterator pointing to pass-the-end of the UTF-32 encoded - string to convert.
- result: an output iterator to the place in the UTF-8 string where to - append the result of conversion.
- Return value: An iterator pointing to the place - after the appended UTF-8 string. -

-

- Example of use: -

-
-int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
-vector<unsigned char> utf8result;
-utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
-assert (utf8result.size() == 9);
-
-

- In case of invalid UTF-32 string, a utf8::invalid_code_point exception - is thrown. -

-

- utf8::utf8to32 -

-

- Available in version 1.0 and later. -

-

- Converts a UTF-8 encoded string to UTF-32. -

-
-template <typename octet_iterator, typename u32bit_iterator>
-u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result);
-   
-
-

- start: an iterator pointing to the beginning of the UTF-8 encoded - string to convert.
- end: an iterator pointing to pass-the-end of the UTF-8 encoded string - to convert.
- result: an output iterator to the place in the UTF-32 string where to - append the result of conversion.
- Return value: An iterator pointing to the place - after the appended UTF-32 string. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-vector<int> utf32result;
-utf8to32(twochars, twochars + 5, back_inserter(utf32result));
-assert (utf32result.size() == 2);
-
-

- In case of an invalid UTF-8 seqence, a utf8::invalid_utf8 exception is - thrown. If end does not point to the past-of-end of a UTF-8 seqence, a - utf8::not_enough_room exception is thrown. -

-

- utf8::find_invalid -

-

- Available in version 1.0 and later. -

-

- Detects an invalid sequence within a UTF-8 string. -

-
-template <typename octet_iterator> 
-octet_iterator find_invalid(octet_iterator start, octet_iterator end);
-
-

- start: an iterator pointing to the beginning of the UTF-8 string to - test for validity.
- end: an iterator pointing to pass-the-end of the UTF-8 string to test - for validity.
- Return value: an iterator pointing to the first - invalid octet in the UTF-8 string. In case none were found, equals - end. -

-

- Example of use: -

-
-char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
-char* invalid = find_invalid(utf_invalid, utf_invalid + 6);
-assert (invalid == utf_invalid + 5);
-
-

- This function is typically used to make sure a UTF-8 string is valid before - processing it with other functions. It is especially important to call it if before - doing any of the unchecked operations on it. -

-

- utf8::is_valid -

-

- Available in version 1.0 and later. -

-

- Checks whether a sequence of octets is a valid UTF-8 string. -

-
-template <typename octet_iterator> 
-bool is_valid(octet_iterator start, octet_iterator end);
-   
-
-

- start: an iterator pointing to the beginning of the UTF-8 string to - test for validity.
- end: an iterator pointing to pass-the-end of the UTF-8 string to test - for validity.
- Return value: true if the sequence - is a valid UTF-8 string; false if not. -

- Example of use: -
-char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
-bool bvalid = is_valid(utf_invalid, utf_invalid + 6);
-assert (bvalid == false);
-
-

- is_valid is a shorthand for find_invalid(start, end) == - end;. You may want to use it to make sure that a byte seqence is a valid - UTF-8 string without the need to know where it fails if it is not valid. -

-

- utf8::replace_invalid -

-

- Available in version 2.0 and later. -

-

- Replaces all invalid UTF-8 sequences within a string with a replacement marker. -

-
-template <typename octet_iterator, typename output_iterator>
-output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement);
-template <typename octet_iterator, typename output_iterator>
-output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out);
-   
-
-

- start: an iterator pointing to the beginning of the UTF-8 string to - look for invalid UTF-8 sequences.
- end: an iterator pointing to pass-the-end of the UTF-8 string to look - for invalid UTF-8 sequences.
- out: An output iterator to the range where the result of replacement - is stored.
- replacement: A Unicode code point for the replacement marker. The - version without this parameter assumes the value 0xfffd
- Return value: An iterator pointing to the place - after the UTF-8 string with replaced invalid sequences. -

-

- Example of use: -

-
-char invalid_sequence[] = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
-vector<char> replace_invalid_result;
-replace_invalid (invalid_sequence, invalid_sequence + sizeof(invalid_sequence), back_inserter(replace_invalid_result), '?');
-bvalid = is_valid(replace_invalid_result.begin(), replace_invalid_result.end());
-assert (bvalid);
-char* fixed_invalid_sequence = "a????z";
-assert (std::equal(replace_invalid_result.begin(), replace_invalid_result.end(), fixed_invalid_sequence));
-
-

- replace_invalid does not perform in-place replacement of invalid - sequences. Rather, it produces a copy of the original string with the invalid - sequences replaced with a replacement marker. Therefore, out must not - be in the [start, end] range. -

-

- If end does not point to the past-of-end of a UTF-8 sequence, a - utf8::not_enough_room exception is thrown. -

-

- utf8::is_bom -

-

- Available in version 1.0 and later. -

-

- Checks whether a sequence of three octets is a UTF-8 byte order mark (BOM) -

-
-template <typename octet_iterator> 
-bool is_bom (octet_iterator it);
-
-

- it: beginning of the 3-octet sequence to check
- Return value: true if the sequence - is UTF-8 byte order mark; false if not. -

-

- Example of use: -

-
-unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
-bool bbom = is_bom(byte_order_mark);
-assert (bbom == true);
-
-

- The typical use of this function is to check the first three bytes of a file. If - they form the UTF-8 BOM, we want to skip them before processing the actual UTF-8 - encoded text. -

-

- Types From utf8 Namespace -

-

- utf8::iterator -

-

- Available in version 2.0 and later. -

-

- Adapts the underlying octet iterator to iterate over the sequence of code points, - rather than raw octets. -

-
-template <typename octet_iterator>
-class iterator;
-
- -
Member functions
-
-
iterator();
the deafult constructor; the underlying octet_iterator is - constructed with its default constructor. -
explicit iterator (const octet_iterator& octet_it, - const octet_iterator& range_start, - const octet_iterator& range_end);
a constructor - that initializes the underlying octet_iterator with octet_it - and sets the range in which the iterator is considered valid. -
octet_iterator base () const;
returns the - underlying octet_iterator. -
uint32_t operator * () const;
decodes the utf-8 sequence - the underlying octet_iterator is pointing to and returns the code point. -
bool operator == (const iterator& rhs) - const;
returns true - if the two underlaying iterators are equal. -
bool operator != (const iterator& rhs) - const;
returns true - if the two underlaying iterators are not equal. -
iterator& operator ++ ();
the prefix increment - moves - the iterator to the next UTF-8 encoded code point. -
iterator operator ++ (int);
- the postfix increment - moves the iterator to the next UTF-8 encoded code point and returns the current one. -
iterator& operator -- ();
the prefix decrement - moves - the iterator to the previous UTF-8 encoded code point. -
iterator operator -- (int);
- the postfix decrement - moves the iterator to the previous UTF-8 encoded code point and returns the current one. -
-

- Example of use: -

-
-char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
-utf8::iterator<char*> it(threechars, threechars, threechars + 9);
-utf8::iterator<char*> it2 = it;
-assert (it2 == it);
-assert (*it == 0x10346);
-assert (*(++it) == 0x65e5);
-assert ((*it++) == 0x65e5);
-assert (*it == 0x0448);
-assert (it != it2);
-utf8::iterator<char*> endit (threechars + 9, threechars, threechars + 9);  
-assert (++it == endit);
-assert (*(--it) == 0x0448);
-assert ((*it--) == 0x0448);
-assert (*it == 0x65e5);
-assert (--it == utf8::iterator<char*>(threechars, threechars, threechars + 9));
-assert (*it == 0x10346);
-
-

- The purpose of utf8::iterator adapter is to enable easy iteration as well as the use of STL - algorithms with UTF-8 encoded strings. Increment and decrement operators are implemented in terms of - utf8::next() and utf8::prior() functions. -

-

- Note that utf8::iterator adapter is a checked iterator. It operates on the range specified in - the constructor; any attempt to go out of that range will result in an exception. Even the comparison operators - require both iterator object to be constructed against the same range - otherwise an exception is thrown. Typically, - the range will be determined by sequence container functions begin and end, i.e.: -

-
-std::string s = "example";
-utf8::iterator i (s.begin(), s.begin(), s.end());
-
-

- Functions From utf8::unchecked Namespace -

-

- utf8::unchecked::append -

-

- Available in version 1.0 and later. -

-

- Encodes a 32 bit code point as a UTF-8 sequence of octets and appends the sequence - to a UTF-8 string. -

-
-template <typename octet_iterator>
-octet_iterator append(uint32_t cp, octet_iterator result);
-   
-
-

- cp: A 32 bit integer representing a code point to append to the - sequence.
- result: An output iterator to the place in the sequence where to - append the code point.
- Return value: An iterator pointing to the place - after the newly appended sequence. -

-

- Example of use: -

-
-unsigned char u[5] = {0,0,0,0,0};
-unsigned char* end = unchecked::append(0x0448, u);
-assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
-
-

- This is a faster but less safe version of utf8::append. It does not - check for validity of the supplied code point, and may produce an invalid UTF-8 - sequence. -

-

- utf8::unchecked::next -

-

- Available in version 1.0 and later. -

-

- Given the iterator to the beginning of a UTF-8 sequence, it returns the code point - and moves the iterator to the next position. -

-
-template <typename octet_iterator>
-uint32_t next(octet_iterator& it);
-   
-
-

- it: a reference to an iterator pointing to the beginning of an UTF-8 - encoded code point. After the function returns, it is incremented to point to the - beginning of the next code point.
- Return value: the 32 bit representation of the - processed UTF-8 code point. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-char* w = twochars;
-int cp = unchecked::next(w);
-assert (cp == 0x65e5);
-assert (w == twochars + 3);
-
-

- This is a faster but less safe version of utf8::next. It does not - check for validity of the supplied UTF-8 sequence. -

-

- utf8::unchecked::peek_next -

-

- Available in version 2.1 and later. -

-

- Given the iterator to the beginning of a UTF-8 sequence, it returns the code point. -

-
-template <typename octet_iterator>
-uint32_t peek_next(octet_iterator it);
-   
-
-

- it: an iterator pointing to the beginning of an UTF-8 - encoded code point.
- Return value: the 32 bit representation of the - processed UTF-8 code point. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-char* w = twochars;
-int cp = unchecked::peek_next(w);
-assert (cp == 0x65e5);
-assert (w == twochars);
-
-

- This is a faster but less safe version of utf8::peek_next. It does not - check for validity of the supplied UTF-8 sequence. -

-

- utf8::unchecked::prior -

-

- Available in version 1.02 and later. -

-

- Given a reference to an iterator pointing to an octet in a UTF-8 seqence, it - decreases the iterator until it hits the beginning of the previous UTF-8 encoded - code point and returns the 32 bits representation of the code point. -

-
-template <typename octet_iterator>
-uint32_t prior(octet_iterator& it);
-   
-
-

- it: a reference pointing to an octet within a UTF-8 encoded string. - After the function returns, it is decremented to point to the beginning of the - previous code point.
- Return value: the 32 bit representation of the - previous code point. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-char* w = twochars + 3;
-int cp = unchecked::prior (w);
-assert (cp == 0x65e5);
-assert (w == twochars);
-
-

- This is a faster but less safe version of utf8::prior. It does not - check for validity of the supplied UTF-8 sequence and offers no boundary checking. -

-

- utf8::unchecked::previous (deprecated, see utf8::unchecked::prior) -

-

- Deprecated in version 1.02 and later. -

-

- Given a reference to an iterator pointing to an octet in a UTF-8 seqence, it - decreases the iterator until it hits the beginning of the previous UTF-8 encoded - code point and returns the 32 bits representation of the code point. -

-
-template <typename octet_iterator>
-uint32_t previous(octet_iterator& it);
-   
-
-

- it: a reference pointing to an octet within a UTF-8 encoded string. - After the function returns, it is decremented to point to the beginning of the - previous code point.
- Return value: the 32 bit representation of the - previous code point. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-char* w = twochars + 3;
-int cp = unchecked::previous (w);
-assert (cp == 0x65e5);
-assert (w == twochars);
-
-

- The reason this function is deprecated is just the consistency with the "checked" - versions, where prior should be used instead of previous. - In fact, unchecked::previous behaves exactly the same as - unchecked::prior -

-

- This is a faster but less safe version of utf8::previous. It does not - check for validity of the supplied UTF-8 sequence and offers no boundary checking. -

-

- utf8::unchecked::advance -

-

- Available in version 1.0 and later. -

-

- Advances an iterator by the specified number of code points within an UTF-8 - sequence. -

-
-template <typename octet_iterator, typename distance_type>
-void advance (octet_iterator& it, distance_type n);
-   
-
-

- it: a reference to an iterator pointing to the beginning of an UTF-8 - encoded code point. After the function returns, it is incremented to point to the - nth following code point.
- n: a positive integer that shows how many code points we want to - advance.
-

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-char* w = twochars;
-unchecked::advance (w, 2);
-assert (w == twochars + 5);
-
-

- This function works only "forward". In case of a negative n, there is - no effect. -

-

- This is a faster but less safe version of utf8::advance. It does not - check for validity of the supplied UTF-8 sequence and offers no boundary checking. -

-

- utf8::unchecked::distance -

-

- Available in version 1.0 and later. -

-

- Given the iterators to two UTF-8 encoded code points in a seqence, returns the - number of code points between them. -

-
-template <typename octet_iterator>
-typename std::iterator_traits<octet_iterator>::difference_type distance (octet_iterator first, octet_iterator last);
-
-

- first: an iterator to a beginning of a UTF-8 encoded code point.
- last: an iterator to a "post-end" of the last UTF-8 encoded code - point in the sequence we are trying to determine the length. It can be the - beginning of a new code point, or not.
- Return value the distance between the iterators, - in code points. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-size_t dist = utf8::unchecked::distance(twochars, twochars + 5);
-assert (dist == 2);
-
-

- This is a faster but less safe version of utf8::distance. It does not - check for validity of the supplied UTF-8 sequence. -

-

- utf8::unchecked::utf16to8 -

-

- Available in version 1.0 and later. -

-

- Converts a UTF-16 encoded string to UTF-8. -

-
-template <typename u16bit_iterator, typename octet_iterator>
-octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result);
-   
-
-

- start: an iterator pointing to the beginning of the UTF-16 encoded - string to convert.
- end: an iterator pointing to pass-the-end of the UTF-16 encoded - string to convert.
- result: an output iterator to the place in the UTF-8 string where to - append the result of conversion.
- Return value: An iterator pointing to the place - after the appended UTF-8 string. -

-

- Example of use: -

-
-unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
-vector<unsigned char> utf8result;
-unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
-assert (utf8result.size() == 10);    
-
-

- This is a faster but less safe version of utf8::utf16to8. It does not - check for validity of the supplied UTF-16 sequence. -

-

- utf8::unchecked::utf8to16 -

-

- Available in version 1.0 and later. -

-

- Converts an UTF-8 encoded string to UTF-16 -

-
-template <typename u16bit_iterator, typename octet_iterator>
-u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result);
-   
-
-

- start: an iterator pointing to the beginning of the UTF-8 encoded - string to convert. < br /> end: an iterator pointing to - pass-the-end of the UTF-8 encoded string to convert.
- result: an output iterator to the place in the UTF-16 string where to - append the result of conversion.
- Return value: An iterator pointing to the place - after the appended UTF-16 string. -

-

- Example of use: -

-
-char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
-vector <unsigned short> utf16result;
-unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
-assert (utf16result.size() == 4);
-assert (utf16result[2] == 0xd834);
-assert (utf16result[3] == 0xdd1e);
-
-

- This is a faster but less safe version of utf8::utf8to16. It does not - check for validity of the supplied UTF-8 sequence. -

-

- utf8::unchecked::utf32to8 -

-

- Available in version 1.0 and later. -

-

- Converts a UTF-32 encoded string to UTF-8. -

-
-template <typename octet_iterator, typename u32bit_iterator>
-octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result);
-   
-
-

- start: an iterator pointing to the beginning of the UTF-32 encoded - string to convert.
- end: an iterator pointing to pass-the-end of the UTF-32 encoded - string to convert.
- result: an output iterator to the place in the UTF-8 string where to - append the result of conversion.
- Return value: An iterator pointing to the place - after the appended UTF-8 string. -

-

- Example of use: -

-
-int utf32string[] = {0x448, 0x65e5, 0x10346, 0};
-vector<unsigned char> utf8result;
-utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
-assert (utf8result.size() == 9);
-
-

- This is a faster but less safe version of utf8::utf32to8. It does not - check for validity of the supplied UTF-32 sequence. -

-

- utf8::unchecked::utf8to32 -

-

- Available in version 1.0 and later. -

-

- Converts a UTF-8 encoded string to UTF-32. -

-
-template <typename octet_iterator, typename u32bit_iterator>
-u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result);
-   
-
-

- start: an iterator pointing to the beginning of the UTF-8 encoded - string to convert.
- end: an iterator pointing to pass-the-end of the UTF-8 encoded string - to convert.
- result: an output iterator to the place in the UTF-32 string where to - append the result of conversion.
- Return value: An iterator pointing to the place - after the appended UTF-32 string. -

-

- Example of use: -

-
-char* twochars = "\xe6\x97\xa5\xd1\x88";
-vector<int> utf32result;
-unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result));
-assert (utf32result.size() == 2);
-
-

- This is a faster but less safe version of utf8::utf8to32. It does not - check for validity of the supplied UTF-8 sequence. -

-

- Types From utf8::unchecked Namespace -

-

- utf8::iterator -

-

- Available in version 2.0 and later. -

-

- Adapts the underlying octet iterator to iterate over the sequence of code points, - rather than raw octets. -

-
-template <typename octet_iterator>
-class iterator;
-
- -
Member functions
-
-
iterator();
the deafult constructor; the underlying octet_iterator is - constructed with its default constructor. -
explicit iterator (const octet_iterator& octet_it); -
a constructor - that initializes the underlying octet_iterator with octet_it -
octet_iterator base () const;
returns the - underlying octet_iterator. -
uint32_t operator * () const;
decodes the utf-8 sequence - the underlying octet_iterator is pointing to and returns the code point. -
bool operator == (const iterator& rhs) - const;
returns true - if the two underlaying iterators are equal. -
bool operator != (const iterator& rhs) - const;
returns true - if the two underlaying iterators are not equal. -
iterator& operator ++ ();
the prefix increment - moves - the iterator to the next UTF-8 encoded code point. -
iterator operator ++ (int);
- the postfix increment - moves the iterator to the next UTF-8 encoded code point and returns the current one. -
iterator& operator -- ();
the prefix decrement - moves - the iterator to the previous UTF-8 encoded code point. -
iterator operator -- (int);
- the postfix decrement - moves the iterator to the previous UTF-8 encoded code point and returns the current one. -
-

- Example of use: -

-
-char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
-utf8::unchecked::iterator<char*> un_it(threechars);
-utf8::unchecked::iterator<char*> un_it2 = un_it;
-assert (un_it2 == un_it);
-assert (*un_it == 0x10346);
-assert (*(++un_it) == 0x65e5);
-assert ((*un_it++) == 0x65e5);
-assert (*un_it == 0x0448);
-assert (un_it != un_it2);
-utf8::::unchecked::iterator<char*> un_endit (threechars + 9);  
-assert (++un_it == un_endit);
-assert (*(--un_it) == 0x0448);
-assert ((*un_it--) == 0x0448);
-assert (*un_it == 0x65e5);
-assert (--un_it == utf8::unchecked::iterator<char*>(threechars));
-assert (*un_it == 0x10346);
-
-

- This is an unchecked version of utf8::iterator. It is faster in many cases, but offers - no validity or range checks. -

-

- Points of interest -

-

- Design goals and decisions -

-

- The library was designed to be: -

-
    -
  1. - Generic: for better or worse, there are many C++ string classes out there, and - the library should work with as many of them as possible. -
    2. -
  2. - Portable: the library should be portable both accross different platforms and - compilers. The only non-portable code is a small section that declares unsigned - integers of different sizes: three typedefs. They can be changed by the users of - the library if they don't match their platform. The default setting should work - for Windows (both 32 and 64 bit), and most 32 bit and 64 bit Unix derivatives. -
    3. -
  3. - Lightweight: follow the "pay only for what you use" guidline. -
    4. -
  4. - Unintrusive: avoid forcing any particular design or even programming style on the - user. This is a library, not a framework. -
    5. -
-

- Alternatives -

-

- In case you want to look into other means of working with UTF-8 strings from C++, - here is the list of solutions I am aware of: -

-
    -
  1. - ICU Library. It is very powerful, - complete, feature-rich, mature, and widely used. Also big, intrusive, - non-generic, and doesn't play well with the Standard Library. I definitelly - recommend looking at ICU even if you don't plan to use it. -
    2. -
  2. - Glib::ustring. - A class specifically made to work with UTF-8 strings, and also feel like - std::string. If you prefer to have yet another string class in your - code, it may be worth a look. Be aware of the licensing issues, though. -
    3. -
  3. - Platform dependent solutions: Windows and POSIX have functions to convert strings - from one encoding to another. That is only a subset of what my library offers, - but if that is all you need it may be good enough, especially given the fact that - these functions are mature and tested in production. -
    4. -
-

- Conclusion -

-

- Until Unicode becomes officially recognized by the C++ Standard Library, we need to - use other means to work with UTF-8 strings. Template functions I describe in this - article may be a good step in this direction. -

- -
    -
  1. - The Unicode Consortium. -
    2. -
  2. - ICU Library. -
    3. -
  3. - UTF-8 at Wikipedia -
    4. -
  4. - UTF-8 and Unicode FAQ for - Unix/Linux -
    5. -
- - diff --git a/ext/utf8/source/utf8.h b/ext/utf8/source/utf8.h deleted file mode 100644 index 4e445140..00000000 --- a/ext/utf8/source/utf8.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2006 Nemanja Trifunovic - -/* -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - - -#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 -#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 - -#include "utf8/checked.h" -#include "utf8/unchecked.h" - -#endif // header guard diff --git a/ext/utf8/source/utf8/checked.h b/ext/utf8/source/utf8/checked.h deleted file mode 100644 index 16bab4db..00000000 --- a/ext/utf8/source/utf8/checked.h +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright 2006 Nemanja Trifunovic - -/* -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - - -#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 -#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 - -#include "core.h" -#include - -namespace utf8 -{ - // Exceptions that may be thrown from the library functions. - class invalid_code_point : public std::exception { - uint32_t cp; - public: - invalid_code_point(uint32_t cp) : cp(cp) {} - virtual const char* what() const throw() { return "Invalid code point"; } - uint32_t code_point() const {return cp;} - }; - - class invalid_utf8 : public std::exception { - uint8_t u8; - public: - invalid_utf8 (uint8_t u) : u8(u) {} - virtual const char* what() const throw() { return "Invalid UTF-8"; } - uint8_t utf8_octet() const {return u8;} - }; - - class invalid_utf16 : public std::exception { - uint16_t u16; - public: - invalid_utf16 (uint16_t u) : u16(u) {} - virtual const char* what() const throw() { return "Invalid UTF-16"; } - uint16_t utf16_word() const {return u16;} - }; - - class not_enough_room : public std::exception { - public: - virtual const char* what() const throw() { return "Not enough space"; } - }; - - /// The library API - functions intended to be called by the users - - template - output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement) - { - while (start != end) { - octet_iterator sequence_start = start; - internal::utf_error err_code = internal::validate_next(start, end); - switch (err_code) { - case internal::UTF8_OK : - for (octet_iterator it = sequence_start; it != start; ++it) - *out++ = *it; - break; - case internal::NOT_ENOUGH_ROOM: - throw not_enough_room(); - case internal::INVALID_LEAD: - append (replacement, out); - ++start; - break; - case internal::INCOMPLETE_SEQUENCE: - case internal::OVERLONG_SEQUENCE: - case internal::INVALID_CODE_POINT: - append (replacement, out); - ++start; - // just one replacement mark for the sequence - while (internal::is_trail(*start) && start != end) - ++start; - break; - } - } - return out; - } - - template - inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) - { - static const uint32_t replacement_marker = internal::mask16(0xfffd); - return replace_invalid(start, end, out, replacement_marker); - } - - template - octet_iterator append(uint32_t cp, octet_iterator result) - { - if (!internal::is_code_point_valid(cp)) - throw invalid_code_point(cp); - - if (cp < 0x80) // one octet - *(result++) = static_cast(cp); - else if (cp < 0x800) { // two octets - *(result++) = static_cast((cp >> 6) | 0xc0); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - else if (cp < 0x10000) { // three octets - *(result++) = static_cast((cp >> 12) | 0xe0); - *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - else if (cp <= internal::CODE_POINT_MAX) { // four octets - *(result++) = static_cast((cp >> 18) | 0xf0); - *(result++) = static_cast(((cp >> 12) & 0x3f) | 0x80); - *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - else - throw invalid_code_point(cp); - - return result; - } - - template - uint32_t next(octet_iterator& it, octet_iterator end) - { - uint32_t cp = 0; - internal::utf_error err_code = internal::validate_next(it, end, &cp); - switch (err_code) { - case internal::UTF8_OK : - break; - case internal::NOT_ENOUGH_ROOM : - throw not_enough_room(); - case internal::INVALID_LEAD : - case internal::INCOMPLETE_SEQUENCE : - case internal::OVERLONG_SEQUENCE : - throw invalid_utf8(*it); - case internal::INVALID_CODE_POINT : - throw invalid_code_point(cp); - } - return cp; - } - - template - uint32_t peek_next(octet_iterator it, octet_iterator end) - { - return next(it, end); - } - - template - uint32_t prior(octet_iterator& it, octet_iterator start) - { - octet_iterator end = it; - while (internal::is_trail(*(--it))) - if (it < start) - throw invalid_utf8(*it); // error - no lead byte in the sequence - octet_iterator temp = it; - return next(temp, end); - } - - /// Deprecated in versions that include "prior" - template - uint32_t previous(octet_iterator& it, octet_iterator pass_start) - { - octet_iterator end = it; - while (internal::is_trail(*(--it))) - if (it == pass_start) - throw invalid_utf8(*it); // error - no lead byte in the sequence - octet_iterator temp = it; - return next(temp, end); - } - - template - void advance (octet_iterator& it, distance_type n, octet_iterator end) - { - for (distance_type i = 0; i < n; ++i) - next(it, end); - } - - template - typename std::iterator_traits::difference_type - distance (octet_iterator first, octet_iterator last) - { - typename std::iterator_traits::difference_type dist; - for (dist = 0; first < last; ++dist) - next(first, last); - return dist; - } - - template - octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) - { - while (start != end) { - uint32_t cp = internal::mask16(*start++); - // Take care of surrogate pairs first - if (internal::is_surrogate(cp)) { - if (start != end) { - uint32_t trail_surrogate = internal::mask16(*start++); - if (trail_surrogate >= internal::TRAIL_SURROGATE_MIN && trail_surrogate <= internal::TRAIL_SURROGATE_MAX) - cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; - else - throw invalid_utf16(static_cast(trail_surrogate)); - } - else - throw invalid_utf16(static_cast(*start)); - - } - result = append(cp, result); - } - return result; - } - - template - u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) - { - while (start != end) { - uint32_t cp = next(start, end); - if (cp > 0xffff) { //make a surrogate pair - *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); - *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); - } - else - *result++ = static_cast(cp); - } - return result; - } - - template - octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) - { - while (start != end) - result = append(*(start++), result); - - return result; - } - - template - u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) - { - while (start < end) - (*result++) = next(start, end); - - return result; - } - - // The iterator class - template - class iterator : public std::iterator { - octet_iterator it; - octet_iterator range_start; - octet_iterator range_end; - public: - iterator () {}; - explicit iterator (const octet_iterator& octet_it, - const octet_iterator& range_start, - const octet_iterator& range_end) : - it(octet_it), range_start(range_start), range_end(range_end) - { - if (it < range_start || it > range_end) - throw std::out_of_range("Invalid utf-8 iterator position"); - } - // the default "big three" are OK - octet_iterator base () const { return it; } - uint32_t operator * () const - { - octet_iterator temp = it; - return next(temp, range_end); - } - bool operator == (const iterator& rhs) const - { - if (range_start != rhs.range_start || range_end != rhs.range_end) - throw std::logic_error("Comparing utf-8 iterators defined with different ranges"); - return (it == rhs.it); - } - bool operator != (const iterator& rhs) const - { - return !(operator == (rhs)); - } - iterator& operator ++ () - { - next(it, range_end); - return *this; - } - iterator operator ++ (int) - { - iterator temp = *this; - next(it, range_end); - return temp; - } - iterator& operator -- () - { - prior(it, range_start); - return *this; - } - iterator operator -- (int) - { - iterator temp = *this; - prior(it, range_start); - return temp; - } - }; // class iterator - -} // namespace utf8 - -#endif //header guard - - diff --git a/ext/utf8/source/utf8/core.h b/ext/utf8/source/utf8/core.h deleted file mode 100644 index 557ab0fe..00000000 --- a/ext/utf8/source/utf8/core.h +++ /dev/null @@ -1,334 +0,0 @@ -// Copyright 2006 Nemanja Trifunovic - -/* -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - - -#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 -#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 - -#include - -namespace utf8 -{ - // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers - // You may need to change them to match your system. - // These typedefs have the same names as ones from cstdint, or boost/cstdint - typedef unsigned char uint8_t; - typedef unsigned short uint16_t; - typedef unsigned int uint32_t; - -// Helper code - not intended to be directly called by the library users. May be changed at any time -namespace internal -{ - // Unicode constants - // Leading (high) surrogates: 0xd800 - 0xdbff - // Trailing (low) surrogates: 0xdc00 - 0xdfff - const uint16_t LEAD_SURROGATE_MIN = 0xd800u; - const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; - const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u; - const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu; - const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); - const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; - - // Maximum valid value for a Unicode code point - const uint32_t CODE_POINT_MAX = 0x0010ffffu; - - template - inline uint8_t mask8(octet_type oc) - { - return static_cast(0xff & oc); - } - template - inline uint16_t mask16(u16_type oc) - { - return static_cast(0xffff & oc); - } - template - inline bool is_trail(octet_type oc) - { - return ((mask8(oc) >> 6) == 0x2); - } - - template - inline bool is_surrogate(u16 cp) - { - return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); - } - - template - inline bool is_code_point_valid(u32 cp) - { - return (cp <= CODE_POINT_MAX && !is_surrogate(cp) && cp != 0xfffe && cp != 0xffff); - } - - template - inline typename std::iterator_traits::difference_type - sequence_length(octet_iterator lead_it) - { - uint8_t lead = mask8(*lead_it); - if (lead < 0x80) - return 1; - else if ((lead >> 5) == 0x6) - return 2; - else if ((lead >> 4) == 0xe) - return 3; - else if ((lead >> 3) == 0x1e) - return 4; - else - return 0; - } - - inline bool is_overlong_sequence(uint32_t cp, int length) - { - if (cp < 0x80) { - if (length != 1) - return true; - } - else if (cp < 0x800) { - if (length != 2) - return true; - } - else if (cp < 0x10000) { - if (length != 3) - return true; - } - - return false; - } - - enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; - - /// get_sequence_x functions decode utf-8 sequences of the length x - - template - utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t* code_point) - { - if (it != end) { - if (code_point) - *code_point = mask8(*it); - return UTF8_OK; - } - return NOT_ENOUGH_ROOM; - } - - template - utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t* code_point) - { - utf_error ret_code = NOT_ENOUGH_ROOM; - - if (it != end) { - uint32_t cp = mask8(*it); - if (++it != end) { - if (is_trail(*it)) { - cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); - - if (code_point) - *code_point = cp; - ret_code = UTF8_OK; - } - else - ret_code = INCOMPLETE_SEQUENCE; - } - else - ret_code = NOT_ENOUGH_ROOM; - } - - return ret_code; - } - - template - utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t* code_point) - { - utf_error ret_code = NOT_ENOUGH_ROOM; - - if (it != end) { - uint32_t cp = mask8(*it); - if (++it != end) { - if (is_trail(*it)) { - cp = ((cp << 12) & 0xffff) + ((mask8(*it) << 6) & 0xfff); - if (++it != end) { - if (is_trail(*it)) { - cp += (*it) & 0x3f; - - if (code_point) - *code_point = cp; - ret_code = UTF8_OK; - } - else - ret_code = INCOMPLETE_SEQUENCE; - } - else - ret_code = NOT_ENOUGH_ROOM; - } - else - ret_code = INCOMPLETE_SEQUENCE; - } - else - ret_code = NOT_ENOUGH_ROOM; - } - - return ret_code; - } - - template - utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t* code_point) - { - utf_error ret_code = NOT_ENOUGH_ROOM; - - if (it != end) { - uint32_t cp = mask8(*it); - if (++it != end) { - if (is_trail(*it)) { - cp = ((cp << 18) & 0x1fffff) + ((mask8(*it) << 12) & 0x3ffff); - if (++it != end) { - if (is_trail(*it)) { - cp += (mask8(*it) << 6) & 0xfff; - if (++it != end) { - if (is_trail(*it)) { - cp += (*it) & 0x3f; - - if (code_point) - *code_point = cp; - ret_code = UTF8_OK; - } - else - ret_code = INCOMPLETE_SEQUENCE; - } - else - ret_code = NOT_ENOUGH_ROOM; - } - else - ret_code = INCOMPLETE_SEQUENCE; - } - else - ret_code = NOT_ENOUGH_ROOM; - } - else - ret_code = INCOMPLETE_SEQUENCE; - } - else - ret_code = NOT_ENOUGH_ROOM; - } - - return ret_code; - } - - template - utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t* code_point) - { - // Save the original value of it so we can go back in case of failure - // Of course, it does not make much sense with i.e. stream iterators - octet_iterator original_it = it; - - uint32_t cp = 0; - // Determine the sequence length based on the lead octet - typedef typename std::iterator_traits::difference_type octet_difference_type; - octet_difference_type length = sequence_length(it); - if (length == 0) - return INVALID_LEAD; - - // Now that we have a valid sequence length, get trail octets and calculate the code point - utf_error err = UTF8_OK; - switch (length) { - case 1: - err = get_sequence_1(it, end, &cp); - break; - case 2: - err = get_sequence_2(it, end, &cp); - break; - case 3: - err = get_sequence_3(it, end, &cp); - break; - case 4: - err = get_sequence_4(it, end, &cp); - break; - } - - if (err == UTF8_OK) { - // Decoding succeeded. Now, security checks... - if (is_code_point_valid(cp)) { - if (!is_overlong_sequence(cp, length)){ - // Passed! Return here. - if (code_point) - *code_point = cp; - ++it; - return UTF8_OK; - } - else - err = OVERLONG_SEQUENCE; - } - else - err = INVALID_CODE_POINT; - } - - // Failure branch - restore the original value of the iterator - it = original_it; - return err; - } - - template - inline utf_error validate_next(octet_iterator& it, octet_iterator end) { - return validate_next(it, end, 0); - } - -} // namespace internal - - /// The library API - functions intended to be called by the users - - // Byte order mark - const uint8_t bom[] = {0xef, 0xbb, 0xbf}; - - template - octet_iterator find_invalid(octet_iterator start, octet_iterator end) - { - octet_iterator result = start; - while (result != end) { - internal::utf_error err_code = internal::validate_next(result, end); - if (err_code != internal::UTF8_OK) - return result; - } - return result; - } - - template - inline bool is_valid(octet_iterator start, octet_iterator end) - { - return (find_invalid(start, end) == end); - } - - template - inline bool is_bom (octet_iterator it) - { - return ( - (internal::mask8(*it++)) == bom[0] && - (internal::mask8(*it++)) == bom[1] && - (internal::mask8(*it)) == bom[2] - ); - } -} // namespace utf8 - -#endif // header guard - - diff --git a/ext/utf8/source/utf8/unchecked.h b/ext/utf8/source/utf8/unchecked.h deleted file mode 100644 index 84207a5f..00000000 --- a/ext/utf8/source/utf8/unchecked.h +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright 2006 Nemanja Trifunovic - -/* -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - - -#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 -#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 - -#include "core.h" - -namespace utf8 -{ - namespace unchecked - { - template - octet_iterator append(uint32_t cp, octet_iterator result) - { - if (cp < 0x80) // one octet - *(result++) = static_cast(cp); - else if (cp < 0x800) { // two octets - *(result++) = static_cast((cp >> 6) | 0xc0); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - else if (cp < 0x10000) { // three octets - *(result++) = static_cast((cp >> 12) | 0xe0); - *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - else { // four octets - *(result++) = static_cast((cp >> 18) | 0xf0); - *(result++) = static_cast(((cp >> 12) & 0x3f)| 0x80); - *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - return result; - } - - template - uint32_t next(octet_iterator& it) - { - uint32_t cp = internal::mask8(*it); - typename std::iterator_traits::difference_type length = utf8::internal::sequence_length(it); - switch (length) { - case 1: - break; - case 2: - it++; - cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); - break; - case 3: - ++it; - cp = ((cp << 12) & 0xffff) + ((internal::mask8(*it) << 6) & 0xfff); - ++it; - cp += (*it) & 0x3f; - break; - case 4: - ++it; - cp = ((cp << 18) & 0x1fffff) + ((internal::mask8(*it) << 12) & 0x3ffff); - ++it; - cp += (internal::mask8(*it) << 6) & 0xfff; - ++it; - cp += (*it) & 0x3f; - break; - } - ++it; - return cp; - } - - template - uint32_t peek_next(octet_iterator it) - { - return next(it); - } - - template - uint32_t prior(octet_iterator& it) - { - while (internal::is_trail(*(--it))) ; - octet_iterator temp = it; - return next(temp); - } - - // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous) - template - inline uint32_t previous(octet_iterator& it) - { - return prior(it); - } - - template - void advance (octet_iterator& it, distance_type n) - { - for (distance_type i = 0; i < n; ++i) - next(it); - } - - template - typename std::iterator_traits::difference_type - distance (octet_iterator first, octet_iterator last) - { - typename std::iterator_traits::difference_type dist; - for (dist = 0; first < last; ++dist) - next(first); - return dist; - } - - template - octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) - { - while (start != end) { - uint32_t cp = internal::mask16(*start++); - // Take care of surrogate pairs first - if (internal::is_surrogate(cp)) { - uint32_t trail_surrogate = internal::mask16(*start++); - cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; - } - result = append(cp, result); - } - return result; - } - - template - u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) - { - while (start != end) { - uint32_t cp = next(start); - if (cp > 0xffff) { //make a surrogate pair - *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); - *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); - } - else - *result++ = static_cast(cp); - } - return result; - } - - template - octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) - { - while (start != end) - result = append(*(start++), result); - - return result; - } - - template - u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) - { - while (start < end) - (*result++) = next(start); - - return result; - } - - // The iterator class - template - class iterator : public std::iterator { - octet_iterator it; - public: - iterator () {}; - explicit iterator (const octet_iterator& octet_it): it(octet_it) {} - // the default "big three" are OK - octet_iterator base () const { return it; } - uint32_t operator * () const - { - octet_iterator temp = it; - return next(temp); - } - bool operator == (const iterator& rhs) const - { - return (it == rhs.it); - } - bool operator != (const iterator& rhs) const - { - return !(operator == (rhs)); - } - iterator& operator ++ () - { - std::advance(it, internal::sequence_length(it)); - return *this; - } - iterator operator ++ (int) - { - iterator temp = *this; - std::advance(it, internal::sequence_length(it)); - return temp; - } - iterator& operator -- () - { - prior(it); - return *this; - } - iterator operator -- (int) - { - iterator temp = *this; - prior(it); - return temp; - } - }; // class iterator - - } // namespace utf8::unchecked -} // namespace utf8 - - -#endif // header guard -