| /* XzEnc.c -- Xz Encode |
| 2023-04-13 : Igor Pavlov : Public domain */ |
| |
| #include "Precomp.h" |
| |
| #include <stdlib.h> |
| #include <string.h> |
| |
| #include "7zCrc.h" |
| #include "Bra.h" |
| #include "CpuArch.h" |
| |
| #ifdef USE_SUBBLOCK |
| #include "Bcj3Enc.c" |
| #include "SbFind.c" |
| #include "SbEnc.c" |
| #endif |
| |
| #include "XzEnc.h" |
| |
| // #define Z7_ST |
| |
| #ifndef Z7_ST |
| #include "MtCoder.h" |
| #else |
| #define MTCODER_THREADS_MAX 1 |
| #define MTCODER_BLOCKS_MAX 1 |
| #endif |
| |
| #define XZ_GET_PAD_SIZE(dataSize) ((4 - ((unsigned)(dataSize) & 3)) & 3) |
| |
| /* max pack size for LZMA2 block + check-64bytrs: */ |
| #define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + 64) |
| |
| #define XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(unpackSize) (XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize)) |
| |
| |
| // #define XzBlock_ClearFlags(p) (p)->flags = 0; |
| #define XzBlock_ClearFlags_SetNumFilters(p, n) (p)->flags = (Byte)((n) - 1); |
| #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE; |
| #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE; |
| |
| |
| static SRes WriteBytes(ISeqOutStreamPtr s, const void *buf, size_t size) |
| { |
| return (ISeqOutStream_Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; |
| } |
| |
| static SRes WriteBytes_UpdateCrc(ISeqOutStreamPtr s, const void *buf, size_t size, UInt32 *crc) |
| { |
| *crc = CrcUpdate(*crc, buf, size); |
| return WriteBytes(s, buf, size); |
| } |
| |
| |
| static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStreamPtr s) |
| { |
| UInt32 crc; |
| Byte header[XZ_STREAM_HEADER_SIZE]; |
| memcpy(header, XZ_SIG, XZ_SIG_SIZE); |
| header[XZ_SIG_SIZE] = (Byte)(f >> 8); |
| header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF); |
| crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE); |
| SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc) |
| return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE); |
| } |
| |
| |
| static SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStreamPtr s) |
| { |
| Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; |
| |
| unsigned pos = 1; |
| unsigned numFilters, i; |
| header[pos++] = p->flags; |
| |
| if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize); |
| if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize); |
| numFilters = XzBlock_GetNumFilters(p); |
| |
| for (i = 0; i < numFilters; i++) |
| { |
| const CXzFilter *f = &p->filters[i]; |
| pos += Xz_WriteVarInt(header + pos, f->id); |
| pos += Xz_WriteVarInt(header + pos, f->propsSize); |
| memcpy(header + pos, f->props, f->propsSize); |
| pos += f->propsSize; |
| } |
| |
| while ((pos & 3) != 0) |
| header[pos++] = 0; |
| |
| header[0] = (Byte)(pos >> 2); |
| SetUi32(header + pos, CrcCalc(header, pos)) |
| return WriteBytes(s, header, pos + 4); |
| } |
| |
| |
| |
| |
| typedef struct |
| { |
| size_t numBlocks; |
| size_t size; |
| size_t allocated; |
| Byte *blocks; |
| } CXzEncIndex; |
| |
| |
| static void XzEncIndex_Construct(CXzEncIndex *p) |
| { |
| p->numBlocks = 0; |
| p->size = 0; |
| p->allocated = 0; |
| p->blocks = NULL; |
| } |
| |
| static void XzEncIndex_Init(CXzEncIndex *p) |
| { |
| p->numBlocks = 0; |
| p->size = 0; |
| } |
| |
| static void XzEncIndex_Free(CXzEncIndex *p, ISzAllocPtr alloc) |
| { |
| if (p->blocks) |
| { |
| ISzAlloc_Free(alloc, p->blocks); |
| p->blocks = NULL; |
| } |
| p->numBlocks = 0; |
| p->size = 0; |
| p->allocated = 0; |
| } |
| |
| |
| static SRes XzEncIndex_ReAlloc(CXzEncIndex *p, size_t newSize, ISzAllocPtr alloc) |
| { |
| Byte *blocks = (Byte *)ISzAlloc_Alloc(alloc, newSize); |
| if (!blocks) |
| return SZ_ERROR_MEM; |
| if (p->size != 0) |
| memcpy(blocks, p->blocks, p->size); |
| if (p->blocks) |
| ISzAlloc_Free(alloc, p->blocks); |
| p->blocks = blocks; |
| p->allocated = newSize; |
| return SZ_OK; |
| } |
| |
| |
| static SRes XzEncIndex_PreAlloc(CXzEncIndex *p, UInt64 numBlocks, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) |
| { |
| UInt64 pos; |
| { |
| Byte buf[32]; |
| unsigned pos2 = Xz_WriteVarInt(buf, totalSize); |
| pos2 += Xz_WriteVarInt(buf + pos2, unpackSize); |
| pos = numBlocks * pos2; |
| } |
| |
| if (pos <= p->allocated - p->size) |
| return SZ_OK; |
| { |
| UInt64 newSize64 = p->size + pos; |
| size_t newSize = (size_t)newSize64; |
| if (newSize != newSize64) |
| return SZ_ERROR_MEM; |
| return XzEncIndex_ReAlloc(p, newSize, alloc); |
| } |
| } |
| |
| |
| static SRes XzEncIndex_AddIndexRecord(CXzEncIndex *p, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) |
| { |
| Byte buf[32]; |
| unsigned pos = Xz_WriteVarInt(buf, totalSize); |
| pos += Xz_WriteVarInt(buf + pos, unpackSize); |
| |
| if (pos > p->allocated - p->size) |
| { |
| size_t newSize = p->allocated * 2 + 16 * 2; |
| if (newSize < p->size + pos) |
| return SZ_ERROR_MEM; |
| RINOK(XzEncIndex_ReAlloc(p, newSize, alloc)) |
| } |
| memcpy(p->blocks + p->size, buf, pos); |
| p->size += pos; |
| p->numBlocks++; |
| return SZ_OK; |
| } |
| |
| |
| static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, ISeqOutStreamPtr s) |
| { |
| Byte buf[32]; |
| UInt64 globalPos; |
| UInt32 crc = CRC_INIT_VAL; |
| unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); |
| |
| globalPos = pos; |
| buf[0] = 0; |
| RINOK(WriteBytes_UpdateCrc(s, buf, pos, &crc)) |
| RINOK(WriteBytes_UpdateCrc(s, p->blocks, p->size, &crc)) |
| globalPos += p->size; |
| |
| pos = XZ_GET_PAD_SIZE(globalPos); |
| buf[1] = 0; |
| buf[2] = 0; |
| buf[3] = 0; |
| globalPos += pos; |
| |
| crc = CrcUpdate(crc, buf + 4 - pos, pos); |
| SetUi32(buf + 4, CRC_GET_DIGEST(crc)) |
| |
| SetUi32(buf + 8 + 4, (UInt32)(globalPos >> 2)) |
| buf[8 + 8] = (Byte)(flags >> 8); |
| buf[8 + 9] = (Byte)(flags & 0xFF); |
| SetUi32(buf + 8, CrcCalc(buf + 8 + 4, 6)) |
| buf[8 + 10] = XZ_FOOTER_SIG_0; |
| buf[8 + 11] = XZ_FOOTER_SIG_1; |
| |
| return WriteBytes(s, buf + 4 - pos, pos + 4 + 12); |
| } |
| |
| |
| |
| /* ---------- CSeqCheckInStream ---------- */ |
| |
| typedef struct |
| { |
| ISeqInStream vt; |
| ISeqInStreamPtr realStream; |
| const Byte *data; |
| UInt64 limit; |
| UInt64 processed; |
| int realStreamFinished; |
| CXzCheck check; |
| } CSeqCheckInStream; |
| |
| static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned checkMode) |
| { |
| p->limit = (UInt64)(Int64)-1; |
| p->processed = 0; |
| p->realStreamFinished = 0; |
| XzCheck_Init(&p->check, checkMode); |
| } |
| |
| static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest) |
| { |
| XzCheck_Final(&p->check, digest); |
| } |
| |
| static SRes SeqCheckInStream_Read(ISeqInStreamPtr pp, void *data, size_t *size) |
| { |
| Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CSeqCheckInStream) |
| size_t size2 = *size; |
| SRes res = SZ_OK; |
| |
| if (p->limit != (UInt64)(Int64)-1) |
| { |
| UInt64 rem = p->limit - p->processed; |
| if (size2 > rem) |
| size2 = (size_t)rem; |
| } |
| if (size2 != 0) |
| { |
| if (p->realStream) |
| { |
| res = ISeqInStream_Read(p->realStream, data, &size2); |
| p->realStreamFinished = (size2 == 0) ? 1 : 0; |
| } |
| else |
| memcpy(data, p->data + (size_t)p->processed, size2); |
| XzCheck_Update(&p->check, data, size2); |
| p->processed += size2; |
| } |
| *size = size2; |
| return res; |
| } |
| |
| |
| /* ---------- CSeqSizeOutStream ---------- */ |
| |
| typedef struct |
| { |
| ISeqOutStream vt; |
| ISeqOutStreamPtr realStream; |
| Byte *outBuf; |
| size_t outBufLimit; |
| UInt64 processed; |
| } CSeqSizeOutStream; |
| |
| static size_t SeqSizeOutStream_Write(ISeqOutStreamPtr pp, const void *data, size_t size) |
| { |
| Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CSeqSizeOutStream) |
| if (p->realStream) |
| size = ISeqOutStream_Write(p->realStream, data, size); |
| else |
| { |
| if (size > p->outBufLimit - (size_t)p->processed) |
| return 0; |
| memcpy(p->outBuf + (size_t)p->processed, data, size); |
| } |
| p->processed += size; |
| return size; |
| } |
| |
| |
| /* ---------- CSeqInFilter ---------- */ |
| |
| #define FILTER_BUF_SIZE (1 << 20) |
| |
| typedef struct |
| { |
| ISeqInStream vt; |
| ISeqInStreamPtr realStream; |
| IStateCoder StateCoder; |
| Byte *buf; |
| size_t curPos; |
| size_t endPos; |
| int srcWasFinished; |
| } CSeqInFilter; |
| |
| |
| static const z7_Func_BranchConv g_Funcs_BranchConv_RISC_Enc[] = |
| { |
| Z7_BRANCH_CONV_ENC(PPC), |
| Z7_BRANCH_CONV_ENC(IA64), |
| Z7_BRANCH_CONV_ENC(ARM), |
| Z7_BRANCH_CONV_ENC(ARMT), |
| Z7_BRANCH_CONV_ENC(SPARC), |
| Z7_BRANCH_CONV_ENC(ARM64) |
| }; |
| |
| static SizeT XzBcFilterStateBase_Filter_Enc(CXzBcFilterStateBase *p, Byte *data, SizeT size) |
| { |
| switch (p->methodId) |
| { |
| case XZ_ID_Delta: |
| Delta_Encode(p->delta_State, p->delta, data, size); |
| break; |
| case XZ_ID_X86: |
| size = (SizeT)(z7_BranchConvSt_X86_Enc(data, size, p->ip, &p->X86_State) - data); |
| break; |
| default: |
| if (p->methodId >= XZ_ID_PPC) |
| { |
| const UInt32 i = p->methodId - XZ_ID_PPC; |
| if (i < Z7_ARRAY_SIZE(g_Funcs_BranchConv_RISC_Enc)) |
| size = (SizeT)(g_Funcs_BranchConv_RISC_Enc[i](data, size, p->ip) - data); |
| } |
| break; |
| } |
| p->ip += (UInt32)size; |
| return size; |
| } |
| |
| |
| static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPtr alloc) |
| { |
| if (!p->buf) |
| { |
| p->buf = (Byte *)ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); |
| if (!p->buf) |
| return SZ_ERROR_MEM; |
| } |
| p->curPos = p->endPos = 0; |
| p->srcWasFinished = 0; |
| RINOK(Xz_StateCoder_Bc_SetFromMethod_Func(&p->StateCoder, props->id, XzBcFilterStateBase_Filter_Enc, alloc)) |
| RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, alloc)) |
| p->StateCoder.Init(p->StateCoder.p); |
| return SZ_OK; |
| } |
| |
| |
| static SRes SeqInFilter_Read(ISeqInStreamPtr pp, void *data, size_t *size) |
| { |
| Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(CSeqInFilter) |
| const size_t sizeOriginal = *size; |
| if (sizeOriginal == 0) |
| return SZ_OK; |
| *size = 0; |
| |
| for (;;) |
| { |
| if (!p->srcWasFinished && p->curPos == p->endPos) |
| { |
| p->curPos = 0; |
| p->endPos = FILTER_BUF_SIZE; |
| RINOK(ISeqInStream_Read(p->realStream, p->buf, &p->endPos)) |
| if (p->endPos == 0) |
| p->srcWasFinished = 1; |
| } |
| { |
| SizeT srcLen = p->endPos - p->curPos; |
| ECoderStatus status; |
| SRes res; |
| *size = sizeOriginal; |
| res = p->StateCoder.Code2(p->StateCoder.p, |
| (Byte *)data, size, |
| p->buf + p->curPos, &srcLen, |
| p->srcWasFinished, CODER_FINISH_ANY, |
| &status); |
| p->curPos += srcLen; |
| if (*size != 0 || srcLen == 0 || res != SZ_OK) |
| return res; |
| } |
| } |
| } |
| |
| static void SeqInFilter_Construct(CSeqInFilter *p) |
| { |
| p->buf = NULL; |
| p->StateCoder.p = NULL; |
| p->vt.Read = SeqInFilter_Read; |
| } |
| |
| static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc) |
| { |
| if (p->StateCoder.p) |
| { |
| p->StateCoder.Free(p->StateCoder.p, alloc); |
| p->StateCoder.p = NULL; |
| } |
| if (p->buf) |
| { |
| ISzAlloc_Free(alloc, p->buf); |
| p->buf = NULL; |
| } |
| } |
| |
| |
| /* ---------- CSbEncInStream ---------- */ |
| |
| #ifdef USE_SUBBLOCK |
| |
| typedef struct |
| { |
| ISeqInStream vt; |
| ISeqInStreamPtr inStream; |
| CSbEnc enc; |
| } CSbEncInStream; |
| |
| static SRes SbEncInStream_Read(ISeqInStreamPtr pp, void *data, size_t *size) |
| { |
| CSbEncInStream *p = Z7_CONTAINER_FROM_VTBL(pp, CSbEncInStream, vt); |
| size_t sizeOriginal = *size; |
| if (sizeOriginal == 0) |
| return SZ_OK; |
| |
| for (;;) |
| { |
| if (p->enc.needRead && !p->enc.readWasFinished) |
| { |
| size_t processed = p->enc.needReadSizeMax; |
| RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed)) |
| p->enc.readPos += processed; |
| if (processed == 0) |
| { |
| p->enc.readWasFinished = True; |
| p->enc.isFinalFinished = True; |
| } |
| p->enc.needRead = False; |
| } |
| |
| *size = sizeOriginal; |
| RINOK(SbEnc_Read(&p->enc, data, size)) |
| if (*size != 0 || !p->enc.needRead) |
| return SZ_OK; |
| } |
| } |
| |
| void SbEncInStream_Construct(CSbEncInStream *p, ISzAllocPtr alloc) |
| { |
| SbEnc_Construct(&p->enc, alloc); |
| p->vt.Read = SbEncInStream_Read; |
| } |
| |
| SRes SbEncInStream_Init(CSbEncInStream *p) |
| { |
| return SbEnc_Init(&p->enc); |
| } |
| |
| void SbEncInStream_Free(CSbEncInStream *p) |
| { |
| SbEnc_Free(&p->enc); |
| } |
| |
| #endif |
| |
| |
| |
| /* ---------- CXzProps ---------- */ |
| |
| |
| void XzFilterProps_Init(CXzFilterProps *p) |
| { |
| p->id = 0; |
| p->delta = 0; |
| p->ip = 0; |
| p->ipDefined = False; |
| } |
| |
| void XzProps_Init(CXzProps *p) |
| { |
| p->checkId = XZ_CHECK_CRC32; |
| p->blockSize = XZ_PROPS_BLOCK_SIZE_AUTO; |
| p->numBlockThreads_Reduced = -1; |
| p->numBlockThreads_Max = -1; |
| p->numTotalThreads = -1; |
| p->reduceSize = (UInt64)(Int64)-1; |
| p->forceWriteSizesInHeader = 0; |
| // p->forceWriteSizesInHeader = 1; |
| |
| XzFilterProps_Init(&p->filterProps); |
| Lzma2EncProps_Init(&p->lzma2Props); |
| } |
| |
| |
| static void XzEncProps_Normalize_Fixed(CXzProps *p) |
| { |
| UInt64 fileSize; |
| int t1, t1n, t2, t2r, t3; |
| { |
| CLzma2EncProps tp = p->lzma2Props; |
| if (tp.numTotalThreads <= 0) |
| tp.numTotalThreads = p->numTotalThreads; |
| Lzma2EncProps_Normalize(&tp); |
| t1n = tp.numTotalThreads; |
| } |
| |
| t1 = p->lzma2Props.numTotalThreads; |
| t2 = p->numBlockThreads_Max; |
| t3 = p->numTotalThreads; |
| |
| if (t2 > MTCODER_THREADS_MAX) |
| t2 = MTCODER_THREADS_MAX; |
| |
| if (t3 <= 0) |
| { |
| if (t2 <= 0) |
| t2 = 1; |
| t3 = t1n * t2; |
| } |
| else if (t2 <= 0) |
| { |
| t2 = t3 / t1n; |
| if (t2 == 0) |
| { |
| t1 = 1; |
| t2 = t3; |
| } |
| if (t2 > MTCODER_THREADS_MAX) |
| t2 = MTCODER_THREADS_MAX; |
| } |
| else if (t1 <= 0) |
| { |
| t1 = t3 / t2; |
| if (t1 == 0) |
| t1 = 1; |
| } |
| else |
| t3 = t1n * t2; |
| |
| p->lzma2Props.numTotalThreads = t1; |
| |
| t2r = t2; |
| |
| fileSize = p->reduceSize; |
| |
| if ((p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) |
| p->lzma2Props.lzmaProps.reduceSize = p->blockSize; |
| |
| Lzma2EncProps_Normalize(&p->lzma2Props); |
| |
| t1 = p->lzma2Props.numTotalThreads; |
| |
| { |
| if (t2 > 1 && fileSize != (UInt64)(Int64)-1) |
| { |
| UInt64 numBlocks = fileSize / p->blockSize; |
| if (numBlocks * p->blockSize != fileSize) |
| numBlocks++; |
| if (numBlocks < (unsigned)t2) |
| { |
| t2r = (int)numBlocks; |
| if (t2r == 0) |
| t2r = 1; |
| t3 = t1 * t2r; |
| } |
| } |
| } |
| |
| p->numBlockThreads_Max = t2; |
| p->numBlockThreads_Reduced = t2r; |
| p->numTotalThreads = t3; |
| } |
| |
| |
| static void XzProps_Normalize(CXzProps *p) |
| { |
| /* we normalize xzProps properties, but we normalize only some of CXzProps::lzma2Props properties. |
| Lzma2Enc_SetProps() will normalize lzma2Props later. */ |
| |
| if (p->blockSize == XZ_PROPS_BLOCK_SIZE_SOLID) |
| { |
| p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; |
| p->numBlockThreads_Reduced = 1; |
| p->numBlockThreads_Max = 1; |
| if (p->lzma2Props.numTotalThreads <= 0) |
| p->lzma2Props.numTotalThreads = p->numTotalThreads; |
| return; |
| } |
| else |
| { |
| CLzma2EncProps *lzma2 = &p->lzma2Props; |
| if (p->blockSize == LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO) |
| { |
| // xz-auto |
| p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; |
| |
| if (lzma2->blockSize == LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID) |
| { |
| // if (xz-auto && lzma2-solid) - we use solid for both |
| p->blockSize = XZ_PROPS_BLOCK_SIZE_SOLID; |
| p->numBlockThreads_Reduced = 1; |
| p->numBlockThreads_Max = 1; |
| if (p->lzma2Props.numTotalThreads <= 0) |
| p->lzma2Props.numTotalThreads = p->numTotalThreads; |
| } |
| else |
| { |
| // if (xz-auto && (lzma2-auto || lzma2-fixed_) |
| // we calculate block size for lzma2 and use that block size for xz, lzma2 uses single-chunk per block |
| CLzma2EncProps tp = p->lzma2Props; |
| if (tp.numTotalThreads <= 0) |
| tp.numTotalThreads = p->numTotalThreads; |
| |
| Lzma2EncProps_Normalize(&tp); |
| |
| p->blockSize = tp.blockSize; // fixed or solid |
| p->numBlockThreads_Reduced = tp.numBlockThreads_Reduced; |
| p->numBlockThreads_Max = tp.numBlockThreads_Max; |
| if (lzma2->blockSize == LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO) |
| lzma2->blockSize = tp.blockSize; // fixed or solid, LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID |
| if (lzma2->lzmaProps.reduceSize > tp.blockSize && tp.blockSize != LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID) |
| lzma2->lzmaProps.reduceSize = tp.blockSize; |
| lzma2->numBlockThreads_Reduced = 1; |
| lzma2->numBlockThreads_Max = 1; |
| return; |
| } |
| } |
| else |
| { |
| // xz-fixed |
| // we can use xz::reduceSize or xz::blockSize as base for lzmaProps::reduceSize |
| |
| p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; |
| { |
| UInt64 r = p->reduceSize; |
| if (r > p->blockSize || r == (UInt64)(Int64)-1) |
| r = p->blockSize; |
| lzma2->lzmaProps.reduceSize = r; |
| } |
| if (lzma2->blockSize == LZMA2_ENC_PROPS_BLOCK_SIZE_AUTO) |
| lzma2->blockSize = LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID; |
| else if (lzma2->blockSize > p->blockSize && lzma2->blockSize != LZMA2_ENC_PROPS_BLOCK_SIZE_SOLID) |
| lzma2->blockSize = p->blockSize; |
| |
| XzEncProps_Normalize_Fixed(p); |
| } |
| } |
| } |
| |
| |
| /* ---------- CLzma2WithFilters ---------- */ |
| |
| typedef struct |
| { |
| CLzma2EncHandle lzma2; |
| CSeqInFilter filter; |
| |
| #ifdef USE_SUBBLOCK |
| CSbEncInStream sb; |
| #endif |
| } CLzma2WithFilters; |
| |
| |
| static void Lzma2WithFilters_Construct(CLzma2WithFilters *p) |
| { |
| p->lzma2 = NULL; |
| SeqInFilter_Construct(&p->filter); |
| |
| #ifdef USE_SUBBLOCK |
| SbEncInStream_Construct(&p->sb, alloc); |
| #endif |
| } |
| |
| |
| static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p, ISzAllocPtr alloc, ISzAllocPtr bigAlloc) |
| { |
| if (!p->lzma2) |
| { |
| p->lzma2 = Lzma2Enc_Create(alloc, bigAlloc); |
| if (!p->lzma2) |
| return SZ_ERROR_MEM; |
| } |
| return SZ_OK; |
| } |
| |
| |
| static void Lzma2WithFilters_Free(CLzma2WithFilters *p, ISzAllocPtr alloc) |
| { |
| #ifdef USE_SUBBLOCK |
| SbEncInStream_Free(&p->sb); |
| #endif |
| |
| SeqInFilter_Free(&p->filter, alloc); |
| if (p->lzma2) |
| { |
| Lzma2Enc_Destroy(p->lzma2); |
| p->lzma2 = NULL; |
| } |
| } |
| |
| |
| typedef struct |
| { |
| UInt64 unpackSize; |
| UInt64 totalSize; |
| size_t headerSize; |
| } CXzEncBlockInfo; |
| |
| |
| static SRes Xz_CompressBlock( |
| CLzma2WithFilters *lzmaf, |
| |
| ISeqOutStreamPtr outStream, |
| Byte *outBufHeader, |
| Byte *outBufData, size_t outBufDataLimit, |
| |
| ISeqInStreamPtr inStream, |
| // UInt64 expectedSize, |
| const Byte *inBuf, // used if (!inStream) |
| size_t inBufSize, // used if (!inStream), it's block size, props->blockSize is ignored |
| |
| const CXzProps *props, |
| ICompressProgressPtr progress, |
| int *inStreamFinished, /* only for inStream version */ |
| CXzEncBlockInfo *blockSizes, |
| ISzAllocPtr alloc, |
| ISzAllocPtr allocBig) |
| { |
| CSeqCheckInStream checkInStream; |
| CSeqSizeOutStream seqSizeOutStream; |
| CXzBlock block; |
| unsigned filterIndex = 0; |
| CXzFilter *filter = NULL; |
| const CXzFilterProps *fp = &props->filterProps; |
| if (fp->id == 0) |
| fp = NULL; |
| |
| *inStreamFinished = False; |
| |
| RINOK(Lzma2WithFilters_Create(lzmaf, alloc, allocBig)) |
| |
| RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, &props->lzma2Props)) |
| |
| // XzBlock_ClearFlags(&block) |
| XzBlock_ClearFlags_SetNumFilters(&block, 1 + (fp ? 1 : 0)) |
| |
| if (fp) |
| { |
| filter = &block.filters[filterIndex++]; |
| filter->id = fp->id; |
| filter->propsSize = 0; |
| |
| if (fp->id == XZ_ID_Delta) |
| { |
| filter->props[0] = (Byte)(fp->delta - 1); |
| filter->propsSize = 1; |
| } |
| else if (fp->ipDefined) |
| { |
| Byte *ptr = filter->props; |
| SetUi32(ptr, fp->ip) |
| filter->propsSize = 4; |
| } |
| } |
| |
| { |
| CXzFilter *f = &block.filters[filterIndex++]; |
| f->id = XZ_ID_LZMA2; |
| f->propsSize = 1; |
| f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); |
| } |
| |
| seqSizeOutStream.vt.Write = SeqSizeOutStream_Write; |
| seqSizeOutStream.realStream = outStream; |
| seqSizeOutStream.outBuf = outBufData; |
| seqSizeOutStream.outBufLimit = outBufDataLimit; |
| seqSizeOutStream.processed = 0; |
| |
| /* |
| if (expectedSize != (UInt64)(Int64)-1) |
| { |
| block.unpackSize = expectedSize; |
| if (props->blockSize != (UInt64)(Int64)-1) |
| if (expectedSize > props->blockSize) |
| block.unpackSize = props->blockSize; |
| XzBlock_SetHasUnpackSize(&block) |
| } |
| */ |
| |
| if (outStream) |
| { |
| RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)) |
| } |
| |
| checkInStream.vt.Read = SeqCheckInStream_Read; |
| SeqCheckInStream_Init(&checkInStream, props->checkId); |
| |
| checkInStream.realStream = inStream; |
| checkInStream.data = inBuf; |
| checkInStream.limit = props->blockSize; |
| if (!inStream) |
| checkInStream.limit = inBufSize; |
| |
| if (fp) |
| { |
| #ifdef USE_SUBBLOCK |
| if (fp->id == XZ_ID_Subblock) |
| { |
| lzmaf->sb.inStream = &checkInStream.vt; |
| RINOK(SbEncInStream_Init(&lzmaf->sb)) |
| } |
| else |
| #endif |
| { |
| lzmaf->filter.realStream = &checkInStream.vt; |
| RINOK(SeqInFilter_Init(&lzmaf->filter, filter, alloc)) |
| } |
| } |
| |
| { |
| SRes res; |
| Byte *outBuf = NULL; |
| size_t outSize = 0; |
| BoolInt useStream = (fp || inStream); |
| // useStream = True; |
| |
| if (!useStream) |
| { |
| XzCheck_Update(&checkInStream.check, inBuf, inBufSize); |
| checkInStream.processed = inBufSize; |
| } |
| |
| if (!outStream) |
| { |
| outBuf = seqSizeOutStream.outBuf; // + (size_t)seqSizeOutStream.processed; |
| outSize = seqSizeOutStream.outBufLimit; // - (size_t)seqSizeOutStream.processed; |
| } |
| |
| res = Lzma2Enc_Encode2(lzmaf->lzma2, |
| outBuf ? NULL : &seqSizeOutStream.vt, |
| outBuf, |
| outBuf ? &outSize : NULL, |
| |
| useStream ? |
| (fp ? |
| ( |
| #ifdef USE_SUBBLOCK |
| (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.vt: |
| #endif |
| &lzmaf->filter.vt) : |
| &checkInStream.vt) : NULL, |
| |
| useStream ? NULL : inBuf, |
| useStream ? 0 : inBufSize, |
| |
| progress); |
| |
| if (outBuf) |
| seqSizeOutStream.processed += outSize; |
| |
| RINOK(res) |
| blockSizes->unpackSize = checkInStream.processed; |
| } |
| { |
| Byte buf[4 + 64]; |
| unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed); |
| UInt64 packSize = seqSizeOutStream.processed; |
| |
| buf[0] = 0; |
| buf[1] = 0; |
| buf[2] = 0; |
| buf[3] = 0; |
| |
| SeqCheckInStream_GetDigest(&checkInStream, buf + 4); |
| RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId))) |
| |
| blockSizes->totalSize = seqSizeOutStream.processed - padSize; |
| |
| if (!outStream) |
| { |
| seqSizeOutStream.outBuf = outBufHeader; |
| seqSizeOutStream.outBufLimit = XZ_BLOCK_HEADER_SIZE_MAX; |
| seqSizeOutStream.processed = 0; |
| |
| block.unpackSize = blockSizes->unpackSize; |
| XzBlock_SetHasUnpackSize(&block) |
| |
| block.packSize = packSize; |
| XzBlock_SetHasPackSize(&block) |
| |
| RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)) |
| |
| blockSizes->headerSize = (size_t)seqSizeOutStream.processed; |
| blockSizes->totalSize += seqSizeOutStream.processed; |
| } |
| } |
| |
| if (inStream) |
| *inStreamFinished = checkInStream.realStreamFinished; |
| else |
| { |
| *inStreamFinished = False; |
| if (checkInStream.processed != inBufSize) |
| return SZ_ERROR_FAIL; |
| } |
| |
| return SZ_OK; |
| } |
| |
| |
| |
| typedef struct |
| { |
| ICompressProgress vt; |
| ICompressProgressPtr progress; |
| UInt64 inOffset; |
| UInt64 outOffset; |
| } CCompressProgress_XzEncOffset; |
| |
| |
| static SRes CompressProgress_XzEncOffset_Progress(ICompressProgressPtr pp, UInt64 inSize, UInt64 outSize) |
| { |
| const CCompressProgress_XzEncOffset *p = Z7_CONTAINER_FROM_VTBL_CONST(pp, CCompressProgress_XzEncOffset, vt); |
| inSize += p->inOffset; |
| outSize += p->outOffset; |
| return ICompressProgress_Progress(p->progress, inSize, outSize); |
| } |
| |
| |
| |
| |
| struct CXzEnc |
| { |
| ISzAllocPtr alloc; |
| ISzAllocPtr allocBig; |
| |
| CXzProps xzProps; |
| UInt64 expectedDataSize; |
| |
| CXzEncIndex xzIndex; |
| |
| CLzma2WithFilters lzmaf_Items[MTCODER_THREADS_MAX]; |
| |
| size_t outBufSize; /* size of allocated outBufs[i] */ |
| Byte *outBufs[MTCODER_BLOCKS_MAX]; |
| |
| #ifndef Z7_ST |
| unsigned checkType; |
| ISeqOutStreamPtr outStream; |
| BoolInt mtCoder_WasConstructed; |
| CMtCoder mtCoder; |
| CXzEncBlockInfo EncBlocks[MTCODER_BLOCKS_MAX]; |
| #endif |
| }; |
| |
| |
| static void XzEnc_Construct(CXzEnc *p) |
| { |
| unsigned i; |
| |
| XzEncIndex_Construct(&p->xzIndex); |
| |
| for (i = 0; i < MTCODER_THREADS_MAX; i++) |
| Lzma2WithFilters_Construct(&p->lzmaf_Items[i]); |
| |
| #ifndef Z7_ST |
| p->mtCoder_WasConstructed = False; |
| { |
| for (i = 0; i < MTCODER_BLOCKS_MAX; i++) |
| p->outBufs[i] = NULL; |
| p->outBufSize = 0; |
| } |
| #endif |
| } |
| |
| |
| static void XzEnc_FreeOutBufs(CXzEnc *p) |
| { |
| unsigned i; |
| for (i = 0; i < MTCODER_BLOCKS_MAX; i++) |
| if (p->outBufs[i]) |
| { |
| ISzAlloc_Free(p->alloc, p->outBufs[i]); |
| p->outBufs[i] = NULL; |
| } |
| p->outBufSize = 0; |
| } |
| |
| |
| static void XzEnc_Free(CXzEnc *p, ISzAllocPtr alloc) |
| { |
| unsigned i; |
| |
| XzEncIndex_Free(&p->xzIndex, alloc); |
| |
| for (i = 0; i < MTCODER_THREADS_MAX; i++) |
| Lzma2WithFilters_Free(&p->lzmaf_Items[i], alloc); |
| |
| #ifndef Z7_ST |
| if (p->mtCoder_WasConstructed) |
| { |
| MtCoder_Destruct(&p->mtCoder); |
| p->mtCoder_WasConstructed = False; |
| } |
| XzEnc_FreeOutBufs(p); |
| #endif |
| } |
| |
| |
| CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) |
| { |
| CXzEnc *p = (CXzEnc *)ISzAlloc_Alloc(alloc, sizeof(CXzEnc)); |
| if (!p) |
| return NULL; |
| XzEnc_Construct(p); |
| XzProps_Init(&p->xzProps); |
| XzProps_Normalize(&p->xzProps); |
| p->expectedDataSize = (UInt64)(Int64)-1; |
| p->alloc = alloc; |
| p->allocBig = allocBig; |
| return (CXzEncHandle)p; |
| } |
| |
| // #define GET_CXzEnc_p CXzEnc *p = (CXzEnc *)(void *)pp; |
| |
| void XzEnc_Destroy(CXzEncHandle p) |
| { |
| // GET_CXzEnc_p |
| XzEnc_Free(p, p->alloc); |
| ISzAlloc_Free(p->alloc, p); |
| } |
| |
| |
| SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props) |
| { |
| // GET_CXzEnc_p |
| p->xzProps = *props; |
| XzProps_Normalize(&p->xzProps); |
| return SZ_OK; |
| } |
| |
| |
| void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize) |
| { |
| // GET_CXzEnc_p |
| p->expectedDataSize = expectedDataSiize; |
| } |
| |
| |
| |
| |
| #ifndef Z7_ST |
| |
| static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, |
| const Byte *src, size_t srcSize, int finished) |
| { |
| CXzEnc *me = (CXzEnc *)pp; |
| SRes res; |
| CMtProgressThunk progressThunk; |
| |
| Byte *dest = me->outBufs[outBufIndex]; |
| |
| UNUSED_VAR(finished) |
| |
| { |
| CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; |
| bInfo->totalSize = 0; |
| bInfo->unpackSize = 0; |
| bInfo->headerSize = 0; |
| } |
| |
| if (!dest) |
| { |
| dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); |
| if (!dest) |
| return SZ_ERROR_MEM; |
| me->outBufs[outBufIndex] = dest; |
| } |
| |
| MtProgressThunk_CreateVTable(&progressThunk); |
| progressThunk.mtProgress = &me->mtCoder.mtProgress; |
| MtProgressThunk_INIT(&progressThunk) |
| |
| { |
| CXzEncBlockInfo blockSizes; |
| int inStreamFinished; |
| |
| res = Xz_CompressBlock( |
| &me->lzmaf_Items[coderIndex], |
| |
| NULL, |
| dest, |
| dest + XZ_BLOCK_HEADER_SIZE_MAX, me->outBufSize - XZ_BLOCK_HEADER_SIZE_MAX, |
| |
| NULL, |
| // srcSize, // expectedSize |
| src, srcSize, |
| |
| &me->xzProps, |
| &progressThunk.vt, |
| &inStreamFinished, |
| &blockSizes, |
| me->alloc, |
| me->allocBig); |
| |
| if (res == SZ_OK) |
| me->EncBlocks[outBufIndex] = blockSizes; |
| |
| return res; |
| } |
| } |
| |
| |
| static SRes XzEnc_MtCallback_Write(void *pp, unsigned outBufIndex) |
| { |
| CXzEnc *me = (CXzEnc *)pp; |
| |
| const CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; |
| const Byte *data = me->outBufs[outBufIndex]; |
| |
| RINOK(WriteBytes(me->outStream, data, bInfo->headerSize)) |
| |
| { |
| UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize); |
| RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize)) |
| } |
| |
| return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc); |
| } |
| |
| #endif |
| |
| |
| |
| SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStreamPtr outStream, ISeqInStreamPtr inStream, ICompressProgressPtr progress) |
| { |
| // GET_CXzEnc_p |
| |
| const CXzProps *props = &p->xzProps; |
| |
| XzEncIndex_Init(&p->xzIndex); |
| { |
| UInt64 numBlocks = 1; |
| UInt64 blockSize = props->blockSize; |
| |
| if (blockSize != XZ_PROPS_BLOCK_SIZE_SOLID |
| && props->reduceSize != (UInt64)(Int64)-1) |
| { |
| numBlocks = props->reduceSize / blockSize; |
| if (numBlocks * blockSize != props->reduceSize) |
| numBlocks++; |
| } |
| else |
| blockSize = (UInt64)1 << 62; |
| |
| RINOK(XzEncIndex_PreAlloc(&p->xzIndex, numBlocks, blockSize, XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(blockSize), p->alloc)) |
| } |
| |
| RINOK(Xz_WriteHeader((CXzStreamFlags)props->checkId, outStream)) |
| |
| |
| #ifndef Z7_ST |
| if (props->numBlockThreads_Reduced > 1) |
| { |
| IMtCoderCallback2 vt; |
| |
| if (!p->mtCoder_WasConstructed) |
| { |
| p->mtCoder_WasConstructed = True; |
| MtCoder_Construct(&p->mtCoder); |
| } |
| |
| vt.Code = XzEnc_MtCallback_Code; |
| vt.Write = XzEnc_MtCallback_Write; |
| |
| p->checkType = props->checkId; |
| p->xzProps = *props; |
| |
| p->outStream = outStream; |
| |
| p->mtCoder.allocBig = p->allocBig; |
| p->mtCoder.progress = progress; |
| p->mtCoder.inStream = inStream; |
| p->mtCoder.inData = NULL; |
| p->mtCoder.inDataSize = 0; |
| p->mtCoder.mtCallback = &vt; |
| p->mtCoder.mtCallbackObject = p; |
| |
| if ( props->blockSize == XZ_PROPS_BLOCK_SIZE_SOLID |
| || props->blockSize == XZ_PROPS_BLOCK_SIZE_AUTO) |
| return SZ_ERROR_FAIL; |
| |
| p->mtCoder.blockSize = (size_t)props->blockSize; |
| if (p->mtCoder.blockSize != props->blockSize) |
| return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ |
| |
| { |
| size_t destBlockSize = XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(p->mtCoder.blockSize); |
| if (destBlockSize < p->mtCoder.blockSize) |
| return SZ_ERROR_PARAM; |
| if (p->outBufSize != destBlockSize) |
| XzEnc_FreeOutBufs(p); |
| p->outBufSize = destBlockSize; |
| } |
| |
| p->mtCoder.numThreadsMax = (unsigned)props->numBlockThreads_Max; |
| p->mtCoder.expectedDataSize = p->expectedDataSize; |
| |
| RINOK(MtCoder_Code(&p->mtCoder)) |
| } |
| else |
| #endif |
| { |
| int writeStartSizes; |
| CCompressProgress_XzEncOffset progress2; |
| Byte *bufData = NULL; |
| size_t bufSize = 0; |
| |
| progress2.vt.Progress = CompressProgress_XzEncOffset_Progress; |
| progress2.inOffset = 0; |
| progress2.outOffset = 0; |
| progress2.progress = progress; |
| |
| writeStartSizes = 0; |
| |
| if (props->blockSize != XZ_PROPS_BLOCK_SIZE_SOLID) |
| { |
| writeStartSizes = (props->forceWriteSizesInHeader > 0); |
| |
| if (writeStartSizes) |
| { |
| size_t t2; |
| size_t t = (size_t)props->blockSize; |
| if (t != props->blockSize) |
| return SZ_ERROR_PARAM; |
| t = XZ_GET_MAX_BLOCK_PACK_SIZE(t); |
| if (t < props->blockSize) |
| return SZ_ERROR_PARAM; |
| t2 = XZ_BLOCK_HEADER_SIZE_MAX + t; |
| if (!p->outBufs[0] || t2 != p->outBufSize) |
| { |
| XzEnc_FreeOutBufs(p); |
| p->outBufs[0] = (Byte *)ISzAlloc_Alloc(p->alloc, t2); |
| if (!p->outBufs[0]) |
| return SZ_ERROR_MEM; |
| p->outBufSize = t2; |
| } |
| bufData = p->outBufs[0] + XZ_BLOCK_HEADER_SIZE_MAX; |
| bufSize = t; |
| } |
| } |
| |
| for (;;) |
| { |
| CXzEncBlockInfo blockSizes; |
| int inStreamFinished; |
| |
| /* |
| UInt64 rem = (UInt64)(Int64)-1; |
| if (props->reduceSize != (UInt64)(Int64)-1 |
| && props->reduceSize >= progress2.inOffset) |
| rem = props->reduceSize - progress2.inOffset; |
| */ |
| |
| blockSizes.headerSize = 0; // for GCC |
| |
| RINOK(Xz_CompressBlock( |
| &p->lzmaf_Items[0], |
| |
| writeStartSizes ? NULL : outStream, |
| writeStartSizes ? p->outBufs[0] : NULL, |
| bufData, bufSize, |
| |
| inStream, |
| // rem, |
| NULL, 0, |
| |
| props, |
| progress ? &progress2.vt : NULL, |
| &inStreamFinished, |
| &blockSizes, |
| p->alloc, |
| p->allocBig)) |
| |
| { |
| UInt64 totalPackFull = blockSizes.totalSize + XZ_GET_PAD_SIZE(blockSizes.totalSize); |
| |
| if (writeStartSizes) |
| { |
| RINOK(WriteBytes(outStream, p->outBufs[0], blockSizes.headerSize)) |
| RINOK(WriteBytes(outStream, bufData, (size_t)totalPackFull - blockSizes.headerSize)) |
| } |
| |
| RINOK(XzEncIndex_AddIndexRecord(&p->xzIndex, blockSizes.unpackSize, blockSizes.totalSize, p->alloc)) |
| |
| progress2.inOffset += blockSizes.unpackSize; |
| progress2.outOffset += totalPackFull; |
| } |
| |
| if (inStreamFinished) |
| break; |
| } |
| } |
| |
| return XzEncIndex_WriteFooter(&p->xzIndex, (CXzStreamFlags)props->checkId, outStream); |
| } |
| |
| |
| #include "Alloc.h" |
| |
| SRes Xz_Encode(ISeqOutStreamPtr outStream, ISeqInStreamPtr inStream, |
| const CXzProps *props, ICompressProgressPtr progress) |
| { |
| SRes res; |
| CXzEncHandle xz = XzEnc_Create(&g_Alloc, &g_BigAlloc); |
| if (!xz) |
| return SZ_ERROR_MEM; |
| res = XzEnc_SetProps(xz, props); |
| if (res == SZ_OK) |
| res = XzEnc_Encode(xz, outStream, inStream, progress); |
| XzEnc_Destroy(xz); |
| return res; |
| } |
| |
| |
| SRes Xz_EncodeEmpty(ISeqOutStreamPtr outStream) |
| { |
| SRes res; |
| CXzEncIndex xzIndex; |
| XzEncIndex_Construct(&xzIndex); |
| res = Xz_WriteHeader((CXzStreamFlags)0, outStream); |
| if (res == SZ_OK) |
| res = XzEncIndex_WriteFooter(&xzIndex, (CXzStreamFlags)0, outStream); |
| XzEncIndex_Free(&xzIndex, NULL); // g_Alloc |
| return res; |
| } |