pax_global_header00006660000000000000000000000064137553075700014526gustar00rootroot0000000000000052 comment=0e323ac1ea7260d73486dc96959ba7f306bbb19a ancient-1.0/000077500000000000000000000000001375530757000130075ustar00rootroot00000000000000ancient-1.0/.gitignore000066400000000000000000000000071375530757000147740ustar00rootroot00000000000000*~ *.o ancient-1.0/LICENSE000066400000000000000000000024461375530757000140220ustar00rootroot00000000000000BSD 2-Clause License Copyright (c) 2017, Teemu Suutari All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ancient-1.0/Makefile000066400000000000000000000033511375530757000144510ustar00rootroot00000000000000# Copyright (C) Teemu Suutari VPATH := src src/Lzh src/Zip src/common fuzzing CXX ?= c++ COMMONFLAGS = -Os -Wall -Wsign-compare -Wnarrowing -Wno-error=multichar -Wno-multichar -Isrc CFLAGS = $(COMMONFLAGS) CXXFLAGS = $(COMMONFLAGS) -std=c++17 -fno-rtti PROG = ancient MAIN ?= main.o OBJS = Buffer.o Common.o MemoryBuffer.o StaticBuffer.o SubBuffer.o CRC16.o CRC32.o \ Decompressor.o XPKDecompressor.o XPKMain.o \ OutputStream.o InputStream.o RangeDecoder.o \ ACCADecompressor.o ARTMDecompressor.o BLZWDecompressor.o BZIP2Decompressor.o \ CBR0Decompressor.o CRMDecompressor.o CYB2Decoder.o DEFLATEDecompressor.o \ DLTADecode.o DMSDecompressor.o FASTDecompressor.o FBR2Decompressor.o \ FRLEDecompressor.o HFMNDecompressor.o HUFFDecompressor.o ILZRDecompressor.o \ IMPDecompressor.o LHLBDecompressor.o LIN1Decompressor.o LIN2Decompressor.o \ LZBSDecompressor.o LZCBDecompressor.o LZW2Decompressor.o LZW4Decompressor.o \ LZW5Decompressor.o LZXDecompressor.o MASHDecompressor.o MMCMPDecompressor.o \ NONEDecompressor.o NUKEDecompressor.o PPDecompressor.o RAKEDecompressor.o \ RDCNDecompressor.o RLENDecompressor.o RNCDecompressor.o SDHCDecompressor.o \ SHR3Decompressor.o SHRIDecompressor.o SLZ3Decompressor.o SMPLDecompressor.o \ StoneCrackerDecompressor.o SQSHDecompressor.o SXSCDecompressor.o TDCSDecompressor.o \ TPWMDecompressor.o ZENODecompressor.o LH1Decompressor.o LH2Decompressor.o \ LH3Decompressor.o LHXDecompressor.o LZ5Decompressor.o LZSDecompressor.o \ PMDecompressor.o LZHDecompressor.o ImplodeDecompressor.o ReduceDecompressor.o \ ShrinkDecompressor.o ZIPDecompressor.o all: $(PROG) .cpp.o: $(CXX) $(CXXFLAGS) -o $@ -c $< $(PROG): $(OBJS) $(MAIN) $(CXX) $(CFLAGS) -o $@ $^ clean: rm -f $(OBJS) $(MAIN) $(PROG) *~ src/*~ .PHONY: ancient-1.0/README.md000066400000000000000000000152731375530757000142760ustar00rootroot00000000000000# Ancient - Modern decompressor for old data compression formats This is a collection of decompression routines for old formats popular in the Amiga, Atari computers and some other systems from 80's and 90's as well as some that are currently used which were used in a some specific way in these old systems. For simple usage both a simple command line application as well as a simple API to use the decompressors are provided. The compression algorithm is automatically detected in most cases, however there are some corner cases where it is not entirely reliable due to weaknesses in the old format used. Please see the main.cpp and Decompressor.hpp to get an idea. Even though most of these algorithms are still available for download, scavenging and using them might prove to be a challenge. Thus the purpose of this project is to: * Provide a clean, modern implementation of the algorithms - Typically the implementations were not meant to be used outside of the original systems they were made for. Some other ported implementations are incomplete, bad quality or direct translations from old M68K assembly code. * Provide a clean BSD-style licensing - Original implementations or their ports might have strange license or no visible license at all. There are also implementations that have been ripped off from some other source thus their legality is questionable at best. * Provide a tested implementation - The code is no good if it does not work properly and the old code have a lot of corner cases. These implementations are tested using a cache of available files (~10k) that used these algorithms. Although it does not offer any guarantee especially when we are talking about undocumented formats, it gives hope that there are less "stupid errors" in the code. This code should compile cleanly on most C++17 capable compilers (C++14 compilers might need tweaking), and it is tested on clang and MSVC. For MSVC please use https://github.com/tronkko/dirent or similar to compile to command line tool Currently there are no plans to add password protected file support nor any kind of decryption capability apart from some very basic password bypassing in some formats that can be done easily Decompression algorithms provided: * CrunchMania by Thomas Schwarz * CrM!: Crunch-Mania standard-mode * Crm!: Crunch-Mania standard-mode, sampled * CrM2: Crunch-Mania LZH-mode * Crm2: Crunch-Mania LZH-mode, sampled * Disk Masher System a.k.a. DMS * Supports all different compression methods (NONE,SIMPLE,QUICK,MEDIUM,DEEP,HEAVY1,HEAVY2) * Supports password bypassing * File Imploder (and most of its clones) * ATN! * BDPI * CHFI * EDAM * IMP! * M.H. * RDC9 * FLT! (verification missing) * Dupa (verification missing) * PARA (verification missing) * PowerPacker * PP 1.1 (verification missing) * PP 2.0 * Rob Northen compressors. * RNC1: Both old and formats utilizing the same header. heuristics for detecting the current one * RNC2: RNC version 2 stream * Turbo Packer by Wolfgang Mayerle. * Standard gzip * Standard bzip2, both normal and randomized * Zip decompressor backend (decompressor only, no Zip file format reading yet) * Shrink * Reduce * Implode * Deflate * Deflate64 * Bzip2 * Lha/Lzh decompressor backend (decompressor only, no Lha file format reading yet) * LH0: Null compressor * LH1: LZRW-compressor with 4kB window * LH2: LZRW-compressor with Dynamic Huffman Encoding (experimental) * LH3: LZRW-compressor (experimental) * LH4: LZRW-compressor with 4kB window * LH5: LZRW-compressor with 8kB window * LH6: LZRW-compressor with 32kB window * LH7: LZRW-compressor with 64kB window * LH8: LZRW-compressor with 64kB window (Joe Jared extension) * LHX: LZRW-compressor with up to 512kB window (UnLHX extension) * LZ4: Null compressor * LZ5: LZ-compressor * LZS: LZ-compressor * PM0: Null compressor * PM1: LZ-compressor * PM2: LZ-compressor * MMCMP: Music Module Compressor * StoneCracker * SC: StoneCracker v2.69 - v2.81 * SC: StoneCracker v2.92, v2.99 * S300: StoneCracker v3.00 * S310: StoneCracker v3.10, v3.11b * S400: StoneCracker pre v4.00 * S401: StoneCracker v4.01 * S403: StoneCracker v4.02a * S404: StoneCracker v4.10 * XPK-encapsulated files * ACCA: Andre's Code Compression Algorithm * ARTM: Arithmetic encoding compressor * BLZW: LZW-compressor * BZP2: Bzip2 backend for XPK * CBR0: RLE compressor * CBR1: RLE compressor * CRM2: CrunchMania backend for XPK * CRMS: CrunchMania backend for XPK, sampled * CYB2: xpkCybPrefs container * DLTA: Delta encoding * DUKE: NUKE with Delta encoding * ELZX: LZX-compressor * FAST: LZ77-compressor * FBR2: CyberYAFA compressor * FRHT: LZ77-compressor * FRLE: RLE compressor * GZIP: Deflate backend for XPK * HUFF: Huffman modeling compressor * HFMN: Huffman modeling compressor * ILZR: Incremental Lempel-Ziv-Renau compressor * IMPL: File Imploder backend for XPK * LHLB: LZRW-compressor * LIN1: Lino packer * LIN2: Lino packer * LIN3: Lino packer * LIN4: Lino packer * LZBS: CyberYAFA compressor * LZCB: LZ-compressor * LZW2: CyberYAFA compressor * LZW3: CyberYAFA compressor * LZW4: CyberYAFA compressor * LZW5: CyberYAFA compressor * MASH: LZRW-compressor * NONE: Null compressor * NUKE: LZ77-compressor * PWPK: PowerPacker backend for XPK * RAKE: LZ77-compressor * RDCN: Ross Data Compression * RLEN: RLE compressor * SASC: LZ-compressor with arithmetic encoding * SDHC: Sample delta huffman compressor * SHR3: LZ-compressor with arithmetic encoding * SHRI: LZ-compressor with arithmetic encoding * SHSC: Context modeling compressor * SLZ3: CyberYAFA compressor * SLZX: LZX-compressor with delta encoding * SMPL: Huffman compressor with delta encoding * SQSH: Compressor for sampled sounds * TDCS: LZ77-compressor * ZENO: LZW-compressor Special thanks go to Cholok for providing me references to many of the XPK-compressors. BZIP2 tables for randomization have been included, they have BZIP2-license. SASC/SHSC decompressors have been re-implemented by using the original HA code from Harri Hirvola as reference. (No code re-used) Some of the rare Lzh-compressors have been re-implemented by using Lhasa as a reference. (No code re-used) I'm slowly adding new stuff. If your favorite is not listed contact me and maybe I can add it. Current work in progress * Completing XPK: DCMB, DMCU (maybe), PPMQ * Disk images support for Warp and DiskSqueeze Not planned: * PPC only XPK compressors Wishlist: * More files for my testbench * More DMS images * XPK DMCI and DMCD compressors. I can't find it anywhere or find anyone who knows anything about it. Would be fun to add it for completeness * XPK TLTA compressor/encoder. ditto. Feedback: tz at iki dot fi ancient-1.0/extra/000077500000000000000000000000001375530757000141325ustar00rootroot00000000000000ancient-1.0/extra/BruteForceRNC1Encoder.cpp000066400000000000000000000354051375530757000206310ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ // This is really quick and dirty. Works though #include #include #include #include #include #include #include #include #include #include #include #include "common/Buffer.hpp" #include "common/SubBuffer.hpp" class VectorBuffer : public Buffer { public: VectorBuffer(); virtual ~VectorBuffer() override final; virtual const uint8_t *data() const noexcept override final; virtual uint8_t *data() override final; virtual size_t size() const noexcept override final; virtual bool isResizable() const noexcept override final; virtual void resize(size_t newSize) override final; private: std::vector _data; }; VectorBuffer::VectorBuffer() { // nothing needed } VectorBuffer::~VectorBuffer() { // nothing needed } const uint8_t *VectorBuffer::data() const noexcept { return _data.data(); } uint8_t *VectorBuffer::data() { return _data.data(); } size_t VectorBuffer::size() const noexcept { return _data.size(); } bool VectorBuffer::isResizable() const noexcept { return true; } void VectorBuffer::resize(size_t newSize) { return _data.resize(newSize); } std::unique_ptr readFile(const std::string &fileName) { std::unique_ptr ret=std::make_unique(); std::ifstream file(fileName.c_str(),std::ios::in|std::ios::binary); bool success=false; if (file.is_open()) { file.seekg(0,std::ios::end); size_t length=size_t(file.tellg()); file.seekg(0,std::ios::beg); ret->resize(length); file.read(reinterpret_cast(ret->data()),length); success=bool(file); if (!success) ret->resize(0); file.close(); } if (!success) { fprintf(stderr,"Could not read file %s\n",fileName.c_str()); } return ret; } bool writeFile(const std::string &fileName,const Buffer &content) { bool ret=false; std::ofstream file(fileName.c_str(),std::ios::out|std::ios::binary|std::ios::trunc); if (file.is_open()) { file.write(reinterpret_cast(content.data()),content.size()); ret=bool(file); file.close(); } if (!ret) { fprintf(stderr,"Could not write file %s\n",fileName.c_str()); } return ret; } uint16_t RNCCRC(const uint8_t *buffer,size_t len) { // bit reversed 16bit CRC with 0x8005 polynomial static const uint16_t CRCTable[256]={ 0x0000,0xc0c1,0xc181,0x0140,0xc301,0x03c0,0x0280,0xc241,0xc601,0x06c0,0x0780,0xc741,0x0500,0xc5c1,0xc481,0x0440, 0xcc01,0x0cc0,0x0d80,0xcd41,0x0f00,0xcfc1,0xce81,0x0e40,0x0a00,0xcac1,0xcb81,0x0b40,0xc901,0x09c0,0x0880,0xc841, 0xd801,0x18c0,0x1980,0xd941,0x1b00,0xdbc1,0xda81,0x1a40,0x1e00,0xdec1,0xdf81,0x1f40,0xdd01,0x1dc0,0x1c80,0xdc41, 0x1400,0xd4c1,0xd581,0x1540,0xd701,0x17c0,0x1680,0xd641,0xd201,0x12c0,0x1380,0xd341,0x1100,0xd1c1,0xd081,0x1040, 0xf001,0x30c0,0x3180,0xf141,0x3300,0xf3c1,0xf281,0x3240,0x3600,0xf6c1,0xf781,0x3740,0xf501,0x35c0,0x3480,0xf441, 0x3c00,0xfcc1,0xfd81,0x3d40,0xff01,0x3fc0,0x3e80,0xfe41,0xfa01,0x3ac0,0x3b80,0xfb41,0x3900,0xf9c1,0xf881,0x3840, 0x2800,0xe8c1,0xe981,0x2940,0xeb01,0x2bc0,0x2a80,0xea41,0xee01,0x2ec0,0x2f80,0xef41,0x2d00,0xedc1,0xec81,0x2c40, 0xe401,0x24c0,0x2580,0xe541,0x2700,0xe7c1,0xe681,0x2640,0x2200,0xe2c1,0xe381,0x2340,0xe101,0x21c0,0x2080,0xe041, 0xa001,0x60c0,0x6180,0xa141,0x6300,0xa3c1,0xa281,0x6240,0x6600,0xa6c1,0xa781,0x6740,0xa501,0x65c0,0x6480,0xa441, 0x6c00,0xacc1,0xad81,0x6d40,0xaf01,0x6fc0,0x6e80,0xae41,0xaa01,0x6ac0,0x6b80,0xab41,0x6900,0xa9c1,0xa881,0x6840, 0x7800,0xb8c1,0xb981,0x7940,0xbb01,0x7bc0,0x7a80,0xba41,0xbe01,0x7ec0,0x7f80,0xbf41,0x7d00,0xbdc1,0xbc81,0x7c40, 0xb401,0x74c0,0x7580,0xb541,0x7700,0xb7c1,0xb681,0x7640,0x7200,0xb2c1,0xb381,0x7340,0xb101,0x71c0,0x7080,0xb041, 0x5000,0x90c1,0x9181,0x5140,0x9301,0x53c0,0x5280,0x9241,0x9601,0x56c0,0x5780,0x9741,0x5500,0x95c1,0x9481,0x5440, 0x9c01,0x5cc0,0x5d80,0x9d41,0x5f00,0x9fc1,0x9e81,0x5e40,0x5a00,0x9ac1,0x9b81,0x5b40,0x9901,0x59c0,0x5880,0x9841, 0x8801,0x48c0,0x4980,0x8941,0x4b00,0x8bc1,0x8a81,0x4a40,0x4e00,0x8ec1,0x8f81,0x4f40,0x8d01,0x4dc0,0x4c80,0x8c41, 0x4400,0x84c1,0x8581,0x4540,0x8701,0x47c0,0x4680,0x8641,0x8201,0x42c0,0x4380,0x8341,0x4100,0x81c1,0x8081,0x4040}; uint16_t retValue=0; for (size_t i=0;i>8)^CRCTable[(retValue&0xff)^buffer[i]]; return retValue; } // this is really really quick 'n dirty // leeway is suspicious. I can't see it from the official RNC ProPack, but seems to be present elsewhere... void packRNC(Buffer &dest,const Buffer &source,uint32_t chunkSize) { if (!chunkSize) chunkSize=65536; if (chunkSize<1024) chunkSize=1024; std::vector stream(20); stream[0]='R'; stream[1]='N'; stream[2]='C'; stream[3]=1; stream[4]=uint8_t(source.size()>>24); stream[5]=uint8_t(source.size()>>16); stream[6]=uint8_t(source.size()>>8); stream[7]=uint8_t(source.size()); stream[8]=stream[9]=stream[10]=stream[11]=0; uint16_t rawCrc=RNCCRC(source.data(),source.size()); stream[12]=uint8_t(rawCrc>>8); stream[13]=uint8_t(rawCrc); stream[14]=stream[15]=stream[16]=stream[17]=0; stream[18]=stream[19]=0; uint32_t bitStreamPosition=18; uint32_t bitAccumContent=0; uint32_t bitAccumCount=2; uint32_t offset=0; uint32_t chunkCount=0; uint32_t leeway=0; while (offset!=source.size()) { auto bitLength=[](uint32_t value)->uint32_t { uint32_t ret=0; while (value) { value>>=1; ret++; } return ret; }; // returns code, extra bits, value auto packValue=[&](uint32_t value)->std::tuple { if (value<2) return std::tuple{value,0,0}; uint32_t bits=bitLength(value)-1; value&=(1<{bits+1,bits,value}; }; // bruteforce!!! // returns length,offset auto findLongestRepeat=[&](const uint8_t *buf,uint32_t offset,uint32_t length)->std::pair { std::pair best{0,0}; uint32_t distance=1; while (distance<=offset) { uint32_t i=0; while (offset+i=2 && i>best.first) { // encoding cost: how efficiently this hit encodes stuff if (i>3 || bitLength(i-2)+bitLength(distance-1)<7 || (i==3 && bitLength(i-2)+bitLength(distance-1)<10)) { best=std::make_pair(i,distance); } } distance++; } return best; }; std::vector litFrequencies(32,0); std::vector distanceFrequencies(32,0); std::vector lengthFrequencies(32,0); // this is what makes this implementaion even more bruteforce // table index (lit=0,distance=1,length=2,byte=3,bits=4), code, extra bits, value std::vector> rawChunk; uint32_t litCountOffset=0; bool litActive=false; // sub count rawChunk.push_back(std::tuple{4,0,16,0}); auto packLit=[&]() { auto pack=packValue(std::get<3>(rawChunk[litCountOffset])); std::get<1>(rawChunk[litCountOffset])=std::get<0>(pack); std::get<2>(rawChunk[litCountOffset])=std::get<1>(pack); std::get<3>(rawChunk[litCountOffset])=std::get<2>(pack); litFrequencies[std::get<0>(pack)]+=1; }; uint32_t foundLength=1; uint32_t currentChunkSize=std::min(uint32_t(source.size())-offset,chunkSize); for (uint32_t i=offset;i{0,0,0,0}); litActive=true; std::get<3>(rawChunk[0])+=1; } auto repeat=findLongestRepeat(source.data(),i,currentChunkSize+offset); // auto repeat=findLongestRepeat(source.data()+offset,i-offset,currentChunkSize); if (repeat.first) { packLit(); litActive=false; auto dist=packValue(repeat.second-1); rawChunk.push_back(std::tuple{1,std::get<0>(dist),std::get<1>(dist),std::get<2>(dist)}); distanceFrequencies[std::get<0>(dist)]+=1; auto count=packValue(repeat.first-2); rawChunk.push_back(std::tuple{2,std::get<0>(count),std::get<1>(count),std::get<2>(count)}); lengthFrequencies[std::get<0>(count)]+=1; foundLength=repeat.first; } else { rawChunk.push_back(std::tuple{3,source.data()[i],0,0}); std::get<3>(rawChunk[litCountOffset])+=1; foundLength=1; } } if (litActive) { packLit(); } else { litCountOffset=uint32_t(rawChunk.size()); rawChunk.push_back(std::tuple{0,0,0,0}); packLit(); std::get<3>(rawChunk[0])+=1; } offset+=currentChunkSize; chunkCount++; std::function writeBits=[&](uint32_t bitCount,uint32_t bits) { if (!bitCount) return; if (bitCount+bitAccumCount>16) { uint32_t bitsToWrite=16-bitAccumCount; writeBits(bitsToWrite,bits&((1<>=bitsToWrite; bitCount-=bitsToWrite; } if (!bitAccumCount) { bitStreamPosition=uint32_t(stream.size()); stream.push_back(0); stream.push_back(0); } bitAccumContent|=bits<>8; bitAccumContent=0; bitAccumCount=0; } }; auto writeByte=[&](uint8_t byte) { stream.push_back(byte); }; // also writes table to stream // result vector is code, length pairs auto createHuffmanCodeTable=[&](std::vector> &codes,const std::vector &frequencies) { std::vector> sortedList; uint32_t totalCount=0; uint32_t totalFreq=0; for (uint32_t i=0;ib.second||(a.second==b.second&&a.first>bitCount; } } // use the full range while (totalUsed!=initialNorm) { uint32_t bestIndex=totalCount; uint32_t bestImprovement=0; for (uint32_t i=0;i>sortedList[i].second)<=initialNorm) { // adding cost factor here too uint32_t improvement=sortedFrequencies[i]<bestImprovement) { bestIndex=i; bestImprovement=improvement; } } } if (bestIndex==totalCount) break; totalUsed+=initialNorm>>sortedList[bestIndex].second; sortedList[bestIndex].second-=1; } // using the full range will sometimes result out-of-order indexes std::sort(sortedList.begin(),sortedList.end(),[&](const auto &a,const auto &b){return a.seconduint32_t { uint32_t ret=0; while (bitCount--) { ret<<=1; ret|=bits&1; bits>>=1; } return ret; }; uint32_t code=sortedList[i].first; if (sortedList[i].second) { codes[code].first=reverseBits(maxDepth,value); codes[code].second=sortedList[i].second; value+=1<<(maxDepth-sortedList[i].second); } } for (uint32_t i=0;i> litCodes(32,std::make_pair(0,0)); std::vector> distanceCodes(32,std::make_pair(0,0)); std::vector> lengthCodes(32,std::make_pair(0,0)); uint32_t streamStart=uint32_t(stream.size()); createHuffmanCodeTable(litCodes,litFrequencies); createHuffmanCodeTable(distanceCodes,distanceFrequencies); createHuffmanCodeTable(lengthCodes,lengthFrequencies); for (uint32_t i=0;i(rawChunk[i])) { // literal case 0: writeBits(litCodes[std::get<1>(rawChunk[i])].second,litCodes[std::get<1>(rawChunk[i])].first); writeBits(std::get<2>(rawChunk[i]),std::get<3>(rawChunk[i])); break; // distance case 1: writeBits(distanceCodes[std::get<1>(rawChunk[i])].second,distanceCodes[std::get<1>(rawChunk[i])].first); writeBits(std::get<2>(rawChunk[i]),std::get<3>(rawChunk[i])); break; // length case 2: writeBits(lengthCodes[std::get<1>(rawChunk[i])].second,lengthCodes[std::get<1>(rawChunk[i])].first); writeBits(std::get<2>(rawChunk[i]),std::get<3>(rawChunk[i])); break; // bytes case 3: writeByte(std::get<1>(rawChunk[i])); break; // bits case 4: writeBits(std::get<2>(rawChunk[i]),std::get<3>(rawChunk[i])); break; default: break; } } uint32_t outputLength=uint32_t(stream.size())-streamStart; if (currentChunkSize>outputLength && outputLength-outputLength>leeway) leeway=outputLength-outputLength; } if (bitAccumCount) { stream[bitStreamPosition]=bitAccumContent; stream[bitStreamPosition+1]=bitAccumContent>>8; if (bitStreamPosition==stream.size()-2 && bitAccumCount<=8) stream.pop_back(); } uint32_t packedSize=uint32_t(stream.size()-18); stream[8]=uint8_t(packedSize>>24); stream[9]=uint8_t(packedSize>>16); stream[10]=uint8_t(packedSize>>8); stream[11]=uint8_t(packedSize); uint16_t packedCrc=RNCCRC(stream.data()+18,packedSize); stream[14]=uint8_t(packedCrc>>8); stream[15]=uint8_t(packedCrc); if (leeway>255) { fprintf(stderr,"Leeway larger than 255\n"); exit(-1); } stream[16]=leeway; stream[17]=chunkCount; dest.resize(stream.size()); std::memcpy(dest.data(),stream.data(),stream.size()); } int main(int argc,char **argv) { auto usage=[]() { fprintf(stderr,"Usage: input_raw output_packed [chunk_size]\n"); }; if (argc<3) { usage(); return -1; } auto raw{readFile(argv[1])}; VectorBuffer packed; packRNC(packed,*raw,(argc>=4)?atoi(argv[3]):0); writeFile(argv[2],packed); return 0; } ancient-1.0/extra/Makefile000066400000000000000000000010171375530757000155710ustar00rootroot00000000000000# Copyright (C) Teemu Suutari VPATH := ../src ../src/common CC = clang CXX = clang++ COMMONFLAGS = -g -Wall -Wsign-compare -Wshorten-64-to-32 -Wno-error=multichar -Wno-multichar -I../src CFLAGS = $(COMMONFLAGS) CXXFLAGS = $(COMMONFLAGS) -std=c++14 -fno-rtti PROG = bruteRNC1 OBJS = Buffer.o SubBuffer.o BruteForceRNC1Encoder.o all: $(PROG) .cpp.o: $(CXX) $(CXXFLAGS) -o $@ -c $< .c.o: $(CC) $(CFLAGS) -o $@ -c $< $(PROG): $(OBJS) $(CXX) $(CFLAGS) -o $(PROG) $(OBJS) clean: rm -f $(OBJS) $(PROG) *~ src/*~ .PHONY: ancient-1.0/fuzzing/000077500000000000000000000000001375530757000145035ustar00rootroot00000000000000ancient-1.0/fuzzing/all_formats.dict000066400000000000000000000010701375530757000176510ustar00rootroot00000000000000"XPKF" "TPWM" "TDCS" "ACCA" "SASC" "SHSC" "S300" "S310" "S400" "S401" "S403" "S404" "SQSH" "SMPL" "SLZ3" "SHRI" "SHR3" "SDHC" "RNC\x01" "RNC\x02" "RLEN" "RDCN" "FRHT" "RAKE" "PP11" "PP20" "PWPK" "NUKE" "DUKE" "NONE" "ziRCONia" "MASH" "ELZX" "SLZX" "LZW5" "LZW4" "LZW2" "LZW3" "LZCB" "LZBS" "LIN2" "LIN4" "LIN1" "LIN3" "LHLB" "ATN!" "EDAM" "IMP!" "M.H." "BDPI" "CHFI" "RDC9" "Dupa" "FLT!" "PARA" "IMPL" "ILZR" "HUFF" "HFMN" "FRLE" "FBR2" "FAST" "DMS!" "TR" "DLTA" "GZIP" "CYB2" "CrM!" "CrM2" "Crm!" "Crm2" "CRM2" "CRMS" "CBR0" "CBR1" "BZP2" "BZh2" "BLZW" "ARTM" "ZENO" ancient-1.0/fuzzing/build.sh000077500000000000000000000002201375530757000161330ustar00rootroot00000000000000#!/usr/bin/env bash cd "${0%/*}" cd .. CXX=fuzzing/afl/afl-clang-fast++ LD=fuzzing/afl/afl-clang-fast++ MAIN=fuzz.o AFL_HARDEN=1 make clean all ancient-1.0/fuzzing/fuzz-main.sh000077500000000000000000000006151375530757000167640ustar00rootroot00000000000000#!/usr/bin/env bash cd "${0%/*}" . ./fuzz-settings.sh # Create tmpfs for storing temporary fuzzing data mkdir $FUZZING_TEMPDIR sudo mount -t tmpfs -o size=100M none $FUZZING_TEMPDIR cp ../ancient $FUZZING_TEMPDIR/ $AFL_BIN -f $FUZZING_TEMPDIR/infile01 -x all_formats.dict -t $FUZZING_TIMEOUT $FUZZING_INPUT -o $FUZZING_FINDINGS_DIR -M fuzzer01 $FUZZING_TEMPDIR/ancient $FUZZING_TEMPDIR/infile01 ancient-1.0/fuzzing/fuzz-secondary.sh000077500000000000000000000003541375530757000200270ustar00rootroot00000000000000#!/usr/bin/env bash cd "${0%/*}" . ./fuzz-settings.sh $AFL_BIN -f $FUZZING_TEMPDIR/infile02 -x all_formats.dict -t $FUZZING_TIMEOUT $FUZZING_INPUT -o $FUZZING_FINDINGS_DIR -S fuzzer02 $FUZZING_TEMPDIR/ancient $FUZZING_TEMPDIR/infile02 ancient-1.0/fuzzing/fuzz-settings.sh000077500000000000000000000012151375530757000176750ustar00rootroot00000000000000#!/usr/bin/env bash # Input data for fuzzer # If you run the fuzzer for the first time, specify a directory with some input # files for the fuzzer, e.g. # FUZZING_INPUT="-i /home/foo/testcases/" # If you want to continue fuzzing using the previous findings, use: # FUZZING_INPUT=-i- FUZZING_INPUT="-i /path/to/initial" # Directory to place temporary fuzzing data into FUZZING_TEMPDIR=~/fuzzers/ancient-temp # Directory to store permanent fuzzing data (e.g. found crashes) into FUZZING_FINDINGS_DIR=~/fuzzers/ancient-fuzzing # Fuzzer timeout in ms, + = don't abort on timeout FUZZING_TIMEOUT=5000+ # Path to afl-fuzz binary AFL_BIN=afl/afl-fuzzancient-1.0/fuzzing/fuzz.cpp000066400000000000000000000033571375530757000162150ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include #include "common/MemoryBuffer.hpp" #include "common/SubBuffer.hpp" #include "common/StaticBuffer.hpp" #include "Decompressor.hpp" std::unique_ptr readFile(const std::string &fileName) { std::unique_ptr ret=std::make_unique(0); std::ifstream file(fileName.c_str(),std::ios::in|std::ios::binary); bool success=false; if (file.is_open()) { file.seekg(0,std::ios::end); size_t length=size_t(file.tellg()); file.seekg(0,std::ios::beg); ret->resize(length); file.read(reinterpret_cast(ret->data()),length); success=bool(file); file.close(); } if (!success) { return std::make_unique>(); } return ret; } int main(int argc,char **argv) { (void)argc; #ifdef __AFL_HAVE_MANUAL_CONTROL __AFL_INIT(); #endif auto packed{readFile(argv[1])}; std::unique_ptr decompressor; try { decompressor=Decompressor::create(*packed,true,true); } catch (const Decompressor::InvalidFormatError&) { return -1; } catch (const Decompressor::VerificationError&) { return -1; } std::unique_ptr raw; try { raw=std::make_unique((decompressor->getRawSize())?decompressor->getRawSize():Decompressor::getMaxRawSize()); } catch (const Buffer::Error&) { return -1; } try { decompressor->decompress(*raw,true); } catch (const Decompressor::DecompressionError&) { return -1; } catch (const Decompressor::VerificationError&) { return -1; } try { raw->resize(decompressor->getRawSize()); } catch (const Buffer::Error&) { return -1; } if (decompressor->getImageOffset() || decompressor->getImageSize()) { return -1; } return 0; } ancient-1.0/fuzzing/get-afl.sh000077500000000000000000000006771375530757000163730ustar00rootroot00000000000000#!/usr/bin/env bash cd "${0%/*}" AFL_VERSION="$(wget --quiet -O - "https://api.github.com/repos/AFLplusplus/AFLplusplus/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')" AFL_FILENAME="$AFL_VERSION.tar.gz" AFL_URL="https://github.com/AFLplusplus/AFLplusplus/archive/$AFL_FILENAME" rm $AFL_FILENAME wget $AFL_URL || exit tar -xzvf $AFL_FILENAME rm $AFL_FILENAME cd AFLplusplus-* make source-only || exit cd .. rm -rf afl mv AFLplusplus-* aflancient-1.0/fuzzing/readme.md000066400000000000000000000045041375530757000162650ustar00rootroot00000000000000ancient fuzz suite ================== In this directory, you can find the necessary tools for fuzzing the ancient decoders with the American Fuzzy Lop fuzzer (afl++). Contents: * `all_formats.dict`: A dictionary containing magic bytes from all supported formats to make the life of the fuzzer a bit easier. * `fuzz-main.sh`: Script to launch the main fuzzing process. If you want to use just one fuzzer instance, run this one. * `fuzz-secondary.sh`: Script to launch the secondary fuzzing process. It is recommended to run at least two fuzzer instances, as the deterministic and random fuzz mode have been found to complement each other really well. * `fuzz-settings.sh`: Set up your preferences and afl settings here before the first run. * `fuzz.cpp`: A tiny C++ program that is used by the fuzzer to test ancient. * `get-afl.sh`: A simple script to obtain the latest version of the fuzzer. Prerequisites ============= * [afl++](https://github.com/AFLplusplus/AFLplusplus) or [afl](https://lcamtuf.coredump.cx/afl/) - the makefile expects this to be installed in `contrib/fuzzing/afl`, as it is automatically done by the `get-afl.sh` install script. * Clang with LLVM dev headers (llvm-config needs to be installed). afl also works with gcc, but our makefile has been set up to make use of afl's faster LLVM mode. How to use ========== * Run `get-afl.sh`, or manually extract afl to `contrib/fuzzing/afl`, use `make` to build afl-fuzz, `cd llvm_mode`, `make` to build afl-clang-fast. If building with either option fails because `llvm-config` cannot be found, try prepending `LLVM_CONFIG=/usr/bin/llvm-config-3.8` or similar, and read the afl manual. * Build ancient with the `build.sh` script in this directory. * Set up `fuzz-settings.sh` to your taste. Most importantly, you will have to specify the input directory for first use. The default setup mounts a tmpfs folder for all temporary files. You may change this behaviour if you do not have root privileges. * Run `fuzz-main.sh` for the first (deterministic) instance of afl-fuzz. * For a secondary instance to run on another core, run `fuzz-secondary.sh`. * If you want to make use of even more cores, make a copy of `fuzz-secondary.sh` and adjust "infile02" / "fuzzer02" to "infile03" / "fuzzer03" (they need to be unique)ancient-1.0/main.cpp000066400000000000000000000163341375530757000144460ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include #include #include #include #include #include #include "common/MemoryBuffer.hpp" #include "common/SubBuffer.hpp" #include "common/StaticBuffer.hpp" #include "Decompressor.hpp" std::unique_ptr readFile(const std::string &fileName) { std::unique_ptr ret=std::make_unique(0); std::ifstream file(fileName.c_str(),std::ios::in|std::ios::binary); bool success=false; if (file.is_open()) { file.seekg(0,std::ios::end); size_t length=size_t(file.tellg()); file.seekg(0,std::ios::beg); ret->resize(length); file.read(reinterpret_cast(ret->data()),length); success=bool(file); file.close(); } if (!success) { fprintf(stderr,"Could not read file %s\n",fileName.c_str()); return std::make_unique>(); } return ret; } bool writeFile(const std::string &fileName,const Buffer &content) { bool ret=false; std::ofstream file(fileName.c_str(),std::ios::out|std::ios::binary|std::ios::trunc); if (file.is_open()) { file.write(reinterpret_cast(content.data()),content.size()); ret=bool(file); file.close(); } if (!ret) { fprintf(stderr,"Could not write file %s\n",fileName.c_str()); } return ret; } int main(int argc,char **argv) { auto usage=[]() { fprintf(stderr,"Usage: ancient identify packed_input_file\n"); fprintf(stderr," - identifies compression used in a file\n"); fprintf(stderr,"Usage: ancient verify packed_input_file unpacked_comparison_file\n"); fprintf(stderr," - verifies decompression against known good unpacked file\n"); fprintf(stderr,"Usage: ancient decompress packed_input_file output_file\n"); fprintf(stderr," - decompresses single file\n"); fprintf(stderr,"Usage: ancient scan input_dir output_dir\n"); fprintf(stderr," - scans input directory recursively and stores all found\n" " - known compressed streams to separate files in output directory\n"); }; if (argc<3) { usage(); return -1; } std::string cmd=argv[1]; if (cmd=="identify") { if (argc!=3) { usage(); return -1; } auto packed{readFile(argv[2])}; std::unique_ptr decompressor; try { decompressor=Decompressor::create(*packed,true,true); } catch (const Decompressor::InvalidFormatError&) { fprintf(stderr,"Unknown or invalid compression format in file %s\n",argv[2]); return -1; } printf("Compression of %s is %s\n",argv[2],decompressor->getName().c_str()); return 0; } else if (cmd=="decompress" || cmd=="verify") { if (argc!=4) { usage(); return -1; } auto packed{readFile(argv[2])}; std::unique_ptr decompressor; try { decompressor=Decompressor::create(*packed,true,true); } catch (const Decompressor::InvalidFormatError&) { fprintf(stderr,"Unknown or invalid compression format in file %s\n",argv[2]); return -1; } catch (const Decompressor::VerificationError&) { fprintf(stderr,"Verify (packed) failed for %s\n",argv[2]); return -1; } std::unique_ptr raw; try { raw=std::make_unique((decompressor->getRawSize())?decompressor->getRawSize():Decompressor::getMaxRawSize()); } catch (const Buffer::Error&) { fprintf(stderr,"Out of memory\n"); return -1; } try { decompressor->decompress(*raw,true); } catch (const Decompressor::DecompressionError&) { fprintf(stderr,"Decompression failed for %s\n",argv[2]); return -1; } catch (const Decompressor::VerificationError&) { fprintf(stderr,"Verify (raw) failed for %s\n",argv[2]); return -1; } try { raw->resize(decompressor->getRawSize()); } catch (const Buffer::Error&) { fprintf(stderr,"Out of memory\n"); return -1; } if (decompressor->getImageOffset() || decompressor->getImageSize()) { printf("File %s is disk image, decompressed stream offset is %zu, full image size is %zu\n",argv[2],decompressor->getImageOffset(),decompressor->getImageSize()); printf("!!! Please note !!!\n!!! The destination will not be padded !!!\n\n"); } if (cmd=="decompress") { writeFile(argv[3],*raw); return 0; } else { auto verify{readFile(argv[3])}; if (raw->size()!=verify->size()) { fprintf(stderr,"Verify failed for %s and %s - sizes differ\n",argv[2],argv[3]); return -1; } for (size_t i=0;isize();i++) { if (raw->data()[i]!=verify->data()[i]) { fprintf(stderr,"Verify failed for %s and %s - contents differ @ %zu\n",argv[2],argv[3],i); return -1; } } printf("Files match!\n"); return 0; } } else if (cmd=="scan") { if (argc!=4) { usage(); return -1; } uint32_t fileIndex=0; std::function processDir=[&](std::string inputDir) { std::unique_ptr dir{::opendir(inputDir.c_str()),::closedir}; if (dir) { while (struct dirent *de=::readdir(dir.get())) { std::string subName(de->d_name); if (subName=="." || subName=="..") continue; std::string name=inputDir+"/"+subName; struct stat st; if (stat(name.c_str(),&st)<0) continue; if (st.st_mode&S_IFDIR) { processDir(name); } else if (st.st_mode&S_IFREG) { auto packed{readFile(name)}; ConstSubBuffer scanBuffer(*packed,0,packed->size()); for (size_t i=0;isize();) { scanBuffer.adjust(i,packed->size()-i); // We will detect first, before trying the format for real if (!Decompressor::detect(scanBuffer)) { i++; continue; } try { auto decompressor{Decompressor::create(scanBuffer,false,true)}; std::unique_ptr raw; try { raw=std::make_unique((decompressor->getRawSize())?decompressor->getRawSize():Decompressor::getMaxRawSize()); } catch (const Buffer::Error&) { fprintf(stderr,"Out of memory\n"); i++; continue; } // for formats that do not encode packed size. // we will get it from decompressor if (!decompressor->getPackedSize()) decompressor->decompress(*raw,true); if (decompressor->getPackedSize()) { // final checks with the limited buffer and fresh decompressor ConstSubBuffer finalBuffer(*packed,i,decompressor->getPackedSize()); auto decompressor2{Decompressor::create(finalBuffer,true,true)}; decompressor2->decompress(*raw,true); std::string outputName=std::string(argv[3])+"/file"+std::to_string(fileIndex++)+".pack"; printf("Found compressed stream at %zu, size %zu in file %s with type '%s', storing it into %s\n",i,decompressor2->getPackedSize(),name.c_str(),decompressor2->getName().c_str(),outputName.c_str()); writeFile(outputName,finalBuffer); i+=finalBuffer.size(); continue; } } catch (const Decompressor::Error&) { // full steam ahead (with next offset) } i++; } } } } else { fprintf(stderr,"Could not process directory %s\n",inputDir.c_str()); } }; processDir(std::string(argv[2])); return 0; } else { fprintf(stderr,"Unknown command\n"); usage(); return -1; } } ancient-1.0/src/000077500000000000000000000000001375530757000135765ustar00rootroot00000000000000ancient-1.0/src/ACCADecompressor.cpp000066400000000000000000000050401375530757000173560ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "ACCADecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool ACCADecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("ACCA"); } std::unique_ptr ACCADecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } ACCADecompressor::ACCADecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } ACCADecompressor::~ACCADecompressor() { // nothing needed } const std::string &ACCADecompressor::getSubName() const noexcept { static std::string name="XPK-ACCA: Andre's code compression algorithm"; return name; } void ACCADecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { static const uint8_t staticBytes[16]={ 0x00,0x01,0x02,0x03,0x04,0x08,0x10,0x20, 0x40,0x55,0x60,0x80,0xaa,0xc0,0xe0,0xff}; uint8_t tmp=readByte(); uint32_t count=tmp&0xf; uint32_t code=tmp>>4; uint32_t distance=0; uint8_t repeatChar=0; bool doRLE=false; switch (code) { case 0: repeatChar=readByte(); case 14: count+=3; doRLE=true; break; case 1: count=(count|(uint32_t(readByte())<<4))+19; repeatChar=readByte(); doRLE=true; break; case 2: repeatChar=staticBytes[count]; count=2; doRLE=true; break; case 15: distance=(count|(uint32_t(readByte())<<4))+3; count=uint32_t(readByte())+14; break; default: /* 3 to 13 */ distance=(count|(uint32_t(readByte())<<4))+3; count=code; break; } if (doRLE) { for (uint32_t i=0;i ACCADecompressor::_XPKregistration; ancient-1.0/src/ACCADecompressor.hpp000066400000000000000000000015541375530757000173710ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef ACCADECOMPRESSOR_HPP #define ACCADECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class ACCADecompressor : public XPKDecompressor { public: ACCADecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~ACCADecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/ARTMDecompressor.cpp000066400000000000000000000063301375530757000174350ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "ARTMDecompressor.hpp" #include "RangeDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool ARTMDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("ARTM"); } std::unique_ptr ARTMDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } ARTMDecompressor::ARTMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); if (packedData.size()<2) throw Decompressor::InvalidFormatError(); } ARTMDecompressor::~ARTMDecompressor() { // nothing needed } const std::string &ARTMDecompressor::getSubName() const noexcept { static std::string name="XPK-ARTM: Arithmetic encoding compressor"; return name; } // There really is not much to see here. // Except maybe for the fact that there is extra symbol defined but never used. // It is used in the original implementation as a terminator. In here it is considered as an error. // Logically it would decode into null-character (practically it would be instant buffer overflow) void ARTMDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { class BitReader : public RangeDecoder::BitReader { public: BitReader(ForwardInputStream &stream) : _reader(stream) { // nothing needed } virtual ~BitReader() { // nothing needed } virtual uint32_t readBit() override final { return _reader.readBits8(1); } private: LSBBitReader _reader; }; ForwardInputStream inputStream(_packedData,0,_packedData.size(),true); ForwardOutputStream outputStream(rawData,0,rawData.size()); BitReader bitReader(inputStream); uint16_t initialValue=0; for (uint32_t i=0;i<16;i++) initialValue=(initialValue<<1)|bitReader.readBit(); RangeDecoder decoder(bitReader,initialValue); uint8_t characters[256]; uint16_t frequencies[256]; uint16_t frequencySums[256]; for (uint32_t i=0;i<256;i++) { characters[i]=i; frequencies[i]=1; frequencySums[i]=256-i; } uint16_t frequencyTotal=257; while (!outputStream.eof()) { uint16_t value=decoder.decode(frequencyTotal); uint16_t symbol; for (symbol=0;symbol<256;symbol++) if (frequencySums[symbol]<=value) break; if (symbol==256) throw Decompressor::DecompressionError(); decoder.scale(frequencySums[symbol],frequencySums[symbol]+frequencies[symbol],frequencyTotal); outputStream.writeByte(characters[symbol]); if (frequencyTotal==0x3fffU) { frequencyTotal=1; for (int32_t i=255;i>=0;i--) { frequencySums[i]=frequencyTotal; frequencyTotal+=frequencies[i]=(frequencies[i]+1)>>1; } } uint16_t i; for (i=symbol;i&&frequencies[i-1]==frequencies[i];i--); if (i!=symbol) std::swap(characters[symbol],characters[i]); frequencies[i]++; while (i--) frequencySums[i]++; frequencyTotal++; } } XPKDecompressor::Registry ARTMDecompressor::_XPKregistration; ancient-1.0/src/ARTMDecompressor.hpp000066400000000000000000000015541375530757000174450ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef ARTMDECOMPRESSOR_HPP #define ARTMDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class ARTMDecompressor : public XPKDecompressor { public: ARTMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~ARTMDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/BLZWDecompressor.cpp000066400000000000000000000057251375530757000174570ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "BLZWDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool BLZWDecompressor::detectHeaderXPK(uint32_t hdr) { return hdr==FourCC("BLZW"); } std::unique_ptr BLZWDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } BLZWDecompressor::BLZWDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); _maxBits=_packedData.readBE16(0); if (_maxBits<9 || _maxBits>20) throw Decompressor::InvalidFormatError();; _stackLength=uint32_t(_packedData.readBE16(2))+5; } BLZWDecompressor::~BLZWDecompressor() { // nothing needed } const std::string &BLZWDecompressor::getSubName() const noexcept { static std::string name="XPK-BLZW: LZW-compressor"; return name; } void BLZWDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,4,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); uint32_t maxCode=1<<_maxBits; auto prefix=std::make_unique(maxCode-259); auto suffix=std::make_unique(maxCode-259); auto stack=std::make_unique(_stackLength); uint32_t freeIndex,codeBits,prevCode,newCode; auto suffixLookup=[&](uint32_t code)->uint32_t { if (code>=freeIndex) throw Decompressor::DecompressionError(); return (code<259)?code:suffix[code-259]; }; auto insert=[&](uint32_t code) { uint32_t stackPos=0; newCode=suffixLookup(code); while (code>=259) { if (stackPos+1>=_stackLength) throw Decompressor::DecompressionError(); stack[stackPos++]=newCode; code=prefix[code-259]; newCode=suffixLookup(code); } stack[stackPos++]=newCode; while (stackPos) outputStream.writeByte(stack[--stackPos]); }; auto init=[&]() { codeBits=9; freeIndex=259; prevCode=readBits(codeBits); insert(prevCode); }; init(); while (!outputStream.eof()) { uint32_t code=readBits(codeBits); switch (code) { case 256: throw Decompressor::DecompressionError(); break; case 257: init(); break; case 258: codeBits++; break; default: if (code>=freeIndex) { uint32_t tmp=newCode; insert(prevCode); outputStream.writeByte(tmp); } else insert(code); if (freeIndex BLZWDecompressor::_XPKregistration; ancient-1.0/src/BLZWDecompressor.hpp000066400000000000000000000016231375530757000174550ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef BLZWDECOMPRESSOR_HPP #define BLZWDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class BLZWDecompressor : public XPKDecompressor { public: BLZWDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~BLZWDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr); static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _maxBits=0; size_t _stackLength=0; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/BZIP2Decompressor.cpp000066400000000000000000000275341375530757000175310ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "BZIP2Decompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/MemoryBuffer.hpp" #include "common/CRC32.hpp" bool BZIP2Decompressor::detectHeader(uint32_t hdr) noexcept { return ((hdr&0xffff'ff00U)==FourCC("BZh\0") && (hdr&0xffU)>='1' && (hdr&0xffU)<='9'); } bool BZIP2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return (hdr==FourCC("BZP2")); } std::unique_ptr BZIP2Decompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_unique(packedData,exactSizeKnown,verify); } std::unique_ptr BZIP2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } BZIP2Decompressor::BZIP2Decompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData(packedData), _packedSize(0) { uint32_t hdr=packedData.readBE32(0); if (!detectHeader(hdr)) throw Decompressor::InvalidFormatError();; _blockSize=((hdr&0xffU)-'0')*100'000; } BZIP2Decompressor::BZIP2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData), _packedSize(_packedData.size()) { uint32_t blockHdr=packedData.readBE32(0); if (!detectHeader(blockHdr)) throw Decompressor::InvalidFormatError();; _blockSize=((blockHdr&0xffU)-'0')*100'000; } BZIP2Decompressor::~BZIP2Decompressor() { // nothing needed } const std::string &BZIP2Decompressor::getName() const noexcept { static std::string name="bz2: bzip2"; return name; } const std::string &BZIP2Decompressor::getSubName() const noexcept { static std::string name="XPK-BZP2: bzip2"; return name; } size_t BZIP2Decompressor::getPackedSize() const noexcept { // no way to know before decompressing return _packedSize; } size_t BZIP2Decompressor::getRawSize() const noexcept { // same thing, decompression needed first return _rawSize; } void BZIP2Decompressor::decompressImpl(Buffer &rawData,bool verify) { size_t packedSize=_packedSize?_packedSize:_packedData.size(); size_t rawSize=_rawSize?_rawSize:rawData.size(); ForwardInputStream inputStream(_packedData,4,packedSize); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawSize); // stream verification // // there is so much wrong in bzip2 CRC-calculation :( // 1. The bit ordering is opposite what everyone else does with CRC32 // 2. The block CRCs are calculated separately, no way of calculating a complete // CRC without knowing the block layout // 3. The CRC is the end of the stream and the stream is bit aligned. You // can't read CRC without decompressing the stream. uint32_t crc=0; auto calculateBlockCRC=[&](size_t blockPos,size_t blockSize) { crc=(crc<<1)|(crc>>31); crc^=CRC32Rev(rawData,blockPos,blockSize,0); }; HuffmanDecoder selectorDecoder { // incomplete Huffman table. errors possible HuffmanCode{1,0b000000,0}, HuffmanCode{2,0b000010,1}, HuffmanCode{3,0b000110,2}, HuffmanCode{4,0b001110,3}, HuffmanCode{5,0b011110,4}, HuffmanCode{6,0b111110,5} }; HuffmanDecoder deltaDecoder { HuffmanCode{1,0b00,0}, HuffmanCode{2,0b10,1}, HuffmanCode{2,0b11,-1} }; MemoryBuffer tmpBuffer(_blockSize); uint8_t *tmpBufferPtr=tmpBuffer.data(); // This is the dark, ancient secret of bzip2. // versions before 0.9.5 had a data randomization for "too regular" // data problematic for the bwt-implementation at that time. // although it is never utilized anymore, the support is still there // And this is exactly the kind of ancient stuff we want to support :) // // On this specific part (since it is a table of magic numbers) // we have no way other than copying it from the original reference // Table has a separate copyright, lets have it as a separate file as well #include "BZIP2Table.hpp" for (;;) { uint32_t blockHdrHigh=readBits(32); uint32_t blockHdrLow=readBits(16); if (blockHdrHigh==0x31415926U && blockHdrLow==0x5359U) { // a block // this is rather spaghetti... readBits(32); // block crc, not interested bool randomized=readBit(); // basically the random inserted is one LSB after n-th bytes // per defined in the table. uint32_t randomPos=1; uint32_t randomCounter=randomTable[0]-1; auto randomBit=[&]()->bool { // Beauty is in the eye of the beholder: this is smallest form to hide the ugliness return (!randomCounter--)?randomCounter=randomTable[randomPos++&511]:false; }; uint32_t currentPtr=readBits(24); uint32_t currentBlockSize=0; { uint32_t numHuffmanItems=2; uint32_t huffmanValues[256]; { // this is just a little bit inefficient but still we reading bit by bit since // reference does it. (bitsream format details do not spill over) std::vector usedMap(16); for (uint32_t i=0;i<16;i++) usedMap[i]=readBit(); std::vector huffmanMap(256); for (uint32_t i=0;i<16;i++) { for (uint32_t j=0;j<16;j++) huffmanMap[i*16+j]=(usedMap[i])?readBit():false; } for (uint32_t i=0;i<256;i++) if (huffmanMap[i]) numHuffmanItems++; if (numHuffmanItems==2) throw DecompressionError(); for (uint32_t currentValue=0,i=0;i<256;i++) if (huffmanMap[i]) huffmanValues[currentValue++]=i; } uint32_t huffmanGroups=readBits(3); if (huffmanGroups<2 || huffmanGroups>6) throw DecompressionError(); uint32_t selectorsUsed=readBits(15); if (!selectorsUsed) throw DecompressionError(); MemoryBuffer huffmanSelectorList(selectorsUsed); auto unMTF=[](uint8_t value,uint8_t map[])->uint8_t { uint8_t ret=map[value]; if (value) { uint8_t tmp=map[value]; for (uint32_t i=value;i;i--) map[i]=map[i-1]; map[0]=tmp; } return ret; }; // create Huffman selectors uint8_t selectorMTFMap[6]={0,1,2,3,4,5}; for (uint32_t i=0;i=huffmanGroups) throw DecompressionError(); huffmanSelectorList[i]=item; } typedef HuffmanDecoder BZIP2Decoder; std::vector dataDecoders(huffmanGroups); // Create all tables for (uint32_t i=0;i20) throw DecompressionError(); bitLengths[j]=currentBits; } dataDecoders[i].createOrderlyHuffmanTable(bitLengths,numHuffmanItems); } // Huffman decode + unRLE + unMTF BZIP2Decoder *currentHuffmanDecoder=nullptr; uint32_t currentHuffmanIndex=0; uint8_t dataMTFMap[256]; for (uint32_t i=0;i_blockSize) throw DecompressionError(); for (uint32_t i=0;i=selectorsUsed) throw DecompressionError(); currentHuffmanDecoder=&dataDecoders[huffmanSelectorList[currentHuffmanIndex++]]; } uint32_t symbolMTF=currentHuffmanDecoder->decode(readBit); // stop marker is referenced only once, and it is the last one // This means we do no have to un-MTF it for detection if (symbolMTF==numHuffmanItems-1) break; if (currentBlockSize>=_blockSize) throw DecompressionError(); if (symbolMTF<2) { currentRunLength+=currentRLEWeight<=_blockSize) throw DecompressionError(); tmpBufferPtr[currentBlockSize++]=huffmanValues[symbol]; } } decodeRLE(); if (currentPtr>=currentBlockSize) throw DecompressionError(); } // inverse BWT + final RLE decoding. // there are a few dark corners here as well // 1. Can the stream end at 4 literals without count? I assume it is a valid optimization (and that this does not spillover to next block) // 2. Can the RLE-step include counts 252 to 255 even if reference does not do them? I assume yes here as here as well // 3. Can the stream be empty? We do not take issue here about that (that should be culled out earlier already) uint32_t sums[256]; for (uint32_t i=0;i<256;i++) sums[i]=0; for (uint32_t i=0;i(); for (uint32_t i=0;i BZIP2Decompressor::_registration; XPKDecompressor::Registry BZIP2Decompressor::_XPKregistration; ancient-1.0/src/BZIP2Decompressor.hpp000066400000000000000000000030401375530757000175200ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef BZIP2DECOMPRESSOR_HPP #define BZIP2DECOMPRESSOR_HPP #include "Decompressor.hpp" #include "XPKDecompressor.hpp" class BZIP2Decompressor : public Decompressor, public XPKDecompressor { public: BZIP2Decompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); BZIP2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~BZIP2Decompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeader(uint32_t hdr) noexcept; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; size_t _blockSize=0; size_t _packedSize=0; size_t _rawSize=0; static Decompressor::Registry _registration; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/BZIP2Table.hpp000066400000000000000000000121121375530757000161020ustar00rootroot00000000000000/* -------------------------------------------------------------------------- This program, "bzip2", the associated library "libbzip2", and all documentation, are copyright (C) 1996-2010 Julian R Seward. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 4. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Julian Seward, jseward@bzip.org bzip2/libbzip2 version 1.0.6 of 6 September 2010 -------------------------------------------------------------------------- */ // content formatted, numbers -1 (i.e. this is altered version by clause 3) static const uint16_t randomTable[512]={ 619-1,720-1,127-1,481-1,931-1,816-1,813-1,233-1,566-1,247-1,985-1,724-1,205-1,454-1,863-1,491-1, 741-1,242-1,949-1,214-1,733-1,859-1,335-1,708-1,621-1,574-1, 73-1,654-1,730-1,472-1,419-1,436-1, 278-1,496-1,867-1,210-1,399-1,680-1,480-1, 51-1,878-1,465-1,811-1,169-1,869-1,675-1,611-1,697-1, 867-1,561-1,862-1,687-1,507-1,283-1,482-1,129-1,807-1,591-1,733-1,623-1,150-1,238-1, 59-1,379-1, 684-1,877-1,625-1,169-1,643-1,105-1,170-1,607-1,520-1,932-1,727-1,476-1,693-1,425-1,174-1,647-1, 73-1,122-1,335-1,530-1,442-1,853-1,695-1,249-1,445-1,515-1,909-1,545-1,703-1,919-1,874-1,474-1, 882-1,500-1,594-1,612-1,641-1,801-1,220-1,162-1,819-1,984-1,589-1,513-1,495-1,799-1,161-1,604-1, 958-1,533-1,221-1,400-1,386-1,867-1,600-1,782-1,382-1,596-1,414-1,171-1,516-1,375-1,682-1,485-1, 911-1,276-1, 98-1,553-1,163-1,354-1,666-1,933-1,424-1,341-1,533-1,870-1,227-1,730-1,475-1,186-1, 263-1,647-1,537-1,686-1,600-1,224-1,469-1, 68-1,770-1,919-1,190-1,373-1,294-1,822-1,808-1,206-1, 184-1,943-1,795-1,384-1,383-1,461-1,404-1,758-1,839-1,887-1,715-1, 67-1,618-1,276-1,204-1,918-1, 873-1,777-1,604-1,560-1,951-1,160-1,578-1,722-1, 79-1,804-1, 96-1,409-1,713-1,940-1,652-1,934-1, 970-1,447-1,318-1,353-1,859-1,672-1,112-1,785-1,645-1,863-1,803-1,350-1,139-1, 93-1,354-1, 99-1, 820-1,908-1,609-1,772-1,154-1,274-1,580-1,184-1, 79-1,626-1,630-1,742-1,653-1,282-1,762-1,623-1, 680-1, 81-1,927-1,626-1,789-1,125-1,411-1,521-1,938-1,300-1,821-1, 78-1,343-1,175-1,128-1,250-1, 170-1,774-1,972-1,275-1,999-1,639-1,495-1, 78-1,352-1,126-1,857-1,956-1,358-1,619-1,580-1,124-1, 737-1,594-1,701-1,612-1,669-1,112-1,134-1,694-1,363-1,992-1,809-1,743-1,168-1,974-1,944-1,375-1, 748-1, 52-1,600-1,747-1,642-1,182-1,862-1, 81-1,344-1,805-1,988-1,739-1,511-1,655-1,814-1,334-1, 249-1,515-1,897-1,955-1,664-1,981-1,649-1,113-1,974-1,459-1,893-1,228-1,433-1,837-1,553-1,268-1, 926-1,240-1,102-1,654-1,459-1, 51-1,686-1,754-1,806-1,760-1,493-1,403-1,415-1,394-1,687-1,700-1, 946-1,670-1,656-1,610-1,738-1,392-1,760-1,799-1,887-1,653-1,978-1,321-1,576-1,617-1,626-1,502-1, 894-1,679-1,243-1,440-1,680-1,879-1,194-1,572-1,640-1,724-1,926-1, 56-1,204-1,700-1,707-1,151-1, 457-1,449-1,797-1,195-1,791-1,558-1,945-1,679-1,297-1, 59-1, 87-1,824-1,713-1,663-1,412-1,693-1, 342-1,606-1,134-1,108-1,571-1,364-1,631-1,212-1,174-1,643-1,304-1,329-1,343-1, 97-1,430-1,751-1, 497-1,314-1,983-1,374-1,822-1,928-1,140-1,206-1, 73-1,263-1,980-1,736-1,876-1,478-1,430-1,305-1, 170-1,514-1,364-1,692-1,829-1, 82-1,855-1,953-1,676-1,246-1,369-1,970-1,294-1,750-1,807-1,827-1, 150-1,790-1,288-1,923-1,804-1,378-1,215-1,828-1,592-1,281-1,565-1,555-1,710-1, 82-1,896-1,831-1, 547-1,261-1,524-1,462-1,293-1,465-1,502-1, 56-1,661-1,821-1,976-1,991-1,658-1,869-1,905-1,758-1, 745-1,193-1,768-1,550-1,608-1,933-1,378-1,286-1,215-1,979-1,792-1,961-1, 61-1,688-1,793-1,644-1, 986-1,403-1,106-1,366-1,905-1,644-1,372-1,567-1,466-1,434-1,645-1,210-1,389-1,550-1,919-1,135-1, 780-1,773-1,635-1,389-1,707-1,100-1,626-1,958-1,165-1,504-1,920-1,176-1,193-1,713-1,857-1,265-1, 203-1, 50-1,668-1,108-1,645-1,990-1,626-1,197-1,510-1,357-1,358-1,850-1,858-1,364-1,936-1,638-1}; ancient-1.0/src/CBR0Decompressor.cpp000066400000000000000000000035531375530757000173640ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "CBR0Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool CBR0Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { // CBR1 is practical joke: it is the same as CBR0 but with ID changed return hdr==FourCC("CBR0") || hdr==FourCC("CBR1"); } std::unique_ptr CBR0Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } CBR0Decompressor::CBR0Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData), _isCBR0(hdr==FourCC("CBR0")) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } CBR0Decompressor::~CBR0Decompressor() { // nothing needed } const std::string &CBR0Decompressor::getSubName() const noexcept { static std::string nameCBR0="XPK-CBR0: RLE-compressor"; static std::string nameCBR1="XPK-CBR1: RLE-compressor"; return (_isCBR0)?nameCBR0:nameCBR1; } void CBR0Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); ForwardOutputStream outputStream(rawData,0,rawData.size()); // barely different than RLEN, however the count is well defined here. while (!outputStream.eof()) { uint32_t count=inputStream.readByte(); if (count<128) { count++; for (uint32_t i=0;i CBR0Decompressor::_XPKregistration; ancient-1.0/src/CBR0Decompressor.hpp000066400000000000000000000015741375530757000173720ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef CBR0DECOMPRESSOR_HPP #define CBR0DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class CBR0Decompressor : public XPKDecompressor { public: CBR0Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~CBR0Decompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; bool _isCBR0; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/CRMDecompressor.cpp000066400000000000000000000151571375530757000173220ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "CRMDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "DLTADecode.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool CRMDecompressor::detectHeader(uint32_t hdr) noexcept { switch (hdr) { case FourCC("CrM!"): case FourCC("CrM2"): case FourCC("Crm!"): case FourCC("Crm2"): return true; default: return false; } } bool CRMDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("CRM2") || hdr==FourCC("CRMS"); } std::unique_ptr CRMDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_unique(packedData,0,verify); } std::unique_ptr CRMDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } CRMDecompressor::CRMDecompressor(const Buffer &packedData,uint32_t recursionLevel,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { uint32_t hdr=packedData.readBE32(0); if (!detectHeader(hdr) || packedData.size()<20) throw Decompressor::InvalidFormatError(); _rawSize=packedData.readBE32(6); _packedSize=packedData.readBE32(10); if (!_rawSize || !_packedSize || _rawSize>getMaxRawSize() || _packedSize>getMaxPackedSize() || size_t(_packedSize)+14>packedData.size()) throw Decompressor::InvalidFormatError(); if (((hdr>>8)&0xff)=='m') _isSampled=true; if ((hdr&0xff)=='2') _isLZH=true; } CRMDecompressor::CRMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : CRMDecompressor(packedData,recursionLevel,verify) { _isXPKDelta=(hdr==FourCC("CRMS")); } CRMDecompressor::~CRMDecompressor() { // nothing needed } const std::string &CRMDecompressor::getName() const noexcept { static std::string names[4]={ "CrM!: Crunch-Mania standard-mode", "Crm!: Crunch-Mania standard-mode, sampled", "CrM2: Crunch-Mania LZH-mode", "Crm2: Crunch-Mania LZH-mode, sampled"}; return names[(_isLZH?2:0)+(_isSampled?1:0)]; } const std::string &CRMDecompressor::getSubName() const noexcept { // the XPK-id is not used in decompressing process, // but there is a real id inside the stream // This means we can have frankenstein configurations, // although in practice we don't static std::string names[2]={ "XPK-CRM2: Crunch-Mania LZH-mode", "XPK-CRMS: Crunch-Mania LZH-mode, sampled"}; return names[(_isXPKDelta?1:0)]; } size_t CRMDecompressor::getPackedSize() const noexcept { return _packedSize+14; } size_t CRMDecompressor::getRawSize() const noexcept { return _rawSize; } void CRMDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw Decompressor::DecompressionError(); BackwardInputStream inputStream(_packedData,14,_packedSize+14-6); LSBBitReader bitReader(inputStream); { // There are empty bits?!? at the start of the stream. take them out size_t bufOffset=_packedSize+14-6; uint32_t originalBitsContent=_packedData.readBE32(bufOffset); uint16_t originalShift=_packedData.readBE16(bufOffset+4); uint8_t bufBitsLength=originalShift+16; uint32_t bufBitsContent=originalBitsContent>>(16-originalShift); bitReader.reset(bufBitsContent,bufBitsLength); } auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; BackwardOutputStream outputStream(rawData,0,_rawSize); if (_isLZH) { typedef HuffmanDecoder CRMHuffmanDecoder; auto readHuffmanTable=[&](CRMHuffmanDecoder &dec,uint32_t codeLength) { uint32_t maxDepth=readBits(4); if (!maxDepth) throw Decompressor::DecompressionError(); uint32_t lengthTable[15]; for (uint32_t i=0;i{depth,code>>(maxDepth-depth),value}); code+=1<<(maxDepth-depth); } } }; do { CRMHuffmanDecoder lengthDecoder,distanceDecoder; readHuffmanTable(lengthDecoder,9); readHuffmanTable(distanceDecoder,4); uint32_t items=readBits(16)+1; for (uint32_t i=0;i lengthDecoder { HuffmanCode{1,0b000,0}, HuffmanCode{2,0b010,1}, HuffmanCode{3,0b110,2}, HuffmanCode{3,0b111,3} }; HuffmanDecoder distanceDecoder { HuffmanCode{1,0b00,0}, HuffmanCode{2,0b10,1}, HuffmanCode{2,0b11,2} }; while (!outputStream.eof()) { if (readBit()) { outputStream.writeByte(readBits(8)); } else { uint8_t lengthIndex=lengthDecoder.decode(readBit); static const uint8_t lengthBits[4]={1,2,4,8}; static const uint32_t lengthAdditions[4]={2,4,8,24}; uint32_t count=readBits(lengthBits[lengthIndex])+lengthAdditions[lengthIndex]; if (count==23) { if (readBit()) { count=readBits(5)+15; } else { count=readBits(14)+15; } for (uint32_t i=0;i23) count--; uint8_t distanceIndex=distanceDecoder.decode(readBit); static const uint8_t distanceBits[3]={9,5,14}; static const uint32_t distanceAdditions[3]={32,0,544}; uint32_t distance=readBits(distanceBits[distanceIndex])+distanceAdditions[distanceIndex]; outputStream.copy(distance,count); } } } } if (!outputStream.eof()) throw Decompressor::DecompressionError(); if (_isSampled) DLTADecode::decode(rawData,rawData,0,_rawSize); } void CRMDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); return decompressImpl(rawData,verify); } Decompressor::Registry CRMDecompressor::_registration; XPKDecompressor::Registry CRMDecompressor::_XPKregistration; ancient-1.0/src/CRMDecompressor.hpp000066400000000000000000000032731375530757000173230ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef CRMDECOMPRESSOR_HPP #define CRMDECOMPRESSOR_HPP #include "Decompressor.hpp" #include "XPKDecompressor.hpp" class CRMDecompressor : public Decompressor, public XPKDecompressor { public: CRMDecompressor(const Buffer &packedData,uint32_t recursionLevel,bool verify); CRMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~CRMDecompressor(); virtual const std::string &getName() const noexcept override final; virtual const std::string &getSubName() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual size_t getRawSize() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeader(uint32_t hdr) noexcept; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _packedSize=0; uint32_t _rawSize=0; bool _isLZH=false; // "normal" compression or LZH compression bool _isSampled=false; // normal or "sampled" i.e. obsfuscated bool _isXPKDelta=false; // If delta encoding defined in XPK static Decompressor::Registry _registration; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/CYB2Decoder.cpp000066400000000000000000000034731375530757000162760ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "common/SubBuffer.hpp" #include "CYB2Decoder.hpp" #include "XPKMain.hpp" bool CYB2Decoder::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("CYB2"); } std::unique_ptr CYB2Decoder::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } CYB2Decoder::CYB2Decoder(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || _packedData.size()<=10) throw Decompressor::InvalidFormatError(); _blockHeader=_packedData.readBE32(0); // after the block header, the next 6 bytes seem to be // 00 64 00 00 00 00 // Those bytes do not seem to be terribly important though... if (verify) { // trigger child checks... ConstSubBuffer blockData(_packedData,10,_packedData.size()-10); std::unique_ptr state; auto sub=XPKMain::createDecompressor(_blockHeader,_recursionLevel+1,blockData,state,true); } } CYB2Decoder::~CYB2Decoder() { // nothing needed } const std::string &CYB2Decoder::getSubName() const noexcept { static std::string name="XPK-CYB2: xpkCybPrefs container"; return name; } void CYB2Decoder::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ConstSubBuffer blockData(_packedData,10,_packedData.size()-10); std::unique_ptr state; auto sub=XPKMain::createDecompressor(_blockHeader,_recursionLevel+1,blockData,state,verify); sub->decompressImpl(rawData,previousData,verify); } XPKDecompressor::Registry CYB2Decoder::_XPKregistration; ancient-1.0/src/CYB2Decoder.hpp000066400000000000000000000015471375530757000163030ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef CYB2DECODER_HPP #define CYB2DECODER_HPP #include "XPKDecompressor.hpp" class CYB2Decoder : public XPKDecompressor { public: CYB2Decoder(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~CYB2Decoder(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _blockHeader; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/DEFLATEDecompressor.cpp000066400000000000000000000245611375530757000177440ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "DEFLATEDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/CRC32.hpp" static uint32_t Adler32(const Buffer &buffer,size_t offset,size_t len) { if (!len || offset+len>buffer.size()) throw Buffer::OutOfBoundsError(); const uint8_t *ptr=buffer.data()+offset; uint32_t s1=1,s2=0; for (size_t i=0;i=65521) s1-=65521; s2+=s1; if (s2>=65521) s2-=65521; } return (s2<<16)|s1; } bool DEFLATEDecompressor::detectHeader(uint32_t hdr) noexcept { return ((hdr>>16)==0x1f8b); } bool DEFLATEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return (hdr==FourCC("GZIP")); } std::unique_ptr DEFLATEDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_unique(packedData,exactSizeKnown,verify); } std::unique_ptr DEFLATEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } bool DEFLATEDecompressor::detectZLib() { if (_packedData.size()<6) return false; // no knowledge about rawSize, before decompression // packedSize told by decompressor _packedSize=uint32_t(_packedData.size()); _packedOffset=2; uint8_t cm=_packedData.read8(0); if ((cm&0xf)!=8) return false; if ((cm&0xf0)>0x70) return false; uint8_t flags=_packedData.read8(1); if (flags&0x20) { if (_packedSize<8) return false; _packedOffset+=4; } if (((uint16_t(cm)<<8)|uint16_t(flags))%31) return false; _type=Type::ZLib; return true; } DEFLATEDecompressor::DEFLATEDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData(packedData), _exactSizeKnown(exactSizeKnown) { if (_packedData.size()<18) throw InvalidFormatError(); uint32_t hdr=_packedData.readBE32(0); if (!detectHeader(hdr)) throw InvalidFormatError(); uint8_t cm=_packedData.read8(2); if (cm!=8) throw InvalidFormatError();; uint8_t flags=_packedData.read8(3); if (flags&0xe0) throw InvalidFormatError();; uint32_t currentOffset=10; if (flags&4) { uint16_t xlen=_packedData.readLE16(currentOffset); currentOffset+=uint32_t(xlen)+2; } auto skipString=[&]() { uint8_t ch; do { ch=_packedData.read8(currentOffset); currentOffset++; } while (ch); }; if (flags&8) skipString(); // FNAME if (flags&16) skipString(); // FCOMMENT if (flags&2) currentOffset+=2; // FHCRC, not using that since it is only for header _packedOffset=currentOffset; if (size_t(currentOffset)+8>_packedData.size()) throw InvalidFormatError(); if (_exactSizeKnown) { _packedSize=_packedData.size(); _rawSize=_packedData.readLE32(_packedData.size()-4); if (!_rawSize) throw InvalidFormatError(); } _type=Type::GZIP; } DEFLATEDecompressor::DEFLATEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectZLib()) { _packedSize=packedData.size(); _packedOffset=0; _type=Type::Raw; } } DEFLATEDecompressor::DEFLATEDecompressor(const Buffer &packedData,size_t packedSize,size_t rawSize,bool isZlib,bool verify,bool deflate64) : _packedData(packedData), _deflate64(deflate64) { _packedSize=packedSize; if (_packedSize>_packedData.size()) throw InvalidFormatError(); if (isZlib) { // if it is not real zlib-stream fail. if (!detectZLib()) throw InvalidFormatError(); } else { // raw stream _packedOffset=0; _rawSize=rawSize; _type=Type::Raw; } } DEFLATEDecompressor::~DEFLATEDecompressor() { // nothing needed } const std::string &DEFLATEDecompressor::getName() const noexcept { static std::string names[3]={ "gzip: Deflate", "zlib: Deflate", "raw: Deflate/Deflate64"}; return names[static_cast(_type)]; } const std::string &DEFLATEDecompressor::getSubName() const noexcept { static std::string name="XPK-GZIP: Deflate"; return name; } size_t DEFLATEDecompressor::getPackedSize() const noexcept { // no way to know before decompressing return _packedSize; } size_t DEFLATEDecompressor::getRawSize() const noexcept { // same thing, decompression needed first return _rawSize; } void DEFLATEDecompressor::decompressImpl(Buffer &rawData,bool verify) { size_t packedSize=_packedSize?_packedSize:_packedData.size(); size_t rawSize=_rawSize?_rawSize:rawData.size(); ForwardInputStream inputStream(_packedData,_packedOffset,packedSize); LSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawSize); bool final; do { final=readBit(); uint8_t blockType=readBits(2); if (!blockType) { bitReader.reset(); uint16_t len=inputStream.readByte(); len|=uint16_t(inputStream.readByte())<<8; uint16_t nlen=inputStream.readByte(); nlen|=uint16_t(inputStream.readByte())<<8; if (len!=(nlen^0xffffU)) throw DecompressionError(); outputStream.produce(inputStream.consume(len),len); } else if (blockType==1 || blockType==2) { typedef HuffmanDecoder DEFLATEDecoder; DEFLATEDecoder llDecoder; DEFLATEDecoder distanceDecoder; if (blockType==1) { for (uint32_t i=0;i<24;i++) llDecoder.insert(HuffmanCode{7,i,int32_t(i+256)}); for (uint32_t i=0;i<144;i++) llDecoder.insert(HuffmanCode{8,i+0x30,int32_t(i)}); for (uint32_t i=0;i<8;i++) llDecoder.insert(HuffmanCode{8,i+0xc0,int32_t(i+280)}); for (uint32_t i=0;i<112;i++) llDecoder.insert(HuffmanCode{9,i+0x190,int32_t(i+144)}); for (uint32_t i=0;i<30;i++) distanceDecoder.insert(HuffmanCode{5,i,int32_t(i)}); } else { uint32_t hlit=readBits(5)+257; // lets just error here, it is simpler (possibly deflate64 stream) if (hlit>=287) throw DecompressionError(); uint32_t hdist=readBits(5)+1; uint32_t hclen=readBits(4)+4; uint8_t lengthTable[19]; for (uint32_t i=0;i<19;i++) lengthTable[i]=0; static const uint8_t lengthTableOrder[19]={ 16,17,18, 0, 8, 7, 9, 6, 10, 5,11, 4,12, 3,13, 2, 14, 1,15}; for (uint32_t i=0;i=hlit+hdist) throw DecompressionError(); if (i>=hlit) distanceTableBits[i-hlit]=value; else llTableBits[i]=value; prevValue=value; i++; }; int32_t code=bitLengthDecoder.decode(readBit); if (code<16) { insert(code); } else switch (code) { case 16: if (i) { uint32_t count=readBits(2)+3; for (uint32_t j=0;j(_deflate64?31:29)) throw DecompressionError(); static const uint32_t distanceAdditions[32]={ 1,2,3,4,5,7,9,13, 0x11,0x19,0x21,0x31,0x41,0x61,0x81,0xc1, 0x101,0x181,0x201,0x301,0x401,0x601,0x801,0xc01, 0x1001,0x1801,0x2001,0x3001,0x4001,0x6001, 0x8001,0xc001}; static const uint32_t distanceBits[32]={ 0,0,0,0,1,1,2,2, 3,3,4,4,5,5,6,6, 7,7,8,8,9,9,10,10, 11,11,12,12,13,13, 14,14}; uint32_t distance=readBits(distanceBits[distCode])+distanceAdditions[distCode]; outputStream.copy(distance,count); } } } else { throw DecompressionError(); } } while (!final); if (!_rawSize) _rawSize=outputStream.getOffset(); if (_type==Type::GZIP) { if (inputStream.getOffset()+8>packedSize) throw DecompressionError(); if (!_packedSize) _packedSize=inputStream.getOffset()+8; } else if (_type==Type::ZLib) { if (inputStream.getOffset()+4>packedSize) throw DecompressionError(); if (!_packedSize) _packedSize=inputStream.getOffset()+4; } else { if (!_packedSize) _packedSize=inputStream.getOffset(); } if (_rawSize!=outputStream.getOffset()) throw DecompressionError(); if (verify) { if (_type==Type::GZIP) { uint32_t crc=_packedData.readLE32(inputStream.getOffset()); if (CRC32(rawData,0,_rawSize,0)!=crc) throw VerificationError(); } else if (_type==Type::ZLib) { uint32_t adler=_packedData.readBE32(inputStream.getOffset()); if (Adler32(rawData,0,_rawSize)!=adler) throw VerificationError(); } } } void DEFLATEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { decompressImpl(rawData,verify); } Decompressor::Registry DEFLATEDecompressor::_registration; XPKDecompressor::Registry DEFLATEDecompressor::_XPKregistration; ancient-1.0/src/DEFLATEDecompressor.hpp000066400000000000000000000035231375530757000177440ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef DEFLATEDECOMPRESSOR_HPP #define DEFLATEDECOMPRESSOR_HPP #include "Decompressor.hpp" #include "XPKDecompressor.hpp" class DEFLATEDecompressor : public Decompressor, public XPKDecompressor { public: DEFLATEDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); DEFLATEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); DEFLATEDecompressor(const Buffer &packedData,size_t packedSize,size_t rawSize,bool isZlib,bool verify,bool deflate64); // zlib or completely raw stream virtual ~DEFLATEDecompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeader(uint32_t hdr) noexcept; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: bool detectZLib(); enum class Type { GZIP=0, ZLib, Raw }; const Buffer &_packedData; size_t _packedSize=0; size_t _packedOffset=0; size_t _rawSize=0; Type _type; bool _exactSizeKnown=true; bool _deflate64=false; static Decompressor::Registry _registration; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/DLTADecode.cpp000066400000000000000000000030511375530757000161310ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "DLTADecode.hpp" bool DLTADecode::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("DLTA"); } std::unique_ptr DLTADecode::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } DLTADecode::DLTADecode(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } DLTADecode::~DLTADecode() { // nothing needed } const std::string &DLTADecode::getSubName() const noexcept { static std::string name="XPK-DLTA: Delta encoding"; return name; } void DLTADecode::decode(Buffer &bufferDest,const Buffer &bufferSrc,size_t offset,size_t size) { if (bufferSrc.size() DLTADecode::_XPKregistration; ancient-1.0/src/DLTADecode.hpp000066400000000000000000000020001375530757000161270ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef DLTADECODE_HPP #define DLTADECODE_HPP #include "XPKDecompressor.hpp" class DLTADecode : public XPKDecompressor { public: DLTADecode(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~DLTADecode(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); // static method for easy external usage. Buffers can be the same for in-place replacement static void decode(Buffer &bufferDest,const Buffer &bufferSrc,size_t offset,size_t size); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/DMSDecompressor.cpp000066400000000000000000000515721375530757000173250ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "DMSDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "DynamicHuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/MemoryBuffer.hpp" #include "common/CRC16.hpp" bool DMSDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("DMS!"); } std::unique_ptr DMSDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_unique(packedData,verify); } DMSDecompressor::DMSDecompressor(const Buffer &packedData,bool verify) : _packedData(packedData) { uint32_t hdr=packedData.readBE32(0); if (!detectHeader(hdr) || packedData.size()<56) throw InvalidFormatError(); if (verify && CRC16(packedData,4,50,0)!=packedData.readBE16(54)) throw VerificationError(); uint16_t info=packedData.readBE16(10); _isObsfuscated=info&2; // using 16 bit key is not encryption, it is obsfuscation _isHD=info&16; if (info&32) throw InvalidFormatError(); // MS-DOS disk // packed data in header is useless, as is rawsize and track numbers // they are not always filled if (packedData.readBE16(50)>6) throw InvalidFormatError(); // either FMS or unknown // now calculate the real packed size, including headers uint32_t offset=56; uint32_t accountedSize=0; uint32_t lastTrackSize=0; uint32_t numTracks=0; uint32_t minTrack=80; uint32_t prevTrack=0; while (offset+206) throw InvalidFormatError(); static const uint32_t contextSizes[7]={0,0,256,16384,16384,4096,8192}; _contextBufferSize=std::max(_contextBufferSize,contextSizes[mode]); uint8_t flags=_packedData.read8(offset+12); if ((mode>=2 && mode<=4) || (mode>=5 && (flags&4))) { _tmpBufferSize=std::max(_tmpBufferSize,uint32_t(_packedData.readBE16(offset+8))); } uint32_t packedChunkLength=packedData.readBE16(offset+6); if (size_t(offset)+20+size_t(packedChunkLength)>packedData.size()) throw InvalidFormatError(); if (verify && CRC16(packedData,offset+20,packedChunkLength,0)!=packedData.readBE16(offset+16)) throw VerificationError(); if (trackNo<80) { if (trackNo>=numTracks) lastTrackSize=_packedData.readBE16(offset+10); minTrack=std::min(minTrack,trackNo); numTracks=std::max(numTracks,trackNo); prevTrack=trackNo; } offset+=packedChunkLength+20; accountedSize+=packedChunkLength; // this is the real exit critea, unfortunately if (trackNo>=79 && trackNo<0x8000U) break; } uint32_t trackSize=(_isHD)?22528:11264; _rawOffset=minTrack*trackSize; _minTrack=minTrack; _rawSize=(numTracks-minTrack)*trackSize+lastTrackSize; _imageSize=trackSize*80; _packedSize=offset; if (_packedSize>getMaxPackedSize()) throw InvalidFormatError(); } DMSDecompressor::~DMSDecompressor() { // nothing needed } const std::string &DMSDecompressor::getName() const noexcept { static std::string name="DMS: Disk Masher System"; return name; } size_t DMSDecompressor::getPackedSize() const noexcept { return _packedSize; } size_t DMSDecompressor::getRawSize() const noexcept { return _rawSize; } size_t DMSDecompressor::getImageSize() const noexcept { return _imageSize; } size_t DMSDecompressor::getImageOffset() const noexcept { return _rawOffset; } void DMSDecompressor::decompressImpl(Buffer &rawData,bool verify) { uint32_t restartPosition=0; if (!_isObsfuscated) { decompressImpl(rawData,verify,restartPosition); } else { while (restartPosition<0x20000U) { // more than single run here is really rare. It means that first track CRC succeeds // but later something else fails try { decompressImpl(rawData,verify,restartPosition); return; } catch (const Buffer::Error &) { // just continue } catch (const Decompressor::Error &) { // just continue } restartPosition++; } throw DecompressionError(); } } // TODO: Too much state for a single method. too convoluted // needs to be split void DMSDecompressor::decompressImpl(Buffer &rawData,bool verify,uint32_t &restartPosition) { if (rawData.size()<_rawSize) throw DecompressionError(); MemoryBuffer contextBuffer(_contextBufferSize); MemoryBuffer tmpBuffer(_tmpBufferSize); uint32_t limitedDecompress=~0U; class UnObsfuscator { public: UnObsfuscator(ForwardInputStream &inputStream) : _inputStream(inputStream) { // nothing needed } uint8_t readByte() { if (_inputStream.eof()) throw ShortInputError(); uint8_t ch=_inputStream.readByte(); if (!_obsfuscate) { return ch; } else { uint8_t ret=ch^_passAccumulator; _passAccumulator=(_passAccumulator>>1)+uint16_t(ch); return ret; } } void setCode(uint16_t passAccumulator) { _passAccumulator=passAccumulator; } void setObsfuscate(bool obsfuscate) { _obsfuscate=obsfuscate; } bool eof() const { return _inputStream.getOffset()==_inputStream.getEndOffset(); } // not cool, but works bool eofMinus1() const { return _inputStream.getOffset()+1==_inputStream.getEndOffset(); } bool eofMinus1Plus() const { return _inputStream.getOffset()+1>=_inputStream.getEndOffset(); } bool eofMinus2Plus() const { return _inputStream.getOffset()+2>=_inputStream.getEndOffset(); } private: ForwardInputStream &_inputStream; bool _obsfuscate=false; uint16_t _passAccumulator=0; }; ForwardInputStream inputStream(_packedData,0,0); UnObsfuscator inputUnObsfuscator(inputStream); MSBBitReader bitReader(inputUnObsfuscator); auto initInputStream=[&](const Buffer &buffer,uint32_t start,uint32_t length,bool obsfuscate) { inputStream=ForwardInputStream(buffer,start,start+length); inputUnObsfuscator.setObsfuscate(obsfuscate); bitReader.reset(); }; auto finishStream=[&]() { if (_isObsfuscated && limitedDecompress==~0U) while (!inputUnObsfuscator.eof()) inputUnObsfuscator.readByte(); }; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,0); auto initOutputStream=[&](Buffer &buffer,uint32_t start,uint32_t length) { outputStream=ForwardOutputStream(buffer,start,start+length); }; // fill unused tracks with zeros std::memset(rawData.data(),0,_rawSize); auto checksum=[](const uint8_t *src,uint32_t srcLength)->uint16_t { uint16_t ret=0; for (uint32_t i=0;iuint32_t { // hacky, hacky, hacky while (!inputUnObsfuscator.eof()) { if (outputStream.getOffset()>=limitedDecompress) return 0; if (lastCharMissing && inputUnObsfuscator.eofMinus1()) { if (outputStream.getOffset()+1!=outputStream.getEndOffset()) throw DecompressionError(); return 1; } uint8_t ch=inputUnObsfuscator.readByte(); uint32_t count=1; if (ch==0x90U) { if (inputUnObsfuscator.eof()) throw DecompressionError(); if (lastCharMissing && inputUnObsfuscator.eofMinus1()) { // only possible way this can work if (outputStream.getOffset()+1!=outputStream.getEndOffset()) throw DecompressionError(); count=0; } else count=inputUnObsfuscator.readByte(); if (!count) { count=1; } else { if (inputUnObsfuscator.eof()) throw DecompressionError(); if (lastCharMissing && inputUnObsfuscator.eofMinus1()) { if (count==0xffU || outputStream.getOffset()+count!=outputStream.getEndOffset()) throw DecompressionError(); return count; } ch=inputUnObsfuscator.readByte(); } if (count==0xffU) { if (inputUnObsfuscator.eofMinus1Plus()) throw DecompressionError(); if (lastCharMissing && inputUnObsfuscator.eofMinus2Plus()) { count=uint32_t(outputStream.getEndOffset()-outputStream.getOffset()); } else { count=uint32_t(inputUnObsfuscator.readByte())<<8; count|=inputUnObsfuscator.readByte(); } } } for (uint32_t i=0;i> deepDecoder; auto initContext=[&]() { if (doInitContext) { if (_contextBufferSize) std::memset(contextBuffer.data(),0,_contextBufferSize); quickContextLocation=251; mediumContextLocation=16318; deepContextLocation=16324; deepDecoder.reset(); heavyContextLocation=0; doInitContext=false; } }; auto unpackQuick=[&]() { initContext(); while (!outputStream.eof()) { if (outputStream.getOffset()>=limitedDecompress) return; if (readBits(1)) { outputStream.writeByte(contextBufferPtr[quickContextLocation++]=readBits(8)); } else { uint32_t count=readBits(2)+2; uint8_t offset=quickContextLocation-readBits(8)-1; for (uint32_t i=0;iuint32_t { return (((code<uint32_t { return (uint32_t(lengthTable[code])<<8)| uint32_t(((code<=limitedDecompress) return; if (readBits(1)) { outputStream.writeByte(contextBufferPtr[mediumContextLocation++]=readBits(8)); mediumContextLocation&=0x3fffU; } else { uint32_t code=readBits(8); uint32_t count=lengthTable[code]+3; uint32_t tmp=decodeLengthValueFull(decodeLengthValueHalf(code)); uint32_t offset=mediumContextLocation-tmp-1; for (uint32_t i=0;i>(); while (!outputStream.eof()) { if (outputStream.getOffset()>=limitedDecompress) return; uint32_t symbol=deepDecoder->decode(readBit); if (deepDecoder->getMaxFrequency()==0x8000U) deepDecoder->halve(); deepDecoder->update(symbol); if (symbol<256) { outputStream.writeByte(contextBufferPtr[deepContextLocation++]=symbol); deepContextLocation&=0x3fffU; } else { uint32_t count=symbol-253; // minimum repeat is 3 uint32_t offset=deepContextLocation-decodeLengthValueFull(readBits(8))-1; for (uint32_t i=0;i> symbolDecoder,offsetDecoder; bool heavyLastInitialized=false; // this is part of initContext on some implementations. screwy!!! uint32_t heavyLastOffset; auto unpackHeavy=[&](bool initTables,bool use8kDict) { initContext(); // well, this works. Why this works? dunno if (!heavyLastInitialized) { heavyLastOffset=use8kDict?0U:~0U; heavyLastInitialized=true; } auto readTable=[&](std::unique_ptr> &decoder,uint32_t countBits,uint32_t valueBits) { decoder=std::make_unique>(); uint32_t count=readBits(countBits); if (count) { uint8_t lengthBuffer[512]; // in order to speed up the deObsfuscation, do not send the hopeless // data into slow CreateOrderlyHuffmanTable uint64_t sum=0; for (uint32_t i=0;i(uint64_t(1U)<<32)) throw DecompressionError(); } lengthBuffer[i]=bits; } decoder->createOrderlyHuffmanTable(lengthBuffer,count); } else { uint32_t index=readBits(countBits); decoder->setEmpty(index); } }; if (initTables) { readTable(symbolDecoder,9,5); readTable(offsetDecoder,5,4); } uint32_t mask=use8kDict?0x1fffU:0xfffU; uint32_t bitLength=use8kDict?14:13; while (!outputStream.eof()) { if (outputStream.getOffset()>=limitedDecompress) return; uint32_t symbol=symbolDecoder->decode(readBit); if (symbol<256) { outputStream.writeByte(contextBufferPtr[heavyContextLocation++]=symbol); heavyContextLocation&=mask; } else { uint32_t count=symbol-253; // minimum repeat is 3 uint32_t offsetLength=offsetDecoder->decode(readBit); uint32_t rawOffset=heavyLastOffset; if (offsetLength!=bitLength) { if (offsetLength) rawOffset=(1<<(offsetLength-1))|readBits(offsetLength-1); else rawOffset=0; heavyLastOffset=rawOffset; } uint32_t offset=heavyContextLocation-rawOffset-1; for (uint32_t i=0;i=0x8000U) { initInputStream(_packedData,packedOffset+20,packedChunkLength,_isObsfuscated); finishStream(); continue; } if (rawChunkLength>trackLength) throw DecompressionError(); if (trackNo>80) throw DecompressionError(); // should not happen, already excluded uint32_t dataOffset=trackNo*trackLength; if (_rawOffset>dataOffset) throw DecompressionError(); // this is screwy, but it is, what it is auto processBlock=[&](bool doRLE,auto func,auto&&... params) { initInputStream(_packedData,packedOffset+20,packedChunkLength,_isObsfuscated); if (doRLE) { try { initOutputStream(tmpBuffer,0,tmpChunkLength); func(params...); finishStream(); initInputStream(tmpBuffer,0,tmpChunkLength,false); initOutputStream(rawData,dataOffset-_rawOffset,rawChunkLength); unRLE(false); } catch (const ShortInputError &) { // if this error happens on repeat/offset instead of char, though luck :( // missing last char on src we can fix :) initInputStream(tmpBuffer,0,tmpChunkLength,false); initOutputStream(rawData,dataOffset-_rawOffset,rawChunkLength); uint32_t missingNo=unRLE(true); if (missingNo) { uint32_t protoSum=checksum(&rawData[dataOffset-_rawOffset],rawChunkLength-missingNo); uint32_t fileSum=_packedData.readBE16(packedOffset+14); uint8_t ch=((fileSum>=protoSum)?fileSum-protoSum:(0x10000U+fileSum-protoSum))/missingNo; for (uint32_t i=0;i DMSDecompressor::_registration; ancient-1.0/src/DMSDecompressor.hpp000066400000000000000000000024421375530757000173220ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef DMSDECOMPRESSOR_HPP #define DMSDECOMPRESSOR_HPP #include "Decompressor.hpp" #include "common/MemoryBuffer.hpp" class DMSDecompressor : public Decompressor { public: DMSDecompressor(const Buffer &packedData,bool verify); virtual ~DMSDecompressor(); virtual const std::string &getName() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual size_t getRawSize() const noexcept override final; virtual size_t getImageSize() const noexcept override final; virtual size_t getImageOffset() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; static bool detectHeader(uint32_t hdr) noexcept; static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: void decompressImpl(Buffer &rawData,bool verify,uint32_t &restartPosition); class ShortInputError : public Error { // nothing needed }; const Buffer &_packedData; uint32_t _packedSize=0; uint32_t _rawSize=0; uint32_t _contextBufferSize=0; uint32_t _tmpBufferSize=0; uint32_t _imageSize; uint32_t _rawOffset; uint32_t _minTrack; bool _isHD; bool _isObsfuscated; static Decompressor::Registry _registration; }; #endif ancient-1.0/src/Decompressor.cpp000066400000000000000000000032351375530757000167520ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "Decompressor.hpp" std::vector(*)(const Buffer&,bool,bool)>> *Decompressor::_decompressors=nullptr; Decompressor::~Decompressor() { // nothing needed } std::unique_ptr Decompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { try { uint32_t hdr=packedData.readBE32(0); for (auto &it : *_decompressors) { if (it.first(hdr)) return it.second(packedData,exactSizeKnown,verify); } throw InvalidFormatError(); } catch (const Buffer::Error&) { throw InvalidFormatError(); } } bool Decompressor::detect(const Buffer &packedData) noexcept { try { uint32_t hdr=packedData.readBE32(0); for (auto &it : *_decompressors) if (it.first(hdr)) return true; return false; } catch (const Buffer::Error&) { return false; } } void Decompressor::registerDecompressor(bool(*detect)(uint32_t),std::unique_ptr(*create)(const Buffer&,bool,bool)) { static std::vector(*)(const Buffer&,bool,bool)>> _list; if (!_decompressors) _decompressors=&_list; _decompressors->emplace_back(detect,create); } void Decompressor::decompress(Buffer &rawData,bool verify) { // Simplifying the implementation of sub-decompressors. Just let the buffer-exception pass here, // and that will get translated into Decompressor exceptions try { decompressImpl(rawData,verify); } catch (const Buffer::Error&) { throw DecompressionError(); } } size_t Decompressor::getImageSize() const noexcept { return 0; } size_t Decompressor::getImageOffset() const noexcept { return 0; } ancient-1.0/src/Decompressor.hpp000066400000000000000000000074331375530757000167630ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef DECOMPRESSOR_HPP #define DECOMPRESSOR_HPP #include #include #include #include #include "common/Common.hpp" #include "common/Buffer.hpp" class Decompressor { public: // just a base class to easily catch all the errors class Error : public std::exception { // nothing needed }; class InvalidFormatError : public Error { // nothing needed }; class DecompressionError : public Error { // nothing needed }; class VerificationError : public Error { // nothing needed }; Decompressor()=default; Decompressor(const Decompressor&)=delete; Decompressor& operator=(const Decompressor&)=delete; virtual ~Decompressor(); // Name returned is human readable long name virtual const std::string &getName() const noexcept=0; // PackedSize or RawSize are taken from the stream if available, 0 otherwise. // for those compressors having 0 sizes, running decompression will update // the values. (make sure to allocate big-enough buffer for decompression) // There are exceptions: Some decompressors need to exact size of the packed data // in order to decompress. For those providing a indefinitely size packed stream // will not work // use the "exactSizeKnown" flag for create to tell whether you know the size or not virtual size_t getPackedSize() const noexcept=0; virtual size_t getRawSize() const noexcept=0; // Actual decompression. // verify checksum if verify==true // can throw DecompressionError if stream cant be unpacked // can throw VerificationError if verify enabled and checksum does not match void decompress(Buffer &rawData,bool verify); // in case of disk image based formats the data does not necessarily start // from logical beginnig of the image but it is offsetted inside the logical image // (f.e. DMS). getDataOffset will return the offset (or 0 if not relevant or if offset does not exist) // getImageSize will return the size of the the logical image, or 0 if not image-based format virtual size_t getImageSize() const noexcept; virtual size_t getImageOffset() const noexcept; // the functions are there to protect against "accidental" large files when parsing headers // a.k.a. 16M should be enough for everybody (sizes do not have to accurate i.e. // compressors can exclude header content for simplification) // This entirely ok for the context of "old computers" and their files, // for other usages these need to be tuned up static constexpr size_t getMaxPackedSize() noexcept { return 0x100'0000U; } static constexpr size_t getMaxRawSize() noexcept { return 0x100'0000U; } // Main entrypoint // if verify=true then check the packedData for errors: CRC or other checksum if available // check exactSizeKnown from size documentation // can throw InvalidFormatError if stream is not recognized or it is invalid // can throw VerificationError if verify enabled and checksum does not match static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); // Detect signature whether it matches to any known compressor // This does not guarantee the data is decompressable though, only signature is read static bool detect(const Buffer &packedData) noexcept; // Registering new decompressors, not really part of public API template class Registry { public: Registry() { Decompressor::registerDecompressor(T::detectHeader,T::create); } ~Registry() { // TODO: no cleanup yet } }; protected: virtual void decompressImpl(Buffer &rawData,bool verify)=0; private: static void registerDecompressor(bool(*detect)(uint32_t),std::unique_ptr(*create)(const Buffer&,bool,bool)); static std::vector(*)(const Buffer&,bool,bool)>> *_decompressors; }; #endif ancient-1.0/src/DynamicHuffmanDecoder.hpp000066400000000000000000000126331375530757000204730ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef DYNAMICHUFFMANDECODER_HPP #define DYNAMICHUFFMANDECODER_HPP #include #include // For exception #include "Decompressor.hpp" template class DynamicHuffmanDecoder { public: DynamicHuffmanDecoder(uint32_t initialCount=maxCount) : _initialCount(initialCount) { if (_initialCount>maxCount) throw Decompressor::DecompressionError(); reset(); } ~DynamicHuffmanDecoder() { // nothing needed } void reset() { _count=_initialCount; if (!_count) return; for (uint32_t i=0;i<_count;i++) { _nodes[i].frequency=1; _nodes[i].index=i+(maxCount-_count)*2; _nodes[i].parent=maxCount*2-_count+(i>>1); _nodes[i].leaves[0]=0; _nodes[i].leaves[1]=0; _codeMap[i+(maxCount-_count)*2]=i; } for (uint32_t i=maxCount*2-_count,j=0;i=_count)?j+(maxCount-_count)*2:j; uint32_t r=(j+1>=_count)?j+1+(maxCount-_count)*2:(j+1); _nodes[i].frequency=_nodes[l].frequency+_nodes[r].frequency; _nodes[i].index=i; _nodes[i].parent=maxCount+(i>>1); _nodes[i].leaves[0]=l; _nodes[i].leaves[1]=r; _codeMap[i]=i; } } template uint32_t decode(F bitReader) const { if (!_count) throw Decompressor::DecompressionError(); if (_count==1) return 0; uint32_t code=maxCount*2-2; while (code>=maxCount) code=_nodes[code].leaves[bitReader()?1:0]; return code; } void update(uint32_t code) { if (code>=_count) throw Decompressor::DecompressionError(); // this is a bug in LH2. Nobody else uses this codepath, so we can let it be... if (_count==1) { _nodes[0].frequency=1; return; } while (code!=maxCount*2-2) { _nodes[code].frequency++; uint32_t index=_nodes[code].index; uint32_t destIndex=index; uint32_t freq=_nodes[code].frequency; while (destIndex!=maxCount*2-2 && freq>_nodes[_codeMap[destIndex+1]].frequency) destIndex++; if (index!=destIndex) { auto getParentLeaf=[&](uint32_t currentCode)->uint32_t& { Node &parent=_nodes[_nodes[currentCode].parent]; return parent.leaves[(parent.leaves[0]==currentCode)?0:1]; }; uint32_t destCode=_codeMap[destIndex]; std::swap(_nodes[code].index,_nodes[destCode].index); std::swap(_codeMap[index],_codeMap[destIndex]); std::swap(getParentLeaf(code),getParentLeaf(destCode)); std::swap(_nodes[code].parent,_nodes[destCode].parent); } code=_nodes[code].parent; } _nodes[code].frequency++; } // halve the frequencies rounding upwards void halve() { if (!_count) return; else if (_count==1) { _nodes[0].frequency=(_nodes[0].frequency+1)>>1; return; } for (uint32_t i=(maxCount-_count)*2,j=(maxCount-_count)*2;i>1; _nodes[i].parent=maxCount+(_nodes[i].index>>1); _codeMap[_nodes[i].index]=i; } for (uint32_t i=maxCount*2-_count,j=(maxCount-_count)*2;i>1); _nodes[i].leaves[0]=l; _nodes[i].leaves[1]=r; _codeMap[i]=i; for (uint32_t k=i;freq<_nodes[_codeMap[k-1]].frequency;k--) { uint32_t &code=_codeMap[k]; uint32_t &destCode=_codeMap[k-1]; std::swap(_nodes[code].index,_nodes[destCode].index); std::swap(_nodes[code].parent,_nodes[destCode].parent); std::swap(code,destCode); } } } // Defined as in LH2 void addCode() { if (_count>=maxCount) throw Decompressor::DecompressionError(); uint32_t newIndex=(maxCount-_count-1)*2; if (!_count) { _nodes[0].frequency=0; _nodes[0].index=newIndex-1; _nodes[0].parent=maxCount*2-2; _nodes[0].leaves[0]=0; _nodes[0].leaves[1]=0; _codeMap[newIndex-1]=0; _count++; } else { _nodes[_count].frequency=0; _nodes[_count].index=newIndex; _nodes[_count].parent=maxCount*2-_count-1; _nodes[_count].leaves[0]=0; _nodes[_count].leaves[1]=0; _codeMap[newIndex]=_count; uint32_t insertIndex=newIndex+2; uint32_t repNode; uint32_t parentNode; uint32_t insertNode=maxCount*2-_count-1; if (_count>1) { _codeMap[insertIndex-1]=_codeMap[insertIndex]; _nodes[_codeMap[insertIndex-1]].index--; repNode=_codeMap[(maxCount-_count)*2]; parentNode=_nodes[repNode].parent; _nodes[parentNode].leaves[(_nodes[parentNode].leaves[0]==repNode)?0:1]=insertNode; _nodes[repNode].parent=insertNode; } else { repNode=0; parentNode=maxCount*2-1; } _nodes[insertNode].frequency=_nodes[repNode].frequency; _nodes[insertNode].index=insertIndex; _nodes[insertNode].parent=parentNode; _nodes[insertNode].leaves[0]=_count; _nodes[insertNode].leaves[1]=repNode; _codeMap[insertIndex]=insertNode; Node &parent=_nodes[parentNode]; if (_count>1 && _nodes[parent.leaves[0]].index>_nodes[parent.leaves[1]].index) std::swap(parent.leaves[0],parent.leaves[1]); _count++; } } uint32_t getMaxFrequency() const { return _nodes[maxCount*2-2].frequency; } private: struct Node { uint32_t frequency; uint32_t index; uint32_t parent; uint32_t leaves[2]; }; uint32_t _initialCount; uint32_t _count; Node _nodes[maxCount*2-1]; uint32_t _codeMap[maxCount*2-1]; }; #endif ancient-1.0/src/FASTDecompressor.cpp000066400000000000000000000041621375530757000174300ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "FASTDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool FASTDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("FAST"); } std::unique_ptr FASTDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } FASTDecompressor::FASTDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } FASTDecompressor::~FASTDecompressor() { // nothing needed } const std::string &FASTDecompressor::getSubName() const noexcept { static std::string name="XPK-FAST: Fast LZ77 compressor"; return name; } void FASTDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream forwardInputStream(_packedData,0,_packedData.size()); BackwardInputStream backwardInputStream(_packedData,0,_packedData.size()); forwardInputStream.link(backwardInputStream); backwardInputStream.link(forwardInputStream); MSBBitReader bitReader(backwardInputStream); auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1); }; auto readByte=[&]()->uint8_t { return forwardInputStream.readByte(); }; auto readShort=[&]()->uint16_t { const uint8_t *buf=backwardInputStream.consume(2); uint16_t ret=uint16_t(buf[0])<<8; return ret|uint16_t(buf[1]); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint16_t ld=readShort(); uint32_t count=std::min(18U-(ld&0xf),uint32_t(outputStream.getEndOffset()-outputStream.getOffset())); uint32_t distance=ld>>4; outputStream.copy(distance,count); } } } XPKDecompressor::Registry FASTDecompressor::_XPKregistration; ancient-1.0/src/FASTDecompressor.hpp000066400000000000000000000015541375530757000174370ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef FASTDECOMPRESSOR_HPP #define FASTDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class FASTDecompressor : public XPKDecompressor { public: FASTDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~FASTDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/FBR2Decompressor.cpp000066400000000000000000000044521375530757000173700ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "FBR2Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool FBR2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("FBR2"); } std::unique_ptr FBR2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } FBR2Decompressor::FBR2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError();; } FBR2Decompressor::~FBR2Decompressor() { // nothing needed } const std::string &FBR2Decompressor::getSubName() const noexcept { static std::string name="XPK-FBR2: FBR2 CyberYAFA compressor"; return name; } void FBR2Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); ForwardOutputStream outputStream(rawData,0,rawData.size()); uint8_t mode=inputStream.readByte(); while (!outputStream.eof()) { bool doCopy=false; uint32_t count=0; switch (mode) { case 33: count=uint32_t(inputStream.readByte())<<24; count|=uint32_t(inputStream.readByte())<<16; count|=uint32_t(inputStream.readByte())<<8; count|=uint32_t(inputStream.readByte()); if (count>=0x8000'0000) { doCopy=true; count=-count; } break; case 67: count=uint32_t(inputStream.readByte())<<8; count|=uint32_t(inputStream.readByte()); if (count>=0x8000) { doCopy=true; count=0x10000-count; } break; case 100: count=uint32_t(inputStream.readByte()); if (count>=0x80) { doCopy=true; count=0x100-count; } break; default: throw Decompressor::DecompressionError(); } count++; if (doCopy) { for (uint32_t i=0;i FBR2Decompressor::_XPKregistration; ancient-1.0/src/FBR2Decompressor.hpp000066400000000000000000000015541375530757000173750ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef FBR2DECOMPRESSOR_HPP #define FBR2DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class FBR2Decompressor : public XPKDecompressor { public: FBR2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~FBR2Decompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/FRLEDecompressor.cpp000066400000000000000000000033251375530757000174230ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "FRLEDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool FRLEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("FRLE"); } std::unique_ptr FRLEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } FRLEDecompressor::FRLEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } FRLEDecompressor::~FRLEDecompressor() { // nothing needed } const std::string &FRLEDecompressor::getSubName() const noexcept { static std::string name="XPK-FRLE: RLE-compressor"; return name; } void FRLEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { auto countMod=[](uint32_t count)->uint32_t { return (32-(count&0x1f))+(count&0x60); }; uint32_t count=uint32_t(inputStream.readByte()); if (count<128) { count=countMod(count); for (uint32_t i=0;i FRLEDecompressor::_XPKregistration; ancient-1.0/src/FRLEDecompressor.hpp000066400000000000000000000015561375530757000174340ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef FRLEDECOMPRESSOR_HPP #define FRLEDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class FRLEDecompressor : public XPKDecompressor { public: FRLEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~FRLEDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool override) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/HFMNDecompressor.cpp000066400000000000000000000050711375530757000174230ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "HFMNDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool HFMNDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("HFMN"); } std::unique_ptr HFMNDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } HFMNDecompressor::HFMNDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || packedData.size()<4) throw Decompressor::InvalidFormatError(); uint16_t tmp=packedData.readBE16(0); if (tmp&3) throw Decompressor::InvalidFormatError(); // header is being written in 4 byte chunks _headerSize=size_t(tmp&0x1ff); // the top 7 bits are flags. No definition what they are and they are ignored in decoder... if (size_t(_headerSize)+4>packedData.size()) throw Decompressor::InvalidFormatError(); tmp=packedData.readBE16(_headerSize+2); _rawSize=size_t(tmp); if (!_rawSize) throw Decompressor::InvalidFormatError(); _headerSize+=4; } HFMNDecompressor::~HFMNDecompressor() { // nothing needed } const std::string &HFMNDecompressor::getSubName() const noexcept { static std::string name="XPK-HFMN: Huffman compressor"; return name; } void HFMNDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); ForwardInputStream inputStream(_packedData,2,_headerSize); MSBBitReader bitReader(inputStream); auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); HuffmanDecoder decoder; uint32_t code=1; uint32_t codeBits=1; for (;;) { if (!readBit()) { uint32_t lit=0; for (uint32_t i=0;i<8;i++) lit|=readBit()<{codeBits,code,lit}); while (!(code&1) && codeBits) { codeBits--; code>>=1; } if (!codeBits) break; code--; } else { code=(code<<1)+1; codeBits++; } } inputStream=ForwardInputStream(_packedData,_headerSize,_packedData.size()); bitReader.reset(); while (!outputStream.eof()) outputStream.writeByte(decoder.decode(readBit)); } XPKDecompressor::Registry HFMNDecompressor::_XPKregistration; ancient-1.0/src/HFMNDecompressor.hpp000066400000000000000000000016261375530757000174320ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef HFMNDECOMPRESSOR_HPP #define HFMNDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class HFMNDecompressor : public XPKDecompressor { public: HFMNDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~HFMNDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; size_t _headerSize; size_t _rawSize; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/HUFFDecompressor.cpp000066400000000000000000000044141375530757000174230ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "HUFFDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool HUFFDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("HUFF"); } std::unique_ptr HUFFDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } HUFFDecompressor::HUFFDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || packedData.size()<6) throw Decompressor::InvalidFormatError(); // version: only 0 is defined uint16_t ver=packedData.readBE16(0); if (ver) throw Decompressor::InvalidFormatError(); // password: we do not support it... uint32_t pwd=packedData.readBE32(2); if (pwd!=0xabadcafe) throw Decompressor::InvalidFormatError(); } HUFFDecompressor::~HUFFDecompressor() { // nothing needed } const std::string &HUFFDecompressor::getSubName() const noexcept { static std::string name="XPK-HUFF: Huffman compressor"; return name; } void HUFFDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,6,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); HuffmanDecoder decoder; for (uint32_t i=0;i<256;i++) { uint8_t codeBits=readByte()+1; if (!codeBits) continue; if (codeBits>32) throw Decompressor::DecompressionError(); uint32_t code=0; int32_t shift=-codeBits; for (uint32_t j=0;j>shift)&((1<{codeBits,code,i}); } while (!outputStream.eof()) outputStream.writeByte(decoder.decode(readBit)); } XPKDecompressor::Registry HUFFDecompressor::_XPKregistration; ancient-1.0/src/HUFFDecompressor.hpp000066400000000000000000000015541375530757000174320ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef HUFFDECOMPRESSOR_HPP #define HUFFDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class HUFFDecompressor : public XPKDecompressor { public: HUFFDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~HUFFDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/HuffmanDecoder.hpp000066400000000000000000000104601375530757000171620ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef HUFFMANDECODER_HPP #define HUFFMANDECODER_HPP #include #include #include #include // For exception #include "Decompressor.hpp" #include "common/MemoryBuffer.hpp" template struct HuffmanCode { uint32_t length; uint32_t code; T value; }; template class OptionalHuffmanDecoder; template class HuffmanDecoder { friend class OptionalHuffmanDecoder; private: struct Node { uint32_t sub[2]; T value; Node(uint32_t _sub0,uint32_t _sub1,T _value) : sub{_sub0,_sub1}, value(_value) { // nothing needed } Node(Node &&source) : sub{source.sub[0],source.sub[1]}, value(source.value) { // nothing needed } Node& operator=(Node &&source) { if (this!=&source) { sub[0]=source.sub[0]; sub[1]=source.sub[1]; value=source.value; } return *this; } }; public: HuffmanDecoder() { // nothing needed } template HuffmanDecoder(const Args&& ...args) : HuffmanDecoder() { const HuffmanCode list[sizeof...(args)]={args...}; for (auto &item : list) insert(item); } ~HuffmanDecoder() { } void reset() { _table.clear(); } template const T &decode(F bitReader) const { if (!_table.size()) throw Decompressor::DecompressionError(); uint32_t i=0; while (_table[i].sub[0] || _table[i].sub[1]) { i=_table[i].sub[bitReader()?1:0]; if (!i) throw Decompressor::DecompressionError(); } return _table[i].value; } void insert(const HuffmanCode &code) { uint32_t i=0,length=uint32_t(_table.size()); for (int32_t currentBit=code.length;currentBit>=0;currentBit--) { uint32_t codeBit=(currentBit && (size_t(code.code)&(1<<(currentBit-1))))?1:0; if (i!=length) { if (!currentBit || (!_table[i].sub[0] && !_table[i].sub[1])) throw Decompressor::DecompressionError(); uint32_t &tmp=_table[i].sub[codeBit]; if (!tmp) tmp=i=length; else i=tmp; } else { _table.emplace_back((currentBit&&!codeBit)?length+1:0,(currentBit&&codeBit)?length+1:0,currentBit?T():code.value); length++; i++; } } } // create orderly Huffman table, as used by Deflate and Bzip2 void createOrderlyHuffmanTable(const uint8_t *bitLengths,uint32_t bitTableLength) { uint8_t minDepth=32,maxDepth=0; // some optimization: more tables uint16_t firstIndex[33],lastIndex[33]; MemoryBuffer nextIndexBuffer(bitTableLength*sizeof(uint16_t)); uint16_t *nextIndex=nextIndexBuffer.cast(); for (uint32_t i=1;i<33;i++) firstIndex[i]=0xffffU; uint32_t realItems=0; for (uint32_t i=0;i32) throw Decompressor::DecompressionError(); if (length) { if (lengthmaxDepth) maxDepth=length; if (firstIndex[length]==0xffffU) { firstIndex[length]=i; lastIndex[length]=i; } else { nextIndex[lastIndex[length]]=i; lastIndex[length]=i; } realItems++; } } if (!maxDepth) throw Decompressor::DecompressionError(); // optimization, the multiple depends how sparse the tree really is. (minimum is *2) // usually it is sparse. _table.reserve(realItems*3); uint32_t code=0; for (uint32_t depth=minDepth;depth<=maxDepth;depth++) { if (firstIndex[depth]!=0xffffU) nextIndex[lastIndex[depth]]=bitTableLength; for (uint32_t i=firstIndex[depth];i{depth,code>>(maxDepth-depth),(T)i}); code+=1<<(maxDepth-depth); } } } private: std::vector _table; }; template class OptionalHuffmanDecoder { public: OptionalHuffmanDecoder() : _base() { // nothing needed } ~OptionalHuffmanDecoder() { // nothing needed } void reset() { _base.reset(); } void setEmpty(T value) { reset(); _emptyValue=value; } template T decode(F bitReader) const { if (!_base._table.size()) return _emptyValue; else return _base.decode(bitReader); } void insert(const HuffmanCode &code) { _base.insert(code); } void createOrderlyHuffmanTable(const uint8_t *bitLengths,uint32_t bitTableLength) { _base.createOrderlyHuffmanTable(bitLengths,bitTableLength); } private: HuffmanDecoder _base; T _emptyValue; }; #endif ancient-1.0/src/ILZRDecompressor.cpp000066400000000000000000000040161375530757000174510ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "ILZRDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool ILZRDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("ILZR"); } std::unique_ptr ILZRDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } ILZRDecompressor::ILZRDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || packedData.size()<2) throw Decompressor::InvalidFormatError(); _rawSize=_packedData.readBE16(0); if (!_rawSize) throw Decompressor::InvalidFormatError(); } ILZRDecompressor::~ILZRDecompressor() { // nothing needed } const std::string &ILZRDecompressor::getSubName() const noexcept { static std::string name="XPK-ILZR: Incremental Lempel-Ziv-Renau compressor"; return name; } void ILZRDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); ForwardInputStream inputStream(_packedData,2,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); uint32_t bits=8; while (!outputStream.eof()) { if (readBits(1)) { outputStream.writeByte(readBits(8)); } else { while (outputStream.getOffset()>(1U<=outputStream.getOffset()) throw Decompressor::DecompressionError(); outputStream.copy(outputStream.getOffset()-position,count); } } } XPKDecompressor::Registry ILZRDecompressor::_XPKregistration; ancient-1.0/src/ILZRDecompressor.hpp000066400000000000000000000016021375530757000174540ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef ILZRDECOMPRESSOR_HPP #define ILZRDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class ILZRDecompressor : public XPKDecompressor { public: ILZRDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~ILZRDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; size_t _rawSize=0; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/IMPDecompressor.cpp000066400000000000000000000163371375530757000173270ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "IMPDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" static bool readIMPHeader(uint32_t hdr,uint32_t &addition) noexcept { switch (hdr) { case FourCC("ATN!"): case FourCC("EDAM"): case FourCC("IMP!"): case FourCC("M.H."): addition=7; return true; case FourCC("BDPI"): addition=0x6e8; return true; case FourCC("CHFI"): addition=0xfe4; return true; case FourCC("RDC9"): // Files do not contain checksum // I haven't got these files to be sure what is the addition case FourCC("Dupa"): case FourCC("FLT!"): case FourCC("PARA"): addition=0; // disable checksum for now return true; default: return false; } } bool IMPDecompressor::detectHeader(uint32_t hdr) noexcept { uint32_t dummy; return readIMPHeader(hdr,dummy); } bool IMPDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("IMPL"); } std::unique_ptr IMPDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_unique(packedData,verify); } std::unique_ptr IMPDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } IMPDecompressor::IMPDecompressor(const Buffer &packedData,bool verify) : _packedData(packedData) { uint32_t hdr=packedData.readBE32(0); uint32_t checksumAddition; if (!readIMPHeader(hdr,checksumAddition) || packedData.size()<0x32) throw InvalidFormatError(); _rawSize=packedData.readBE32(4); _endOffset=packedData.readBE32(8); if ((_endOffset&1) || _endOffset<0xc || _endOffset+0x32>packedData.size() || !_rawSize || !_endOffset || _rawSize>getMaxRawSize() || _endOffset>getMaxPackedSize()) throw InvalidFormatError(); uint32_t checksum=packedData.readBE32(_endOffset+0x2e); if (verify && checksumAddition) { // size is divisible by 2 uint32_t sum=checksumAddition; for (uint32_t i=0;i<_endOffset+0x2e;i+=2) { // TODO: slow, optimize uint16_t tmp=_packedData.readBE16(i); sum+=uint32_t(tmp); } if (checksum!=sum) throw InvalidFormatError(); } } IMPDecompressor::IMPDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || packedData.size()<0x2e) throw InvalidFormatError(); _rawSize=packedData.readBE32(4); _endOffset=packedData.readBE32(8); if ((_endOffset&1) || _endOffset<0xc || size_t(_endOffset)+0x2e>packedData.size()) throw InvalidFormatError(); _isXPK=true; } IMPDecompressor::~IMPDecompressor() { // nothing needed } const std::string &IMPDecompressor::getName() const noexcept { static std::string name="IMP: File Imploder"; return name; } const std::string &IMPDecompressor::getSubName() const noexcept { static std::string name="XPK-IMPL: File Imploder"; return name; } size_t IMPDecompressor::getPackedSize() const noexcept { return _endOffset+0x32; } size_t IMPDecompressor::getRawSize() const noexcept { return _rawSize; } void IMPDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); class IMPInputStream { public: IMPInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset) : _bufPtr(buffer.data()), _currentOffset(endOffset), _endOffset(startOffset), _refOffset(endOffset) { if (_currentOffset<_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); uint8_t markerByte=buffer.read8(_currentOffset+16); if (!(markerByte&0x80)) { if (_currentOffset==_endOffset) throw Decompressor::DecompressionError(); _currentOffset--; } } ~IMPInputStream() { // nothing needed } uint8_t readByte() { // streamreader with funny ordering auto sourceOffset=[&](size_t i)->size_t { if (i>=12) { return i; } else { if (i<4) { return i+_refOffset+8; } else if (i<8) { return i+_refOffset; } else { return i+_refOffset-8; } } }; if (_currentOffset<=_endOffset) throw Decompressor::DecompressionError(); return _bufPtr[sourceOffset(--_currentOffset)]; } bool eof() const { return _currentOffset==_endOffset; } private: const uint8_t *_bufPtr; size_t _currentOffset; size_t _endOffset; size_t _refOffset; }; IMPInputStream inputStream(_packedData,0,_endOffset); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; // the anchor-bit does not seem always to be at the correct place { uint8_t halfByte=_packedData.read8(_endOffset+17); for (uint32_t i=0;i<7;i++) if (halfByte&(1<>(i+1),7-i); break; } } BackwardOutputStream outputStream(rawData,0,_rawSize); // tables uint16_t distanceValues[2][4]; for (uint32_t i=0;i<8;i++) distanceValues[i>>2][i&3]=_packedData.readBE16(_endOffset+18+i*2); uint8_t distanceBits[3][4]; for (uint32_t i=0;i<12;i++) distanceBits[i>>2][i&3]=_packedData.read8(_endOffset+34+i); // length, distance & literal counts are all intertwined HuffmanDecoder lldDecoder { HuffmanCode{1,0b00000,0}, HuffmanCode{2,0b00010,1}, HuffmanCode{3,0b00110,2}, HuffmanCode{4,0b01110,3}, HuffmanCode{5,0b11110,4}, HuffmanCode{5,0b11111,5} }; HuffmanDecoder lldDecoder2 { HuffmanCode{1,0b00,0}, HuffmanCode{2,0b10,1}, HuffmanCode{2,0b11,2} }; // finally loop uint32_t litLength=_packedData.readBE32(_endOffset+12); for (;;) { for (uint32_t i=0;i IMPDecompressor::_registration; XPKDecompressor::Registry IMPDecompressor::_XPKregistration; ancient-1.0/src/IMPDecompressor.hpp000066400000000000000000000027671375530757000173360ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef IMPDECOMPRESSOR_HPP #define IMPDECOMPRESSOR_HPP #include "Decompressor.hpp" #include "XPKDecompressor.hpp" class IMPDecompressor : public Decompressor, public XPKDecompressor { public: IMPDecompressor(const Buffer &packedData,bool verify); IMPDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~IMPDecompressor(); virtual const std::string &getName() const noexcept override final; virtual const std::string &getSubName() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual size_t getRawSize() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeader(uint32_t hdr) noexcept; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _rawSize=0; uint32_t _endOffset=0; bool _isXPK=false; static Decompressor::Registry _registration; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/InputStream.cpp000066400000000000000000000050531375530757000165600ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "InputStream.hpp" // for exceptions #include "Decompressor.hpp" ForwardInputStream::ForwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset,bool allowOverrun) : _bufPtr(buffer.data()), _currentOffset(startOffset), _endOffset(endOffset), _allowOverrun(allowOverrun) { if (_currentOffset>_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); } ForwardInputStream::~ForwardInputStream() { // nothing needed } uint8_t ForwardInputStream::readByte() { if (_currentOffset>=_endOffset) { if (_allowOverrun) { _currentOffset++; return 0; } throw Decompressor::DecompressionError(); } uint8_t ret=_bufPtr[_currentOffset++]; if (_linkedInputStream) _linkedInputStream->setOffset(_currentOffset); return ret; } const uint8_t *ForwardInputStream::consume(size_t bytes,uint8_t *buffer) { if (_currentOffset+bytes>_endOffset) { if (_allowOverrun && buffer) { for (size_t i=0;isetOffset(_currentOffset); return ret; } BackwardInputStream::BackwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset,bool allowOverrun) : _bufPtr(buffer.data()), _currentOffset(endOffset), _endOffset(startOffset), _allowOverrun(allowOverrun) { if (_currentOffset<_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); } BackwardInputStream::~BackwardInputStream() { // nothing needed } uint8_t BackwardInputStream::readByte() { if (_currentOffset<=_endOffset) { if (_allowOverrun) { --_currentOffset; return 0; } throw Decompressor::DecompressionError(); } uint8_t ret=_bufPtr[--_currentOffset]; if (_linkedInputStream) _linkedInputStream->setOffset(_currentOffset); return ret; } const uint8_t *BackwardInputStream::consume(size_t bytes,uint8_t *buffer) { if (_currentOffset<_endOffset+bytes) { if (_allowOverrun && buffer) { for (size_t i=bytes;i;i--) { buffer[i-1]=(_currentOffset>_endOffset)?_bufPtr[_currentOffset-1]:0; --_currentOffset; } return buffer; } throw Decompressor::DecompressionError(); } _currentOffset-=bytes; if (_linkedInputStream) _linkedInputStream->setOffset(_currentOffset); return &_bufPtr[_currentOffset]; } ancient-1.0/src/InputStream.hpp000066400000000000000000000111711375530757000165630ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef INPUTSTREAM_HPP #define INPUTSTREAM_HPP #include #include #include #include "common/Buffer.hpp" class BackwardInputStream; class ForwardInputStream { friend class BackwardInputStream; public: ForwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset,bool allowOverrun=false); ~ForwardInputStream(); uint8_t readByte(); const uint8_t *consume(size_t bytes,uint8_t *buffer=nullptr); bool eof() const { return _currentOffset==_endOffset; } size_t getOffset() const { return _currentOffset; } size_t getEndOffset() const { return _endOffset; } void link(BackwardInputStream &stream) { _linkedInputStream=&stream; } private: void setOffset(size_t offset) { _endOffset=offset; } const uint8_t *_bufPtr; size_t _currentOffset; size_t _endOffset; bool _allowOverrun; BackwardInputStream *_linkedInputStream=nullptr; }; class BackwardInputStream { friend class ForwardInputStream; public: BackwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset,bool allowOverrun=false); ~BackwardInputStream(); uint8_t readByte(); const uint8_t *consume(size_t bytes,uint8_t *buffer=nullptr); bool eof() const { return _currentOffset==_endOffset; } size_t getOffset() const { return _currentOffset; } void link(ForwardInputStream &stream) { _linkedInputStream=&stream; } private: void setOffset(size_t offset) { _endOffset=offset; } const uint8_t *_bufPtr; size_t _currentOffset; size_t _endOffset; bool _allowOverrun; ForwardInputStream *_linkedInputStream=nullptr; }; template class LSBBitReader { public: LSBBitReader(T &inputStream) : _inputStream(inputStream) { // nothing needed } ~LSBBitReader() { // nothing needed } uint32_t readBits8(uint32_t count) { return readBitsInternal(count,[&](){ _bufContent=_inputStream.readByte(); _bufLength=8; }); } uint32_t readBitsBE16(uint32_t count) { return readBitsInternal(count,[&](){ uint8_t tmp[2]; const uint8_t *buf=_inputStream.consume(2,tmp); _bufContent=(uint32_t(buf[0])<<8)|uint32_t(buf[1]); _bufLength=16; }); } uint32_t readBitsBE32(uint32_t count) { return readBitsInternal(count,[&](){ uint8_t tmp[4]; const uint8_t *buf=_inputStream.consume(4,tmp); _bufContent=(uint32_t(buf[0])<<24)|(uint32_t(buf[1])<<16)| (uint32_t(buf[2])<<8)|uint32_t(buf[3]); _bufLength=32; }); } // RNC uint32_t readBits16Limit(uint32_t count) { return readBitsInternal(count,[&](){ _bufContent=_inputStream.readByte(); if (_inputStream.eof()) { _bufLength=8; } else { _bufContent=_bufContent|(uint32_t(_inputStream.readByte())<<8); _bufLength=16; } }); } void reset(uint32_t bufContent=0,uint8_t bufLength=0) { _bufContent=bufContent; _bufLength=bufLength; } private: template uint32_t readBitsInternal(uint32_t count,F readWord) { uint32_t ret=0,pos=0; while (count) { if (!_bufLength) readWord(); uint8_t maxCount=std::min(uint8_t(count),_bufLength); ret|=(_bufContent&((1<>=maxCount; _bufLength-=maxCount; count-=maxCount; pos+=maxCount; } return ret; } T &_inputStream; uint32_t _bufContent=0; uint8_t _bufLength=0; }; template class MSBBitReader { public: MSBBitReader(T &inputStream) : _inputStream(inputStream) { // nothing needed } ~MSBBitReader() { // nothing needed } uint32_t readBits8(uint32_t count) { return readBitsInternal(count,[&](){ _bufContent=_inputStream.readByte(); _bufLength=8; }); } uint32_t readBitsBE16(uint32_t count) { return readBitsInternal(count,[&](){ uint8_t tmp[2]; const uint8_t *buf=_inputStream.consume(2,tmp); _bufContent=(uint32_t(buf[0])<<8)|uint32_t(buf[1]); _bufLength=16; }); } uint32_t readBitsBE32(uint32_t count) { return readBitsInternal(count,[&](){ uint8_t tmp[4]; const uint8_t *buf=_inputStream.consume(4,tmp); _bufContent=(uint32_t(buf[0])<<24)|(uint32_t(buf[1])<<16)| (uint32_t(buf[2])<<8)|uint32_t(buf[3]); _bufLength=32; }); } void reset(uint32_t bufContent=0,uint8_t bufLength=0) { _bufContent=bufContent; _bufLength=bufLength; } private: template uint32_t readBitsInternal(uint32_t count,F readWord) { uint32_t ret=0; while (count) { if (!_bufLength) readWord(); uint8_t maxCount=std::min(uint8_t(count),_bufLength); _bufLength-=maxCount; ret=(ret<>_bufLength)&((1< LHLBDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } LHLBDecompressor::LHLBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } LHLBDecompressor::~LHLBDecompressor() { // nothing needed } const std::string &LHLBDecompressor::getSubName() const noexcept { static std::string name="XPK-LHLB: LZRW-compressor"; return name; } void LHLBDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); // Same logic as in Choloks pascal implementation // Differences to LH1: // - LHLB does not halve probabilities at 32k // - 314 vs. 317 sized huffman entry // - no end code // - different distance/count logic DynamicHuffmanDecoder<317> decoder; while (!outputStream.eof()) { uint32_t code=decoder.decode(readBit); if (code==316) break; if (decoder.getMaxFrequency()<0x8000U) decoder.update(code); if (code<256) { outputStream.writeByte(code); } else { static const uint8_t distanceHighBits[256]={ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11, 12,12,12,12,13,13,13,13, 14,14,14,14,15,15,15,15, 16,16,16,16,17,17,17,17, 18,18,18,18,19,19,19,19, 20,20,20,20,21,21,21,21, 22,22,22,22,23,23,23,23, 24,24,25,25,26,26,27,27, 28,28,29,29,30,30,31,31, 32,32,33,33,34,34,35,35, 36,36,37,37,38,38,39,39, 40,40,41,41,42,42,43,43, 44,44,45,45,46,46,47,47, 48,49,50,51,52,53,54,55, 56,57,58,59,60,61,62,63}; static const uint8_t distanceBits[16]={1,1,2,2,2,3,3,3,3,4,4,4,5,5,5,6}; uint32_t tmp=readBits(8); uint32_t distance=uint32_t(distanceHighBits[tmp])<<6; uint32_t bits=distanceBits[tmp>>4]; tmp=(tmp< LHLBDecompressor::_XPKregistration; ancient-1.0/src/LHLBDecompressor.hpp000066400000000000000000000015541375530757000174230ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LHLBDECOMPRESSOR_HPP #define LHLBDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class LHLBDecompressor : public XPKDecompressor { public: LHLBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~LHLBDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/LIN1Decompressor.cpp000066400000000000000000000056311375530757000174000ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LIN1Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool LIN1Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LIN1") || hdr==FourCC("LIN3"); } std::unique_ptr LIN1Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } LIN1Decompressor::LIN1Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); _ver=(hdr==FourCC("LIN1"))?1:3; if (packedData.size()<5) throw Decompressor::InvalidFormatError(); uint32_t tmp=packedData.readBE32(0); if (tmp) throw Decompressor::InvalidFormatError(); // password set } LIN1Decompressor::~LIN1Decompressor() { // nothing needed } const std::string &LIN1Decompressor::getSubName() const noexcept { static std::string name1="XPK-LIN1: LIN1 LINO packer"; static std::string name3="XPK-LIN3: LIN3 LINO packer"; return (_ver==1)?name1:name3; } void LIN1Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,5,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; size_t rawSize=rawData.size(); ForwardOutputStream outputStream(rawData,0,rawSize); while (!outputStream.eof()) { if (!readBits(1)) { outputStream.writeByte(readByte()^0x55); } else { uint32_t count=3; if (readBits(1)) { count=readBits(2); if (count==3) { count=readBits(3); if (count==7) { count=readBits(4); if (count==15) { count=readByte(); if (count==0xff) throw Decompressor::DecompressionError(); count+=3; } else count+=14; } else count+=7; } else count+=4; } uint32_t distance = 0; switch (readBits(2)) { case 0: distance=readByte()+1; break; case 1: distance=uint32_t(readBits(2))<<8; distance|=readByte(); distance+=0x101; break; case 2: distance=uint32_t(readBits(4))<<8; distance|=readByte(); distance+=0x501; break; case 3: distance=uint32_t(readBits(6))<<8; distance|=readByte(); distance+=0x1501; break; } // buggy compressors count=std::min(count,uint32_t(rawSize-outputStream.getOffset())); if (!count) throw Decompressor::DecompressionError(); outputStream.copy(distance,count); } } } XPKDecompressor::Registry LIN1Decompressor::_XPKregistration; ancient-1.0/src/LIN1Decompressor.hpp000066400000000000000000000015771375530757000174120ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LIN1DECOMPRESSOR_HPP #define LIN1DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class LIN1Decompressor : public XPKDecompressor { public: LIN1Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~LIN1Decompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _ver=0; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/LIN2Decompressor.cpp000066400000000000000000000151341375530757000174000ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LIN2Decompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool LIN2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LIN2") || hdr==FourCC("LIN4"); } std::unique_ptr LIN2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } LIN2Decompressor::LIN2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); _ver=(hdr==FourCC("LIN2"))?2:4; if (packedData.size()<10) throw Decompressor::InvalidFormatError(); uint32_t tmp=packedData.readBE32(0); if (tmp) throw Decompressor::InvalidFormatError(); // password set // LIN4 is very similar to LIN2 - it only has 5 bit literals instead of 4 bit literals // (and thus larger table at the end of the stream) // Also, the huffman decoder for length is different _endStreamOffset=packedData.size()-1; const uint8_t *bufPtr=_packedData.data(); while (_endStreamOffset && bufPtr[--_endStreamOffset]!=0xffU); // end stream // 1 byte, byte before 0xff // 0x10 bytes/0x20 for table if (_endStreamOffset<0x11+0xa) throw Decompressor::InvalidFormatError(); _endStreamOffset-=(_ver==2)?0x11:0x21; size_t midStreamOffset=(_ver==2)?0x16:0x26; // midstream // from endstream without // add 0x10/0x20 byte back to point after table // add 6 bytes to point to correct place tmp=packedData.readBE32(4); if (size_t(_endStreamOffset)+size_t(midStreamOffset) bitsReader(forwardInputStream); MSBBitReader bitReader(middleInputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitsReader.readBits8(count); }; { uint8_t tmp=middleInputStream.readByte(); if (tmp>8) throw Decompressor::DecompressionError(); bitReader.reset(middleInputStream.readByte()>>tmp,8-tmp); } auto readBit=[&]()->uint8_t { return bitReader.readBits8(1); }; bool buf4Incomplete=false; uint8_t nibbleContent=0; { uint8_t tmp=_packedData.read8(9); buf4Incomplete=!!tmp; if (buf4Incomplete) nibbleContent=backwardInputStream.readByte(); } // this is a rather strange thing... auto read4Bits=[&](bool fullByte)->uint8_t { if (!fullByte) { buf4Incomplete=!buf4Incomplete; if (!buf4Incomplete) { return nibbleContent&0xf; } else { nibbleContent=backwardInputStream.readByte(); return nibbleContent>>4; } } else { if (buf4Incomplete) { uint8_t ret=nibbleContent&0xf; nibbleContent=backwardInputStream.readByte(); ret|=nibbleContent&0xf0U; return ret; } else { return backwardInputStream.readByte(); } } }; const uint8_t *literalTable=&_packedData[_endStreamOffset]; size_t rawSize=rawData.size(); ForwardOutputStream outputStream(rawData,0,rawSize); // little meh to initialize both (intentionally deleted copy/assign) HuffmanDecoder lengthDecoder2 { HuffmanCode{1,0b000000,3}, HuffmanCode{3,0b000100,4}, HuffmanCode{3,0b000101,5}, HuffmanCode{3,0b000110,6}, HuffmanCode{6,0b111000,7}, HuffmanCode{6,0b111001,8}, HuffmanCode{6,0b111010,9}, HuffmanCode{6,0b111011,10}, HuffmanCode{6,0b111100,11}, HuffmanCode{6,0b111101,12}, HuffmanCode{6,0b111110,13}, HuffmanCode{6,0b111111,0} }; HuffmanDecoder lengthDecoder4 { HuffmanCode{2,0b0000000,3}, HuffmanCode{2,0b0000001,4}, HuffmanCode{2,0b0000010,5}, HuffmanCode{4,0b0001100,6}, HuffmanCode{4,0b0001101,7}, HuffmanCode{4,0b0001110,8}, HuffmanCode{7,0b1111000,9}, HuffmanCode{7,0b1111001,10}, HuffmanCode{7,0b1111010,11}, HuffmanCode{7,0b1111011,12}, HuffmanCode{7,0b1111100,13}, HuffmanCode{7,0b1111101,14}, HuffmanCode{7,0b1111110,15}, HuffmanCode{7,0b1111111,0} }; auto &lengthDecoder=(_ver==2)?lengthDecoder2:lengthDecoder4; uint32_t minBits=1; while (!outputStream.eof()) { if (!readBits(1)) { if (readBit()) { outputStream.writeByte(read4Bits(true)); } else { if (_ver==4) { outputStream.writeByte(literalTable[(read4Bits(false)<<1)+readBit()]); } else outputStream.writeByte(literalTable[read4Bits(false)]); } } else { uint32_t count=lengthDecoder.decode([&](){return readBits(1);}); if (!count) { count=readBits(4); if (count==0xfU) { count=readBits(8); if (count==0xffU) throw Decompressor::DecompressionError(); else count+=3; } else count+=(_ver==2)?14:16; } uint32_t distance; bool isMax=false; do { uint32_t bits=readBits(3)+minBits; distance=readBits(bits); isMax=(distance==((1U< LIN2Decompressor::_XPKregistration; ancient-1.0/src/LIN2Decompressor.hpp000066400000000000000000000016711375530757000174060ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LIN2DECOMPRESSOR_HPP #define LIN2DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class LIN2Decompressor : public XPKDecompressor { public: LIN2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~LIN2Decompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _ver=0; size_t _endStreamOffset=0; size_t _midStreamOffset=0; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/LZBSDecompressor.cpp000066400000000000000000000037701375530757000174510ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZBSDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool LZBSDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LZBS"); } std::unique_ptr LZBSDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } LZBSDecompressor::LZBSDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || _packedData.size()<1) throw Decompressor::InvalidFormatError(); } LZBSDecompressor::~LZBSDecompressor() { // nothing needed } const std::string &LZBSDecompressor::getSubName() const noexcept { static std::string name="XPK-LZBS: LZBS CyberYAFA compressor"; return name; } void LZBSDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,1,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return rotateBits(bitReader.readBits8(count),count); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); uint32_t bits=0,maxBits=uint32_t(_packedData[0]); while (!outputStream.eof()) { if (!readBits(1)) { outputStream.writeByte(readBits(8)); } else { uint32_t count=readBits(8)+2; if (count==2) { count=readBits(12); if (!count) throw Decompressor::DecompressionError(); for (uint32_t i=0;i=(1U< LZBSDecompressor::_XPKregistration; ancient-1.0/src/LZBSDecompressor.hpp000066400000000000000000000015541375530757000174540ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZBSDECOMPRESSOR_HPP #define LZBSDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class LZBSDecompressor : public XPKDecompressor { public: LZBSDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~LZBSDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/LZCBDecompressor.cpp000066400000000000000000000161061375530757000174260ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "LZCBDecompressor.hpp" #include "RangeDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" template class FrequencyTree { public: FrequencyTree() { for (uint32_t i=0;i<_size;i++) _tree[i]=0; } ~FrequencyTree() { // nothing needed } uint16_t decode(uint16_t value,uint16_t &low,uint16_t &freq) const { if (value>=_tree[_size-1]) throw Decompressor::DecompressionError(); uint16_t symbol=0; low=0; for (uint32_t i=_levels-2;;i--) { uint16_t tmp=_tree[_levelOffsets[i]+symbol]; if (uint32_t(symbol+1)<_levelSizes[i] && value>=tmp) { symbol++; low+=tmp; value-=tmp; } if (!i) break; symbol<<=1; } freq=_tree[symbol]; return symbol; } bool exists(uint16_t symbol) const { return _tree[symbol]; } void increment(uint16_t symbol) { for (uint16_t i=0;i<_levels;i++) { _tree[_levelOffsets[i]+symbol]++; symbol>>=1; } } void halve() { // non-standard way for (uint32_t i=0;i>=1; for (uint32_t i=T;i<_size;i++) _tree[i]=0; for (uint32_t i=0,length=T;i<_levels-1;i++,length=(length+1)>>1) { for (uint32_t j=0;j>1)]+=_tree[_levelOffsets[i]+j]; } } uint32_t getTotal() const { return _tree[_size-1]; } private: static constexpr uint32_t levelSize(uint32_t level) { uint32_t ret=T; for (uint32_t i=0;i>1; } return ret; } static constexpr uint32_t levels() { uint32_t ret=0; while (levelSize(ret)!=1) ret++; return ret+1; } static constexpr uint32_t size() { uint32_t ret=0; for (uint32_t i=0;i static constexpr auto makeLevelOffsetSequence(std::integer_sequence) { return std::integer_sequence{}; } template static constexpr auto makeLevelSizeSequence(std::integer_sequence) { return std::integer_sequence{}; } template static constexpr std::array makeArray(std::integer_sequence) { return std::array{{I...}}; } static constexpr uint32_t _size=size(); static constexpr uint32_t _levels=levels(); static constexpr std::array _levelOffsets=makeArray(makeLevelOffsetSequence(std::make_integer_sequence{})); static constexpr std::array _levelSizes=makeArray(makeLevelSizeSequence(std::make_integer_sequence{})); uint16_t _tree[size()]; }; template class FrequencyDecoder { public: FrequencyDecoder(RangeDecoder &decoder) : _decoder(decoder) { // nothing needed } ~FrequencyDecoder() { // nothing needed } template uint16_t decode(F readFunc) { uint16_t freq=0,symbol,value=_decoder.decode(_threshold+_tree.getTotal()); if (value>=_threshold) { uint16_t low; symbol=_tree.decode(value-_threshold,low,freq); _decoder.scale(_threshold+low,_threshold+low+freq,_threshold+_tree.getTotal()); if (freq==1 && _threshold>1) _threshold--; } else { _decoder.scale(0,_threshold,_threshold+_tree.getTotal()); symbol=readFunc(); // A bug in the encoder if (!symbol && _tree.exists(symbol)) symbol=T; _threshold++; } _tree.increment(symbol); if (_threshold+_tree.getTotal()>=0x3ffdU) { _tree.halve(); _threshold=(_threshold>>1)+1; } return symbol; } private: RangeDecoder &_decoder; FrequencyTree _tree; uint16_t _threshold=1; }; bool LZCBDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LZCB"); } std::unique_ptr LZCBDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } LZCBDecompressor::LZCBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (packedData.size()<2) throw Decompressor::InvalidFormatError(); } LZCBDecompressor::~LZCBDecompressor() { // nothing needed } const std::string &LZCBDecompressor::getSubName() const noexcept { static std::string name="XPK-LZCB: LZ-compressor"; return name; } void LZCBDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { class BitReader : public RangeDecoder::BitReader { public: BitReader(ForwardInputStream &stream) : _reader(stream) { // nothing needed } virtual ~BitReader() { // nothing needed } virtual uint32_t readBit() override final { return _reader.readBitsBE32(1); } uint32_t readBits(uint32_t bitCount) { return _reader.readBitsBE32(bitCount); } private: MSBBitReader _reader; }; ForwardInputStream inputStream(_packedData,0,_packedData.size(),true); ForwardOutputStream outputStream(rawData,0,rawData.size()); BitReader bitReader(inputStream); RangeDecoder rangeDecoder(bitReader,bitReader.readBits(16)); // Ugly duplicates auto readByte=[&]()->uint16_t { uint16_t ret=rangeDecoder.decode(0x100U); rangeDecoder.scale(ret,ret+1,0x100U); return ret; }; auto readCount=[&]()->uint16_t { uint16_t ret=rangeDecoder.decode(0x101U); rangeDecoder.scale(ret,ret+1,0x101U); return ret; }; FrequencyDecoder<256> baseLiteralDecoder(rangeDecoder); FrequencyDecoder<257> repeatCountDecoder(rangeDecoder); FrequencyDecoder<257> literalCountDecoder(rangeDecoder); FrequencyDecoder<256> distanceDecoder(rangeDecoder); std::unique_ptr> literalDecoders[256]; uint8_t ch=baseLiteralDecoder.decode(readByte); outputStream.writeByte(ch); bool lastIsLiteral=true; while (!outputStream.eof()) { uint32_t count=repeatCountDecoder.decode(readCount); if (count) { if (count==0x100U) { uint32_t tmp; do { tmp=readByte(); count+=tmp; } while (tmp==0xffU); } count+=lastIsLiteral?5:4; uint32_t distance=distanceDecoder.decode(readByte)<<8; distance|=readByte(); ch=outputStream.copy(distance,count); lastIsLiteral=false; } else { uint16_t literalCount; do { literalCount=literalCountDecoder.decode(readCount); if (!literalCount) throw Decompressor::DecompressionError(); for (uint32_t i=0;i>(rangeDecoder); ch=literalDecoder->decode([&]() { return baseLiteralDecoder.decode(readByte); }); outputStream.writeByte(ch); } } while (literalCount==0x100U); lastIsLiteral=true; } } } XPKDecompressor::Registry LZCBDecompressor::_XPKregistration; ancient-1.0/src/LZCBDecompressor.hpp000066400000000000000000000015541375530757000174340ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZCBDECOMPRESSOR_HPP #define LZCBDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class LZCBDecompressor : public XPKDecompressor { public: LZCBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~LZCBDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/LZW2Decompressor.cpp000066400000000000000000000037251375530757000174350ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZW2Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool LZW2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LZW2") || hdr==FourCC("LZW3"); } std::unique_ptr LZW2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } LZW2Decompressor::LZW2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); _ver=(hdr==FourCC("LZW2"))?2:3; } LZW2Decompressor::~LZW2Decompressor() { // nothing needed } const std::string &LZW2Decompressor::getSubName() const noexcept { static std::string name2="XPK-LZW2: LZW2 CyberYAFA compressor"; static std::string name3="XPK-LZW3: LZW3 CyberYAFA compressor"; return (_ver==2)?name2:name3; } void LZW2Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); LSBBitReader bitReader(inputStream); auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint32_t distance=uint32_t(readByte())<<8; distance|=uint32_t(readByte()); if (!distance) throw Decompressor::DecompressionError(); distance=65536-distance; uint32_t count=uint32_t(readByte())+4; outputStream.copy(distance,count); } } } XPKDecompressor::Registry LZW2Decompressor::_XPKregistration; ancient-1.0/src/LZW2Decompressor.hpp000066400000000000000000000015771375530757000174450ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZW2DECOMPRESSOR_HPP #define LZW2DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class LZW2Decompressor : public XPKDecompressor { public: LZW2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~LZW2Decompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _ver=0; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/LZW4Decompressor.cpp000066400000000000000000000035121375530757000174310ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZW4Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool LZW4Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LZW4"); } std::unique_ptr LZW4Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } LZW4Decompressor::LZW4Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } LZW4Decompressor::~LZW4Decompressor() { // nothing needed } const std::string &LZW4Decompressor::getSubName() const noexcept { static std::string name="XPK-LZW4: LZW4 CyberYAFA compressor"; return name; } void LZW4Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint32_t distance=uint32_t(readByte())<<8; distance|=uint32_t(readByte()); if (!distance) throw Decompressor::DecompressionError(); distance=65536-distance; uint32_t count=uint32_t(readByte())+3; outputStream.copy(distance,count); } } } XPKDecompressor::Registry LZW4Decompressor::_XPKregistration; ancient-1.0/src/LZW4Decompressor.hpp000066400000000000000000000015541375530757000174420ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZW4DECOMPRESSOR_HPP #define LZW4DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class LZW4Decompressor : public XPKDecompressor { public: LZW4Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~LZW4Decompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/LZW5Decompressor.cpp000066400000000000000000000044361375530757000174400ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZW5Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool LZW5Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LZW5"); } std::unique_ptr LZW5Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } LZW5Decompressor::LZW5Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } LZW5Decompressor::~LZW5Decompressor() { // nothing needed } const std::string &LZW5Decompressor::getSubName() const noexcept { static std::string name="XPK-LZW5: LZW5 CyberYAFA compressor"; return name; } void LZW5Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto read2Bits=[&]()->uint32_t { return bitReader.readBitsBE32(2); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { uint32_t distance,count; auto readld=[&]()->uint32_t { uint32_t ret=uint32_t(readByte())<<8; ret|=uint32_t(readByte()); if (!ret) throw Decompressor::DecompressionError(); return ret; }; switch (read2Bits()) { case 0: outputStream.writeByte(readByte()); break; case 1: distance=readld(); count=(distance&3)+2; distance=0x4000-(distance>>2); outputStream.copy(distance,count); break; case 2: distance=readld(); count=(distance&15)+2; distance=0x1000-(distance>>4); outputStream.copy(distance,count); break; case 3: distance=readld(); count=uint32_t(readByte())+3; distance=0x10000-distance; outputStream.copy(distance,count); break; default: throw Decompressor::DecompressionError(); } } } XPKDecompressor::Registry LZW5Decompressor::_XPKregistration; ancient-1.0/src/LZW5Decompressor.hpp000066400000000000000000000015541375530757000174430ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZW5DECOMPRESSOR_HPP #define LZW5DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class LZW5Decompressor : public XPKDecompressor { public: LZW5Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~LZW5Decompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/LZXDecompressor.cpp000066400000000000000000000155701375530757000173550ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "LZXDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "DLTADecode.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/CRC32.hpp" bool LZXDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("ELZX") || hdr==FourCC("SLZX"); } std::unique_ptr LZXDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } LZXDecompressor::LZXDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); if (hdr==FourCC("SLZX")) _isSampled=true; // There is no good spec on the LZX header content -> lots of unknowns here if (_packedData.size()<41) throw Decompressor::InvalidFormatError(); // XPK LZX compression is embedded single file of LZX -> read first file. Ignore rest // this will include flags, which need to be zero anyway uint32_t streamHdr=_packedData.readBE32(0); if (streamHdr!=FourCC("LZX\0")) throw Decompressor::InvalidFormatError(); _rawSize=_packedData.readLE32(12); _packedSize=_packedData.readLE32(16); _rawCRC=_packedData.readLE32(32); uint32_t headerCRC=_packedData.readLE32(36); uint8_t tmp=_packedData.read8(21); if (tmp && tmp!=2) throw Decompressor::InvalidFormatError(); if (tmp==2) _isCompressed=true; _packedOffset=41+size_t(_packedData.read8(40)); _packedOffset+=size_t(_packedData.read8(24)); _packedSize+=_packedOffset; if (_packedSize>_packedData.size()) throw Decompressor::InvalidFormatError(); if (verify) { uint32_t crc=CRC32(_packedData,10,26,0); for (uint32_t i=0;i<4;i++) crc=CRC32Byte(0,crc); crc=CRC32(_packedData,40,_packedOffset-40,crc); if (crc!=headerCRC) throw Decompressor::VerificationError(); } } LZXDecompressor::~LZXDecompressor() { // nothing needed } const std::string &LZXDecompressor::getSubName() const noexcept { static std::string nameE="XPK-ELZX: LZX-compressor"; static std::string nameS="XPK-SLZX: LZX-compressor with delta encoding"; return (_isSampled)?nameS:nameE; } void LZXDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); if (!_isCompressed) { if (_packedSize!=_rawSize) throw Decompressor::DecompressionError(); std::memcpy(rawData.data(),_packedData.data()+_packedOffset,_rawSize); return; } ForwardInputStream inputStream(_packedData,_packedOffset,_packedSize); LSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBitsBE16(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); typedef HuffmanDecoder LZXDecoder; // possibly padded/reused later if multiple blocks uint8_t literalTable[768]; for (uint32_t i=0;i<768;i++) literalTable[i]=0; LZXDecoder literalDecoder; uint32_t previousDistance=1; while (!outputStream.eof()) { auto createHuffmanTable=[&](LZXDecoder &dec,const uint8_t *bitLengths,uint32_t bitTableLength) { uint8_t minDepth=16,maxDepth=0; for (uint32_t i=0;imaxDepth) maxDepth=bitLengths[i]; } if (!maxDepth) return; dec.createOrderlyHuffmanTable(bitLengths,bitTableLength); }; uint32_t method=readBits(3); if (method<1 || method>3) throw Decompressor::DecompressionError(); LZXDecoder distanceDecoder; if (method==3) { uint8_t bitLengths[8]; for (uint32_t i=0;i<8;i++) bitLengths[i]=readBits(3); createHuffmanTable(distanceDecoder,bitLengths,8); } size_t blockLength=readBits(8)<<16; blockLength|=readBits(8)<<8; blockLength|=readBits(8); if (size_t(blockLength)+outputStream.getOffset()>size_t(_rawSize)) throw Decompressor::DecompressionError(); if (method!=1) { literalDecoder.reset(); for (uint32_t pos=0,block=0;block<2;block++) { uint32_t adjust=(block)?0:1; uint32_t maxPos=(block)?768:256; LZXDecoder bitLengthDecoder; { uint8_t lengthTable[20]; for (uint32_t i=0;i<20;i++) lengthTable[i]=readBits(4); createHuffmanTable(bitLengthDecoder,lengthTable,20); } while (posmaxPos-pos) count=maxPos-pos; while (count--) literalTable[pos++]=value; }; auto symDecode=[&](uint32_t value)->uint32_t { return (literalTable[pos]+17-value)%17; }; switch (symbol) { case 17: doRepeat(readBits(4)+3+adjust,0); break; case 18: doRepeat(readBits(6-adjust)+19+adjust,0); break; case 19: { uint32_t count=readBit()+3+adjust; doRepeat(count,symDecode(bitLengthDecoder.decode(readBit))); } break; default: literalTable[pos++]=symDecode(symbol); break; } } } createHuffmanTable(literalDecoder,literalTable,768); } while (blockLength) { uint32_t symbol=literalDecoder.decode(readBit); if (symbol<256) { outputStream.writeByte(symbol); blockLength--; } else { // both of these tables are almost too regular to be tables... static const uint8_t ldBits[32]={ 0,0,0,0,1,1,2,2, 3,3,4,4,5,5,6,6, 7,7,8,8,9,9,10,10, 11,11,12,12,13,13,14,14}; static const uint32_t ldAdditions[32]={ 0x0, 0x1, 0x2, 0x3, 0x4, 0x6, 0x8, 0xc, 0x10, 0x18, 0x20, 0x30, 0x40, 0x60, 0x80, 0xc0, 0x100, 0x180, 0x200, 0x300, 0x400, 0x600, 0x800, 0xc00,0x1000, 0x1800,0x2000,0x3000,0x4000,0x6000,0x8000,0xc000}; symbol-=256; uint32_t bits=ldBits[symbol&0x1f]; uint32_t distance=ldAdditions[symbol&0x1f]; if (bits>=3 && method==3) { distance+=readBits(bits-3)<<3; uint32_t tmp=distanceDecoder.decode(readBit); distance+=tmp; } else { distance+=readBits(bits); if (!distance) distance=previousDistance; } previousDistance=distance; uint32_t count=ldAdditions[symbol>>5]+readBits(ldBits[symbol>>5])+3; if (count>blockLength) throw Decompressor::DecompressionError(); outputStream.copy(distance,count); blockLength-=count; } } } if (verify) { uint32_t crc=CRC32(rawData,0,_rawSize,0); if (crc!=_rawCRC) throw Decompressor::VerificationError(); } if (_isSampled) DLTADecode::decode(rawData,rawData,0,_rawSize); } XPKDecompressor::Registry LZXDecompressor::_XPKregistration; ancient-1.0/src/LZXDecompressor.hpp000066400000000000000000000017671375530757000173650ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZXDECOMPRESSOR_HPP #define LZXDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class LZXDecompressor : public XPKDecompressor { public: LZXDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~LZXDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; bool _isSampled=false; bool _isCompressed=false; size_t _packedSize=0; size_t _packedOffset=0; size_t _rawSize=0; uint32_t _rawCRC=0; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/Lzh/000077500000000000000000000000001375530757000143335ustar00rootroot00000000000000ancient-1.0/src/Lzh/LH1Decompressor.cpp000066400000000000000000000034211375530757000200110ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LH1Decompressor.hpp" #include "../HuffmanDecoder.hpp" #include "../DynamicHuffmanDecoder.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" LH1Decompressor::LH1Decompressor(const Buffer &packedData) : _packedData(packedData) { // nothing needed } LH1Decompressor::~LH1Decompressor() { // nothing needed } size_t LH1Decompressor::getRawSize() const noexcept { // N/A return 0; } size_t LH1Decompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &LH1Decompressor::getName() const noexcept { static std::string name="LHA: LH1"; return name; } void LH1Decompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); DynamicHuffmanDecoder<314> decoder; static const uint8_t distanceHighBits[64]={ 3,4,4,4,5,5,5,5, 5,5,5,5,6,6,6,6, 6,6,6,6,6,6,6,6, 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8 }; HuffmanDecoder distanceDecoder; distanceDecoder.createOrderlyHuffmanTable(distanceHighBits,64); while (!outputStream.eof()) { uint32_t code=decoder.decode(readBit); if (decoder.getMaxFrequency()==0x8000U) decoder.halve(); decoder.update(code); if (code<256) { outputStream.writeByte(code); } else { uint32_t distance=distanceDecoder.decode(readBit); distance=(distance<<6)|readBits(6); distance++; uint32_t count=code-253; outputStream.copy(distance,count,0x20); } } } ancient-1.0/src/Lzh/LH1Decompressor.hpp000066400000000000000000000010711375530757000200150ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LH1DECOMPRESSOR_HPP #define LH1DECOMPRESSOR_HPP #include "../Decompressor.hpp" class LH1Decompressor : public Decompressor { public: LH1Decompressor(const Buffer &packedData); virtual ~LH1Decompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; }; #endif ancient-1.0/src/Lzh/LH2Decompressor.cpp000066400000000000000000000060241375530757000200140ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LH2Decompressor.hpp" #include "../HuffmanDecoder.hpp" #include "../DynamicHuffmanDecoder.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" LH2Decompressor::LH2Decompressor(const Buffer &packedData) : _packedData(packedData) { // nothing needed } LH2Decompressor::~LH2Decompressor() { // nothing needed } size_t LH2Decompressor::getRawSize() const noexcept { // N/A return 0; } size_t LH2Decompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &LH2Decompressor::getName() const noexcept { static std::string name="LHA: LH2"; return name; } // This is probably fishiest of the fishy formats I've worked with // Basically it uses Dynamic Huffman decoder but instead of static // size, it is dynamically growing. However, that part is not well // defined. Thus // a. It is very hard to use LH2 using generic implementation // instead of the specific one used by LHA // b. There are bugs in encoder which need to be baked in the decoder // as well (Probably there is lots of unneccesary stuff in addCode, // but better safe than sorry) // c. LH 1.9x and UNLHA32 refuse to use LH2 beyond 8k files. Thus // we can only guess if the wraparound is correct, since nothing // should use it // jLHA in theory supports LH2 without limitations, but it produces // broken bitstream. // // So far this works with the files I've tried from "official" LHA void LH2Decompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); DynamicHuffmanDecoder<286> valueDecoder; DynamicHuffmanDecoder<128> distanceDecoder(0); uint32_t distancePosition=0; uint32_t distCount=0; while (!outputStream.eof()) { uint32_t code=valueDecoder.decode(readBit); if (valueDecoder.getMaxFrequency()==0x8000U) valueDecoder.halve(); valueDecoder.update(code); if (code<256) { outputStream.writeByte(code); } else { if (code==285) code+=readBits(8); uint32_t count=code-253; // Bug in LH2 where count does not match frequency. // Makes things more complicated auto updateDist=[&](uint32_t code) { if (distCount==0x8000U) { distanceDecoder.halve(); distCount=distanceDecoder.getMaxFrequency(); } distanceDecoder.update(code); distCount++; }; uint32_t maxDist=std::min(uint32_t((outputStream.getOffset()+63)>>6),128U); if (distancePosition!=maxDist) { for (uint32_t i=distancePosition;i bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); OptionalHuffmanDecoder decoder; static const uint8_t distanceHighBits[128]={ 2,4,4,5,5,5,6,6, 6,6,6,6,6,7,7,7, 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,9,9, 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9 }; OptionalHuffmanDecoder distanceDecoder; uint32_t blockRemaining=0; while (!outputStream.eof()) { if (!blockRemaining) { blockRemaining=readBits(16); if (!blockRemaining) blockRemaining=0x10000; // not strictly needed as a lambda, but cleaner this way auto createDecoderTable=[&]() { decoder.reset(); uint8_t symbolBits[286]; uint32_t oneCount=0; for (uint32_t i=0;i<286;i++) { if (readBit()) symbolBits[i]=readBits(4)+1; else symbolBits[i]=0; if (symbolBits[i]==1) oneCount++; if (i==2 && oneCount==3) { decoder.setEmpty(readBits(9)); return; } } decoder.createOrderlyHuffmanTable(symbolBits,286); }; // ditto auto createDistanceDecoderTable=[&]() { distanceDecoder.reset(); if (readBit()) { uint8_t symbolBits[128]; uint32_t oneCount=0; for (uint32_t i=0;i<128;i++) { symbolBits[i]=readBits(4); if (symbolBits[i]==1) oneCount++; if (i==2 && oneCount==3) { // 7 bits would be fine, but whatever distanceDecoder.setEmpty(readBits(9)); return; } } distanceDecoder.createOrderlyHuffmanTable(symbolBits,128); } else distanceDecoder.createOrderlyHuffmanTable(distanceHighBits,128); }; createDecoderTable(); createDistanceDecoderTable(); } blockRemaining--; uint32_t code=decoder.decode(readBit); if (code<256) { outputStream.writeByte(code); } else { if (code==285) code+=readBits(8); uint32_t distance=distanceDecoder.decode(readBit); distance=(distance<<6)|readBits(6); distance++; uint32_t count=code-253; outputStream.copy(distance,count,0x20); } } } ancient-1.0/src/Lzh/LH3Decompressor.hpp000066400000000000000000000010711375530757000200170ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LH3DECOMPRESSOR_HPP #define LH3DECOMPRESSOR_HPP #include "../Decompressor.hpp" class LH3Decompressor : public Decompressor { public: LH3Decompressor(const Buffer &packedData); virtual ~LH3Decompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; }; #endif ancient-1.0/src/Lzh/LHXDecompressor.cpp000066400000000000000000000111041375530757000200550ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LHXDecompressor.hpp" #include "../HuffmanDecoder.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" LHXDecompressor::LHXDecompressor(const Buffer &packedData) : _packedData(packedData), _method(0) { // nothing needed } LHXDecompressor::LHXDecompressor(const Buffer &packedData,uint32_t method) : _packedData(packedData), _method(method-3) { if (method<4 || method>8) throw InvalidFormatError(); } LHXDecompressor::~LHXDecompressor() { // nothing needed } size_t LHXDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t LHXDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &LHXDecompressor::getName() const noexcept { static std::string name="LHA: LH4, LH5, LH6, LH7, LH8, LHX"; return name; } void LHXDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); OptionalHuffmanDecoder decoder; OptionalHuffmanDecoder distanceDecoder; static const struct { uint32_t mask; uint32_t distanceTableSize; uint32_t distanceBits; } methodTable[6] = { {0x7ffffU,20,5}, // LHX {0xfffU,14,4}, // LH4 {0x1fffU,14,4}, // LH5 {0x7fffU,16,5}, // LH6 {0xffffU,17,5}, // LH7 {0xffffU,17,5} // LH8 // LH8 is the only Joe Jared method seen. // It exists in early versions of LH7-tool // In those versions LH8 is just synonym for LH7 // However, LH9+ is something I have not seen at // all (i.e. not in DLH7021Q/WLH7021Q.) // There would be one more version (0.21R) but // that seems lost. Most likely minor bump from Q->R // do not enable those formats either and the LH9+ // formats are nothing but hopes for "future work" // that never materialized (until I'm proven wrong ofc) }; uint32_t blockRemaining=0; while (!outputStream.eof()) { if (!blockRemaining) { blockRemaining=readBits(16); if (!blockRemaining) blockRemaining=0x10000; auto createTable=[&](OptionalHuffmanDecoder &dest,uint32_t count,uint32_t bits,bool enableHole) { uint8_t symbolBits[20]; uint32_t length=readBits(bits); if (!length) { dest.setEmpty(readBits(bits)); } else if (length<=count) { for (uint32_t i=0;i32) throw DecompressionError(); symbolBits[i++]=value; if (i==3 && enableHole) { uint32_t zeros=readBits(2); if (i+zeros>length) throw DecompressionError(); for (uint32_t j=0;j tmpDecoder; createTable(tmpDecoder,19,5,true); decoder.reset(); uint8_t symbolBits[511]; uint32_t length=readBits(9); if (!length) { decoder.setEmpty(readBits(9)); } else if (length<=511) { for (uint32_t i=0;ilength) throw DecompressionError(); for (uint32_t j=0;j bitReader(inputStream); auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint32_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); StaticBuffer<4096> prevBuffer; { uint8_t *bufPtr=prevBuffer.data(); for (uint32_t i=0;i<18;i++) *(bufPtr++)=0; for (uint32_t i=0;i<256;i++) for (uint32_t j=0;j<13;j++) *(bufPtr++)=i; for (uint32_t i=0;i<256;i++) *(bufPtr++)=i; for (uint32_t i=0;i<256;i++) *(bufPtr++)=255-i; for (uint32_t i=0;i<128;i++) *(bufPtr++)=0; for (uint32_t i=0;i<110;i++) *(bufPtr++)=' '; } while (!outputStream.eof()) { if (readBit()) { outputStream.writeByte(readByte()); } else { uint32_t byte1=readByte(); uint32_t byte2=readByte(); uint32_t distance=((outputStream.getOffset()-byte1-((byte2&0xf0U)<<4)-19)&0xfffU)+1; uint32_t count=(byte2&0xfU)+3; outputStream.copy(distance,count,prevBuffer); } } } ancient-1.0/src/Lzh/LZ5Decompressor.hpp000066400000000000000000000010711375530757000200430ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZ5DECOMPRESSOR_HPP #define LZ5DECOMPRESSOR_HPP #include "../Decompressor.hpp" class LZ5Decompressor : public Decompressor { public: LZ5Decompressor(const Buffer &packedData); virtual ~LZ5Decompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; }; #endif ancient-1.0/src/Lzh/LZHDecompressor.cpp000066400000000000000000000060061375530757000200640ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include "LZHDecompressor.hpp" #include "LH1Decompressor.hpp" #include "LH2Decompressor.hpp" #include "LH3Decompressor.hpp" #include "LHXDecompressor.hpp" #include "LZ5Decompressor.hpp" #include "LZSDecompressor.hpp" #include "PMDecompressor.hpp" LZHDecompressor::LZHDecompressor(const Buffer &packedData,const std::string &method) : _packedData(packedData), _method(method) { // nothing needed } LZHDecompressor::~LZHDecompressor() { // nothing needed } size_t LZHDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t LZHDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &LZHDecompressor::getName() const noexcept { static std::string name="Lzh"; return name; } void LZHDecompressor::decompressImpl(Buffer &rawData,bool verify) { enum class Compressor { LH0=0, LH1, LH2, LH3, LH4, LH5, LH6, LH7, LH8, LHX, LZ4, LZ5, LZS, PM0, PM1, PM2 }; static std::map compressorMap{ {"-lh0-",Compressor::LH0}, {"-lh1-",Compressor::LH1}, {"-lh2-",Compressor::LH2}, {"-lh3-",Compressor::LH3}, {"-lh4-",Compressor::LH4}, {"-lh5-",Compressor::LH5}, {"-lh6-",Compressor::LH6}, {"-lh7-",Compressor::LH7}, {"-lh8-",Compressor::LH8}, {"-lhx-",Compressor::LHX}, {"-lz4-",Compressor::LZ4}, {"-lz5-",Compressor::LZ5}, {"-lzs-",Compressor::LZS}, {"-pm0-",Compressor::PM0}, {"-pm1-",Compressor::PM1}, {"-pm2-",Compressor::PM2} }; auto it=compressorMap.find(_method); if (it==compressorMap.end()) throw DecompressionError(); switch (it->second) { case Compressor::LH0: case Compressor::LZ4: case Compressor::PM0: if (rawData.size()!=_packedData.size()) throw DecompressionError(); std::memcpy(rawData.data(),_packedData.data(),rawData.size()); break; case Compressor::LH1: { LH1Decompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::LH2: { LH2Decompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::LH3: { LH3Decompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::LH4: case Compressor::LH5: case Compressor::LH6: case Compressor::LH7: case Compressor::LH8: { LHXDecompressor dec(_packedData,static_cast(it->second)-static_cast(Compressor::LH4)+4); dec.decompress(rawData,verify); } break; case Compressor::LHX: { LHXDecompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::LZ5: { LZ5Decompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::LZS: { LZSDecompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::PM1: case Compressor::PM2: { PMDecompressor dec(_packedData,static_cast(it->second)-static_cast(Compressor::PM1)+1); dec.decompress(rawData,verify); } break; } } ancient-1.0/src/Lzh/LZHDecompressor.hpp000066400000000000000000000011521375530757000200660ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZHDECOMPRESSOR_HPP #define LZHDECOMPRESSOR_HPP #include "../Decompressor.hpp" class LZHDecompressor : public Decompressor { public: LZHDecompressor(const Buffer &packedData,const std::string &method); virtual ~LZHDecompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; std::string _method; }; #endif ancient-1.0/src/Lzh/LZSDecompressor.cpp000066400000000000000000000023441375530757000201000ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZSDecompressor.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" LZSDecompressor::LZSDecompressor(const Buffer &packedData) : _packedData(packedData) { // nothing needed } LZSDecompressor::~LZSDecompressor() { // nothing needed } size_t LZSDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t LZSDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &LZSDecompressor::getName() const noexcept { static std::string name="LHA: LZS"; return name; } void LZSDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { if (readBit()) { outputStream.writeByte(readBits(8)); } else { uint32_t distance=((outputStream.getOffset()-readBits(11)-18)&0x7ffU)+1; uint32_t count=readBits(4)+2; outputStream.copy(distance,count,0x20); } } } ancient-1.0/src/Lzh/LZSDecompressor.hpp000066400000000000000000000010711375530757000201010ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZSDECOMPRESSOR_HPP #define LZSDECOMPRESSOR_HPP #include "../Decompressor.hpp" class LZSDecompressor : public Decompressor { public: LZSDecompressor(const Buffer &packedData); virtual ~LZSDecompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; }; #endif ancient-1.0/src/Lzh/PMDecompressor.cpp000066400000000000000000000263541375530757000177530ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "PMDecompressor.hpp" #include "../HuffmanDecoder.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" PMDecompressor::PMDecompressor(const Buffer &packedData,uint32_t version) : _packedData(packedData), _version(version) { if (version!=1 && version!=2) throw InvalidFormatError(); } PMDecompressor::~PMDecompressor() { // nothing needed } size_t PMDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t PMDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &PMDecompressor::getName() const noexcept { static std::string name="LHA: PM1, PM2"; return name; } uint8_t PMDecompressor::decodeMTF(uint8_t value,uint8_t map[]) { return map[value]; } void PMDecompressor::updateMTF(uint8_t value,uint8_t map[]) { for (uint32_t i=0;;i++) { if (map[i]==value) { value=i; break; } } if (value) { uint8_t tmp=map[value]; for (uint32_t i=value;i;i--) map[i]=map[i-1]; map[0]=tmp; } } void PMDecompressor::createMTFMap(uint8_t map[]) { for (uint32_t i=0,j=0x20;j<0x80;i++,j++) map[i]=j; for (uint32_t i=0x60,j=0;j<0x20;i++,j++) map[i]=j; for (uint32_t i=0x80,j=0xa0;j<0xe0;i++,j++) map[i]=j; for (uint32_t i=0xc0,j=0x80;j<0xa0;i++,j++) map[i]=j; for (uint32_t i=0xe0,j=0xe0;j<0x100;i++,j++) map[i]=j; } void PMDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (_version==1) decompressImplPM1(rawData,verify); else decompressImplPM2(rawData,verify); } void PMDecompressor::decompressImplPM1(Buffer &rawData,bool verify) { static const struct { uint8_t length; uint8_t code; } treeDefinitions[32][6] { // This is madness, I had to write a special program to decode this. // Also, in the original there is a bug @ index 17, as identified by // lhasa (we fix it when creating the decoder) {{4, 0b0000},{4, 0b0001},{3, 0b001},{2, 0b01},{2, 0b10},{2, 0b11}}, {{3, 0b000},{3, 0b001},{3, 0b010},{2, 0b10},{2, 0b11},{3, 0b011}}, {{3, 0b000},{3, 0b001},{2, 0b01},{2, 0b10},{3, 0b110},{3, 0b111}}, {{2, 0b00},{3, 0b010},{3, 0b011},{2, 0b10},{3, 0b110},{3, 0b111}}, {{2, 0b00},{3, 0b010},{2, 0b10},{3, 0b011},{3, 0b110},{3, 0b111}}, {{2, 0b00},{3, 0b010},{2, 0b10},{2, 0b11},{4, 0b0110},{4, 0b0111}}, {{2, 0b00},{2, 0b01},{3, 0b100},{3, 0b101},{3, 0b110},{3, 0b111}}, {{2, 0b00},{2, 0b01},{3, 0b100},{2, 0b11},{4, 0b1010},{4, 0b1011}}, {{2, 0b00},{2, 0b01},{2, 0b10},{3, 0b110},{4, 0b1110},{4, 0b1111}}, {{1, 0b0},{4, 0b1000},{3, 0b101},{3, 0b110},{3, 0b111},{4, 0b1001}}, {{1, 0b0},{4, 0b1000},{3, 0b101},{2, 0b11},{5,0b10010},{5,0b10011}}, {{1, 0b0},{4, 0b1000},{4, 0b1001},{3, 0b101},{3, 0b110},{3, 0b111}}, {{1, 0b0},{3, 0b100},{4, 0b1010},{3, 0b110},{3, 0b111},{4, 0b1011}}, {{1, 0b0},{3, 0b100},{3, 0b101},{3, 0b110},{4, 0b1110},{4, 0b1111}}, {{1, 0b0},{3, 0b100},{2, 0b11},{4, 0b1010},{5,0b10110},{5,0b10111}}, {{1, 0b0},{2, 0b10},{4, 0b1100},{4, 0b1101},{4, 0b1110},{4, 0b1111}}, {{1, 0b0},{2, 0b10},{3, 0b110},{4, 0b1110},{5,0b11110},{5,0b11111}}, {{3, 0b000},{3, 0b001},{2, 0b01},{2, 0b10},{2, 0b11},{0, 0b0}}, {{2, 0b00},{3, 0b010},{2, 0b10},{2, 0b11},{3, 0b011},{0, 0b0}}, {{2, 0b00},{2, 0b01},{2, 0b10},{3, 0b110},{3, 0b111},{0, 0b0}}, {{1, 0b0},{4, 0b1000},{3, 0b101},{2, 0b11},{4, 0b1001},{0, 0b0}}, {{1, 0b0},{3, 0b100},{3, 0b101},{3, 0b110},{3, 0b111},{0, 0b0}}, {{1, 0b0},{3, 0b100},{2, 0b11},{4, 0b1010},{4, 0b1011},{0, 0b0}}, {{1, 0b0},{2, 0b10},{3, 0b110},{4, 0b1110},{4, 0b1111},{0, 0b0}}, {{3, 0b000},{3, 0b001},{2, 0b01},{1, 0b1},{0, 0b0},{0, 0b0}}, {{2, 0b00},{3, 0b010},{1, 0b1},{3, 0b011},{0, 0b0},{0, 0b0}}, {{2, 0b00},{2, 0b01},{2, 0b10},{2, 0b11},{0, 0b0},{0, 0b0}}, {{1, 0b0},{3, 0b100},{2, 0b11},{3, 0b101},{0, 0b0},{0, 0b0}}, {{1, 0b0},{2, 0b10},{3, 0b110},{3, 0b111},{0, 0b0},{0, 0b0}}, {{1, 0b0},{2, 0b10},{2, 0b11},{0, 0b0},{0, 0b0},{0, 0b0}}, {{1, 0b0},{1, 0b1},{0, 0b0},{0, 0b0},{0, 0b0},{0, 0b0}}, {{0, 0b0},{1, 0b1},{0, 0b0},{0, 0b0},{0, 0b0},{0, 0b0}} }; ForwardInputStream inputStream(_packedData,0,_packedData.size(),true); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; size_t rawSize=rawData.size(); ForwardOutputStream outputStream(rawData,0,rawSize); OptionalHuffmanDecoder decoder; { uint32_t treeIndex=readBits(5); for (uint32_t i=0;i<6;i++) { uint32_t length=treeDefinitions[treeIndex][i].length; uint32_t code=treeDefinitions[treeIndex][i].code; if (!length) { if (!i) decoder.setEmpty(0); break; } if (treeIndex==17 && i<2) decoder.insert(HuffmanCode{length,code,i+3}); else decoder.insert(HuffmanCode{length,code,i}); } } uint8_t dataMTFMap[256]; createMTFMap(dataMTFMap); auto processOutput=[&](uint8_t value)->uint8_t { updateMTF(value,dataMTFMap); return value; }; while (!outputStream.eof()) { bool doCopy=true; if (readBit()) { uint32_t count=readBits(2)+1; if (count==4) { count=readBits(3)+4; if (count==11) { count=readBits(4)+11; if (count==25) { count=readBits(6)+25; } else if (count==26) { count=readBits(7)+89; } } } count=std::min(count,uint32_t(rawSize-outputStream.getOffset())); for (uint32_t i=0;i=0x240?readBit():0) { code=4; if (offset<0x340) code=7; else if (offset<0x440) code=8; else if (offset<0x640) code=9; } else { code=offset>=0x40?readBit():0; count=2; } } else { if (offset>=0x40?!readBit():0) { code=3; if (offset<0x140) code=6; } else if (offset>=0xa40?readBit():1) { code=2; } else { code=5; if (offset<0xb40) code=10; else if (offset<0xc40) code=11; else if (offset<0xe40) code=12; else if (offset<0x1240) code=13; else if (offset<0x1a40) code=14; } } if (code>=2) { count=readBits(2)+3; if (count==6) { count=readBits(3)+6; if (count==11) { count=readBits(2)+11; } else if (count==12) { count=readBits(3)+15; } else if (count==13) { count=readBits(6)+23; if (count==85) { count=readBits(5)+85; } else if (count==86) count=readBits(7)+117; } } } static const uint32_t distanceAdditions[15]={ 1,0x41,1,0x41,0x241,0xa41,0x41,0x241,0x241,0x241,0xa41,0xa41,0xa41,0xa41,0xa41}; static const uint32_t distanceBits[15]={ 6,8,6,9,11,13,8,8,9,10,8,9,10,11,12}; uint32_t distance=readBits(distanceBits[code])+distanceAdditions[code]; count=std::min(count,uint32_t(rawSize-outputStream.getOffset())); outputStream.copy(distance,count,0x20); const uint8_t *block=outputStream.history(count); for (uint32_t i=0;i bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); OptionalHuffmanDecoder decoder; OptionalHuffmanDecoder distanceDecoder; auto createDecoder=[&]()->bool { decoder.reset(); // codes beyond 29 are going to fault anyway, but maybe they are not used? uint8_t symbols[31]; uint32_t numCodes=readBits(5); uint32_t minLength=readBits(3); bool ret=(numCodes>=10)&&(numCodes!=29||minLength); if (!minLength) { if (!numCodes) throw DecompressionError(); decoder.setEmpty(numCodes-1); } else { uint32_t codeLength=readBits(3); for (uint32_t i=0;iuint8_t { offset++; // we are interested offset after the update updateMTF(value,dataMTFMap); if (!(offset&0x3ffU)) { switch (offset>>10) { case 1: if (distanceTreeRequired) createDistanceDecoder(6); break; case 2: if (distanceTreeRequired) createDistanceDecoder(7); break; case 4: if (readBit()) distanceTreeRequired=createDecoder(); if (distanceTreeRequired) createDistanceDecoder(8); break; default: if (!(offset&0xfffU) && (offset>>12)>=2) { if (readBit()) { distanceTreeRequired=createDecoder(); if (distanceTreeRequired) createDistanceDecoder(8); } } break; } } return value; }; readBit(); // ignore first bit distanceTreeRequired=createDecoder(); if (distanceTreeRequired) createDistanceDecoder(5); while (!outputStream.eof()) { uint32_t code=decoder.decode(readBit); if (code<8) { static const uint32_t symbolAdditions[8]={0,8,16,32,64,96,128,192}; static const uint32_t symbolBits[8]={3,3,4,5,5,5,6,6}; outputStream.writeByte(processOutput(outputStream.getOffset(),decodeMTF(readBits(symbolBits[code])+symbolAdditions[code],dataMTFMap))); } else { code-=8; uint32_t count; if (code<15) { count=code+2; } else { if (code>=21) throw DecompressionError(); static const uint32_t countAdditions[6]={17,25,33,65,129,256}; static const uint32_t countBits[6]={3,3,5,6,7,0}; count=readBits(countBits[code-15])+countAdditions[code-15]; } uint32_t distance; if (!code) { distance=readBits(6)+1; } else if (code<20) { uint32_t tmp=distanceDecoder.decode(readBit); if (!tmp) distance=readBits(6)+1; else distance=readBits(tmp+5)+(1<<(tmp+5))+1; } else distance=1; outputStream.copy(distance,count,0x20); const uint8_t *block=outputStream.history(count); size_t offset=outputStream.getOffset()-count; for (uint32_t i=0;i MASHDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } MASHDecompressor::MASHDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } MASHDecompressor::~MASHDecompressor() { // nothing needed } const std::string &MASHDecompressor::getSubName() const noexcept { static std::string name="XPK-MASH: LZRW-compressor"; return name; } void MASHDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; size_t rawSize=rawData.size(); ForwardOutputStream outputStream(rawData,0,rawSize); HuffmanDecoder litDecoder { HuffmanCode{1,0b000000,0}, HuffmanCode{2,0b000010,1}, HuffmanCode{3,0b000110,2}, HuffmanCode{4,0b001110,3}, HuffmanCode{5,0b011110,4}, HuffmanCode{6,0b111110,5}, HuffmanCode{6,0b111111,6} }; while (!outputStream.eof()) { uint32_t litLength=litDecoder.decode(readBit); if (litLength==6) { uint32_t litBits; for (litBits=1;litBits<=17;litBits++) if (!readBit()) break; if (litBits==17) throw Decompressor::DecompressionError(); litLength=readBits(litBits)+(1< MASHDecompressor::_XPKregistration; ancient-1.0/src/MASHDecompressor.hpp000066400000000000000000000015541375530757000174320ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef MASHDECOMPRESSOR_HPP #define MASHDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class MASHDecompressor : public XPKDecompressor { public: MASHDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~MASHDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/MMCMPDecompressor.cpp000066400000000000000000000152331375530757000175450ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "MMCMPDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool MMCMPDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("ziRC"); } std::unique_ptr MMCMPDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_unique(packedData,exactSizeKnown,verify); } MMCMPDecompressor::MMCMPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData(packedData) { if (!detectHeader(packedData.readBE32(0)) || packedData.readBE32(4U)!=FourCC("ONia") || packedData.readLE16(8U)!=14U || packedData.size()<24U) throw InvalidFormatError(); _blocks=packedData.readLE16(12U); _blocksOffset=packedData.readLE32(18U); _rawSize=packedData.readLE32(14U); if (size_t(_blocksOffset)+size_t(_blocks)*4U>packedData.size()) throw InvalidFormatError(); _packedSize=0; for (uint32_t i=0;i<_blocks;i++) { uint32_t blockAddr=packedData.readLE32(_blocksOffset+i*4U); if (size_t(blockAddr)+20U>=packedData.size()) throw InvalidFormatError(); uint32_t blockSize=packedData.readLE32(blockAddr+4U)+uint32_t(packedData.readLE16(blockAddr+12U))*8U+20U; _packedSize=std::max(_packedSize,blockAddr+blockSize); } if (_packedSize>packedData.size()) throw InvalidFormatError(); } MMCMPDecompressor::~MMCMPDecompressor() { // nothing needed } const std::string &MMCMPDecompressor::getName() const noexcept { static std::string name="MMCMP: Music Module Compressor"; return name; } size_t MMCMPDecompressor::getPackedSize() const noexcept { return _packedSize; } size_t MMCMPDecompressor::getRawSize() const noexcept { return _rawSize; } void MMCMPDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); // MMCMP allows gaps in data. Although not used in practice still we memset before decompressing to be sure std::memset(rawData.data(),0,rawData.size()); uint8_t *rawDataPtr=rawData.data(); for (uint32_t i=0;i<_blocks;i++) { uint32_t blockAddr=_packedData.readLE32(_blocksOffset+i*4U); uint32_t unpackedBlockSize=_packedData.readLE32(blockAddr); uint32_t packedBlockSize=_packedData.readLE32(blockAddr+4U); uint32_t fileChecksum=_packedData.readLE32(blockAddr+8U); uint32_t subBlocks=_packedData.readLE16(blockAddr+12U); uint16_t flags=_packedData.readLE16(blockAddr+14U); uint32_t packTableSize=_packedData.readLE16(blockAddr+16U); if (packTableSize>packedBlockSize) throw DecompressionError(); uint16_t bitCount=_packedData.readLE16(blockAddr+18U); ForwardInputStream inputStream(_packedData,blockAddr+subBlocks*8U+20U+packTableSize,blockAddr+subBlocks*8U+20U+packedBlockSize); LSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; uint32_t currentSubBlock=0; uint32_t outputOffset=0,outputSize=0; auto readNextSubBlock=[&]() { if (currentSubBlock>=subBlocks) throw DecompressionError(); outputOffset=_packedData.readLE32(blockAddr+currentSubBlock*8U+20U); outputSize=_packedData.readLE32(blockAddr+currentSubBlock*8U+24U); if (size_t(outputOffset)+size_t(outputSize)>size_t(_rawSize)) throw DecompressionError(); currentSubBlock++; }; uint32_t checksum=0; auto writeByte=[&](uint8_t value) { while (!outputSize) readNextSubBlock(); outputSize--; rawDataPtr[outputOffset++]=value; if (verify) { checksum^=value; checksum=(checksum<<1)|(checksum>>31); } }; // flags are // 0 = compressed // 1 = delta mode // 2 = 16 bit mode // 8 = stereo // 9 = abs16 // 10 = endian // flags do not combine nicely // no compress - no other flags // compressed 8 bit - only delta (and presumably stereo matters) // compressed 16 bit - all flags matter if (!(flags&0x1U)) { // not compressed for (uint32_t j=0;j=8) throw DecompressionError(); uint8_t oldValue[2]={0,0}; uint32_t chIndex=0; const uint8_t *tablePtr=&_packedData[blockAddr+subBlocks*8U+20U]; for (uint32_t j=0;j=valueThresholds[bitCount]) { uint32_t newBitCount=readBits(extraBits[bitCount])+((value-valueThresholds[bitCount])<=packTableSize) throw DecompressionError(); value=tablePtr[value]; if (flags&0x2U) { // delta value+=oldValue[chIndex]; oldValue[chIndex]=value; if (flags&0x100U) chIndex^=1U; // stereo } writeByte(value); j++; } } else { // 16 bit compression // shameless copy-paste from 8-bit variant, with minor changes static const uint16_t valueThresholds[16]={ 0x1U, 0x3U, 0x7U, 0xfU, 0x1eU, 0x3cU, 0x78U, 0xf0U, 0x1f0U, 0x3f0U, 0x7f0U, 0xff0U,0x1ff0U,0x3ff0U,0x7ff0U,0xfff0U }; static const uint8_t extraBits[16]={4,4,4,4, 3,2,1,0, 0,0,0,0, 0,0,0,0}; if (bitCount>=16) throw DecompressionError(); int16_t oldValue[2]={0,0}; uint32_t chIndex=0; for (uint32_t j=0;j=valueThresholds[bitCount]) { uint32_t newBitCount=readBits(extraBits[bitCount])+((value-valueThresholds[bitCount])<>=1; if (flags&0x2U) { // delta value+=oldValue[chIndex]; oldValue[chIndex]=value; if (flags&0x100U) chIndex^=1U; // stereo } if (flags&0x200U) value^=0x8000U; // abs16 if (flags&0x400U) { // big ending writeByte(value>>8U); writeByte(value); } else { // little endian writeByte(value); writeByte(value>>8U); } j+=2; } } if (verify && checksum!=fileChecksum) throw VerificationError(); } } Decompressor::Registry MMCMPDecompressor::_registration; ancient-1.0/src/MMCMPDecompressor.hpp000066400000000000000000000016351375530757000175530ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef MMCMPDECOMPRESSOR_HPP #define MMCMPDECOMPRESSOR_HPP #include "Decompressor.hpp" class MMCMPDecompressor : public Decompressor { public: MMCMPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); virtual ~MMCMPDecompressor(); virtual const std::string &getName() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual size_t getRawSize() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; static bool detectHeader(uint32_t hdr) noexcept; static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: const Buffer &_packedData; uint32_t _packedSize=0; uint32_t _rawSize=0; uint32_t _blocksOffset=0; uint32_t _blocks=0; static Decompressor::Registry _registration; }; #endif ancient-1.0/src/NONEDecompressor.cpp000066400000000000000000000024011375530757000174240ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "NONEDecompressor.hpp" bool NONEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("NONE"); } std::unique_ptr NONEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } NONEDecompressor::NONEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } NONEDecompressor::~NONEDecompressor() { // nothing needed } const std::string &NONEDecompressor::getSubName() const noexcept { static std::string name="XPK-NONE: Null compressor"; return name; } void NONEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_packedData.size()) throw Decompressor::DecompressionError(); std::memcpy(rawData.data(),_packedData.data(),_packedData.size()); } XPKDecompressor::Registry NONEDecompressor::_XPKregistration; ancient-1.0/src/NONEDecompressor.hpp000066400000000000000000000015541375530757000174410ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef NONEDECOMPRESSOR_HPP #define NONEDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class NONEDecompressor : public XPKDecompressor { public: NONEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~NONEDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/NUKEDecompressor.cpp000066400000000000000000000070121375530757000174320ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "NUKEDecompressor.hpp" #include "DLTADecode.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool NUKEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("NUKE") || hdr==FourCC("DUKE"); } std::unique_ptr NUKEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } NUKEDecompressor::NUKEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); if (hdr==FourCC("DUKE")) _isDUKE=true; } NUKEDecompressor::~NUKEDecompressor() { // nothing needed } const std::string &NUKEDecompressor::getSubName() const noexcept { static std::string nameN="XPK-NUKE: LZ77-compressor"; static std::string nameD="XPK-DUKE: LZ77-compressor with delta encoding"; return (_isDUKE)?nameD:nameN; } void NUKEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { // there are 2 streams, reverse stream for bytes and // normal stream for bits, the bit stream is divided // into single bit, 2 bit, 4 bit and random accumulator ForwardInputStream forwardInputStream(_packedData,0,_packedData.size()); BackwardInputStream backwardInputStream(_packedData,0,_packedData.size()); forwardInputStream.link(backwardInputStream); backwardInputStream.link(forwardInputStream); MSBBitReader bit1Reader(forwardInputStream); MSBBitReader bit2Reader(forwardInputStream); LSBBitReader bit4Reader(forwardInputStream); MSBBitReader bitXReader(forwardInputStream); auto readBit=[&]()->uint32_t { return bit1Reader.readBitsBE16(1); }; auto read2Bits=[&]()->uint32_t { return bit2Reader.readBitsBE16(2); }; auto read4Bits=[&]()->uint32_t { return bit4Reader.readBitsBE32(4); }; auto readBits=[&](uint32_t count)->uint32_t { return bitXReader.readBitsBE16(count); }; auto readByte=[&]()->uint8_t { return backwardInputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); for (;;) { if (!readBit()) { uint32_t count=0; if (readBit()) { count=1; } else { uint32_t tmp; do { tmp=read2Bits(); if (tmp) count+=5-tmp; else count+=3; } while (!tmp); } for (uint32_t i=0;i NUKEDecompressor::_XPKregistration; ancient-1.0/src/NUKEDecompressor.hpp000066400000000000000000000016031375530757000174370ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef NUKEDECOMPRESSOR_HPP #define NUKEDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class NUKEDecompressor : public XPKDecompressor { public: NUKEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~NUKEDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; bool _isDUKE=false; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/OutputStream.cpp000066400000000000000000000072341375530757000167640ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "OutputStream.hpp" // for exceptions #include "Decompressor.hpp" ForwardOutputStream::ForwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset) : _bufPtr(buffer.data()), _startOffset(startOffset), _currentOffset(startOffset), _endOffset(endOffset) { if (_startOffset>_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); } ForwardOutputStream::~ForwardOutputStream() { // nothing needed } void ForwardOutputStream::writeByte(uint8_t value) { if (_currentOffset>=_endOffset) throw Decompressor::DecompressionError(); _bufPtr[_currentOffset++]=value; } uint8_t ForwardOutputStream::copy(size_t distance,size_t count) { if (!distance || _startOffset+distance>_currentOffset || _currentOffset+count>_endOffset) throw Decompressor::DecompressionError(); uint8_t ret=0; for (size_t i=0;i_endOffset) throw Decompressor::DecompressionError(); size_t prevCount=0; uint8_t ret=0; if (_startOffset+distance>_currentOffset) { size_t prevSize=prevBuffer.size(); if (_startOffset+distance>_currentOffset+prevSize) throw Decompressor::DecompressionError(); size_t prevDist=_startOffset+distance-_currentOffset; prevCount=std::min(count,prevDist); const uint8_t *prev=&prevBuffer[prevSize-prevDist]; for (size_t i=0;i_endOffset) throw Decompressor::DecompressionError(); size_t prevCount=0; uint8_t ret=0; if (_startOffset+distance>_currentOffset) { prevCount=std::min(count,_startOffset+distance-_currentOffset); for (size_t i=0;i_currentOffset) throw Decompressor::DecompressionError(); return &_bufPtr[_currentOffset-distance]; } void ForwardOutputStream::produce(const uint8_t *src,size_t bytes) { if (_currentOffset+bytes>_endOffset) throw Decompressor::DecompressionError(); std::memcpy(&_bufPtr[_currentOffset],src,bytes); _currentOffset+=bytes; } BackwardOutputStream::BackwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset) : _bufPtr(buffer.data()), _startOffset(startOffset), _currentOffset(endOffset), _endOffset(endOffset) { if (_startOffset>_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); } BackwardOutputStream::~BackwardOutputStream() { // nothing needed } void BackwardOutputStream::writeByte(uint8_t value) { if (_currentOffset<=_startOffset) throw Decompressor::DecompressionError(); _bufPtr[--_currentOffset]=value; } uint8_t BackwardOutputStream::copy(size_t distance,size_t count) { if (!distance || _startOffset+count>_currentOffset || _currentOffset+distance>_endOffset) throw Decompressor::DecompressionError(); uint8_t ret=0; for (size_t i=0;i #include #include "common/Buffer.hpp" class ForwardOutputStream { public: ForwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset); ~ForwardOutputStream(); void writeByte(uint8_t value); uint8_t copy(size_t distance,size_t count); uint8_t copy(size_t distance,size_t count,const Buffer &prevBuffer); uint8_t copy(size_t distance,size_t count,uint8_t defaultChar); const uint8_t *history(size_t distance) const; void produce(const uint8_t *src,size_t bytes); bool eof() const { return _currentOffset==_endOffset; } size_t getOffset() const { return _currentOffset; } size_t getEndOffset() const { return _endOffset; } private: uint8_t *_bufPtr; size_t _startOffset; size_t _currentOffset; size_t _endOffset; }; class BackwardOutputStream { public: BackwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset); ~BackwardOutputStream(); void writeByte(uint8_t value); uint8_t copy(size_t distance,size_t count); bool eof() const { return _currentOffset==_startOffset; } size_t getOffset() const { return _currentOffset; } private: uint8_t *_bufPtr; size_t _startOffset; size_t _currentOffset; size_t _endOffset; }; #endif ancient-1.0/src/PPDecompressor.cpp000066400000000000000000000111261375530757000172100ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "PPDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" PPDecompressor::PPState::PPState(uint32_t mode) : _cachedMode(mode) { // nothing needed } PPDecompressor::PPState::~PPState() { // nothing needed } bool PPDecompressor::detectHeader(uint32_t hdr) noexcept { return (hdr==FourCC("PP11") || hdr==FourCC("PP20")); } bool PPDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("PWPK"); } std::unique_ptr PPDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_unique(packedData,exactSizeKnown,verify); } std::unique_ptr PPDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } PPDecompressor::PPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData(packedData) { if (!exactSizeKnown || packedData.size()<0x10) throw InvalidFormatError(); // no scanning support _dataStart=_packedData.size()-4; uint32_t hdr=packedData.readBE32(0); if (!detectHeader(hdr)) throw InvalidFormatError(); uint32_t mode=packedData.readBE32(4); if (mode!=0x9090909 && mode!=0x90a0a0a && mode!=0x90a0b0b && mode!=0x90a0c0c && mode!=0x90a0c0d) throw InvalidFormatError(); for (uint32_t i=0;i<4;i++) { _modeTable[i]=mode>>24; mode<<=8; } uint32_t tmp=packedData.readBE32(_dataStart); _rawSize=tmp>>8; _startShift=tmp&0xff; if (!_rawSize || _startShift>=0x20 || _rawSize>getMaxRawSize()) throw InvalidFormatError(); } PPDecompressor::PPDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || packedData.size()<0x10) throw InvalidFormatError(); _dataStart=_packedData.size()-4; uint32_t mode; if (state.get()) { mode=static_cast(state.get())->_cachedMode; } else { mode=packedData.readBE32(_dataStart); if (mode>4) throw InvalidFormatError(); state.reset(new PPState(mode)); _dataStart-=4; } static const uint32_t modeMap[5]={0x9090909,0x90a0a0a,0x90a0b0b,0x90a0c0c,0x90a0c0d}; mode=modeMap[mode]; for (uint32_t i=0;i<4;i++) { _modeTable[i]=mode>>24; mode<<=8; } uint32_t tmp=packedData.readBE32(_dataStart); _rawSize=tmp>>8; _startShift=tmp&0xff; if (!_rawSize || _startShift>=0x20 || _rawSize>getMaxRawSize()) throw InvalidFormatError(); _isXPK=true; } PPDecompressor::~PPDecompressor() { // nothing needed } const std::string &PPDecompressor::getName() const noexcept { static std::string name="PP: PowerPacker"; return name; } const std::string &PPDecompressor::getSubName() const noexcept { static std::string name="XPK-PWPK: PowerPacker"; return name; } size_t PPDecompressor::getPackedSize() const noexcept { return 0; } size_t PPDecompressor::getRawSize() const noexcept { return _rawSize; } void PPDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); BackwardInputStream inputStream(_packedData,_isXPK?0:8,_dataStart); LSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return rotateBits(bitReader.readBitsBE32(count),count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1); }; readBits(_startShift); BackwardOutputStream outputStream(rawData,0,_rawSize); for (;;) { if (!readBit()) { uint32_t count=1; // This does not make much sense I know. But it is what it is... for (;;) { uint32_t tmp=readBits(2); count+=tmp; if (tmp<3) break; } for (uint32_t i=0;i PPDecompressor::_registration; XPKDecompressor::Registry PPDecompressor::_XPKregistration; ancient-1.0/src/PPDecompressor.hpp000066400000000000000000000033061375530757000172160ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef PPDECOMPRESSOR_HPP #define PPDECOMPRESSOR_HPP #include "Decompressor.hpp" #include "XPKDecompressor.hpp" class PPDecompressor : public Decompressor, public XPKDecompressor { private: class PPState : public XPKDecompressor::State { public: PPState(uint32_t mode); virtual ~PPState(); uint32_t _cachedMode; }; public: PPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); PPDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~PPDecompressor(); virtual const std::string &getName() const noexcept override final; virtual const std::string &getSubName() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual size_t getRawSize() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeader(uint32_t hdr) noexcept; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; size_t _dataStart=0; size_t _rawSize=0; uint8_t _startShift=0; uint8_t _modeTable[4]; bool _isXPK=false; static Decompressor::Registry _registration; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/RAKEDecompressor.cpp000066400000000000000000000134371375530757000174220ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "RAKEDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool RAKEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return (hdr==FourCC("FRHT") || hdr==FourCC("RAKE")); } std::unique_ptr RAKEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } RAKEDecompressor::RAKEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData), _isRAKE(hdr==FourCC("RAKE")) { if (!detectHeaderXPK(hdr) || packedData.size()<4) throw Decompressor::InvalidFormatError(); _midStreamOffset=packedData.readBE16(2); if (_midStreamOffset>=packedData.size()) throw Decompressor::InvalidFormatError(); } RAKEDecompressor::~RAKEDecompressor() { // nothing needed } const std::string &RAKEDecompressor::getSubName() const noexcept { static std::string nameFRHT="XPK-FRHT: LZ77-compressor"; static std::string nameRAKE="XPK-RAKE: LZ77-compressor"; return (_isRAKE)?nameRAKE:nameFRHT; } void RAKEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { // 2 streams // 1st: bit stream starting from _midStreamOffset(+1) going to packedSize // 2nd: byte stream starting from _midStreamOffset going backwards to 4 ForwardInputStream forwardInputStream(_packedData,_midStreamOffset+(_midStreamOffset&1),_packedData.size()); BackwardInputStream backwardInputStream(_packedData,4,_midStreamOffset); MSBBitReader bitReader(forwardInputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBitsBE32(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1); }; auto readByte=[&]()->uint8_t { return backwardInputStream.readByte(); }; { uint16_t tmp=_packedData.readBE16(0); if (tmp>32) throw Decompressor::DecompressionError(); const uint8_t *buf=forwardInputStream.consume(4); uint32_t content=(uint32_t(buf[0])<<24)|(uint32_t(buf[1])<<16)| (uint32_t(buf[2])<<8)|uint32_t(buf[3]); bitReader.reset(content>>tmp,32-tmp); } BackwardOutputStream outputStream(rawData,0,rawData.size()); HuffmanDecoder lengthDecoder; // is there some logic into this? static const uint8_t decTable[255][2]={ { 1,0x01},{ 3,0x03},{ 5,0x05},{ 6,0x09},{ 7,0x0c},{ 9,0x13},{12,0x34},{18,0xc0}, {18,0xc2},{18,0xc3},{18,0xc6},{16,0x79},{18,0xc7},{18,0xd6},{18,0xd7},{18,0xd8}, {17,0xa8},{17,0x92},{17,0x8a},{17,0x82},{16,0x6c},{17,0x94},{18,0xda},{18,0xca}, {16,0x7b},{13,0x36},{13,0x39},{13,0x48},{14,0x49},{14,0x50},{15,0x62},{15,0x5e}, {16,0x6f},{17,0x83},{17,0x87},{15,0x56},{11,0x21},{12,0x31},{13,0x38},{13,0x3d}, { 8,0x0f},{ 4,0x04},{ 6,0x08},{10,0x1c},{12,0x27},{13,0x42},{13,0x3a},{12,0x30}, {12,0x32},{ 9,0x16},{ 8,0x11},{ 7,0x0b},{ 5,0x06},{10,0x19},{10,0x1a},{10,0x18}, {11,0x26},{17,0x98},{17,0x99},{17,0x9b},{17,0x9e},{17,0x9f},{17,0xa6},{16,0x73}, {17,0x7f},{17,0x81},{17,0x84},{17,0x85},{15,0x5d},{14,0x4d},{14,0x4f},{13,0x45}, {13,0x3c},{ 9,0x17},{10,0x1d},{12,0xff},{13,0x41},{17,0x8c},{18,0xaa},{19,0xdb}, {19,0xdc},{16,0x77},{15,0x63},{16,0x7c},{16,0x76},{16,0x71},{16,0x7d},{12,0x2c}, {13,0x3b},{16,0x7a},{16,0x75},{15,0x55},{15,0x60},{16,0x74},{17,0xa4},{18,0xab}, {18,0xac},{ 7,0x0a},{ 6,0x07},{ 9,0x15},{11,0x20},{11,0x24},{10,0x1b},{ 8,0x10}, { 9,0x12},{12,0x33},{14,0x4b},{15,0x53},{19,0xdd},{19,0xde},{18,0xad},{19,0xdf}, {19,0xe0},{18,0xae},{17,0x88},{18,0xaf},{19,0xe1},{19,0xe2},{13,0x37},{12,0x2e}, {18,0xb0},{18,0xb1},{19,0xe3},{19,0xe4},{18,0xb2},{18,0xb3},{19,0xe5},{19,0xe6}, {19,0xe7},{19,0xe8},{18,0xb4},{17,0x9a},{18,0xb5},{18,0xb6},{18,0xb7},{19,0xe9}, {19,0xea},{18,0xb8},{19,0xeb},{19,0xec},{19,0xed},{19,0xee},{18,0xb9},{19,0xef}, {19,0xf0},{18,0xbb},{18,0xbc},{19,0xf1},{19,0xf2},{18,0xbd},{18,0xbe},{19,0xf3}, {19,0xf4},{18,0xbf},{18,0xc1},{19,0xf5},{19,0xf6},{18,0xc4},{18,0xc5},{17,0x95}, {18,0xc8},{18,0xc9},{19,0xf7},{19,0xf8},{18,0xcb},{18,0xcc},{19,0xf9},{19,0xfa}, {18,0xcd},{18,0xce},{17,0x96},{18,0xcf},{18,0xd0},{19,0xfb},{19,0xfc},{18,0xd1}, {18,0xd2},{18,0xd3},{17,0x9c},{17,0x9d},{18,0xd4},{18,0xd5},{17,0xa0},{17,0xa1}, {17,0xa2},{17,0xa3},{17,0xa5},{19,0xfd},{19,0xfe},{18,0xd9},{17,0xa7},{16,0x66}, {15,0x54},{15,0x57},{16,0x6b},{16,0x68},{14,0x4c},{14,0x4e},{12,0x28},{11,0x23}, { 8,0x0e},{ 7,0x0d},{10,0x1f},{13,0x47},{15,0x64},{15,0x58},{15,0x59},{15,0x5a}, {12,0x29},{13,0x3e},{15,0x5f},{17,0x8e},{18,0xba},{18,0xa9},{16,0x70},{14,0x4a}, {12,0x2a},{ 9,0x14},{11,0x22},{12,0x2f},{16,0x7e},{16,0x67},{16,0x69},{16,0x65}, {15,0x51},{16,0x78},{16,0x6a},{13,0x46},{11,0x25},{16,0x72},{16,0x6e},{15,0x5b}, {15,0x61},{15,0x52},{13,0x40},{13,0x43},{13,0x44},{13,0x3f},{15,0x5c},{17,0x93}, {17,0x80},{17,0x8d},{17,0x8b},{17,0x86},{17,0x89},{17,0x97},{17,0x8f},{17,0x90}, {17,0x91},{16,0x6d},{12,0x2b},{12,0x2d},{12,0x35},{10,0x1e},{ 3,0x02}}; uint32_t hufCode=0; for (auto &it: decTable) { lengthDecoder.insert(HuffmanCode{it[0],hufCode>>(32-it[0]),it[1]}); hufCode+=1<<(32-it[0]); } while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint32_t count=lengthDecoder.decode(readBit); count+=2; uint32_t distance; if (!readBit()) { distance=uint32_t(readByte())+1; } else { if (!readBit()) { distance=((readBits(3)<<8)|uint32_t(readByte()))+0x101; } else { distance=((readBits(6)<<8)|uint32_t(readByte()))+0x901; } } outputStream.copy(distance,count); } } } XPKDecompressor::Registry RAKEDecompressor::_XPKregistration; ancient-1.0/src/RAKEDecompressor.hpp000066400000000000000000000016321375530757000174210ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef RAKEDECOMPRESSOR_HPP #define RAKEDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class RAKEDecompressor : public XPKDecompressor { public: RAKEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~RAKEDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; bool _isRAKE; size_t _midStreamOffset=0; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/RDCNDecompressor.cpp000066400000000000000000000044471375530757000174270ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "RDCNDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool RDCNDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("RDCN"); } std::unique_ptr RDCNDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } RDCNDecompressor::RDCNDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } RDCNDecompressor::~RDCNDecompressor() { // nothing needed } const std::string &RDCNDecompressor::getSubName() const noexcept { static std::string name="XPK-RDCN: Ross data compression"; return name; } void RDCNDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint8_t tmp=readByte(); uint32_t count=tmp&0xf; uint32_t code=tmp>>4; uint32_t distance=0; uint8_t repeatChar=0; bool doRLE=false; switch (code) { case 0: repeatChar=readByte(); count+=3; doRLE=true; break; case 1: count=(count|(uint32_t(readByte())<<4))+19; repeatChar=readByte(); doRLE=true; break; case 2: distance=(count|(uint32_t(readByte())<<4))+3; count=uint32_t(readByte())+16; break; default: /* 3 to 15 */ distance=(count|(uint32_t(readByte())<<4))+3; count=code; break; } if (doRLE) { for (uint32_t i=0;i RDCNDecompressor::_XPKregistration; ancient-1.0/src/RDCNDecompressor.hpp000066400000000000000000000015541375530757000174300ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef RDCNDECOMPRESSOR_HPP #define RDCNDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class RDCNDecompressor : public XPKDecompressor { public: RDCNDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~RDCNDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/RLENDecompressor.cpp000066400000000000000000000034501375530757000174320ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "RLENDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool RLENDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("RLEN"); } std::unique_ptr RLENDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } RLENDecompressor::RLENDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } RLENDecompressor::~RLENDecompressor() { // nothing needed } const std::string &RLENDecompressor::getSubName() const noexcept { static std::string name="XPK-RLEN: RLE-compressor"; return name; } void RLENDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { uint32_t count=uint32_t(inputStream.readByte()); if (count<128) { if (!count) throw Decompressor::DecompressionError(); // lets have this as error... for (uint32_t i=0;i RLENDecompressor::_XPKregistration; ancient-1.0/src/RLENDecompressor.hpp000066400000000000000000000015541375530757000174420ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef RLENDECOMPRESSOR_HPP #define RLENDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class RLENDecompressor : public XPKDecompressor { public: RLENDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~RLENDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/RNCDecompressor.cpp000066400000000000000000000261241375530757000173170ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "RNCDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/CRC16.hpp" bool RNCDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("RNC\001") || hdr==FourCC("RNC\002"); } std::unique_ptr RNCDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_unique(packedData,verify); } RNCDecompressor::RNCDecompressor(const Buffer &packedData,bool verify) : _packedData(packedData) { uint32_t hdr=packedData.readBE32(0); _rawSize=packedData.readBE32(4); _packedSize=packedData.readBE32(8); if (!_rawSize || !_packedSize || _rawSize>getMaxRawSize() || _packedSize>getMaxPackedSize()) throw InvalidFormatError(); bool verified=false; if (hdr==FourCC("RNC\001")) { // now detect between old and new version // since the old and the new version share the same id, there is no foolproof way // to tell them apart. It is easier to prove that it is not something by finding // specific invalid bitstream content. // well, this is silly though but lets assume someone has made old format RNC1 with total size less than 19 if (packedData.size()<19) { _ver=Version::RNC1Old; } else { uint8_t newStreamStart=packedData.read8(18); uint8_t oldStreamStart=packedData.read8(_packedSize+11); // Check that stream starts with a literal(s) if (!(oldStreamStart&0x80)) _ver=Version::RNC1New; // New stream have two bits in start as a filler on new stream. Those are always 0 // (although this is not strictly mandated) // + // Even though it is possible to make new RNC1 stream which starts with zero literal table size, // it is extremely unlikely else if ((newStreamStart&3) || !(newStreamStart&0x7c)) _ver=Version::RNC1Old; // now the last resort: check CRC. else if (_packedData.size()>=_packedSize+18 && CRC16(_packedData,18,_packedSize,0)==packedData.readBE16(14)) { _ver=Version::RNC1New; verified=true; } else _ver=Version::RNC1Old; } } else if (hdr==FourCC("RNC\002")) { _ver=Version::RNC2; } else throw InvalidFormatError(); size_t hdrSize=(_ver==Version::RNC1Old)?12:18; if (size_t(_packedSize)+size_t(hdrSize)>packedData.size()) throw InvalidFormatError(); if (_ver!=Version::RNC1Old) { _rawCRC=packedData.readBE16(12); _chunks=packedData.read8(17); if (verify && !verified) { if (CRC16(_packedData,18,_packedSize,0)!=packedData.readBE16(14)) throw VerificationError(); } } } RNCDecompressor::~RNCDecompressor() { // nothing needed } const std::string &RNCDecompressor::getName() const noexcept { static std::string names[3]={ "RNC1: Rob Northen RNC1 Compressor (old)", "RNC1: Rob Northen RNC1 Compressor ", "RNC2: Rob Northen RNC2 Compressor"}; return names[static_cast(_ver)]; } size_t RNCDecompressor::getPackedSize() const noexcept { if (_ver==Version::RNC1Old) return _packedSize+12; else return _packedSize+18; } size_t RNCDecompressor::getRawSize() const noexcept { return _rawSize; } void RNCDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); switch (_ver) { case Version::RNC1Old: return RNC1DecompressOld(rawData,verify); case Version::RNC1New: return RNC1DecompressNew(rawData,verify); case Version::RNC2: return RNC2Decompress(rawData,verify); default: throw DecompressionError(); } } void RNCDecompressor::RNC1DecompressOld(Buffer &rawData,bool verify) { BackwardInputStream inputStream(_packedData,12,_packedSize+12); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; // the anchor-bit does not seem always to be at the correct place { uint8_t halfByte=readByte(); for (uint32_t i=0;i<7;i++) if (halfByte&(1<>(i+1),7-i); break; } } BackwardOutputStream outputStream(rawData,0,_rawSize); HuffmanDecoder litDecoder { HuffmanCode{1,0b00,0}, HuffmanCode{2,0b10,1}, HuffmanCode{2,0b11,2} }; HuffmanDecoder lengthDecoder { HuffmanCode{1,0b0000,0}, HuffmanCode{2,0b0010,1}, HuffmanCode{3,0b0110,2}, HuffmanCode{4,0b1110,3}, HuffmanCode{4,0b1111,4} }; HuffmanDecoder distanceDecoder { HuffmanCode{1,0b00,0}, HuffmanCode{2,0b10,1}, HuffmanCode{2,0b11,2} }; for (;;) { uint32_t litLength=litDecoder.decode(readBit); if (litLength==2) { static const uint32_t litBitLengths[4]={2,2,3,10}; static const uint32_t litAdditions[4]={2,5,8,15}; for (uint32_t i=0;i<4;i++) { litLength=readBits(litBitLengths[i]); if (litLength!=(1U< bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits16Limit(count); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,_rawSize); typedef HuffmanDecoder RNC1HuffmanDecoder; // helpers auto readHuffmanTable=[&](RNC1HuffmanDecoder &dec) { uint32_t length=readBits(5); if (!length) return; uint32_t maxDepth=0; uint8_t lengthTable[31]; for (uint32_t i=0;imaxDepth) maxDepth=lengthTable[i]; } dec.createOrderlyHuffmanTable(lengthTable,length); }; auto huffmanDecode=[&](const RNC1HuffmanDecoder &dec)->int32_t { // this is kind of non-specced uint32_t ret=dec.decode([&]()->uint32_t{return readBits(1);}); if (ret>=2) ret=(1<<(ret-1))|readBits(ret-1); return ret; }; auto processLiterals=[&](const RNC1HuffmanDecoder &dec) { uint32_t litLength=huffmanDecode(dec); for (uint32_t i=0;i bitReader(inputStream); auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,_rawSize); // Huffman decoding enum class Cmd { LIT=0, // 0, Literal MOV, // 10, Move bytes + length + distance, Get bytes if length=9 + 4bits MV2, // 110, Move 2 bytes MV3, // 1110, Move 3 bytes CND // 1111, Conditional copy, or EOF }; HuffmanDecoder cmdDecoder { HuffmanCode{1,0b0000,Cmd::LIT}, HuffmanCode{2,0b0010,Cmd::MOV}, HuffmanCode{3,0b0110,Cmd::MV2}, HuffmanCode{4,0b1110,Cmd::MV3}, HuffmanCode{4,0b1111,Cmd::CND} }; /* length of 9 is a marker for literals */ HuffmanDecoder lengthDecoder { HuffmanCode{2,0b000,4}, HuffmanCode{2,0b010,5}, HuffmanCode{3,0b010,6}, HuffmanCode{3,0b011,7}, HuffmanCode{3,0b110,8}, HuffmanCode{3,0b111,9} }; HuffmanDecoder distanceDecoder { HuffmanCode{1,0b000000,0}, HuffmanCode{3,0b000110,1}, HuffmanCode{4,0b001000,2}, HuffmanCode{4,0b001001,3}, HuffmanCode{5,0b010101,4}, HuffmanCode{5,0b010111,5}, HuffmanCode{5,0b011101,6}, HuffmanCode{5,0b011111,7}, HuffmanCode{6,0b101000,8}, HuffmanCode{6,0b101001,9}, HuffmanCode{6,0b101100,10}, HuffmanCode{6,0b101101,11}, HuffmanCode{6,0b111000,12}, HuffmanCode{6,0b111001,13}, HuffmanCode{6,0b111100,14}, HuffmanCode{6,0b111101,15} }; // helpers auto readDistance=[&]()->uint32_t { int8_t distMult=distanceDecoder.decode(readBit); if (distMult<0) throw DecompressionError(); uint8_t distByte=readByte(); return (uint32_t(distByte)|(uint32_t(distMult)<<8))+1; }; auto moveBytes=[&](uint32_t distance,uint32_t count)->void { if (!count) throw DecompressionError(); outputStream.copy(distance,count); }; readBit(); readBit(); uint8_t foundChunks=0; bool done=false; while (!done && foundChunks<_chunks) { Cmd cmd=cmdDecoder.decode(readBit); switch (cmd) { case Cmd::LIT: outputStream.writeByte(readByte()); break; case Cmd::MOV: { uint8_t count=lengthDecoder.decode(readBit); if (count!=9) moveBytes(readDistance(),count); else { uint32_t rep=0; for (uint32_t i=0;i<4;i++) rep=(rep<<1)|readBit(); rep=(rep+3)*4; for (uint32_t i=0;i RNCDecompressor::_registration; ancient-1.0/src/RNCDecompressor.hpp000066400000000000000000000021431375530757000173170ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef RNCDECOMPRESSOR_HPP #define RNCDECOMPRESSOR_HPP #include "Decompressor.hpp" class RNCDecompressor : public Decompressor { public: RNCDecompressor(const Buffer &packedData,bool verify); virtual ~RNCDecompressor(); virtual const std::string &getName() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual size_t getRawSize() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; static bool detectHeader(uint32_t hdr) noexcept; static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: enum class Version { RNC1Old=0, RNC1New, RNC2 }; void RNC1DecompressOld(Buffer &rawData,bool verify); void RNC1DecompressNew(Buffer &rawData,bool verify); void RNC2Decompress(Buffer &rawData,bool verify); const Buffer &_packedData; uint32_t _rawSize=0; uint32_t _packedSize=0; uint16_t _rawCRC=0; uint8_t _chunks=0; Version _ver; static Decompressor::Registry _registration; }; #endif ancient-1.0/src/RangeDecoder.cpp000066400000000000000000000021041375530757000166210ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "RangeDecoder.hpp" RangeDecoder::BitReader::BitReader() { // nothing needed } RangeDecoder::BitReader::~BitReader() { // nothing needed } RangeDecoder::RangeDecoder(BitReader &bitReader,uint16_t initialValue) : _bitReader(bitReader), _stream(initialValue) { // nothing needed } RangeDecoder::~RangeDecoder() { // nothing needed } uint16_t RangeDecoder::decode(uint16_t length) { return ((uint32_t(_stream-_low)+1)*length-1)/(uint32_t(_high-_low)+1); } void RangeDecoder::scale(uint16_t newLow,uint16_t newHigh,uint16_t newRange) { uint32_t range=uint32_t(_high-_low)+1; _high=(range*newHigh)/newRange+_low-1; _low=(range*newLow)/newRange+_low; auto doubleContext=[&](uint16_t decr) { _low-=decr; _high-=decr; _stream-=decr; _low<<=1; _high=(_high<<1)|1U; _stream=(_stream<<1)|_bitReader.readBit(); }; for (;;) { if (_high<0x8000U) { doubleContext(0U); } else if (_low>=0x8000U) { doubleContext(0x8000U); } else if (_low>=0x4000U && _high<0xc000U) { doubleContext(0x4000U); } else break; } } ancient-1.0/src/RangeDecoder.hpp000066400000000000000000000010741375530757000166330ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef RANGEDECODER_HPP #define RANGEDECODER_HPP #include // used by too many compressors... class RangeDecoder { public: class BitReader { public: BitReader(); virtual ~BitReader(); virtual uint32_t readBit()=0; }; RangeDecoder(BitReader &bitReader,uint16_t initialValue); ~RangeDecoder(); uint16_t decode(uint16_t length); void scale(uint16_t newLow,uint16_t newHigh,uint16_t newRange); private: BitReader &_bitReader; uint16_t _low=0; uint16_t _high=0xffffU; uint16_t _stream; }; #endif ancient-1.0/src/SDHCDecompressor.cpp000066400000000000000000000053721375530757000174200ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "common/SubBuffer.hpp" #include "SDHCDecompressor.hpp" #include "XPKMain.hpp" #include "DLTADecode.hpp" bool SDHCDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SDHC"); } std::unique_ptr SDHCDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } SDHCDecompressor::SDHCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || _packedData.size()<2) throw Decompressor::InvalidFormatError(); _mode=_packedData.readBE16(0); if (verify && (_mode&0x8000U)) { ConstSubBuffer src(_packedData,2,_packedData.size()-2); XPKMain main(src,_recursionLevel+1,true); } } SDHCDecompressor::~SDHCDecompressor() { // nothing needed } const std::string &SDHCDecompressor::getSubName() const noexcept { static std::string name="XPK-SDHC: Sample delta huffman compressor"; return name; } void SDHCDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ConstSubBuffer src(_packedData,2,_packedData.size()-2); if (_mode&0x8000U) { XPKMain main(src,_recursionLevel+1,false); main.decompress(rawData,verify); } else { if (src.size()!=rawData.size()) throw Decompressor::DecompressionError(); std::memcpy(rawData.data(),src.data(),src.size()); } size_t length=rawData.size()&~3U; auto deltaDecodeMono=[&]() { uint8_t *buf=rawData.data(); uint16_t ctr=0; for (size_t i=0;i>8; buf[i+1]=ctr; } }; auto deltaDecodeStereo=[&]() { uint8_t *buf=rawData.data(); uint16_t ctr1=0,ctr2=0; for (size_t i=0;i>8; buf[i+1]=ctr1; buf[i+2]=ctr2>>8; buf[i+3]=ctr2; } }; switch (_mode&15) { case 1: DLTADecode::decode(rawData,rawData,0,length); // intentional fall through case 0: DLTADecode::decode(rawData,rawData,0,length); break; case 3: deltaDecodeMono(); // intentional fall through case 2: deltaDecodeMono(); break; case 11: deltaDecodeStereo(); // intentional fall through case 10: deltaDecodeStereo(); break; default: throw Decompressor::DecompressionError(); } } XPKDecompressor::Registry SDHCDecompressor::_XPKregistration; ancient-1.0/src/SDHCDecompressor.hpp000066400000000000000000000016001375530757000174130ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SDHCDECOMPRESSOR_HPP #define SDHCDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class SDHCDecompressor : public XPKDecompressor { public: SDHCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~SDHCDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint16_t _mode=0; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/SHR3Decompressor.cpp000066400000000000000000000160351375530757000174140ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SHR3Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" SHR3Decompressor::SHR3State::SHR3State() noexcept { for (uint32_t i=0;i<999;i++) ar[i]=0; } SHR3Decompressor::SHR3State::~SHR3State() { // nothing needed } bool SHR3Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SHR3"); } std::unique_ptr SHR3Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } SHR3Decompressor::SHR3Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData), _state(state) { if (!detectHeaderXPK(hdr) || _packedData.size()<6) throw Decompressor::InvalidFormatError(); _ver=_packedData.read8(0); if (!_ver || _ver>2) throw Decompressor::InvalidFormatError(); if (!_state) { if (_ver==2) throw Decompressor::InvalidFormatError(); _state.reset(new SHR3State()); } } SHR3Decompressor::~SHR3Decompressor() { // nothing needed } const std::string &SHR3Decompressor::getSubName() const noexcept { static std::string name="XPK-SHR3: LZ-compressor with arithmetic encoding"; return name; } void SHR3Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,1,_packedData.size()); auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); // This follows quite closely Choloks pascal reference uint32_t ar[999]; auto resum=[&]() { for (uint32_t i=498;i;i--) ar[i]=ar[i*2]+ar[i*2+1]; }; auto init=[&]() { for (uint32_t i=0;i<499;i++) ar[i]=0; for (uint32_t i=0;i<256;i++) ar[i+499]=(i<32||i>126)?1:3; for (uint32_t i=256+499;i<999;i++) ar[i]=0; resum(); }; auto update=[&](uint32_t updateIndex,uint32_t increment) { if (updateIndex>=499) return; updateIndex+=499; while (updateIndex) { ar[updateIndex]+=increment; updateIndex>>=1; } if (ar[1]>=0x2000) { for (uint32_t i=499;i<998;i++) if (ar[i]) ar[i]=(ar[i]>>1)+1; resum(); } }; auto scale=[&](uint32_t b,uint32_t mult)->uint32_t { if (!b) throw Decompressor::DecompressionError(); uint32_t tmp=(0x10000U/b); uint32_t tmp2=((0x10000U%b)<<16)/b; return ((mult&0xffffU)*tmp>>16)+((mult>>16)*tmp2>>16)+(mult>>16)*tmp; }; uint32_t vlen=0,vnext=0; auto upgrade=[&]() { if (vnext>=65532) { vnext=~0U; } else if (!vlen) { vnext=1; } else { uint32_t vvalue=vnext-1; if (vvalue<48) update(vvalue+256,1); uint32_t bits=0,compare=4; while (vvalue>=compare) { vvalue-=compare; compare<<=1; bits++; } if (bits>=14) { vnext=~0U; } else { if (!vvalue) { if (bits<7) { for (uint32_t i=304;i<=307;i++) update((bits<<2)+i,1); } if (bits<13) { for (uint32_t i=332;i<=333;i++) update((bits<<1)+i,1); } static const uint32_t updates1[6]={358,359,386,387,414,415}; static const uint32_t updates2[4]={442,456,470,484}; for (auto it : updates1) update((bits<<1)+it,1); for (auto it : updates2) update(bits+it,1); } if (vnext<49) { vnext++; } else if (vnext==49) { vnext=61; } else { vnext=(vnext<<1)+3; } } } }; uint32_t stream=0,shift=0; auto refillStream=[&]() { while (shift<0x100'0000) { stream=(stream<<8)|uint32_t(readByte()); shift<<=8; } }; auto getSymbol=[&]()->uint32_t { if (!(shift>>16)) throw Decompressor::DecompressionError(); uint32_t vvalue=(stream/(shift>>16))&0xffff; uint32_t threshold=(ar[1]*vvalue)>>16; uint32_t arIndex=1; uint32_t result=0; do { arIndex<<=1; uint32_t tmp=ar[arIndex]+result; if (threshold>=tmp) { result=tmp; arIndex++; } } while (arIndex<499); uint32_t rawValue=scale(ar[1],shift); uint32_t newValue=rawValue*result; if (newValue>stream) { while (newValue>stream) { if (--arIndex<499) arIndex+=499; result-=ar[arIndex]; newValue=rawValue*result; } } else { result+=ar[arIndex]; while (result=998) arIndex-=499; result+=ar[arIndex]; newValue=compare; } } stream-=newValue; shift=rawValue*ar[arIndex]; uint32_t addition=(ar[1]>>10)+3; arIndex-=499; update(arIndex,addition); refillStream(); return arIndex; }; auto getCode=[&](uint32_t size)->uint32_t { uint32_t ret=0; while (size--) { ret<<=1; shift>>=1; if (stream>=shift) { ret++; stream-=shift; } refillStream(); } return ret; }; if (_ver==1) { init(); update(498,1); shift=0x8000'0000U; } else { SHR3State *state=static_cast(_state.get()); vlen=state->vlen; vnext=state->vnext; shift=state->shift; for (uint32_t i=0;i<999;i++) ar[i]=state->ar[i]; } { const uint8_t *buf=inputStream.consume(4); stream=(uint32_t(buf[0])<<24)|(uint32_t(buf[1])<<16)| (uint32_t(buf[2])<<8)|uint32_t(buf[3]); } while (!outputStream.eof()) { while (vlen>=vnext) upgrade(); uint32_t code=getSymbol(); if (code<256) { outputStream.writeByte(code); vlen++; } else { auto distanceAddition=[](uint32_t i)->uint32_t { return ((1<<(i+2))-1)&~0x3U; }; uint32_t count,distance; if (code<304) { count=2; distance=code-255; } else if (code<332) { uint32_t tmp=code-304; uint32_t extra=getCode(tmp>>2); distance=((extra<<2)|(tmp&3))+distanceAddition(tmp>>2)+1; count=3; } else if (code<358) { uint32_t tmp=code-332; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=4; } else if (code<386) { uint32_t tmp=code-358; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=5; } else if (code<414) { uint32_t tmp=code-386; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=6; } else if (code<442) { uint32_t tmp=code-414; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=7; } else if (code<498) { uint32_t tmp=code-442; uint32_t d=tmp/14; uint32_t m=tmp%14; count=getCode(d+2)+distanceAddition(d)+8; distance=getCode(m+2)+distanceAddition(m)+1; } else { count=getCode(16); distance=getCode(16); } vlen+=count; if (!count) throw Decompressor::DecompressionError(); outputStream.copy(distance,count,previousData); } } SHR3State *state=static_cast(_state.get()); state->vlen=vlen; state->vnext=vnext; state->shift=shift; for (uint32_t i=0;i<999;i++) state->ar[i]=ar[i]; } XPKDecompressor::Registry SHR3Decompressor::_XPKregistration; ancient-1.0/src/SHR3Decompressor.hpp000066400000000000000000000022231375530757000174130ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SHR3DECOMPRESSOR_HPP #define SHR3DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class SHR3Decompressor : public XPKDecompressor { private: class SHR3State : public XPKDecompressor::State { public: SHR3State() noexcept; virtual ~SHR3State(); uint32_t vlen=0; uint32_t vnext=0; uint32_t shift=0; uint32_t ar[999]; }; public: SHR3Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~SHR3Decompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _ver=0; std::unique_ptr &_state; // reference!!! static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/SHRIDecompressor.cpp000066400000000000000000000165721375530757000174500ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SHRIDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" SHRIDecompressor::SHRIState::SHRIState() noexcept { for (uint32_t i=0;i<999;i++) ar[i]=0; } SHRIDecompressor::SHRIState::~SHRIState() { // nothing needed } bool SHRIDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SHRI"); } std::unique_ptr SHRIDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } SHRIDecompressor::SHRIDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData), _state(state) { if (!detectHeaderXPK(hdr) || _packedData.size()<6) throw Decompressor::InvalidFormatError(); _ver=_packedData.read8(0); if (!_ver || _ver>2) throw Decompressor::InvalidFormatError(); // second byte defines something that does not seem to be terribly important... uint8_t tmp=_packedData.read8(2); if (tmp<0x80) { _rawSize=_packedData.readBE16(2); _startOffset=4; } else { _rawSize=-_packedData.readBE32(2); _startOffset=6; } if (!_state) { if (_ver==2) throw Decompressor::InvalidFormatError(); _state.reset(new SHRIState()); } } SHRIDecompressor::~SHRIDecompressor() { // nothing needed } const std::string &SHRIDecompressor::getSubName() const noexcept { static std::string name="XPK-SHRI: LZ-compressor with arithmetic encoding"; return name; } void SHRIDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); ForwardInputStream inputStream(_packedData,_startOffset,_packedData.size()); auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); // This follows quite closely Choloks pascal reference uint32_t ar[999]; auto resum=[&]() { for (uint32_t i=498;i;i--) ar[i]=ar[i*2]+ar[i*2+1]; }; auto init=[&]() { for (uint32_t i=0;i<499;i++) ar[i]=0; for (uint32_t i=0;i<256;i++) ar[i+499]=(i<32||i>126)?1:3; for (uint32_t i=256+499;i<999;i++) ar[i]=0; resum(); }; auto update=[&](uint32_t updateIndex,uint32_t increment) { if (updateIndex>=499) return; updateIndex+=499; while (updateIndex) { ar[updateIndex]+=increment; updateIndex>>=1; } if (ar[1]>=0x2000) { for (uint32_t i=499;i<998;i++) if (ar[i]) ar[i]=(ar[i]>>1)+1; resum(); } }; auto scale=[&](uint32_t a,uint32_t b,uint32_t mult)->uint32_t { if (!b) throw Decompressor::DecompressionError(); uint32_t tmp=(a<<16)/b; uint32_t tmp2=(((a<<16)%b)<<16)/b; return ((mult&0xffffU)*tmp>>16)+((mult>>16)*tmp2>>16)+(mult>>16)*tmp; }; uint32_t vlen=0,vnext=0; auto upgrade=[&]() { if (vnext>=65532) { vnext=~0U; } else if (!vlen) { vnext=1; } else { uint32_t vvalue=vnext-1; if (vvalue<48) update(vvalue+256,1); uint32_t bits=0,compare=4; while (vvalue>=compare) { vvalue-=compare; compare<<=1; bits++; } if (bits>=14) { vnext=~0U; } else { if (!vvalue) { if (bits<7) { for (uint32_t i=304;i<=307;i++) update((bits<<2)+i,1); } if (bits<13) { for (uint32_t i=332;i<=333;i++) update((bits<<1)+i,1); } static const uint32_t updates1[6]={358,359,386,387,414,415}; static const uint32_t updates2[4]={442,456,470,484}; for (auto it : updates1) update((bits<<1)+it,1); for (auto it : updates2) update(bits+it,1); } if (vnext<49) { vnext++; } else if (vnext==49) { vnext=61; } else { vnext=(vnext<<1)+3; } } } }; uint32_t stream=0,shift=0; auto refillStream=[&]() { while (shift<0x100'0000) { stream=(stream<<8)|uint32_t(readByte()); shift<<=8; } }; auto getSymbol=[&]()->uint32_t { if (!(shift>>16)) throw Decompressor::DecompressionError(); uint32_t vvalue=(stream/(shift>>16))&0xffff; uint32_t threshold=(ar[1]*vvalue)>>16; uint32_t arIndex=1; uint32_t result=0; do { arIndex<<=1; uint32_t tmp=ar[arIndex]+result; if (threshold>=tmp) { result=tmp; arIndex++; } } while (arIndex<499); uint32_t newValue=scale(result,ar[1],shift); if (newValue>stream) { while (newValue>stream) { if (--arIndex<499) arIndex+=499; result-=ar[arIndex]; newValue=scale(result,ar[1],shift); } } else { result+=ar[arIndex]; while (result=998) arIndex-=499; result+=ar[arIndex]; newValue=compare; } } stream-=newValue; shift=scale(ar[arIndex],ar[1],shift); uint32_t addition=(ar[1]>>10)+3; arIndex-=499; update(arIndex,addition); refillStream(); return arIndex; }; auto getCode=[&](uint32_t size)->uint32_t { uint32_t ret=0; while (size--) { ret<<=1; shift>>=1; if (stream>=shift) { ret++; stream-=shift; } refillStream(); } return ret; }; if (_ver==1) { init(); update(498,1); shift=0x8000'0000U; } else { SHRIState *state=static_cast(_state.get()); vlen=state->vlen; vnext=state->vnext; shift=state->shift; for (uint32_t i=0;i<999;i++) ar[i]=state->ar[i]; } { const uint8_t *buf=inputStream.consume(4); stream=(uint32_t(buf[0])<<24)|(uint32_t(buf[1])<<16)| (uint32_t(buf[2])<<8)|uint32_t(buf[3]); } while (!outputStream.eof()) { while (vlen>=vnext) upgrade(); uint32_t code=getSymbol(); if (code<256) { outputStream.writeByte(code); vlen++; } else { auto distanceAddition=[](uint32_t i)->uint32_t { return ((1<<(i+2))-1)&~0x3U; }; uint32_t count,distance; if (code<304) { count=2; distance=code-255; } else if (code<332) { uint32_t tmp=code-304; uint32_t extra=getCode(tmp>>2); distance=((extra<<2)|(tmp&3))+distanceAddition(tmp>>2)+1; count=3; } else if (code<358) { uint32_t tmp=code-332; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=4; } else if (code<386) { uint32_t tmp=code-358; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=5; } else if (code<414) { uint32_t tmp=code-386; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=6; } else if (code<442) { uint32_t tmp=code-414; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=7; } else if (code<498) { uint32_t tmp=code-442; uint32_t d=tmp/14; uint32_t m=tmp%14; count=getCode(d+2)+distanceAddition(d)+8; distance=getCode(m+2)+distanceAddition(m)+1; } else { count=getCode(16); distance=getCode(16); } vlen+=count; if (!count) throw Decompressor::DecompressionError(); outputStream.copy(distance,count,previousData); } } SHRIState *state=static_cast(_state.get()); state->vlen=vlen; state->vnext=vnext; state->shift=shift; for (uint32_t i=0;i<999;i++) state->ar[i]=ar[i]; } XPKDecompressor::Registry SHRIDecompressor::_XPKregistration; ancient-1.0/src/SHRIDecompressor.hpp000066400000000000000000000023071375530757000174440ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SHRIDECOMPRESSOR_HPP #define SHRIDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class SHRIDecompressor : public XPKDecompressor { private: class SHRIState : public XPKDecompressor::State { public: SHRIState() noexcept; virtual ~SHRIState(); uint32_t vlen=0; uint32_t vnext=0; uint32_t shift=0; uint32_t ar[999]; }; public: SHRIDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~SHRIDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _ver=0; size_t _startOffset=0; size_t _rawSize=0; std::unique_ptr &_state; // reference!!! static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/SLZ3Decompressor.cpp000066400000000000000000000034731375530757000174320ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SLZ3Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool SLZ3Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SLZ3"); } std::unique_ptr SLZ3Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } SLZ3Decompressor::SLZ3Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } SLZ3Decompressor::~SLZ3Decompressor() { // nothing needed } const std::string &SLZ3Decompressor::getSubName() const noexcept { static std::string name="XPK-SLZ3: SLZ3 CyberYAFA compressor"; return name; } void SLZ3Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint8_t tmp=readByte(); if (!tmp) throw Decompressor::DecompressionError(); uint32_t distance=uint32_t(tmp&0xf0)<<4; distance|=uint32_t(readByte()); uint32_t count=uint32_t(tmp&0xf)+2; outputStream.copy(distance,count); } } } XPKDecompressor::Registry SLZ3Decompressor::_XPKregistration; ancient-1.0/src/SLZ3Decompressor.hpp000066400000000000000000000015541375530757000174350ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SLZ3DECOMPRESSOR_HPP #define SLZ3DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class SLZ3Decompressor : public XPKDecompressor { public: SLZ3Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~SLZ3Decompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/SMPLDecompressor.cpp000066400000000000000000000040501375530757000174420ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SMPLDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "HuffmanDecoder.hpp" bool SMPLDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SMPL"); } std::unique_ptr SMPLDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } SMPLDecompressor::SMPLDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || packedData.size()<2) throw Decompressor::InvalidFormatError(); if (packedData.readBE16(0)!=1) throw Decompressor::InvalidFormatError(); } SMPLDecompressor::~SMPLDecompressor() { // nothing needed } const std::string &SMPLDecompressor::getSubName() const noexcept { static std::string name="XPK-SMPL: Huffman compressor with delta encoding"; return name; } void SMPLDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,2,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); HuffmanDecoder decoder; for (uint32_t i=0;i<256;i++) { uint32_t codeLength=readBits(4); if (!codeLength) continue; if (codeLength==15) codeLength=readBits(4)+15; uint32_t code=readBits(codeLength); decoder.insert(HuffmanCode{codeLength,code,i}); } uint8_t accum=0; while (!outputStream.eof()) { uint32_t code=decoder.decode(readBit); accum+=code; outputStream.writeByte(accum); } } XPKDecompressor::Registry SMPLDecompressor::_XPKregistration; ancient-1.0/src/SMPLDecompressor.hpp000066400000000000000000000015541375530757000174550ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SMPLDECOMPRESSOR_HPP #define SMPLDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class SMPLDecompressor : public XPKDecompressor { public: SMPLDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~SMPLDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/SQSHDecompressor.cpp000066400000000000000000000117151375530757000174530ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SQSHDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "HuffmanDecoder.hpp" bool SQSHDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SQSH"); } std::unique_ptr SQSHDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } SQSHDecompressor::SQSHDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || packedData.size()<3) throw Decompressor::InvalidFormatError(); _rawSize=packedData.readBE16(0); if (!_rawSize) throw Decompressor::InvalidFormatError(); } SQSHDecompressor::~SQSHDecompressor() { // nothing needed } const std::string &SQSHDecompressor::getSubName() const noexcept { static std::string name="XPK-SQSH: Compressor for sampled sounds"; return name; } void SQSHDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); ForwardInputStream inputStream(_packedData,2,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readSignedBits=[&](uint8_t bits)->int32_t { int32_t ret=readBits(bits); if (ret&(1<<(bits-1))) ret|=~0U<uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,_rawSize); HuffmanDecoder modDecoder { HuffmanCode{1,0b0001,0}, HuffmanCode{2,0b0000,1}, HuffmanCode{3,0b0010,2}, HuffmanCode{4,0b0110,3}, HuffmanCode{4,0b0111,4} }; HuffmanDecoder lengthDecoder { HuffmanCode{1,0b0000,0}, HuffmanCode{2,0b0010,1}, HuffmanCode{3,0b0110,2}, HuffmanCode{4,0b1110,3}, HuffmanCode{4,0b1111,4} }; HuffmanDecoder distanceDecoder { HuffmanCode{1,0b01,0}, HuffmanCode{2,0b00,1}, HuffmanCode{2,0b01,2} }; // first byte is special uint8_t currentSample=readByte(); outputStream.writeByte(currentSample); uint32_t accum1=0,accum2=0,prevBits=0; while (!outputStream.eof()) { uint8_t bits=0; uint32_t count=0; bool doRepeat=false; if (accum1>=8) { static const uint8_t bitLengthTable[7][8]={ {2,3,4,5,6,7,8,0}, {3,2,4,5,6,7,8,0}, {4,3,5,2,6,7,8,0}, {5,4,6,2,3,7,8,0}, {6,5,7,2,3,4,8,0}, {7,6,8,2,3,4,5,0}, {8,7,6,2,3,4,5,0}}; auto handleCondCase=[&]() { if (bits==8) { if (accum2<20) { count=1; } else { count=2; accum2+=8; } } else { count=5; accum2+=8; } }; auto handleTable=[&](uint32_t newBits) { if (prevBits<2 || !newBits) throw Decompressor::DecompressionError(); bits=bitLengthTable[prevBits-2][newBits-1]; if (!bits) throw Decompressor::DecompressionError(); handleCondCase(); }; uint32_t mod=modDecoder.decode(readBit); switch (mod) { case 0: if (prevBits==8) { bits=8; handleCondCase(); } else { bits=prevBits; count=5; accum2+=8; } break; case 1: doRepeat=true; break; case 2: handleTable(2); break; case 3: handleTable(3); break; case 4: handleTable(readBits(2)+4); break; default: throw Decompressor::DecompressionError(); } } else { if (readBit()) { doRepeat=true; } else { count=1; bits=8; } } if (doRepeat) { uint32_t lengthIndex=lengthDecoder.decode(readBit); static const uint8_t lengthBits[5]={1,1,1,3,5}; static const uint32_t lengthAdditions[5]={2,4,6,8,16}; count=readBits(lengthBits[lengthIndex])+lengthAdditions[lengthIndex]; if (count>=3) { if (accum1) accum1--; if (count>3 && accum1) accum1--; } uint32_t distanceIndex=distanceDecoder.decode(readBit); static const uint8_t distanceBits[3]={12,8,14}; static const uint32_t distanceAdditions[3]={0x101,1,0x1101}; uint32_t distance=readBits(distanceBits[distanceIndex])+distanceAdditions[distanceIndex]; count=std::min(count,uint32_t(_rawSize-outputStream.getOffset())); currentSample=outputStream.copy(distance,count); } else { count=std::min(count,uint32_t(_rawSize-outputStream.getOffset())); for (uint32_t i=0;i>3; } } XPKDecompressor::Registry SQSHDecompressor::_XPKregistration; ancient-1.0/src/SQSHDecompressor.hpp000066400000000000000000000016031375530757000174530ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SQSHDECOMPRESSOR_HPP #define SQSHDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class SQSHDecompressor : public XPKDecompressor { public: SQSHDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~SQSHDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _rawSize=0; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/SXSCDecompressor.cpp000066400000000000000000000540501375530757000174540ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SXSCDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "DLTADecode.hpp" #include "common/MemoryBuffer.hpp" SXSCDecompressor::SXSCReader::SXSCReader(ForwardInputStream &stream) : _reader(stream) { // nothing needed } SXSCDecompressor::SXSCReader::~SXSCReader() { // nothing needed } uint32_t SXSCDecompressor::SXSCReader::readBit() { return _reader.readBits8(1); } bool SXSCDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SASC")||hdr==FourCC("SHSC"); } std::unique_ptr SXSCDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } SXSCDecompressor::SXSCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData), _isHSC(hdr==FourCC("SHSC")) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } SXSCDecompressor::~SXSCDecompressor() { // nothing needed } const std::string &SXSCDecompressor::getSubName() const noexcept { static std::string nameASC="XPK-SASC: LZ-compressor with arithmetic and delta encoding"; static std::string nameHSC="XPK-SHSC: Context modeling compressor"; return _isHSC?nameHSC:nameASC; } void SXSCDecompressor::decompressASC(Buffer &rawData,ForwardInputStream &inputStream) { ForwardOutputStream outputStream(rawData,0,rawData.size()); uint16_t bitReaderInitialValue; bitReaderInitialValue=inputStream.readByte()<<8; bitReaderInitialValue|=inputStream.readByte(); SXSCReader bitReader(inputStream); RangeDecoder arithDecoder(bitReader,bitReaderInitialValue); // decoder for literal, copy, end decision // two thresholds -> 3 symbols, last symbol is break with size of 1 uint16_t bitThreshold1[4]={40,40,40,40}; uint16_t bitThreshold2[4]={40,40,40,40}; uint32_t bitPos=0; // generics for the other decoder auto tableElements=[](auto &table)->uint16_t { return (sizeof(table)/sizeof(*table)+1)>>1; }; auto initTable=[](auto &table,uint16_t initialValue) { constexpr uint32_t length=(sizeof(table)/sizeof(*table)+1)>>1; for (uint32_t i=0;i>1; for (uint32_t i=index;i>1)+length) table[i]+=value; if (table[length*2-2]>=max) { for (uint32_t i=0;i1) table[i]>>=1; for (uint32_t j=0,i=length;iuint16_t { constexpr uint32_t length=(sizeof(table)/sizeof(*table)+1)>>1; return table[length*2-2]; }; auto decodeSymbol=[](auto &table,uint16_t &value)->uint16_t { constexpr uint32_t length=(sizeof(table)/sizeof(*table)+1)>>1; uint32_t threshold=0; uint32_t i=length*2-4; while (i>=length) { uint32_t child=(i-length)<<1; if (value-threshold>=table[i]) { threshold+=table[i]; child+=2; } i=child; } if (value-threshold>=table[i]) { threshold+=table[i]; i++; } value=threshold; return i; }; // literal decoder uint16_t litInitial[256*2-1]; uint16_t litDynamic[256*2-1]; uint16_t litThreshold=1; initTable(litInitial,1); initTable(litDynamic,0); // distance / length decoder uint16_t distanceCodes[16*2-1]; uint16_t countInitial[64*2-1]; uint16_t countDynamic[64*2-1]; uint16_t countThreshold=8; initTable(distanceCodes,0); initTable(countInitial,1); initTable(countDynamic,0); updateTable(distanceCodes,6000,0,24); uint32_t distanceIndex=0; auto twoStepArithDecoder=[&](auto &initialTable,auto &dynamicTable,uint16_t &threshold,uint16_t max,uint16_t step,uint16_t updateRange)->uint16_t { uint16_t value=arithDecoder.decode(tableSize(dynamicTable)+threshold); uint16_t ret; if (valueupdateRange?ret-updateRange:0;istep?threshold-step:1; return ret; }; for (;;) { uint16_t bitSize=bitThreshold1[bitPos]+bitThreshold2[bitPos]; uint16_t bitValue=arithDecoder.decode(bitSize+1); if (bitValue==bitSize) break; bool bit=bitValue=6000) { if (!(bitThreshold1[bitPos]>>=1)) bitThreshold1[bitPos]=1; if (!(bitThreshold2[bitPos]>>=1)) bitThreshold2[bitPos]=1; } bitPos=(bitPos<<1&2)|(bit?0:1); if (bit) { // literal outputStream.writeByte(twoStepArithDecoder(litInitial,litDynamic,litThreshold,1000,1,8)); } else { // copy while (outputStream.getOffset()>size_t(1<=2) { uint16_t minRange=1<<(distanceBits-1); uint16_t range=distanceIndex==distanceBits?std::min(outputStream.getOffset(),size_t(31200U))-minRange:minRange; distance=arithDecoder.decode(range); arithDecoder.scale(distance,distance+1,range); distance+=minRange; } distance++; uint32_t count=twoStepArithDecoder(countInitial,countDynamic,countThreshold,6000,8,4); if (count==15) { count=783; } else if (count>=16) { uint16_t value=arithDecoder.decode(16); arithDecoder.scale(value,value+1,16); count=((count-16)<<4)+value+15; } count+=3; outputStream.copy(distance,count); } } if (!outputStream.eof()) throw Decompressor::DecompressionError(); } template class CheckedArray { public: CheckedArray() : _memory(length*sizeof(T)) { // nothing needed } ~CheckedArray() { // nothing needed } T &operator[](size_t i) { if (i>=length) throw Decompressor::DecompressionError(); return _memory.cast()[i]; } const T &operator[](size_t i) const { if (i>=length) throw Decompressor::DecompressionError(); return _memory.cast()[i]; } private: MemoryBuffer _memory; }; // The horror. It needs to follow exactly the original logic, even if that logic is not very good. void SXSCDecompressor::decompressHSC(Buffer &rawData,ForwardInputStream &inputStream) { struct Model { uint8_t context[4]; uint16_t hashPointer; uint16_t expiryPrevious; uint16_t expiryNext; uint16_t frequencyTotal; uint16_t escapeFrequency; uint8_t contextLength; uint8_t characterCount; uint8_t refreshCounter; }; struct Frequency { uint16_t frequency; uint16_t next; uint8_t character; }; struct HashItem { uint16_t data; uint16_t random; }; ForwardOutputStream outputStream(rawData,0,rawData.size()); uint16_t bitReaderInitialValue; bitReaderInitialValue=inputStream.readByte()<<8; bitReaderInitialValue|=inputStream.readByte(); SXSCReader bitReader(inputStream); RangeDecoder arithDecoder(bitReader,bitReaderInitialValue); uint8_t maxContextLength=4; int16_t dropCount=2500; int16_t contextSearchLength=0; CheckedArray models{}; for (uint32_t i=0;i<10000;i++) { auto &m=models[i]; for (uint32_t j=0;j<4;j++) m.context[j]=0; m.hashPointer=0; m.expiryPrevious=i-1; m.expiryNext=i+1; m.frequencyTotal=0; m.escapeFrequency=0; m.contextLength=0xffU; m.characterCount=0; m.refreshCounter=0; } uint8_t currentContext[4]; for (uint32_t i=0;i<4;i++) currentContext[i]=0; uint16_t firstExpiry=0,lastExpiry=9999; uint16_t freeBlockPointer=10000; uint16_t releaseBlock=0; CheckedArray frequencies{}; for (uint32_t i=0;i<32760;i++) { auto &f=frequencies[i]; f.frequency=0; f.next=(i>=10000&&i<32759)?i+1:0xffffU; f.character=0; } CheckedArray hashes{}; for (uint32_t i=0,j=10;i<0x4000U;i++) { auto &h=hashes[i]; h.data=0xffffU; // constants used are 2147483647 % / 16807 int32_t integerPart=j/127773U; int32_t fractionalPart=j%127773U; int32_t tmp=16807*fractionalPart-2836*integerPart; j=tmp<0?tmp+0x7fff'ffff:tmp; h.random=j&0x3fffU; } uint16_t hashStack[5]; for (uint32_t i=0;i<5;i++) hashStack[i]=0; bool characterMask[256]; uint8_t characterMaskStack[256]; for (uint32_t i=0;i<256;i++) { characterMask[i]=false; characterMaskStack[i]=0; } int16_t characterMaskStackPointer=0; uint8_t initialEscapeChar[5]; for (uint32_t i=0;i<5;i++) initialEscapeChar[i]=i?15:16; uint8_t escapeCharacterCounter=0; uint16_t stackPointer=0; uint16_t contextPointer[5]; uint16_t frequencyArrayIndex[5]; for (uint32_t i=0;i<5;i++) { contextPointer[i]=0; frequencyArrayIndex[i]=0; } auto loopBreaker=[](uint16_t i) { if (i>=0x8000U) throw Decompressor::DecompressionError(); }; auto findNext=[&]()->uint16_t { for (int32_t i=contextSearchLength-1;i>=0;i--) { for (uint32_t lb=0,j=hashes[hashStack[i]].data;j!=0xffffU;j=models[j].hashPointer,loopBreaker(lb++)) { if (i==models[j].contextLength) { if ([&]()->bool { for (int32_t k=0;k=value) break; i++; } arithDecoder.scale(i,i+1,size); break; } // madness!!! auto getEscFrequency=[&](uint16_t value,uint16_t i)->uint16_t { auto &model=models[i]; if (model.frequencyTotal==1) return initialEscapeChar[model.contextLength]>=16?2:1; if (model.characterCount==0xffU) return 1; uint16_t tmp=uint16_t(model.characterCount)*2+2; if (model.characterCount && tmp>=model.frequencyTotal) { value=int32_t(value)*tmp/model.frequencyTotal; if (model.characterCount+1==model.frequencyTotal) value+=tmp>>2; } if (!value) value++; return value; }; int16_t freq=0,escapeFreq=0; int16_t currentFrequency=0,frequencyTotal=0; auto decodeCf=[&](uint16_t shift,bool cmCondition)->uint16_t { freq<<=shift; uint16_t i,value=arithDecoder.decode(freq+escapeFreq)>>shift; uint32_t lb=0; for (i=index;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) { auto &frequency=frequencies[i]; if (cmCondition||!characterMask[frequency.character]) { if (frequencyTotal+frequency.frequency<=value) { frequencyTotal+=frequency.frequency; } else { currentFrequency=frequency.frequency<bool { if (chPos==0xffffU) { arithDecoder.scale(freq,freq+escapeFreq,freq+escapeFreq); if (models[index].frequencyTotal==1 && initialEscapeChar[models[index].contextLength]<32) initialEscapeChar[models[index].contextLength]++; uint16_t prevI=0; for (uint16_t lb=0,i=index;i!=0xffffU;prevI=i,i=frequencies[i].next,loopBreaker(lb++)) { auto &frequency=frequencies[i]; if (cmCondition||!characterMask[frequency.character]) { if (characterMaskStackPointer==256) throw Decompressor::DecompressionError(); characterMaskStack[characterMaskStackPointer++]=frequency.character; characterMask[frequency.character]=true; } } contextPointer[insertPos]=index|0x8000U; frequencyArrayIndex[insertPos]=prevI; ch=256; return true; } else { arithDecoder.scale(frequencyTotal,frequencyTotal+currentFrequency,freq+escapeFreq); if (models[index].frequencyTotal==1 && initialEscapeChar[models[index].contextLength]) initialEscapeChar[models[index].contextLength]--; contextPointer[insertPos]=index; frequencyArrayIndex[insertPos]=chPos; ch=frequencies[chPos].character; return false; } }; if (characterMaskStackPointer) { for (uint16_t lb=0,i=index;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) { auto &frequency=frequencies[i]; if (!characterMask[frequency.character]) { freq+=frequency.frequency; if (frequency.frequency<3) escapeFreq++; } } escapeFreq=getEscFrequency(escapeFreq,index); uint16_t chPos=decodeCf(0,false); if (stackPointer==5) throw Decompressor::DecompressionError(); if (!decodeCh(chPos,stackPointer,false)) { if (escapeCharacterCounter==10) throw Decompressor::DecompressionError(); escapeCharacterCounter++; } stackPointer++; } else { freq=models[index].frequencyTotal; escapeFreq=getEscFrequency(models[index].escapeFrequency,index); uint16_t chPos=decodeCf((escapeCharacterCounter>=5)?(freq<5 && escapeCharacterCounter==10)?2:1:0,true); stackPointer=1; if (decodeCh(chPos,0,true)) { escapeCharacterCounter=0; } else { if (escapeCharacterCounter<10) escapeCharacterCounter++; } } if (ch!=256) { if (index!=firstExpiry) { auto &model=models[index]; if (index==lastExpiry) { lastExpiry=model.expiryPrevious; } else { models[model.expiryNext].expiryPrevious=model.expiryPrevious; models[model.expiryPrevious].expiryNext=model.expiryNext; } models[firstExpiry].expiryPrevious=index; model.expiryNext=firstExpiry; firstExpiry=index; } break; } } if (ch==256) break; while (stackPointer) { uint16_t freqIndex=frequencyArrayIndex[--stackPointer]; uint16_t pointer=contextPointer[stackPointer]; auto &model=models[pointer&0x7fffU]; if (pointer&0x8000U) { if (freeBlockPointer==0xffffU) { for (uint16_t i=0;i<=stackPointer;) { // yuck uint32_t lb=0; do { releaseBlock=(releaseBlock!=9999U)?releaseBlock+1:0; loopBreaker(lb++); } while (frequencies[releaseBlock].next==0xffffU); for (i=0;i<=stackPointer;i++) if ((contextPointer[i]&0x7fffU)==releaseBlock) break; } auto &frequencyRB=frequencies[releaseBlock]; auto &modelRB=models[releaseBlock]; uint16_t f=frequencyRB.frequency; for (uint16_t lb=0,i=frequencyRB.next;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) if (frequencies[i].frequencyfrequencies[freqIndex].frequency*2) { model.refreshCounter--; } else if (model.refreshCounter<4) { model.refreshCounter++; } // one ugly scaler if (!model.refreshCounter||model.frequencyTotal>=8000) { model.refreshCounter++; model.escapeFrequency=0; model.frequencyTotal=0; for (uint16_t lb=0,i=pointer&0x7fffU;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) { if (frequencies[i].frequency>1) { uint16_t tmp=frequencies[i].frequency>>=1; model.frequencyTotal+=tmp; if (tmp<3) model.escapeFrequency++; } else { model.escapeFrequency++; model.frequencyTotal++; } } } } uint8_t maxLength=maxContextLength+1; while (maxLength-->minLength) { auto hash=[&](const uint8_t *block,uint32_t length)->uint16_t { uint16_t ret=0; for (uint32_t i=0;i=2); std::unique_ptr tmpBuffer; if (needsTmpBuffer) tmpBuffer=std::make_unique(rawData.size()); if (_isHSC) decompressHSC(needsTmpBuffer?*tmpBuffer:rawData,inputStream); else decompressASC(needsTmpBuffer?*tmpBuffer:rawData,inputStream); // Mono, High byte only // also includes de-interleaving auto deltaDecode16BE=[&]() { size_t length=rawData.size(); const uint8_t *src=tmpBuffer->data(); const uint8_t *midSrc=&src[length>>1]; uint8_t *dest=rawData.data(); uint8_t ctr=0; for (size_t i=0,j=0;jdata(); const uint8_t *midSrc=&src[length>>1]; uint8_t *dest=rawData.data(); uint8_t ctr=0; for (size_t i=0,j=0;j SXSCDecompressor::_XPKregistration; ancient-1.0/src/SXSCDecompressor.hpp000066400000000000000000000025071375530757000174610ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SXSCDECOMPRESSOR_HPP #define SXSCDECOMPRESSOR_HPP #include #include "XPKDecompressor.hpp" #include "InputStream.hpp" #include "RangeDecoder.hpp" class SXSCDecompressor : public XPKDecompressor { public: SXSCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~SXSCDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: class SXSCReader : public RangeDecoder::BitReader { public: SXSCReader(ForwardInputStream &stream); virtual ~SXSCReader(); virtual uint32_t readBit() override final; private: MSBBitReader _reader; }; void decompressASC(Buffer &rawData,ForwardInputStream &inputStream); void decompressHSC(Buffer &rawData,ForwardInputStream &inputStream); const Buffer &_packedData; bool _isHSC; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/StoneCrackerDecompressor.cpp000066400000000000000000000404651375530757000212640ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "StoneCrackerDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "HuffmanDecoder.hpp" bool StoneCrackerDecompressor::detectHeaderAndGeneration(uint32_t hdr,uint32_t &generation) noexcept { // Stonecracker 2.71 (and others from 2.69 - 2.8 series) do not have any sensible identification value // first 3 bytes are constants for RLE-encoder (byte values for RLE-encoder, at least they are unique) // last byte is bit length. // // 2.92 and 2.99 do not have either any proper identification word either, however its // bit lengts for decompressor are stored in the first 4 bytes, which forms identifiable // value. // // Thus for detecting 2.71 and friends, we are creating lots of false positives here. // At least we can rule those out later when detecting the actual header content // Final complication is that values for 2.71/2.9X overlap, this we need to handle // later as well if (hdr>=0x08090a08U&&hdr<=0x08090a0eU&&hdr!=0x08090a09U) { // can be generation 1 as well. needs to be determined later generation=2; return true; } if ((hdr&0xffU)>=0x08U&&(hdr&0xffU)<=0x0eU) { uint8_t byte0=hdr>>24U; uint8_t byte1=hdr>>16U; uint8_t byte2=hdr>>8U; // only limiter I can think of apart from the last byte is that the rle-bytes are unique if (byte0!=byte1 && byte0!=byte2 && byte1!=byte2) { generation=1; return true; } } // From 3.00 and onwards we can be certain of the format switch (hdr) { case FourCC("S300"): generation=3; return true; case FourCC("S310"): generation=4; return true; case FourCC("S400"): generation=5; return true; case FourCC("S401"): generation=6; return true; case FourCC("S403"): generation=7; return true; case FourCC("S404"): generation=8; return true; default: return false; } } bool StoneCrackerDecompressor::detectHeader(uint32_t hdr) noexcept { uint32_t dummy; return detectHeaderAndGeneration(hdr,dummy); } std::unique_ptr StoneCrackerDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_unique(packedData,exactSizeKnown,verify); } void StoneCrackerDecompressor::initialize(const Buffer &packedData,uint32_t hdr) { auto readModes=[&](uint32_t value) { for (uint32_t i=0;i<4U;i++) { _modes[i]=value>>24U; if (_modes[i]<8U || _modes[i]>14U) throw InvalidFormatError(); value<<=8U; } }; switch (_generation) { case 1: _dataOffset=18U; _rle[0]=hdr>>24U; _rle[1]=hdr>>16U; _rle[2]=hdr>>8U; _modes[0]=hdr; if (packedData.size()<_dataOffset) throw InvalidFormatError(); for (uint32_t i=1;i<3U;i++) { _modes[i]=packedData.read8(i+15U); if (_modes[i]<4U || _modes[i]>7U) throw InvalidFormatError(); } _rleSize=packedData.readBE32(4U); _rawSize=packedData.readBE32(8U); _packedSize=packedData.readBE32(12U); break; case 2: readModes(hdr); case 4: case 5: case 6: _dataOffset=12U; if (packedData.size()<_dataOffset) throw InvalidFormatError(); _rawSize=packedData.readBE32(4U); _packedSize=packedData.readBE32(8U); break; case 3: _dataOffset=16U; if (packedData.size()<_dataOffset) throw InvalidFormatError(); readModes(packedData.readBE32(4)); _rawSize=packedData.readBE32(8U); _packedSize=packedData.readBE32(12U); break; case 7: case 8: _dataOffset=16U; if (packedData.size()<_dataOffset+2U) throw InvalidFormatError(); _rawSize=packedData.readBE32(8U); _packedSize=packedData.readBE32(12U)+2U; break; default: throw InvalidFormatError(); break; } // Final sanity checks on old formats, especially on 2.71 which can still be false positive // For a sanity check we assume the compressor is actually compressing, which is not a exactly wrong thing to do // (of course there could be expanding files, but it is doubtful someone would actually used them) // Also, the compressor seem to crash on large files. Lets cap the filesize to 1M if (_generation==1 && (_rleSize>_rawSize || _packedSize>_rleSize || _rawSize>1048'576U)) throw InvalidFormatError(); _packedSize+=_dataOffset; if (_packedSize>packedData.size()) throw InvalidFormatError(); } StoneCrackerDecompressor::StoneCrackerDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData(packedData) { uint32_t hdr=packedData.readBE32(0); if (!detectHeaderAndGeneration(hdr,_generation)) throw InvalidFormatError(); bool initialized=false; // this is the fallback if we have accidentally identified the wrong version if (_generation==2) { try { initialize(packedData,hdr); initialized=true; } catch (const Error &e) { _generation=1; } } if (!initialized) initialize(packedData,hdr); } StoneCrackerDecompressor::~StoneCrackerDecompressor() { // nothing needed } const std::string &StoneCrackerDecompressor::getName() const noexcept { switch (_generation) { case 1: { static std::string name="SC: StoneCracker v2.69 - v2.81"; return name; } case 2: { static std::string name="SC: StoneCracker v2.92, v2.99"; return name; } case 3: { static std::string name="S300: StoneCracker v3.00"; return name; } case 4: { static std::string name="S310: StoneCracker v3.10, v3.11b"; return name; } case 5: { static std::string name="S400: StoneCracker pre v4.00"; return name; } case 6: { static std::string name="S401: StoneCracker v4.01"; return name; } case 7: { static std::string name="S403: StoneCracker v4.02a"; return name; } case 8: { static std::string name="S404: StoneCracker v4.10"; return name; } } static std::string dummy=""; return dummy; } size_t StoneCrackerDecompressor::getPackedSize() const noexcept { return _packedSize; } size_t StoneCrackerDecompressor::getRawSize() const noexcept { return _rawSize; } // v2.71 void StoneCrackerDecompressor::decompressGen1(Buffer &rawData) { BackwardInputStream inputStream(_packedData,_dataOffset,_packedSize); MSBBitReader bitReader(inputStream); MemoryBuffer tmpBuffer(_rleSize); BackwardOutputStream outputStream(tmpBuffer,0,_rleSize); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBitsBE32(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1); }; // anchor-bit handling { uint32_t value=0; for (uint32_t i=0;i<4U;i++) { value>>=8U; value|=uint32_t(inputStream.readByte())<<24U; } uint32_t tmp=value; uint32_t count=0; while (tmp) { tmp<<=1; count++; } if (count) count--; if (count) bitReader.reset(value>>(32U-count),count); } struct SCItem { uint32_t distanceBits; uint32_t countBits; uint32_t addition; bool isLiteral; }; // logic is very hard to un-convolute... HuffmanDecoder scDecoder { HuffmanCode{2,0b000,SCItem{0,3U,0,true}}, HuffmanCode{3,0b010,SCItem{0,_modes[2]+1U,8U,true}}, HuffmanCode{3,0b011,SCItem{_modes[0]+1U,_modes[1]+1U,5U,false}}, HuffmanCode{2,0b010,SCItem{8U,0,2U,false}}, HuffmanCode{3,0b110,SCItem{9U,0,3U,false}}, HuffmanCode{3,0b111,SCItem{10U,0,4U,false}} }; while (!outputStream.eof()) { const auto &item=scDecoder.decode(readBit); if (item.isLiteral) { uint32_t count=readBits(item.countBits); if (!item.addition) count=!count?8U:count; // why?!? else count+=item.addition; for (uint32_t i=0;i bitReader(inputStream); BackwardOutputStream outputStream(rawData,0,_rawSize); // anchor-bit handling { uint32_t value=0; for (uint32_t i=0;i<4U;i++) { value>>=8U; value|=uint32_t(inputStream.readByte())<<24U; } if (value) for (uint32_t i=31U;i>0;i--) { if (value&(1U<uint32_t { return bitReader.readBitsBE32(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1); }; auto readCount=[&](uint32_t threshold,uint32_t bits)->uint32_t { uint32_t ret=0; uint32_t tmp; do { tmp=rotateBits(readBits(bits),bits); ret+=tmp; } while (tmp==threshold); return ret; }; bool gen3=_generation>=3; while (!outputStream.eof()) { if (readBit()) { uint32_t count=readCount(7U,3U); if (gen3) count++; // for 2.92 zero count could meant count of 65536 // for 2.99 it would be 4G // nevertheless it is an error // 3.00 fixes this by +1 if (!count) throw DecompressionError(); for (uint32_t i=0;i bitReader(inputStream); ForwardOutputStream outputStream(rawData,0,_rawSize); auto readBits=[&](uint32_t count)->uint32_t { if (_generation==4) return bitReader.readBitsBE32(count); else return bitReader.readBitsBE16(count); }; auto readBit=[&]()->uint32_t { return readBits(1); }; auto readCount=[&](uint32_t threshold,uint32_t bits)->uint32_t { uint32_t ret=0; uint32_t tmp; do { tmp=readBits(bits); ret+=tmp; } while (tmp==threshold); return ret; }; static const uint8_t modes[4]={10U,11U,12U,8U}; while (!outputStream.eof()) { if (readBit()) { uint32_t mode=readBits(2U); uint32_t distance=readBits(modes[mode]); // will obviously throw if distance is 0 uint32_t count; if (mode>=2U) { if (_generation==4) count=readCount(15U,4U)+3U; else count=readCount(7U,3U)+3U; } else { count=mode+3U; } // yet another bug if (count>_rawSize-uint32_t(outputStream.getOffset())) count=_rawSize-uint32_t(outputStream.getOffset()); outputStream.copy(distance,count); } else { uint32_t count=readCount(7U,3U); // A regression in 3.10 that is not fixed until 4.01 if (_generation==6) count++; if (!count) throw DecompressionError(); for (uint32_t i=0;i bitReader(inputStream); BackwardOutputStream outputStream(rawData,0,_rawSize); // incomplete value at start { uint16_t bitCount=_packedData.readBE16(_packedSize-2U); if (bitCount>16U) throw DecompressionError(); uint16_t value=inputStream.readByte(); value|=uint16_t(inputStream.readByte())<<8U; bitReader.reset(value,bitCount); } auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBitsBE16(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1); }; while (!outputStream.eof()) { if (readBit()) { static const uint8_t distanceBits[4]={5U,8U,10U,12U}; static const uint32_t distanceAdditions[4]={1U,0x21,0x121U,0x521U}; uint32_t mode=readBits(2U); uint32_t distance=readBits(distanceBits[mode])+distanceAdditions[mode]; // christmas tree! uint32_t count=2U; if (!readBit()) { count++; if (!readBit()) { count++; if (!readBit()) { count++; uint32_t tmp; do { tmp=readBits(3U); count+=tmp; } while (tmp==7U); } } } outputStream.copy(distance,count); } else { outputStream.writeByte(readBits(8)); } } } // v4.10 void StoneCrackerDecompressor::decompressGen8(Buffer &rawData) { BackwardInputStream inputStream(_packedData,_dataOffset,_packedSize-2U); MSBBitReader bitReader(inputStream); BackwardOutputStream outputStream(rawData,0,_rawSize); // incomplete value at start uint16_t modeBits=0; { uint16_t bitCount=_packedData.readBE16(_packedSize-2U)&15U; uint16_t value=inputStream.readByte(); value|=uint16_t(inputStream.readByte())<<8U; bitReader.reset(value>>(16U-bitCount),bitCount); modeBits=inputStream.readByte(); modeBits|=uint16_t(inputStream.readByte())<<8U; if (modeBits<10U || modeBits>14U) throw DecompressionError(); } auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBitsBE16(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1); }; struct CountItem { uint32_t bits; uint32_t addition; bool isLiteral; }; struct DistanceItem { uint32_t bits; uint32_t addition; }; HuffmanDecoder countDecoder { HuffmanCode{1,0b00000000,CountItem{0,1U,true}}, HuffmanCode{2,0b00000011,CountItem{1U,2U,false}}, HuffmanCode{3,0b00000101,CountItem{2U,4U,false}}, HuffmanCode{4,0b00001000,CountItem{8U,23U,false}}, HuffmanCode{5,0b00010010,CountItem{3U,8U,false}}, HuffmanCode{6,0b00100110,CountItem{2U,16U,false}}, HuffmanCode{7,0b01001110,CountItem{1U,20U,false}}, HuffmanCode{8,0b10011110,CountItem{0,22U,false}}, HuffmanCode{8,0b10011111,CountItem{5U,14U,true}} }; HuffmanDecoder distanceDecoder { HuffmanCode{1,0b01,DistanceItem{modeBits,0x221U}}, HuffmanCode{2,0b00,DistanceItem{9U,0x21U}}, HuffmanCode{2,0b01,DistanceItem{5U,0x1U}} }; while (!outputStream.eof()) { const auto &countItem=countDecoder.decode(readBit); uint32_t count=countItem.addition; uint32_t tmp; do { tmp=readBits(countItem.bits); count+=tmp; } while (tmp==0xffU); if (countItem.isLiteral) { for (uint32_t i=0;i StoneCrackerDecompressor::_registration; ancient-1.0/src/StoneCrackerDecompressor.hpp000066400000000000000000000025321375530757000212620ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef STONECRACKERDECOMPRESSOR_HPP #define STONECRACKERDECOMPRESSOR_HPP #include "Decompressor.hpp" class StoneCrackerDecompressor : public Decompressor { public: StoneCrackerDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); virtual ~StoneCrackerDecompressor(); virtual const std::string &getName() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual size_t getRawSize() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; static bool detectHeader(uint32_t hdr) noexcept; static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: static bool detectHeaderAndGeneration(uint32_t hdr,uint32_t &generation) noexcept; void initialize(const Buffer &packedData,uint32_t hdr); void decompressGen1(Buffer &rawData); void decompressGen23(Buffer &rawData); void decompressGen456(Buffer &rawData); void decompressGen7(Buffer &rawData); void decompressGen8(Buffer &rawData); const Buffer &_packedData; uint32_t _rawSize=0; uint32_t _packedSize=0; uint32_t _rleSize=0; uint8_t _modes[4]; uint8_t _rle[3]; uint32_t _generation; uint32_t _dataOffset; static Decompressor::Registry _registration; }; #endif ancient-1.0/src/TDCSDecompressor.cpp000066400000000000000000000043751375530757000174360ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "TDCSDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool TDCSDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("TDCS"); } std::unique_ptr TDCSDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } TDCSDecompressor::TDCSDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } TDCSDecompressor::~TDCSDecompressor() { // nothing needed } const std::string &TDCSDecompressor::getSubName() const noexcept { static std::string name="XPK-TDCS: LZ77-compressor"; return name; } void TDCSDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto read2Bits=[&]()->uint32_t { return bitReader.readBitsBE32(2); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { uint32_t distance=0; uint32_t count=0; uint32_t tmp; switch (read2Bits()) { case 0: outputStream.writeByte(readByte()); break; case 1: tmp=uint32_t(readByte())<<8; tmp|=uint32_t(readByte()); count=(tmp&3)+3; distance=((tmp>>2)^0x3fff)+1; break; case 2: tmp=uint32_t(readByte())<<8; tmp|=uint32_t(readByte()); count=(tmp&0xf)+3; distance=((tmp>>4)^0xfff)+1; break; case 3: distance=uint32_t(readByte())<<8; distance|=uint32_t(readByte()); count=uint32_t(readByte())+3; if (!distance) throw Decompressor::DecompressionError(); distance=(distance^0xffff)+1; break; default: throw Decompressor::DecompressionError(); } if (count && distance) outputStream.copy(distance,count); } } XPKDecompressor::Registry TDCSDecompressor::_XPKregistration; ancient-1.0/src/TDCSDecompressor.hpp000066400000000000000000000015541375530757000174370ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef TDCSDECOMPRESSOR_HPP #define TDCSDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class TDCSDecompressor : public XPKDecompressor { public: TDCSDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~TDCSDecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/TPWMDecompressor.cpp000066400000000000000000000041371375530757000174640ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "TPWMDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool TPWMDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("TPWM"); } std::unique_ptr TPWMDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_unique(packedData,verify); } TPWMDecompressor::TPWMDecompressor(const Buffer &packedData,bool verify) : _packedData(packedData) { uint32_t hdr=packedData.readBE32(0); if (!detectHeader(hdr) || packedData.size()<12) throw InvalidFormatError(); _rawSize=packedData.readBE32(4); if (!_rawSize || _rawSize>getMaxRawSize()) throw InvalidFormatError(); } TPWMDecompressor::~TPWMDecompressor() { // nothing needed } const std::string &TPWMDecompressor::getName() const noexcept { static std::string name="TPWM: Turbo Packer"; return name; } size_t TPWMDecompressor::getPackedSize() const noexcept { // No packed size in the stream :( // After decompression, we can tell how many bytes were actually used return _decompressedPackedSize; } size_t TPWMDecompressor::getRawSize() const noexcept { return _rawSize; } void TPWMDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); ForwardInputStream inputStream(_packedData,8,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,_rawSize); while (!outputStream.eof()) { if (readBit()) { uint8_t byte1=readByte(); uint8_t byte2=readByte(); uint32_t distance=(uint32_t(byte1&0xf0)<<4)|byte2; uint32_t count=uint32_t(byte1&0xf)+3; count=std::min(count,uint32_t(_rawSize-outputStream.getOffset())); outputStream.copy(distance,count); } else { outputStream.writeByte(readByte()); } } _decompressedPackedSize=inputStream.getOffset(); } Decompressor::Registry TPWMDecompressor::_registration; ancient-1.0/src/TPWMDecompressor.hpp000066400000000000000000000015361375530757000174710ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef TPWMDECOMPRESSOR_HPP #define TPWMDECOMPRESSOR_HPP #include "Decompressor.hpp" class TPWMDecompressor : public Decompressor { public: TPWMDecompressor(const Buffer &packedData,bool verify); virtual ~TPWMDecompressor(); virtual const std::string &getName() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual size_t getRawSize() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; static bool detectHeader(uint32_t hdr) noexcept; static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: const Buffer &_packedData; uint32_t _rawSize=0; size_t _decompressedPackedSize=0; static Decompressor::Registry _registration; }; #endif ancient-1.0/src/XPKDecompressor.cpp000066400000000000000000000010641375530757000173330ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "XPKDecompressor.hpp" #include "XPKMain.hpp" XPKDecompressor::State::~State() { // nothing needed } XPKDecompressor::XPKDecompressor(uint32_t recursionLevel) : _recursionLevel(recursionLevel) { // nothing needed } XPKDecompressor::~XPKDecompressor() { // nothing needed } void XPKDecompressor::registerDecompressor(bool(*detect)(uint32_t),std::unique_ptr(*create)(uint32_t,uint32_t,const Buffer&,std::unique_ptr&,bool)) { XPKMain::registerDecompressor(detect,create); } ancient-1.0/src/XPKDecompressor.hpp000066400000000000000000000022761375530757000173460ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef XPKDECOMPRESSOR_HPP #define XPKDECOMPRESSOR_HPP #include #include #include #include "common/Buffer.hpp" #include "Decompressor.hpp" class XPKDecompressor { public: class State { public: State(const State&)=delete; State& operator=(const State&)=delete; State()=default; virtual ~State(); uint32_t getRecursionLevel() const; }; XPKDecompressor(const XPKDecompressor&)=delete; XPKDecompressor& operator=(const XPKDecompressor&)=delete; XPKDecompressor(uint32_t recursionLevel=0); virtual ~XPKDecompressor(); virtual const std::string &getSubName() const noexcept=0; // Actual decompression virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify)=0; template class Registry { public: Registry() { XPKDecompressor::registerDecompressor(T::detectHeaderXPK,T::create); } ~Registry() { // TODO: no cleanup yet } }; private: static void registerDecompressor(bool(*detect)(uint32_t),std::unique_ptr(*create)(uint32_t,uint32_t,const Buffer&,std::unique_ptr&,bool)); protected: uint32_t _recursionLevel; }; #endif ancient-1.0/src/XPKMain.cpp000066400000000000000000000162771375530757000155660ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include "common/SubBuffer.hpp" #include "XPKMain.hpp" #include "XPKDecompressor.hpp" bool XPKMain::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("XPKF"); } std::unique_ptr XPKMain::create(const Buffer &packedData,bool verify,bool exactSizeKnown) { return std::make_unique(packedData,verify,0); } std::vector(*)(uint32_t,uint32_t,const Buffer&,std::unique_ptr&,bool)>> *XPKMain::_XPKDecompressors=nullptr; void XPKMain::registerDecompressor(bool(*detect)(uint32_t),std::unique_ptr(*create)(uint32_t,uint32_t,const Buffer&,std::unique_ptr&,bool)) { static std::vector(*)(uint32_t,uint32_t,const Buffer&,std::unique_ptr&,bool)>> _list; if (!_XPKDecompressors) _XPKDecompressors=&_list; _XPKDecompressors->emplace_back(detect,create); } XPKMain::XPKMain(const Buffer &packedData,bool verify,uint32_t recursionLevel) : _packedData(packedData) { if (packedData.size()<44) throw InvalidFormatError(); uint32_t hdr=packedData.readBE32(0); if (!detectHeader(hdr)) throw InvalidFormatError(); _packedSize=packedData.readBE32(4); _type=packedData.readBE32(8); _rawSize=packedData.readBE32(12); if (!_rawSize || !_packedSize) throw InvalidFormatError(); if (_rawSize>getMaxRawSize() || _packedSize>getMaxPackedSize()) throw InvalidFormatError(); uint8_t flags=packedData.read8(32); _longHeaders=(flags&1)?true:false; if (flags&2) throw InvalidFormatError(); // needs password. we do not support that if (flags&4) // extra header { _headerSize=38+uint32_t(packedData.readBE16(36)); } else { _headerSize=36; } if (size_t(_packedSize)+8>packedData.size()) throw InvalidFormatError(); bool found=false; for (auto &it : *_XPKDecompressors) { if (it.first(_type)) { if (recursionLevel>=getMaxRecursionLevel()) throw InvalidFormatError(); else { found=true; break; } } } if (!found) throw InvalidFormatError(); auto headerChecksum=[](const Buffer &buffer,size_t offset,size_t len)->bool { if (!len || offset+len>buffer.size()) return false; const uint8_t *ptr=buffer.data()+offset; uint8_t tmp=0; for (size_t i=0;ibool { if (!len || offset+len>buffer.size()) return false; const uint8_t *ptr=buffer.data()+offset; uint8_t tmp[2]={0,0}; for (size_t i=0;i>8) && tmp[1]==(checkValue&0xff); }; if (verify) { if (!headerChecksum(_packedData,0,36)) throw VerificationError(); std::unique_ptr state; forEachChunk([&](const Buffer &header,const Buffer &chunk,uint32_t rawChunkSize,uint8_t chunkType)->bool { if (!headerChecksum(header,0,header.size())) throw VerificationError(); uint16_t hdrCheck=header.readBE16(2); if (chunk.size() && !chunkChecksum(chunk,0,chunk.size(),hdrCheck)) throw VerificationError(); if (chunkType==1) { auto sub=createDecompressor(_type,_recursionLevel,chunk,state,true); } else if (chunkType!=0 && chunkType!=15) throw InvalidFormatError(); return true; }); } } XPKMain::~XPKMain() { // nothing needed } const std::string &XPKMain::getName() const noexcept { std::unique_ptr sub; std::unique_ptr state; try { forEachChunk([&](const Buffer &header,const Buffer &chunk,uint32_t rawChunkSize,uint8_t chunkType)->bool { try { sub=createDecompressor(_type,_recursionLevel,chunk,state,false); } catch (const Error&) { // should not happen since the code is already tried out, // however, lets handle the case gracefully } return false; }); } catch (const Buffer::Error&) { // ditto } static std::string invName=""; return (sub)?sub->getSubName():invName; } size_t XPKMain::getPackedSize() const noexcept { return _packedSize+8; } size_t XPKMain::getRawSize() const noexcept { return _rawSize; } void XPKMain::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); uint32_t destOffset=0; std::unique_ptr state; forEachChunk([&](const Buffer &header,const Buffer &chunk,uint32_t rawChunkSize,uint8_t chunkType)->bool { if (size_t(destOffset)+size_t(rawChunkSize)>rawData.size()) throw DecompressionError(); if (!rawChunkSize) return true; ConstSubBuffer previousBuffer(rawData,0,destOffset); SubBuffer DestBuffer(rawData,destOffset,rawChunkSize); switch (chunkType) { case 0: if (rawChunkSize!=chunk.size()) throw DecompressionError();; std::memcpy(DestBuffer.data(),chunk.data(),rawChunkSize); break; case 1: { try { auto sub=createDecompressor(_type,_recursionLevel,chunk,state,false); sub->decompressImpl(DestBuffer,previousBuffer,verify); } catch (const InvalidFormatError&) { // we should throw a correct error throw DecompressionError(); } } break; case 15: break; default: return false; } destOffset+=rawChunkSize; return true; }); if (destOffset!=_rawSize) throw DecompressionError(); if (verify) { if (std::memcmp(_packedData.data()+16,rawData.data(),std::min(_rawSize,16U))) throw DecompressionError(); } } std::unique_ptr XPKMain::createDecompressor(uint32_t type,uint32_t recursionLevel,const Buffer &buffer,std::unique_ptr &state,bool verify) { // since this method is used externally, better check recursion level if (recursionLevel>=getMaxRecursionLevel()) throw InvalidFormatError(); for (auto &it : *_XPKDecompressors) { if (it.first(type)) return it.second(type,recursionLevel,buffer,state,verify); } throw InvalidFormatError(); } template void XPKMain::forEachChunk(F func) const { uint32_t currentOffset=0,rawSize,packedSize; bool isLast=false; while (currentOffset<_packedSize+8 && !isLast) { auto readDualValue=[&](uint32_t offsetShort,uint32_t offsetLong,uint32_t &value) { if (_longHeaders) { value=_packedData.readBE32(currentOffset+offsetLong); } else { value=uint32_t(_packedData.readBE16(currentOffset+offsetShort)); } }; uint32_t chunkHeaderLen=_longHeaders?12:8; if (!currentOffset) { // return first; currentOffset=_headerSize; } else { uint32_t tmp; readDualValue(4,4,tmp); tmp=((tmp+3U)&~3U); if (size_t(tmp)+size_t(currentOffset)+chunkHeaderLen>_packedSize) throw InvalidFormatError(); currentOffset+=chunkHeaderLen+tmp; } readDualValue(4,4,packedSize); readDualValue(6,8,rawSize); ConstSubBuffer hdr(_packedData,currentOffset,chunkHeaderLen); ConstSubBuffer chunk(_packedData,currentOffset+chunkHeaderLen,packedSize); uint8_t type=_packedData.read8(currentOffset); if (!func(hdr,chunk,rawSize,type)) return; if (type==15) isLast=true; } if (!isLast) throw InvalidFormatError(); } Decompressor::Registry XPKMain::_registration; ancient-1.0/src/XPKMain.hpp000066400000000000000000000033051375530757000155570ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef XPKMAIN_HPP #define XPKMAIN_HPP #include "Decompressor.hpp" #include "XPKDecompressor.hpp" class XPKMain : public Decompressor { friend class XPKDecompressor; public: XPKMain(const Buffer &packedData,bool verify,uint32_t recursionLevel); virtual ~XPKMain(); virtual const std::string &getName() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual size_t getRawSize() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; static bool detectHeader(uint32_t hdr) noexcept; static std::unique_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); // Can be used to create directly decoder for chunk (needed by CYB2) static std::unique_ptr createDecompressor(uint32_t type,uint32_t recursionLevel,const Buffer &buffer,std::unique_ptr &state,bool verify); private: static void registerDecompressor(bool(*detect)(uint32_t),std::unique_ptr(*create)(uint32_t,uint32_t,const Buffer&,std::unique_ptr&,bool)); static constexpr uint32_t getMaxRecursionLevel() noexcept { return 4; } template void forEachChunk(F func) const; const Buffer &_packedData; uint32_t _packedSize=0; uint32_t _rawSize=0; uint32_t _headerSize=0; uint32_t _type=0; bool _longHeaders=false; uint32_t _recursionLevel=0; static std::vector(*)(uint32_t,uint32_t,const Buffer&,std::unique_ptr&,bool)>> *_XPKDecompressors; static Decompressor::Registry _registration; }; #endif ancient-1.0/src/ZENODecompressor.cpp000066400000000000000000000064561375530757000174560ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "ZENODecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" bool ZENODecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("ZENO"); } std::unique_ptr ZENODecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) { return std::make_unique(hdr,recursionLevel,packedData,state,verify); } ZENODecompressor::ZENODecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || _packedData.size()<6) throw Decompressor::InvalidFormatError(); // first 4 bytes is checksum for password. It needs to be zero if (_packedData.readBE32(0)) throw Decompressor::InvalidFormatError(); _maxBits=_packedData.read8(4); if (_maxBits<9 || _maxBits>20) throw Decompressor::InvalidFormatError(); _startOffset=uint32_t(_packedData.read8(5))+6; if (_startOffset>=_packedData.size()) throw Decompressor::InvalidFormatError(); } ZENODecompressor::~ZENODecompressor() { // nothing needed } const std::string &ZENODecompressor::getSubName() const noexcept { static std::string name="XPK-ZENO: LZW-compressor"; return name; } void ZENODecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream(_packedData,_startOffset,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); uint32_t maxCode=1<<_maxBits; uint32_t stackLength=5000; // magic constant auto prefix=std::make_unique(maxCode-258); auto suffix=std::make_unique(maxCode-258); auto stack=std::make_unique(stackLength); uint32_t freeIndex,codeBits,prevCode,newCode; auto init=[&]() { codeBits=9; freeIndex=258; }; init(); prevCode=readBits(9); newCode=prevCode; suffix[freeIndex-258]=0; prefix[freeIndex-258]=0; freeIndex++; outputStream.writeByte(newCode); while (!outputStream.eof()) { if (freeIndex+3>=(1U<=258) { do { if (stackPos+1>=stackLength || tmp>=freeIndex) throw Decompressor::DecompressionError(); stack[stackPos++]=suffix[tmp-258]; tmp=prefix[tmp-258]; } while (tmp>=258); stack[stackPos++]=newCode=tmp; while (stackPos) outputStream.writeByte(stack[--stackPos]); } else { newCode=tmp; outputStream.writeByte(tmp); if (stackPos) outputStream.writeByte(stack[0]); } } if (freeIndex ZENODecompressor::_XPKregistration; ancient-1.0/src/ZENODecompressor.hpp000066400000000000000000000016341375530757000174540ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef ZENODECOMPRESSOR_HPP #define ZENODECOMPRESSOR_HPP #include "XPKDecompressor.hpp" class ZENODecompressor : public XPKDecompressor { public: ZENODecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); virtual ~ZENODecompressor(); virtual const std::string &getSubName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) override final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::unique_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::unique_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _maxBits=0; size_t _startOffset=0; static XPKDecompressor::Registry _XPKregistration; }; #endif ancient-1.0/src/Zip/000077500000000000000000000000001375530757000143405ustar00rootroot00000000000000ancient-1.0/src/Zip/ImplodeDecompressor.cpp000066400000000000000000000056141375530757000210310ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "ImplodeDecompressor.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" #include "../HuffmanDecoder.hpp" ImplodeDecompressor::ImplodeDecompressor(const Buffer &packedData,uint32_t flags) : _packedData(packedData), _flags(flags) { } ImplodeDecompressor::~ImplodeDecompressor() { // nothing needed } size_t ImplodeDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t ImplodeDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &ImplodeDecompressor::getName() const noexcept { static std::string name="Zip: Implode"; return name; } void ImplodeDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); LSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); // Although Implode uses Fannon-Shano, Huffman decoder can be used typedef HuffmanDecoder ImplodeHuffmanDecoder; auto createDecoder=[&](ImplodeHuffmanDecoder &dec,uint32_t maxValue) { uint32_t length=readBits(8)+1; uint8_t bitLengths[257]; uint8_t counts[257]; uint8_t offsets[257]; uint8_t minDepth=32,maxDepth=0; for (uint32_t i=0;imaxDepth) maxDepth=bitLengths[i]; } if (offsets[length-1]+counts[length-1]>maxValue) throw DecompressionError(); uint32_t code=0; for (uint32_t depth=maxDepth;depth>=minDepth;depth--) { for (int32_t i=length-1;i>=0;i--) { if (depth!=bitLengths[i]) continue; for (uint32_t j=counts[i];j>0;j--) { dec.insert(HuffmanCode{depth,code>>(maxDepth-depth),uint8_t(offsets[i]+j-1)}); code+=1<<(maxDepth-depth); } } } }; ImplodeHuffmanDecoder lengthDecoder,distanceDecoder,litDecoder; uint32_t extraDistanceBits=(_flags&2)?7:6; bool literalDecPresent=_flags&4; uint32_t lengthAddition=2; if (literalDecPresent) { createDecoder(litDecoder,256); lengthAddition++; } createDecoder(lengthDecoder,64); createDecoder(distanceDecoder,64); while (!outputStream.eof()) { if (readBit()) { if (literalDecPresent) outputStream.writeByte(litDecoder.decode(readBit)); else outputStream.writeByte(readBits(8)); } else { uint32_t distance=readBits(extraDistanceBits); distance|=uint32_t(distanceDecoder.decode(readBit))< #include #include "ReduceDecompressor.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" ReduceDecompressor::ReduceDecompressor(const Buffer &packedData,uint32_t mode) : _packedData(packedData), _mode(mode) { if (mode<2 || mode>5) throw InvalidFormatError(); } ReduceDecompressor::~ReduceDecompressor() { // nothing needed } size_t ReduceDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t ReduceDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &ReduceDecompressor::getName() const noexcept { static std::string name="Zip: Reduce"; return name; } void ReduceDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); LSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); uint8_t SLengths[256]; auto SBuf=std::make_unique(256*64); uint8_t (&S)[256][64]=*reinterpret_cast(SBuf.get()); static const uint8_t bitLengths[64]={ 8,1,1,2,2,3,3,3, 3,4,4,4,4,4,4,4, 4,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5, 5,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6}; for (int32_t i=255;i>=0;i--) { uint32_t length=SLengths[i]=readBits(6); for (uint32_t j=0;juint8_t { uint8_t ret; if (!SLengths[lastChar] || readBits(1)) { ret=readBits(8); } else { uint8_t tmp=readBits(bitLengths[SLengths[lastChar]]); if (tmp>=SLengths[lastChar]) throw DecompressionError(); ret=S[lastChar][tmp]; } lastChar=ret; return ret; }; uint8_t ch=readByte(); if (ch!=0x90U) { outputStream.writeByte(ch); } else { ch=readByte(); if (!ch) { outputStream.writeByte(0x90U); } else { uint8_t lengthMask=0xffU>>(8U-lengthBits); uint32_t count=ch&lengthMask; if (count==lengthMask) count+=uint32_t(readByte()); count+=3; uint32_t distance=(uint32_t(ch>>lengthBits)<<8)|uint32_t(readByte()); distance++; outputStream.copy(distance,count,0); } } } } ancient-1.0/src/Zip/ReduceDecompressor.hpp000066400000000000000000000011501375530757000206430ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef REDUCEDECOMPRESSOR_HPP #define REDUCEDECOMPRESSOR_HPP #include "../Decompressor.hpp" class ReduceDecompressor : public Decompressor { public: ReduceDecompressor(const Buffer &packedData,uint32_t mode); virtual ~ReduceDecompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; uint32_t _mode; }; #endif ancient-1.0/src/Zip/ShrinkDecompressor.cpp000066400000000000000000000056601375530757000206770ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "ShrinkDecompressor.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" ShrinkDecompressor::ShrinkDecompressor(const Buffer &packedData) : _packedData(packedData) { } ShrinkDecompressor::~ShrinkDecompressor() { // nothing needed } size_t ShrinkDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t ShrinkDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &ShrinkDecompressor::getName() const noexcept { static std::string name="Zip: Shrink"; return name; } void ShrinkDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); LSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); const uint32_t maxCode=0x2000U; auto prefix=std::make_unique(maxCode-257); auto suffix=std::make_unique(maxCode-257); auto stack=std::make_unique(maxCode-257); uint32_t freeIndex,codeBits,prevCode,newCode; auto suffixLookup=[&](uint32_t code)->uint32_t { // main protection against negative values if (code>=maxCode) throw DecompressionError(); return (code<257)?code:suffix[code-257]; }; auto insert=[&](uint32_t code) { uint32_t stackPos=0; newCode=suffixLookup(code); while (code>=257) { if (stackPos+1>=maxCode-257) throw DecompressionError(); stack[stackPos++]=newCode; code=prefix[code-257]; newCode=suffixLookup(code); } stack[stackPos++]=newCode; while (stackPos) outputStream.writeByte(stack[--stackPos]); }; for (uint32_t i=0;i usageMap(freeIndex-257,false); for (uint32_t i=257;i=257) usageMap[tmp-257]=true; } uint32_t firstEmpty=freeIndex; for (uint32_t i=257;i=257 && (prefix[code-257]&0x8000'0000U)) { uint32_t tmp=newCode; insert(prevCode); outputStream.writeByte(tmp); } else insert(code); while (!(prefix[freeIndex-257]&0x8000'0000U) && freeIndex #include #include "ZIPDecompressor.hpp" #include "../BZIP2Decompressor.hpp" #include "../DEFLATEDecompressor.hpp" #include "ImplodeDecompressor.hpp" #include "ReduceDecompressor.hpp" #include "ShrinkDecompressor.hpp" ZIPDecompressor::ZIPDecompressor(const Buffer &packedData,uint32_t method,uint32_t flags) : _packedData(packedData), _method(method), _flags(flags) { } ZIPDecompressor::~ZIPDecompressor() { // nothing needed } size_t ZIPDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t ZIPDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &ZIPDecompressor::getName() const noexcept { static std::string name="Zip"; return name; } void ZIPDecompressor::decompressImpl(Buffer &rawData,bool verify) { switch (_method) { case 0: if (rawData.size()!=_packedData.size()) throw DecompressionError(); std::memcpy(rawData.data(),_packedData.data(),rawData.size()); break; case 1: { ShrinkDecompressor dec(_packedData); dec.decompress(rawData,verify); } break; case 2: case 3: case 4: case 5: { ReduceDecompressor dec(_packedData,_method); dec.decompress(rawData,verify); } break; case 6: { ImplodeDecompressor dec(_packedData,_flags); dec.decompress(rawData,verify); } break; case 8: case 9: { DEFLATEDecompressor dec(_packedData,_packedData.size(),rawData.size(),false,verify,_method==9); dec.decompress(rawData,verify); } break; case 12: { BZIP2Decompressor dec(_packedData,true,verify); dec.decompress(rawData,verify); } break; case 14: // LZMA throw DecompressionError(); break; case 97: // WavPack throw DecompressionError(); break; case 98: // PPMd throw DecompressionError(); break; default: throw DecompressionError(); } } ancient-1.0/src/Zip/ZIPDecompressor.hpp000066400000000000000000000011761375530757000201060ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef ZIPDECOMPRESSOR_HPP #define ZIPDECOMPRESSOR_HPP #include "../Decompressor.hpp" class ZIPDecompressor : public Decompressor { public: ZIPDecompressor(const Buffer &packedData,uint32_t method,uint32_t flags); virtual ~ZIPDecompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; uint32_t _method; uint32_t _flags; }; #endif ancient-1.0/src/common/000077500000000000000000000000001375530757000150665ustar00rootroot00000000000000ancient-1.0/src/common/Buffer.cpp000066400000000000000000000033521375530757000170060ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "Buffer.hpp" Buffer::Buffer() noexcept { // nothing needed } Buffer::~Buffer() { // nothing needed } void Buffer::resize(size_t newSize) { throw InvalidOperationError(); } uint8_t &Buffer::operator[](size_t i) { if (i>=size()) throw OutOfBoundsError(); return data()[i]; } const uint8_t &Buffer::operator[](size_t i) const { if (i>=size()) throw OutOfBoundsError(); return data()[i]; } uint32_t Buffer::readBE32(size_t offset) const { if (offset+4>size()) throw OutOfBoundsError(); const uint8_t *ptr=data()+offset; return (uint32_t(ptr[0])<<24)|(uint32_t(ptr[1])<<16)|(uint32_t(ptr[2])<<8)|uint32_t(ptr[3]); } uint16_t Buffer::readBE16(size_t offset) const { if (offset+2>size()) throw OutOfBoundsError(); const uint8_t *ptr=data()+offset; return (uint16_t(ptr[0])<<8)|uint16_t(ptr[1]); } uint64_t Buffer::readLE64(size_t offset) const { if (offset+8>size()) throw OutOfBoundsError(); const uint8_t *ptr=data()+offset; return (uint64_t(ptr[7])<<56)|(uint64_t(ptr[6])<<48)|(uint64_t(ptr[5])<<40)|(uint64_t(ptr[4])<<32)| (uint64_t(ptr[3])<<24)|(uint64_t(ptr[2])<<16)|(uint64_t(ptr[1])<<8)|uint64_t(ptr[0]); } uint32_t Buffer::readLE32(size_t offset) const { if (offset+4>size()) throw OutOfBoundsError(); const uint8_t *ptr=data()+offset; return (uint32_t(ptr[3])<<24)|(uint32_t(ptr[2])<<16)|(uint32_t(ptr[1])<<8)|uint32_t(ptr[0]); } uint16_t Buffer::readLE16(size_t offset) const { if (offset+2>size()) throw OutOfBoundsError(); const uint8_t *ptr=data()+offset; return (uint16_t(ptr[1])<<8)|uint16_t(ptr[0]); } uint8_t Buffer::read8(size_t offset) const { if (offset>=size()) throw OutOfBoundsError(); const uint8_t *ptr=reinterpret_cast(data())+offset; return ptr[0]; } ancient-1.0/src/common/Buffer.hpp000066400000000000000000000023751375530757000170170ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef BUFFER_HPP #define BUFFER_HPP #include #include #include class Buffer { protected: Buffer() noexcept; public: class Error : public std::exception { // nothing needed }; class OutOfBoundsError : public Error { // nothing needed }; class InvalidOperationError : public Error { // nothing needed }; class OutOfMemoryError : public Error { // nothing needed }; Buffer(const Buffer&)=delete; Buffer& operator=(const Buffer&)=delete; virtual ~Buffer(); virtual const uint8_t *data() const noexcept=0; virtual uint8_t *data()=0; virtual size_t size() const noexcept=0; template const T *cast() const noexcept { return reinterpret_cast(data()); } template T *cast() { return reinterpret_cast(data()); } virtual bool isResizable() const noexcept=0; virtual void resize(size_t newSize); uint8_t &operator[](size_t i); const uint8_t &operator[](size_t i) const; uint32_t readBE32(size_t offset) const; uint16_t readBE16(size_t offset) const; uint64_t readLE64(size_t offset) const; uint32_t readLE32(size_t offset) const; uint16_t readLE16(size_t offset) const; uint8_t read8(size_t offset) const; }; #endif ancient-1.0/src/common/CRC16.cpp000066400000000000000000000045271375530757000163600ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "Buffer.hpp" static const uint16_t CRC16Table[256]={ 0x0000,0xc0c1,0xc181,0x0140,0xc301,0x03c0,0x0280,0xc241,0xc601,0x06c0,0x0780,0xc741,0x0500,0xc5c1,0xc481,0x0440, 0xcc01,0x0cc0,0x0d80,0xcd41,0x0f00,0xcfc1,0xce81,0x0e40,0x0a00,0xcac1,0xcb81,0x0b40,0xc901,0x09c0,0x0880,0xc841, 0xd801,0x18c0,0x1980,0xd941,0x1b00,0xdbc1,0xda81,0x1a40,0x1e00,0xdec1,0xdf81,0x1f40,0xdd01,0x1dc0,0x1c80,0xdc41, 0x1400,0xd4c1,0xd581,0x1540,0xd701,0x17c0,0x1680,0xd641,0xd201,0x12c0,0x1380,0xd341,0x1100,0xd1c1,0xd081,0x1040, 0xf001,0x30c0,0x3180,0xf141,0x3300,0xf3c1,0xf281,0x3240,0x3600,0xf6c1,0xf781,0x3740,0xf501,0x35c0,0x3480,0xf441, 0x3c00,0xfcc1,0xfd81,0x3d40,0xff01,0x3fc0,0x3e80,0xfe41,0xfa01,0x3ac0,0x3b80,0xfb41,0x3900,0xf9c1,0xf881,0x3840, 0x2800,0xe8c1,0xe981,0x2940,0xeb01,0x2bc0,0x2a80,0xea41,0xee01,0x2ec0,0x2f80,0xef41,0x2d00,0xedc1,0xec81,0x2c40, 0xe401,0x24c0,0x2580,0xe541,0x2700,0xe7c1,0xe681,0x2640,0x2200,0xe2c1,0xe381,0x2340,0xe101,0x21c0,0x2080,0xe041, 0xa001,0x60c0,0x6180,0xa141,0x6300,0xa3c1,0xa281,0x6240,0x6600,0xa6c1,0xa781,0x6740,0xa501,0x65c0,0x6480,0xa441, 0x6c00,0xacc1,0xad81,0x6d40,0xaf01,0x6fc0,0x6e80,0xae41,0xaa01,0x6ac0,0x6b80,0xab41,0x6900,0xa9c1,0xa881,0x6840, 0x7800,0xb8c1,0xb981,0x7940,0xbb01,0x7bc0,0x7a80,0xba41,0xbe01,0x7ec0,0x7f80,0xbf41,0x7d00,0xbdc1,0xbc81,0x7c40, 0xb401,0x74c0,0x7580,0xb541,0x7700,0xb7c1,0xb681,0x7640,0x7200,0xb2c1,0xb381,0x7340,0xb101,0x71c0,0x7080,0xb041, 0x5000,0x90c1,0x9181,0x5140,0x9301,0x53c0,0x5280,0x9241,0x9601,0x56c0,0x5780,0x9741,0x5500,0x95c1,0x9481,0x5440, 0x9c01,0x5cc0,0x5d80,0x9d41,0x5f00,0x9fc1,0x9e81,0x5e40,0x5a00,0x9ac1,0x9b81,0x5b40,0x9901,0x59c0,0x5880,0x9841, 0x8801,0x48c0,0x4980,0x8941,0x4b00,0x8bc1,0x8a81,0x4a40,0x4e00,0x8ec1,0x8f81,0x4f40,0x8d01,0x4dc0,0x4c80,0x8c41, 0x4400,0x84c1,0x8581,0x4540,0x8701,0x47c0,0x4680,0x8641,0x8201,0x42c0,0x4380,0x8341,0x4100,0x81c1,0x8081,0x4040}; uint16_t CRC16(const Buffer &buffer,size_t offset,size_t len,uint16_t accumulator) { if (!len || offset+len>buffer.size()) throw Buffer::OutOfBoundsError(); const uint8_t *ptr=buffer.data()+offset; for (size_t i=0;i>8)^CRC16Table[(accumulator&0xff)^ptr[i]]; return accumulator; } uint16_t CRC16Byte(uint8_t ch,uint16_t accumulator) noexcept { return (accumulator>>8)^CRC16Table[(accumulator&0xff)^ch]; } ancient-1.0/src/common/CRC16.hpp000066400000000000000000000004501375530757000163540ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef CRC16_HPP #define CRC16_HPP #include #include "Buffer.hpp" // The most common CRC16 uint16_t CRC16(const Buffer &buffer,size_t offset,size_t len,uint16_t accumulator); uint16_t CRC16Byte(uint8_t ch,uint16_t accumulator) noexcept; #endif ancient-1.0/src/common/CRC32.cpp000066400000000000000000000165611375530757000163570ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "Buffer.hpp" static const uint32_t CRC32Table[256]={ 0x00000000U,0x77073096U,0xee0e612cU,0x990951baU,0x076dc419U,0x706af48fU,0xe963a535U,0x9e6495a3U, 0x0edb8832U,0x79dcb8a4U,0xe0d5e91eU,0x97d2d988U,0x09b64c2bU,0x7eb17cbdU,0xe7b82d07U,0x90bf1d91U, 0x1db71064U,0x6ab020f2U,0xf3b97148U,0x84be41deU,0x1adad47dU,0x6ddde4ebU,0xf4d4b551U,0x83d385c7U, 0x136c9856U,0x646ba8c0U,0xfd62f97aU,0x8a65c9ecU,0x14015c4fU,0x63066cd9U,0xfa0f3d63U,0x8d080df5U, 0x3b6e20c8U,0x4c69105eU,0xd56041e4U,0xa2677172U,0x3c03e4d1U,0x4b04d447U,0xd20d85fdU,0xa50ab56bU, 0x35b5a8faU,0x42b2986cU,0xdbbbc9d6U,0xacbcf940U,0x32d86ce3U,0x45df5c75U,0xdcd60dcfU,0xabd13d59U, 0x26d930acU,0x51de003aU,0xc8d75180U,0xbfd06116U,0x21b4f4b5U,0x56b3c423U,0xcfba9599U,0xb8bda50fU, 0x2802b89eU,0x5f058808U,0xc60cd9b2U,0xb10be924U,0x2f6f7c87U,0x58684c11U,0xc1611dabU,0xb6662d3dU, 0x76dc4190U,0x01db7106U,0x98d220bcU,0xefd5102aU,0x71b18589U,0x06b6b51fU,0x9fbfe4a5U,0xe8b8d433U, 0x7807c9a2U,0x0f00f934U,0x9609a88eU,0xe10e9818U,0x7f6a0dbbU,0x086d3d2dU,0x91646c97U,0xe6635c01U, 0x6b6b51f4U,0x1c6c6162U,0x856530d8U,0xf262004eU,0x6c0695edU,0x1b01a57bU,0x8208f4c1U,0xf50fc457U, 0x65b0d9c6U,0x12b7e950U,0x8bbeb8eaU,0xfcb9887cU,0x62dd1ddfU,0x15da2d49U,0x8cd37cf3U,0xfbd44c65U, 0x4db26158U,0x3ab551ceU,0xa3bc0074U,0xd4bb30e2U,0x4adfa541U,0x3dd895d7U,0xa4d1c46dU,0xd3d6f4fbU, 0x4369e96aU,0x346ed9fcU,0xad678846U,0xda60b8d0U,0x44042d73U,0x33031de5U,0xaa0a4c5fU,0xdd0d7cc9U, 0x5005713cU,0x270241aaU,0xbe0b1010U,0xc90c2086U,0x5768b525U,0x206f85b3U,0xb966d409U,0xce61e49fU, 0x5edef90eU,0x29d9c998U,0xb0d09822U,0xc7d7a8b4U,0x59b33d17U,0x2eb40d81U,0xb7bd5c3bU,0xc0ba6cadU, 0xedb88320U,0x9abfb3b6U,0x03b6e20cU,0x74b1d29aU,0xead54739U,0x9dd277afU,0x04db2615U,0x73dc1683U, 0xe3630b12U,0x94643b84U,0x0d6d6a3eU,0x7a6a5aa8U,0xe40ecf0bU,0x9309ff9dU,0x0a00ae27U,0x7d079eb1U, 0xf00f9344U,0x8708a3d2U,0x1e01f268U,0x6906c2feU,0xf762575dU,0x806567cbU,0x196c3671U,0x6e6b06e7U, 0xfed41b76U,0x89d32be0U,0x10da7a5aU,0x67dd4accU,0xf9b9df6fU,0x8ebeeff9U,0x17b7be43U,0x60b08ed5U, 0xd6d6a3e8U,0xa1d1937eU,0x38d8c2c4U,0x4fdff252U,0xd1bb67f1U,0xa6bc5767U,0x3fb506ddU,0x48b2364bU, 0xd80d2bdaU,0xaf0a1b4cU,0x36034af6U,0x41047a60U,0xdf60efc3U,0xa867df55U,0x316e8eefU,0x4669be79U, 0xcb61b38cU,0xbc66831aU,0x256fd2a0U,0x5268e236U,0xcc0c7795U,0xbb0b4703U,0x220216b9U,0x5505262fU, 0xc5ba3bbeU,0xb2bd0b28U,0x2bb45a92U,0x5cb36a04U,0xc2d7ffa7U,0xb5d0cf31U,0x2cd99e8bU,0x5bdeae1dU, 0x9b64c2b0U,0xec63f226U,0x756aa39cU,0x026d930aU,0x9c0906a9U,0xeb0e363fU,0x72076785U,0x05005713U, 0x95bf4a82U,0xe2b87a14U,0x7bb12baeU,0x0cb61b38U,0x92d28e9bU,0xe5d5be0dU,0x7cdcefb7U,0x0bdbdf21U, 0x86d3d2d4U,0xf1d4e242U,0x68ddb3f8U,0x1fda836eU,0x81be16cdU,0xf6b9265bU,0x6fb077e1U,0x18b74777U, 0x88085ae6U,0xff0f6a70U,0x66063bcaU,0x11010b5cU,0x8f659effU,0xf862ae69U,0x616bffd3U,0x166ccf45U, 0xa00ae278U,0xd70dd2eeU,0x4e048354U,0x3903b3c2U,0xa7672661U,0xd06016f7U,0x4969474dU,0x3e6e77dbU, 0xaed16a4aU,0xd9d65adcU,0x40df0b66U,0x37d83bf0U,0xa9bcae53U,0xdebb9ec5U,0x47b2cf7fU,0x30b5ffe9U, 0xbdbdf21cU,0xcabac28aU,0x53b39330U,0x24b4a3a6U,0xbad03605U,0xcdd70693U,0x54de5729U,0x23d967bfU, 0xb3667a2eU,0xc4614ab8U,0x5d681b02U,0x2a6f2b94U,0xb40bbe37U,0xc30c8ea1U,0x5a05df1bU,0x2d02ef8dU}; uint32_t CRC32(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator) { if (!len || offset+len>buffer.size()) throw Buffer::OutOfBoundsError(); const uint8_t *ptr=buffer.data()+offset; accumulator=~accumulator; for (size_t i=0;i>8)^CRC32Table[(accumulator&0xff)^ptr[i]]; return ~accumulator; } uint32_t CRC32Byte(uint8_t ch,uint32_t accumulator) noexcept { return ~((~accumulator>>8)^CRC32Table[(~accumulator&0xff)^ch]); } // instead of bit-twiddling lets have a separate implementation for reverse // same table as the previous one, but reflected static const uint32_t CRC32RevTable[256]={ 0x00000000U,0x04c11db7U,0x09823b6eU,0x0d4326d9U,0x130476dcU,0x17c56b6bU,0x1a864db2U,0x1e475005U, 0x2608edb8U,0x22c9f00fU,0x2f8ad6d6U,0x2b4bcb61U,0x350c9b64U,0x31cd86d3U,0x3c8ea00aU,0x384fbdbdU, 0x4c11db70U,0x48d0c6c7U,0x4593e01eU,0x4152fda9U,0x5f15adacU,0x5bd4b01bU,0x569796c2U,0x52568b75U, 0x6a1936c8U,0x6ed82b7fU,0x639b0da6U,0x675a1011U,0x791d4014U,0x7ddc5da3U,0x709f7b7aU,0x745e66cdU, 0x9823b6e0U,0x9ce2ab57U,0x91a18d8eU,0x95609039U,0x8b27c03cU,0x8fe6dd8bU,0x82a5fb52U,0x8664e6e5U, 0xbe2b5b58U,0xbaea46efU,0xb7a96036U,0xb3687d81U,0xad2f2d84U,0xa9ee3033U,0xa4ad16eaU,0xa06c0b5dU, 0xd4326d90U,0xd0f37027U,0xddb056feU,0xd9714b49U,0xc7361b4cU,0xc3f706fbU,0xceb42022U,0xca753d95U, 0xf23a8028U,0xf6fb9d9fU,0xfbb8bb46U,0xff79a6f1U,0xe13ef6f4U,0xe5ffeb43U,0xe8bccd9aU,0xec7dd02dU, 0x34867077U,0x30476dc0U,0x3d044b19U,0x39c556aeU,0x278206abU,0x23431b1cU,0x2e003dc5U,0x2ac12072U, 0x128e9dcfU,0x164f8078U,0x1b0ca6a1U,0x1fcdbb16U,0x018aeb13U,0x054bf6a4U,0x0808d07dU,0x0cc9cdcaU, 0x7897ab07U,0x7c56b6b0U,0x71159069U,0x75d48ddeU,0x6b93dddbU,0x6f52c06cU,0x6211e6b5U,0x66d0fb02U, 0x5e9f46bfU,0x5a5e5b08U,0x571d7dd1U,0x53dc6066U,0x4d9b3063U,0x495a2dd4U,0x44190b0dU,0x40d816baU, 0xaca5c697U,0xa864db20U,0xa527fdf9U,0xa1e6e04eU,0xbfa1b04bU,0xbb60adfcU,0xb6238b25U,0xb2e29692U, 0x8aad2b2fU,0x8e6c3698U,0x832f1041U,0x87ee0df6U,0x99a95df3U,0x9d684044U,0x902b669dU,0x94ea7b2aU, 0xe0b41de7U,0xe4750050U,0xe9362689U,0xedf73b3eU,0xf3b06b3bU,0xf771768cU,0xfa325055U,0xfef34de2U, 0xc6bcf05fU,0xc27dede8U,0xcf3ecb31U,0xcbffd686U,0xd5b88683U,0xd1799b34U,0xdc3abdedU,0xd8fba05aU, 0x690ce0eeU,0x6dcdfd59U,0x608edb80U,0x644fc637U,0x7a089632U,0x7ec98b85U,0x738aad5cU,0x774bb0ebU, 0x4f040d56U,0x4bc510e1U,0x46863638U,0x42472b8fU,0x5c007b8aU,0x58c1663dU,0x558240e4U,0x51435d53U, 0x251d3b9eU,0x21dc2629U,0x2c9f00f0U,0x285e1d47U,0x36194d42U,0x32d850f5U,0x3f9b762cU,0x3b5a6b9bU, 0x0315d626U,0x07d4cb91U,0x0a97ed48U,0x0e56f0ffU,0x1011a0faU,0x14d0bd4dU,0x19939b94U,0x1d528623U, 0xf12f560eU,0xf5ee4bb9U,0xf8ad6d60U,0xfc6c70d7U,0xe22b20d2U,0xe6ea3d65U,0xeba91bbcU,0xef68060bU, 0xd727bbb6U,0xd3e6a601U,0xdea580d8U,0xda649d6fU,0xc423cd6aU,0xc0e2d0ddU,0xcda1f604U,0xc960ebb3U, 0xbd3e8d7eU,0xb9ff90c9U,0xb4bcb610U,0xb07daba7U,0xae3afba2U,0xaafbe615U,0xa7b8c0ccU,0xa379dd7bU, 0x9b3660c6U,0x9ff77d71U,0x92b45ba8U,0x9675461fU,0x8832161aU,0x8cf30badU,0x81b02d74U,0x857130c3U, 0x5d8a9099U,0x594b8d2eU,0x5408abf7U,0x50c9b640U,0x4e8ee645U,0x4a4ffbf2U,0x470cdd2bU,0x43cdc09cU, 0x7b827d21U,0x7f436096U,0x7200464fU,0x76c15bf8U,0x68860bfdU,0x6c47164aU,0x61043093U,0x65c52d24U, 0x119b4be9U,0x155a565eU,0x18197087U,0x1cd86d30U,0x029f3d35U,0x065e2082U,0x0b1d065bU,0x0fdc1becU, 0x3793a651U,0x3352bbe6U,0x3e119d3fU,0x3ad08088U,0x2497d08dU,0x2056cd3aU,0x2d15ebe3U,0x29d4f654U, 0xc5a92679U,0xc1683bceU,0xcc2b1d17U,0xc8ea00a0U,0xd6ad50a5U,0xd26c4d12U,0xdf2f6bcbU,0xdbee767cU, 0xe3a1cbc1U,0xe760d676U,0xea23f0afU,0xeee2ed18U,0xf0a5bd1dU,0xf464a0aaU,0xf9278673U,0xfde69bc4U, 0x89b8fd09U,0x8d79e0beU,0x803ac667U,0x84fbdbd0U,0x9abc8bd5U,0x9e7d9662U,0x933eb0bbU,0x97ffad0cU, 0xafb010b1U,0xab710d06U,0xa6322bdfU,0xa2f33668U,0xbcb4666dU,0xb8757bdaU,0xb5365d03U,0xb1f740b4U}; uint32_t CRC32Rev(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator) { if (!len || offset+len>buffer.size()) throw Buffer::OutOfBoundsError(); const uint8_t *ptr=buffer.data()+offset; accumulator=~accumulator; for (size_t i=0;i>24)^ptr[i]]; return ~accumulator; } uint32_t CRC32RevByte(uint8_t ch,uint32_t &accumulator) noexcept { return ~((~accumulator<<8)^CRC32RevTable[(~accumulator>>24)^ch]); } ancient-1.0/src/common/CRC32.hpp000066400000000000000000000007511375530757000163560ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef CRC32_HPP #define CRC32_HPP #include #include "Buffer.hpp" // The most common CRC32 uint32_t CRC32(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator); uint32_t CRC32Byte(uint8_t ch,uint32_t accumulator) noexcept; // Same polynomial, but in reverse... uint32_t CRC32Rev(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator); uint32_t CRC32RevByte(uint8_t ch,uint32_t accumulator) noexcept; #endif ancient-1.0/src/common/Common.cpp000066400000000000000000000005701375530757000170240ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "Common.hpp" uint32_t rotateBits(uint32_t value,uint32_t count) { static const uint8_t rotateNibble[16]={ 0x0,0x8,0x4,0xc, 0x2,0xa,0x6,0xe, 0x1,0x9,0x5,0xd, 0x3,0xb,0x7,0xf }; uint32_t ret=0; for (uint32_t i=0;i>=4; } ret>>=(4-count)&3; return ret; } ancient-1.0/src/common/Common.hpp000066400000000000000000000013101375530757000170220ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef COMMON_HPP #define COMMON_HPP #include #include #include constexpr uint32_t FourCC(const char (&cc)[5]) noexcept { return static_cast((static_cast(cc[0]) << 24) | (static_cast(cc[1]) << 16) | (static_cast(cc[2]) << 8) | static_cast(cc[3])); } constexpr bool isValidSize(uint64_t &value) noexcept { #if INTPTR_MAX == INT32_MAX return value<0x1'0000'0000ULL; #else return true; #endif } constexpr bool isValidSize(off_t &value) noexcept { #if INTPTR_MAX == INT32_MAX return value<0x1'0000'0000ULL; #else return true; #endif } uint32_t rotateBits(uint32_t value,uint32_t count); #endif ancient-1.0/src/common/MemoryBuffer.cpp000066400000000000000000000017111375530757000201740ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include "MemoryBuffer.hpp" MemoryBuffer::MemoryBuffer(size_t size) : _data(reinterpret_cast(std::malloc(size))), _size(size) { if (!_data) throw OutOfMemoryError(); } MemoryBuffer::MemoryBuffer(const Buffer &src,size_t offset,size_t size) : MemoryBuffer(size) { if(offset+size>src.size()) throw InvalidOperationError(); std::memcpy(_data,src.data()+offset,size); } MemoryBuffer::~MemoryBuffer() { std::free(_data); } const uint8_t *MemoryBuffer::data() const noexcept { return _data; } uint8_t *MemoryBuffer::data() { return _data; } size_t MemoryBuffer::size() const noexcept { return _size; } bool MemoryBuffer::isResizable() const noexcept { return true; } void MemoryBuffer::resize(size_t newSize) { _data=reinterpret_cast(std::realloc(_data,newSize)); _size=newSize; if (!_data) { _size=0; throw OutOfMemoryError(); } } ancient-1.0/src/common/MemoryBuffer.hpp000066400000000000000000000011671375530757000202060ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef MEMORYBUFFER_HPP #define MEMORYBUFFER_HPP #include #include "Buffer.hpp" class MemoryBuffer : public Buffer { public: MemoryBuffer(size_t size); MemoryBuffer(const Buffer &src,size_t offset,size_t size); virtual ~MemoryBuffer() override final; virtual const uint8_t *data() const noexcept override final; virtual uint8_t *data() override final; virtual size_t size() const noexcept override final; virtual bool isResizable() const noexcept override final; virtual void resize(size_t newSize) override final; private: uint8_t* _data; size_t _size; }; #endif ancient-1.0/src/common/StaticBuffer.cpp000066400000000000000000000010441375530757000201520ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "StaticBuffer.hpp" ConstStaticBuffer::ConstStaticBuffer(const uint8_t *data,size_t length) : _data(data), _length(length) { // nothing needed } ConstStaticBuffer::~ConstStaticBuffer() { // nothing needed } const uint8_t *ConstStaticBuffer::data() const noexcept { return _data; } uint8_t *ConstStaticBuffer::data() { throw InvalidOperationError(); } size_t ConstStaticBuffer::size() const noexcept { return _length; } bool ConstStaticBuffer::isResizable() const noexcept { return false; } ancient-1.0/src/common/StaticBuffer.hpp000066400000000000000000000022371375530757000201640ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef STATICBUFFER_HPP #define STATICBUFFER_HPP #include #include #include "Buffer.hpp" template class StaticBuffer : public Buffer { public: StaticBuffer(const StaticBuffer&)=delete; StaticBuffer& operator=(const StaticBuffer&)=delete; StaticBuffer() { } virtual ~StaticBuffer() { } virtual const uint8_t *data() const noexcept override { return _data; } virtual uint8_t *data() override { return _data; } virtual size_t size() const noexcept override { return N; } virtual bool isResizable() const noexcept override { return false; } private: uint8_t _data[N]; }; class ConstStaticBuffer : public Buffer { public: ConstStaticBuffer(const ConstStaticBuffer&)=delete; ConstStaticBuffer& operator=(const ConstStaticBuffer&)=delete; ConstStaticBuffer(const uint8_t *data,size_t length); virtual ~ConstStaticBuffer(); virtual const uint8_t *data() const noexcept override; virtual uint8_t *data() override; virtual size_t size() const noexcept override; virtual bool isResizable() const noexcept override; private: const uint8_t *_data; size_t _length; }; #endif ancient-1.0/src/common/SubBuffer.cpp000066400000000000000000000003651375530757000174610ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SubBuffer.hpp" template <> uint8_t *GenericSubBuffer::data() { return _base.data()+_start; } template <> uint8_t *GenericSubBuffer::data() { throw InvalidOperationError(); } ancient-1.0/src/common/SubBuffer.hpp000066400000000000000000000024461375530757000174700ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SUBBUFFER_HPP #define SUBBUFFER_HPP #include #include #include "Buffer.hpp" // helpers to splice Buffer template class GenericSubBuffer : public Buffer { public: GenericSubBuffer(const GenericSubBuffer&)=delete; GenericSubBuffer& operator=(const GenericSubBuffer&)=delete; GenericSubBuffer(T &base,size_t start,size_t length) : _base(base), _start(start), _length(length) { // if the sub-buffer is invalid, we set both _start and _length to 0 if (start+length>_base.size()) { _start=0; _length=0; } } virtual ~GenericSubBuffer() { } virtual const uint8_t *data() const noexcept override { return _base.data()+_start; } virtual uint8_t *data() override; virtual size_t size() const noexcept override { return _length; } virtual bool isResizable() const noexcept override { return false; } // can only make the buffer smaller, can't run away from the current bounds void adjust(size_t start,size_t length) { if (start<_start || start+length>_start+_length) throw OutOfBoundsError(); _start=start; _length=length; } private: T &_base; size_t _start; size_t _length; }; typedef GenericSubBuffer SubBuffer; typedef GenericSubBuffer ConstSubBuffer; #endif