pax_global_header00006660000000000000000000000064146306326260014522gustar00rootroot0000000000000052 comment=55817c6ea8c2a443aa9f7f87e81a953ea7b13e3f ancient-2.2.0/000077500000000000000000000000001463063262600131445ustar00rootroot00000000000000ancient-2.2.0/.gitattributes000066400000000000000000000000261463063262600160350ustar00rootroot00000000000000testing export-ignore ancient-2.2.0/.github/000077500000000000000000000000001463063262600145045ustar00rootroot00000000000000ancient-2.2.0/.github/workflows/000077500000000000000000000000001463063262600165415ustar00rootroot00000000000000ancient-2.2.0/.github/workflows/codeql.yml000066400000000000000000000046061463063262600205410ustar00rootroot00000000000000# For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: [ "master" ] pull_request: # The branches below must be a subset of the branches above branches: [ "master" ] schedule: - cron: '23 22 * * 1' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'cpp' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Use only 'java' to analyze code written in Java, Kotlin or both # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild run: make -f Makefile.unix all shell: bash - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 with: category: "/language:${{matrix.language}}" ancient-2.2.0/.github/workflows/msbuild-latest.yml000066400000000000000000000022411463063262600222140ustar00rootroot00000000000000# This workflow uses actions that are not certified by GitHub. # They are provided by a third-party and are governed by # separate terms of service, privacy policy, and support # documentation. name: MSBuild latest on: push: branches: [ "master" ] pull_request: branches: [ "master" ] env: # Path to the solution file relative to the root of the project. SOLUTION_FILE_PATH: Ancient.sln permissions: contents: read jobs: build: strategy: matrix: build_configuration: [Release, Release_Clang] runs-on: windows-latest steps: - uses: actions/checkout@v3 - name: Add MSBuild to PATH uses: microsoft/setup-msbuild@v1.0.2 - name: Restore NuGet packages working-directory: ${{env.GITHUB_WORKSPACE}} run: nuget restore ${{env.SOLUTION_FILE_PATH}} - name: Build working-directory: ${{env.GITHUB_WORKSPACE}} # Add additional options to the MSBuild command line here (like platform or verbosity level). # See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference run: msbuild /m /p:Configuration=${{matrix.build_configuration}} ${{env.SOLUTION_FILE_PATH}} ancient-2.2.0/.github/workflows/ubuntu-20.04-autotools.yml000066400000000000000000000011631463063262600232770ustar00rootroot00000000000000name: Ubuntu 20.04 Autotools on: push: pull_request: jobs: build: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: install dependencies run: sudo apt-get update && sudo apt-get install -y build-essential autoconf autoconf-archive automake libtool make pkg-config - name: autogen run: ./autogen.sh - name: configure run: ./configure - name: make run: make - name: make check run: make check - name: make dist run: make dist - name: make distcheck run: make distcheck - name: make distclean run: make distclean ancient-2.2.0/.github/workflows/ubuntu-20.04-extra.yml000066400000000000000000000004521463063262600223710ustar00rootroot00000000000000name: Ubuntu 20.04 extra on: push: pull_request: jobs: build: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: install dependencies run: sudo apt-get update && sudo apt-get install -y build-essential make - name: make run: cd extra && make ancient-2.2.0/.github/workflows/ubuntu-20.04-fuzzing-build.yml000066400000000000000000000006311463063262600240360ustar00rootroot00000000000000name: Ubuntu 20.04 fuzzing (build only) on: push: pull_request: jobs: build: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: install dependencies run: sudo apt-get update && sudo apt-get install -y build-essential make clang llvm llvm-dev llvm-runtime - name: get-afl run: ./fuzzing/get-afl.sh - name: build fuzzing run: ./fuzzing/build.sh ancient-2.2.0/.github/workflows/ubuntu-20.04-make.yml000066400000000000000000000005561463063262600221700ustar00rootroot00000000000000name: Ubuntu 20.04 Makefile on: push: pull_request: jobs: build: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: install dependencies run: sudo apt-get update && sudo apt-get install -y build-essential make - name: make run: make -f Makefile.unix EXTRA_CFLAGS=-Werror - name: test run: ./obj/test ancient-2.2.0/.gitignore000066400000000000000000000005331463063262600151350ustar00rootroot00000000000000*~ /.deps/ /.libs/ /aclocal.m4 /ancient /test /ancient-*.tar.gz /autom4te.cache/ /build-aux/ /config.log /config.status /configure /libancient.pc /libtool /m4/ /Makefile /Makefile.in # /src/.deps/ /src/.dirstamp /src/Lzh/.deps/ /src/Lzh/.dirstamp /src/Zip/.deps/ /src/Zip/.dirstamp /src/common/.deps/ /src/common/.dirstamp /test_vectors /obj /.vs ancient-2.2.0/Ancient.sln000077500000000000000000000035241463063262600152520ustar00rootroot00000000000000 Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.6.33829.357 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Ancient", "Ancient.vcxproj", "{9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 Debug|x86 = Debug|x86 Release_Clang|x64 = Release_Clang|x64 Release_Clang|x86 = Release_Clang|x86 Release|x64 = Release|x64 Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Debug|x64.ActiveCfg = Debug|x64 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Debug|x64.Build.0 = Debug|x64 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Debug|x86.ActiveCfg = Debug|Win32 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Debug|x86.Build.0 = Debug|Win32 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Release_Clang|x64.ActiveCfg = Release_Clang|x64 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Release_Clang|x64.Build.0 = Release_Clang|x64 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Release_Clang|x86.ActiveCfg = Release_Clang|Win32 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Release_Clang|x86.Build.0 = Release_Clang|Win32 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Release|x64.ActiveCfg = Release|x64 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Release|x64.Build.0 = Release|x64 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Release|x86.ActiveCfg = Release|Win32 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9E9F4192-A9FA-4938-95EE-1CB72564517A} EndGlobalSection EndGlobal ancient-2.2.0/Ancient.vcxproj000077500000000000000000000441611463063262600161530ustar00rootroot00000000000000 Debug Win32 Release_Clang Win32 Release_Clang x64 Release Win32 Debug x64 Release x64 17.0 {9EA0A49E-A5DB-4A8A-B63D-FED0ACD8E059} Win32Proj 10.0 Application true v143 Application false v143 Application false ClangCL Application true v143 Application false v143 Application false ClangCL false false false false false false WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) Level3 stdcpp17 api;api/ancient MultiThreadedDebug true true Console _DEBUG;_CONSOLE;%(PreprocessorDefinitions) Level3 stdcpp17 api;api/ancient MultiThreadedDebug true true Console WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) Level3 stdcpp17 api;api/ancient MultiThreaded true true Console true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) Level3 stdcpp17 api;api/ancient MultiThreaded true true Console true true NDEBUG;_CONSOLE;%(PreprocessorDefinitions) Level3 stdcpp17 api;api/ancient MultiThreaded true true Console true true NDEBUG;_CONSOLE;%(PreprocessorDefinitions) Level3 stdcpp17 api;api/ancient MultiThreaded true true Console true true ancient-2.2.0/LICENSE000066400000000000000000000024531463063262600141550ustar00rootroot00000000000000BSD 2-Clause License Copyright (c) 2017-2024, Teemu Suutari All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ancient-2.2.0/Makefile.am000066400000000000000000000207301463063262600152020ustar00rootroot00000000000000ACLOCAL_AMFLAGS = -I m4 --install EXTRA_DIST = EXTRA_DIST += .gitignore EXTRA_DIST += libancient.pc.in EXTRA_DIST += LICENSE EXTRA_DIST += Makefile.unix EXTRA_DIST += README.md EXTRA_DIST += extra/BruteForceRNC1Encoder.cpp EXTRA_DIST += extra/Makefile EXTRA_DIST += fuzzing/all_formats.dict EXTRA_DIST += fuzzing/build.sh EXTRA_DIST += fuzzing/fuzz-main.sh EXTRA_DIST += fuzzing/fuzz-secondary.sh EXTRA_DIST += fuzzing/fuzz-settings.sh EXTRA_DIST += fuzzing/fuzz.cpp EXTRA_DIST += fuzzing/get-afl.sh EXTRA_DIST += fuzzing/readme.md MOSTLYCLEANFILES = dist_doc_DATA = dist_doc_DATA += LICENSE dist_doc_DATA += README.md nobase_dist_doc_DATA = bin_PROGRAMS = check_PROGRAMS = lib_LTLIBRARIES = TESTS = pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = nobase_include_HEADERS = LIBANCIENT_FILES_SRC = LIBANCIENT_FILES_SRC += src/ACCADecompressor.cpp LIBANCIENT_FILES_SRC += src/ACCADecompressor.hpp LIBANCIENT_FILES_SRC += src/API.cpp LIBANCIENT_FILES_SRC += src/ARTMDecompressor.cpp LIBANCIENT_FILES_SRC += src/ARTMDecompressor.hpp LIBANCIENT_FILES_SRC += src/BLZWDecompressor.cpp LIBANCIENT_FILES_SRC += src/BLZWDecompressor.hpp LIBANCIENT_FILES_SRC += src/BZIP2Decompressor.cpp LIBANCIENT_FILES_SRC += src/BZIP2Decompressor.hpp LIBANCIENT_FILES_SRC += src/BZIP2Table.hpp LIBANCIENT_FILES_SRC += src/CompactDecompressor.cpp LIBANCIENT_FILES_SRC += src/CompactDecompressor.hpp LIBANCIENT_FILES_SRC += src/CBR0Decompressor.cpp LIBANCIENT_FILES_SRC += src/CBR0Decompressor.hpp LIBANCIENT_FILES_SRC += src/CompressDecompressor.cpp LIBANCIENT_FILES_SRC += src/CompressDecompressor.hpp LIBANCIENT_FILES_SRC += src/CRMDecompressor.cpp LIBANCIENT_FILES_SRC += src/CRMDecompressor.hpp LIBANCIENT_FILES_SRC += src/CYB2Decoder.cpp LIBANCIENT_FILES_SRC += src/CYB2Decoder.hpp LIBANCIENT_FILES_SRC += src/DEFLATEDecompressor.cpp LIBANCIENT_FILES_SRC += src/DEFLATEDecompressor.hpp LIBANCIENT_FILES_SRC += src/DLTADecode.cpp LIBANCIENT_FILES_SRC += src/DLTADecode.hpp LIBANCIENT_FILES_SRC += src/DMSDecompressor.cpp LIBANCIENT_FILES_SRC += src/DMSDecompressor.hpp LIBANCIENT_FILES_SRC += src/Decompressor.cpp LIBANCIENT_FILES_SRC += src/Decompressor.hpp LIBANCIENT_FILES_SRC += src/DynamicHuffmanDecoder.hpp LIBANCIENT_FILES_SRC += src/FASTDecompressor.cpp LIBANCIENT_FILES_SRC += src/FASTDecompressor.hpp LIBANCIENT_FILES_SRC += src/FBR2Decompressor.cpp LIBANCIENT_FILES_SRC += src/FBR2Decompressor.hpp LIBANCIENT_FILES_SRC += src/FreezeDecompressor.cpp LIBANCIENT_FILES_SRC += src/FreezeDecompressor.hpp LIBANCIENT_FILES_SRC += src/FRLEDecompressor.cpp LIBANCIENT_FILES_SRC += src/FRLEDecompressor.hpp LIBANCIENT_FILES_SRC += src/FrequencyTree.hpp LIBANCIENT_FILES_SRC += src/HFMNDecompressor.cpp LIBANCIENT_FILES_SRC += src/HFMNDecompressor.hpp LIBANCIENT_FILES_SRC += src/HUFFDecompressor.cpp LIBANCIENT_FILES_SRC += src/HUFFDecompressor.hpp LIBANCIENT_FILES_SRC += src/HuffmanDecoder.hpp LIBANCIENT_FILES_SRC += src/ILZRDecompressor.cpp LIBANCIENT_FILES_SRC += src/ILZRDecompressor.hpp LIBANCIENT_FILES_SRC += src/IMPDecompressor.cpp LIBANCIENT_FILES_SRC += src/IMPDecompressor.hpp LIBANCIENT_FILES_SRC += src/InputStream.cpp LIBANCIENT_FILES_SRC += src/InputStream.hpp LIBANCIENT_FILES_SRC += src/LHLBDecompressor.cpp LIBANCIENT_FILES_SRC += src/LHLBDecompressor.hpp LIBANCIENT_FILES_SRC += src/LIN1Decompressor.cpp LIBANCIENT_FILES_SRC += src/LIN1Decompressor.hpp LIBANCIENT_FILES_SRC += src/LIN2Decompressor.cpp LIBANCIENT_FILES_SRC += src/LIN2Decompressor.hpp LIBANCIENT_FILES_SRC += src/LOBDecompressor.cpp LIBANCIENT_FILES_SRC += src/LOBDecompressor.hpp LIBANCIENT_FILES_SRC += src/LZBSDecompressor.cpp LIBANCIENT_FILES_SRC += src/LZBSDecompressor.hpp LIBANCIENT_FILES_SRC += src/LZCBDecompressor.cpp LIBANCIENT_FILES_SRC += src/LZCBDecompressor.hpp LIBANCIENT_FILES_SRC += src/LZWDecoder.cpp LIBANCIENT_FILES_SRC += src/LZWDecoder.hpp LIBANCIENT_FILES_SRC += src/LZW2Decompressor.cpp LIBANCIENT_FILES_SRC += src/LZW2Decompressor.hpp LIBANCIENT_FILES_SRC += src/LZW4Decompressor.cpp LIBANCIENT_FILES_SRC += src/LZW4Decompressor.hpp LIBANCIENT_FILES_SRC += src/LZW5Decompressor.cpp LIBANCIENT_FILES_SRC += src/LZW5Decompressor.hpp LIBANCIENT_FILES_SRC += src/LZXDecompressor.cpp LIBANCIENT_FILES_SRC += src/LZXDecompressor.hpp LIBANCIENT_FILES_SRC += src/MASHDecompressor.cpp LIBANCIENT_FILES_SRC += src/MASHDecompressor.hpp LIBANCIENT_FILES_SRC += src/MMCMPDecompressor.cpp LIBANCIENT_FILES_SRC += src/MMCMPDecompressor.hpp LIBANCIENT_FILES_SRC += src/NONEDecompressor.cpp LIBANCIENT_FILES_SRC += src/NONEDecompressor.hpp LIBANCIENT_FILES_SRC += src/NUKEDecompressor.cpp LIBANCIENT_FILES_SRC += src/NUKEDecompressor.hpp LIBANCIENT_FILES_SRC += src/OutputStream.cpp LIBANCIENT_FILES_SRC += src/OutputStream.hpp LIBANCIENT_FILES_SRC += src/PackDecompressor.cpp LIBANCIENT_FILES_SRC += src/PackDecompressor.hpp LIBANCIENT_FILES_SRC += src/PPDecompressor.cpp LIBANCIENT_FILES_SRC += src/PPDecompressor.hpp LIBANCIENT_FILES_SRC += src/PPMQDecompressor.cpp LIBANCIENT_FILES_SRC += src/PPMQDecompressor.hpp LIBANCIENT_FILES_SRC += src/RAKEDecompressor.cpp LIBANCIENT_FILES_SRC += src/RAKEDecompressor.hpp LIBANCIENT_FILES_SRC += src/RDCNDecompressor.cpp LIBANCIENT_FILES_SRC += src/RDCNDecompressor.hpp LIBANCIENT_FILES_SRC += src/RLENDecompressor.cpp LIBANCIENT_FILES_SRC += src/RLENDecompressor.hpp LIBANCIENT_FILES_SRC += src/RNCDecompressor.cpp LIBANCIENT_FILES_SRC += src/RNCDecompressor.hpp LIBANCIENT_FILES_SRC += src/RangeDecoder.cpp LIBANCIENT_FILES_SRC += src/RangeDecoder.hpp LIBANCIENT_FILES_SRC += src/SCOCompressDecompressor.cpp LIBANCIENT_FILES_SRC += src/SCOCompressDecompressor.hpp LIBANCIENT_FILES_SRC += src/SDHCDecompressor.cpp LIBANCIENT_FILES_SRC += src/SDHCDecompressor.hpp LIBANCIENT_FILES_SRC += src/SHRXDecompressor.cpp LIBANCIENT_FILES_SRC += src/SHRXDecompressor.hpp LIBANCIENT_FILES_SRC += src/SLZ3Decompressor.cpp LIBANCIENT_FILES_SRC += src/SLZ3Decompressor.hpp LIBANCIENT_FILES_SRC += src/SMPLDecompressor.cpp LIBANCIENT_FILES_SRC += src/SMPLDecompressor.hpp LIBANCIENT_FILES_SRC += src/SQSHDecompressor.cpp LIBANCIENT_FILES_SRC += src/SQSHDecompressor.hpp LIBANCIENT_FILES_SRC += src/SXSCDecompressor.cpp LIBANCIENT_FILES_SRC += src/SXSCDecompressor.hpp LIBANCIENT_FILES_SRC += src/StoneCrackerDecompressor.cpp LIBANCIENT_FILES_SRC += src/StoneCrackerDecompressor.hpp LIBANCIENT_FILES_SRC += src/TDCSDecompressor.cpp LIBANCIENT_FILES_SRC += src/TDCSDecompressor.hpp LIBANCIENT_FILES_SRC += src/TPWMDecompressor.cpp LIBANCIENT_FILES_SRC += src/TPWMDecompressor.hpp LIBANCIENT_FILES_SRC += src/VariableLengthCodeDecoder.hpp LIBANCIENT_FILES_SRC += src/VicXDecompressor.cpp LIBANCIENT_FILES_SRC += src/VicXDecompressor.hpp LIBANCIENT_FILES_SRC += src/XPKDecompressor.cpp LIBANCIENT_FILES_SRC += src/XPKDecompressor.hpp LIBANCIENT_FILES_SRC += src/XPKMain.cpp LIBANCIENT_FILES_SRC += src/XPKMain.hpp LIBANCIENT_FILES_SRC += src/XPKUnimplemented.cpp LIBANCIENT_FILES_SRC += src/XPKUnimplemented.hpp LIBANCIENT_FILES_SRC += src/ZENODecompressor.cpp LIBANCIENT_FILES_SRC += src/ZENODecompressor.hpp LIBANCIENT_FILES_SRC += src/common/Buffer.cpp LIBANCIENT_FILES_SRC += src/common/Buffer.hpp LIBANCIENT_FILES_SRC += src/common/CRC16.cpp LIBANCIENT_FILES_SRC += src/common/CRC16.hpp LIBANCIENT_FILES_SRC += src/common/CRC32.cpp LIBANCIENT_FILES_SRC += src/common/CRC32.hpp LIBANCIENT_FILES_SRC += src/common/Common.cpp LIBANCIENT_FILES_SRC += src/common/Common.hpp LIBANCIENT_FILES_SRC += src/common/MemoryBuffer.cpp LIBANCIENT_FILES_SRC += src/common/MemoryBuffer.hpp LIBANCIENT_FILES_SRC += src/common/OverflowCheck.hpp LIBANCIENT_FILES_SRC += src/common/StaticBuffer.cpp LIBANCIENT_FILES_SRC += src/common/StaticBuffer.hpp LIBANCIENT_FILES_SRC += src/common/SubBuffer.cpp LIBANCIENT_FILES_SRC += src/common/SubBuffer.hpp LIBANCIENT_FILES_SRC += src/common/WrappedVectorBuffer.cpp LIBANCIENT_FILES_SRC += src/common/WrappedVectorBuffer.hpp pkgconfig_DATA += libancient.pc lib_LTLIBRARIES += libancient.la libancient_la_LDFLAGS = -version-info $(LIBANCIENT_LTVER_CURRENT):$(LIBANCIENT_LTVER_REVISION):$(LIBANCIENT_LTVER_AGE) -no-undefined includeancientdir = $(includedir)/ancient includeancient_HEADERS = includeancient_HEADERS += api/ancient/ancient.hpp #nobase_include_HEADERS += api/ancient/ancient.hpp libancient_la_CPPFLAGS = -I$(srcdir)/src -I$(srcdir)/api -I$(srcdir)/api/ancient libancient_la_CXXFLAGS = libancient_la_CFLAGS = libancient_la_LIBADD = libancient_la_SOURCES = libancient_la_SOURCES += $(LIBANCIENT_FILES_SRC) bin_PROGRAMS += ancient ancient_CPPFLAGS = -I$(srcdir)/api ancient_CXXFLAGS = ancient_LDADD = libancient.la ancient_SOURCES = ancient_SOURCES += main.cpp dist-hook: rm -rf $(distdir)/testingancient-2.2.0/Makefile.unix000066400000000000000000000045011463063262600155660ustar00rootroot00000000000000# Copyright (C) Teemu Suutari VPATH := src src/common fuzzing testing CXX ?= c++ COMMONFLAGS = -Os -Wall -Wsign-compare -Wnarrowing -Isrc -Iapi -Iapi/ancient CFLAGS = $(COMMONFLAGS) CXXFLAGS = $(COMMONFLAGS) -std=c++17 -fno-rtti -fvisibility=hidden -DANCIENT_API_VISIBILITY_DEFAULT $(EXTRA_CFLAGS) LIBNAME = ancient.dylib PROG = obj/ancient MAIN_N ?= main.o OBJS_N = API.o Buffer.o Common.o MemoryBuffer.o StaticBuffer.o SubBuffer.o WrappedVectorBuffer.o CRC16.o CRC32.o \ Decompressor.o LZWDecoder.o XPKDecompressor.o XPKMain.o \ OutputStream.o InputStream.o RangeDecoder.o \ ACCADecompressor.o ARTMDecompressor.o BLZWDecompressor.o BZIP2Decompressor.o \ CBR0Decompressor.o CompactDecompressor.o CompressDecompressor.o CRMDecompressor.o CYB2Decoder.o \ DEFLATEDecompressor.o DLTADecode.o DMSDecompressor.o FASTDecompressor.o \ FBR2Decompressor.o FreezeDecompressor.o FRLEDecompressor.o HFMNDecompressor.o \ HUFFDecompressor.o ILZRDecompressor.o IMPDecompressor.o LHLBDecompressor.o \ LIN1Decompressor.o LIN2Decompressor.o LOBDecompressor.o LZBSDecompressor.o \ LZCBDecompressor.o LZW2Decompressor.o LZW4Decompressor.o LZW5Decompressor.o \ LZXDecompressor.o MASHDecompressor.o MMCMPDecompressor.o NONEDecompressor.o \ NUKEDecompressor.o PackDecompressor.o PPDecompressor.o PPMQDecompressor.o RAKEDecompressor.o \ RDCNDecompressor.o RLENDecompressor.o RNCDecompressor.o SCOCompressDecompressor.o \ SDHCDecompressor.o SHRXDecompressor.o SLZ3Decompressor.o SMPLDecompressor.o \ StoneCrackerDecompressor.o SQSHDecompressor.o SXSCDecompressor.o TDCSDecompressor.o \ TPWMDecompressor.o VicXDecompressor.o XPKUnimplemented.o ZENODecompressor.o TEST_N = test.o TESTBIN = obj/test LIB = $(addprefix obj/,$(LIBNAME)) OBJS = $(addprefix obj/,$(OBJS_N)) MAIN = $(addprefix obj/,$(MAIN_N)) TEST = $(addprefix obj/,$(TEST_N)) all: $(PROG) $(TESTBIN) obj: mkdir -p obj obj/%.o: %.cpp | obj $(CXX) $(CXXFLAGS) -o $@ -c $< ifeq ($(BUILD_LIBRARY),1) $(LIB): $(OBJS) $(CXX) -Wl,-dylib -Wl,-install_name,@executable_path/$(LIBNAME) -shared -o $@ $^ strip -X $@ $(PROG): $(MAIN) $(LIB) | obj $(CXX) $(CFLAGS) -o $@ $(MAIN) $(LIB) else $(PROG): $(MAIN) $(OBJS) | obj $(CXX) $(CFLAGS) -o $@ $(MAIN) $(OBJS) endif $(TESTBIN): $(OBJS) $(TEST) | obj $(CXX) $(CFLAGS) -o $@ $^ clean: rm -f $(OBJS) $(MAIN) $(PROG) $(LIB) $(TEST) $(TEST_BIN) *~ src/*~ .PHONY: ancient-2.2.0/README.md000066400000000000000000000177001463063262600144300ustar00rootroot00000000000000# Ancient - Modern decompressor for old data compression formats This is a collection of decompression routines for old formats popular in the Amiga, Atari computers and some other systems from 80's and 90's as well as some that are currently used which were used in a some specific way in these old systems. Even though most of these algorithms are still available for download, scavenging and using them might prove to be a challenge. Thus the purpose of this project is to: * Provide a clean, modern implementation of the algorithms - Typically the implementations were not meant to be used outside of the original systems they were made for. Some other ported implementations are incomplete, bad quality or direct translations from old M68K assembly code. * Provide a clean BSD-style licensing - Original implementations or their ports might have strange license or no visible license at all. There are also implementations that have been ripped off from some other source thus their legality is questionable at best. * Provide a tested implementation - The code is no good if it does not work properly and the old code have a lot of corner cases. These implementations are tested using a cache of available files (~10k) that used these algorithms. Although it does not offer any guarantee especially when we are talking about undocumented formats, it gives hope that there are less "stupid errors" in the code. I have also generated a small batch of test files for different formats for testing. The source files are known public domain sources For simple usage both a simple command line application as well as a simple API to use the decompressors are provided. The compression algorithm is automatically detected in most cases, however there are some corner cases where it is not entirely reliable due to weaknesses in the old format used. Please see the [main.cpp](main.cpp) and [ancient.hpp](api/ancient/ancient.hpp) to get an idea. This code should compile cleanly on most C++17 capable compilers, and it is tested on clang and MSVC. Some formats have incorporated weak password protection on them which can be bypassed. However, this project does not attempt to do any real cryptograpy. Currently the project does not support any archival files nor self extracting executables. Decompression algorithms provided: * bzip2 * both normal and randomized bitstreams * Compact (Unix) * Compress (Unix) * Supports both old and new formats * CrunchMania by Thomas Schwarz * CrM!: Crunch-Mania standard-mode * Crm!: Crunch-Mania standard-mode, sampled * CrM2: Crunch-Mania LZH-mode * Crm2: Crunch-Mania LZH-mode, sampled * ID 0x18051973 (CrunchMania CrM2 Clone) * ID CD³¹ (CrunchMania CrM2 Clone) * ID DCS! (CrunchMania CrM! Clone) * ID Iron (CrunchMania CrM2 Clone) * ID MSS! (CrunchMania CrM2 Clone) * ID mss! (CrunchMania Crm2 Clone) * Disk Masher System a.k.a. DMS * Supports all different compression methods (NONE,SIMPLE,QUICK,MEDIUM,DEEP,HEAVY1,HEAVY2) * Supports password bypassing * File Imploder * ID ATN! (Imploder Clone) * ID BDPI (Imploder Clone) * ID CHFI (Imploder Clone) * ID EDAM (Imploder Clone) * ID M.H. (Imploder Clone) * ID RDC9 (Imploder Clone) * ID FLT! (Imploder Clone) (verification missing) * ID Dupa (Imploder Clone) (verification missing) * ID PARA (Imploder Clone) (verification missing) * Freeze/Melt * Supports both old and new formats * gzip * LOB's File Compressor (Also known as a Multipak) * Supports all original 6 modes and their combinations (BMC, HUF, LZW, LZB, MSP, MSS) * Does not support mode 8 (as defined by some game files) * Pack (Unix) * Supports both old and new formats * PowerPacker * PP 1.1 (verification missing) * PP 2.0 * PX20: Supports bypassing password protected files. * ID CHFC (PowerPacker Clone) * ID DEN! (PowerPacker Clone) * ID DXS9 (PowerPacker Clone) * ID H.D. (PowerPacker Clone) * ID RVV! (PowerPacker Clone) * Quasijarus Strong Compression * Rob Northen compressors. * RNC1: Supports both new and old format of RNC1 * RNC2: Supports both new and old format of RNC2 * ID ...1 (RNC1 Clone) * Turbo Packer by Wolfgang Mayerle. * MMCMP: Music Module Compressor * SCO Compress LZH * StoneCracker * SC: StoneCracker v2.69 - v2.81 * SC: StoneCracker v2.92, v2.99 * S300: StoneCracker v3.00 * S310: StoneCracker v3.10, v3.11b * S400: StoneCracker pre v4.00 * S401: StoneCracker v4.01 * S403: StoneCracker v4.02a * S404: StoneCracker v4.10 * ID 1AM (StoneCracker S300 Clone) * ID 2AM (StoneCracker S401 Clone) * ID AYS! (StoneCracker S404 Clone) * ID Z&G! (StoneCracker S403 Clone) * ID ZULU (StoneCracker S403 Clone) * Vice / Vic2 Huffman compressor with RLE * XPK-encapsulated files * ACCA: Andre's Code Compression Algorithm * ARTM: Arithmetic encoding compressor * BLZW: LZW-compressor * BZP2: Bzip2 backend for XPK * CBR0: RLE compressor * CBR1: RLE compressor * CRM2: CrunchMania backend for XPK * CRMS: CrunchMania backend for XPK, sampled * CYB2: xpkCybPrefs container * DLTA: Delta encoding * DUKE: NUKE with Delta encoding * ELZX: LZX-compressor * FAST: LZ77-compressor * FBR2: CyberYAFA compressor * FRHT: LZ77-compressor * FRLE: RLE compressor * GZIP: Deflate backend for XPK * HUFF: Huffman modeling compressor * HFMN: Huffman modeling compressor * ILZR: Incremental Lempel-Ziv-Renau compressor * IMPL: File Imploder backend for XPK * LHLB: LZRW-compressor * LIN1: Lino packer * LIN2: Lino packer * LIN3: Lino packer * LIN4: Lino packer * LZBS: CyberYAFA compressor * LZCB: LZ-compressor * LZW2: CyberYAFA compressor * LZW3: CyberYAFA compressor * LZW4: CyberYAFA compressor * LZW5: CyberYAFA compressor * MASH: LZRW-compressor * NONE: Null compressor * NUKE: LZ77-compressor * PPMQ: PPM compressor * PWPK: PowerPacker backend for XPK * RAKE: LZ77-compressor * RDCN: Ross Data Compression * RLEN: RLE compressor * SASC: LZ-compressor with arithmetic encoding * SDHC: Sample delta huffman compressor * SHR3: LZ-compressor with arithmetic encoding * SHRI: LZ-compressor with arithmetic encoding * SHSC: Context modeling compressor * SLZ3: CyberYAFA compressor * SLZX: LZX-compressor with delta encoding * SMPL: Huffman compressor with delta encoding * SQSH: Compressor for sampled sounds * TDCS: LZ77-compressor * ZENO: LZW-compressor There is some support for archival decompressors: However, these are not built in at the moment but the code can be as a reference * Zip decompressor backend (decompressor only, no Zip file format reading yet) * Shrink * Reduce * Implode * Deflate * Deflate64 * Bzip2 * Lha/Lzh decompressor backend (decompressor only, no Lha file format reading yet) * LH0: Null compressor * LH1: LZRW-compressor with 4kB window * LH2: LZRW-compressor with Dynamic Huffman Encoding (experimental) * LH3: LZRW-compressor (experimental) * LH4: LZRW-compressor with 4kB window * LH5: LZRW-compressor with 8kB window * LH6: LZRW-compressor with 32kB window * LH7: LZRW-compressor with 64kB window * LH8: LZRW-compressor with 64kB window (Joe Jared extension) * LHX: LZRW-compressor with up to 512kB window (UnLHX extension) * LZ4: Null compressor * LZ5: LZ-compressor * LZS: LZ-compressor * PM0: Null compressor * PM1: LZ-compressor * PM2: LZ-compressor Special thanks go to Cholok for providing me references to many of the XPK-compressors. BZIP2 tables for randomization have been included, they have BZIP2-license. SASC/SHSC decompressors have been re-implemented by using the original HA code from Harri Hirvola as reference. (No code re-used) Some of the rare Lzh-compressors have been re-implemented by using Lhasa as a reference. (No code re-used) I'm slowly adding new stuff. If your favorite is not listed contact me and maybe I can add it. Currently not planned to be supported: * PPC only XPK compressors. XPK implementation is now considered complete in practical terms for classic Amiga. Wishlist: * More files for my testbench. Feedback: tz at iki dot fi ancient-2.2.0/api/000077500000000000000000000000001463063262600137155ustar00rootroot00000000000000ancient-2.2.0/api/ancient/000077500000000000000000000000001463063262600153365ustar00rootroot00000000000000ancient-2.2.0/api/ancient/ancient.hpp000066400000000000000000000105271463063262600174750ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef ANCIENT_ANCIENT_HPP #define ANCIENT_ANCIENT_HPP #ifndef ANCIENT_API #if defined(ANCIENT_API_DECLSPEC_DLLEXPORT) #define ANCIENT_API __declspec(dllexport) #endif #if defined(ANCIENT_API_DECLSPEC_DLLIMPORT) #define ANCIENT_API __declspec(dllimport) #endif #if defined(ANCIENT_API_VISIBILITY_DEFAULT_USED) #define ANCIENT_API __attribute__((visibility("default"))) __attribute__((used)) #endif #if defined(ANCIENT_API_VISIBILITY_DEFAULT) #define ANCIENT_API __attribute__((visibility("default"))) #endif #if defined(ANCIENT_API_VISIBILITY_HIDDEN) #define ANCIENT_API __attribute__((visibility("hidden"))) #endif #ifndef ANCIENT_API #define ANCIENT_API #endif #endif #include #include #include #include #include #include #include namespace ancient { namespace internal { namespace APIv2 { class DecompressorImpl; } } inline namespace APIv2 { // just a base class to easily catch all the errors class ANCIENT_API Error : public std::exception { public: Error() noexcept; virtual ~Error() noexcept; }; class ANCIENT_API InvalidFormatError : public Error { public: InvalidFormatError() noexcept; ~InvalidFormatError() noexcept; }; class ANCIENT_API DecompressionError : public Error { public: DecompressionError() noexcept; ~DecompressionError() noexcept; }; class ANCIENT_API VerificationError : public Error { public: VerificationError() noexcept; ~VerificationError() noexcept; }; class ANCIENT_API Decompressor final { public: // Detect signature whether it matches to any known compressor // This does not guarantee the data is decompressable though, only signature is read static bool detect(const std::vector &packedData) noexcept; static bool detect(const uint8_t *packedData,size_t packedSize) noexcept; // Main entrypoint // if verify=true then check the packedData for errors: CRC or other checksum if available // check exactSizeKnown from size documentation // can throw InvalidFormatError if stream is not recognized or it is invalid // can throw VerificationError if verify enabled and checksum does not match explicit Decompressor(const std::vector &packedData,bool exactSizeKnown,bool verify); explicit Decompressor(const uint8_t *packedData,size_t packedSize,bool exactSizeKnown,bool verify); // Name returned is human readable long name const std::string &getName() const noexcept; // the functions are there to protect against "accidental" large files when parsing headers // a.k.a. 16M should be enough for everybody (sizes do not have to accurate i.e. // compressors can exclude header content for simplification) // This entirely ok for the context of "old computers" and their files, // for other usages these need to be tuned up static size_t getMaxPackedSize() noexcept; static size_t getMaxRawSize() noexcept; // PackedSize or RawSize are taken from the stream if available, std::nullopt otherwise. // for those compressors having no known sizes, running decompression will update // the values. // There are exceptions: Some decompressors need to exact size of the packed data // in order to decompress. For those providing a indefinitely size packed stream // will not work // use the "exactSizeKnown" flag for create to tell whether you know the size or not std::optional getPackedSize() const noexcept; std::optional getRawSize() const noexcept; // in case of disk image based formats the data does not necessarily start // from logical beginnig of the image but it is offsetted inside the logical image (f.e. DMS). // getDataOffset will return the offset which can also be 0, or std::nullopt if not image-based format. // getImageSize will return the size of the the logical image, or std::nullopt if not image-based format. std::optional getImageSize() const noexcept; std::optional getImageOffset() const noexcept; // Actual decompression. // verify checksum if verify==true // can throw DecompressionError if stream cant be unpacked // can throw VerificationError if verify enabled and checksum does not match std::vector decompress(bool verify); ~Decompressor() noexcept; private: std::unique_ptr m_impl; private: Decompressor(const Decompressor&)=delete; Decompressor& operator=(const Decompressor&)=delete; }; } } #endif ancient-2.2.0/autogen.sh000077500000000000000000000000401463063262600151370ustar00rootroot00000000000000#!/usr/bin/env sh autoreconf -i ancient-2.2.0/configure.ac000066400000000000000000000060621463063262600154360ustar00rootroot00000000000000 AC_INIT([ancient], [2.2.0], [https://github.com/temisu/ancient/issues], [ancient], [https://github.com/temisu/ancient/]) # ^^^^^^^^^^^ # Package Version # # MAJOR.MINOR.PATCH[-PRE][+META] # # * increment MAJOR and set MINOR=0 and PATCH=0 for incompatible ABI/API changes (changed/removed API elements) # * increment MINOR and set PATCH=0 for compatible ABI/API changes (added API elements) # * increment PATCH and set PRE=-pre.0 immediately after every release, always # * increment PRE at free will # * for a release, set PRE="", i.e. the version number should look like "2.32.42" or maybe "2.23.42+autotools" # * eveything appended after "+" is considered build/variant metadata is not part of version ordering # For details, see . # # # Libtool version-info (used to determine soname) # # * increment REVISION immediately after every release, always # * increment CURRENT when the next release contains any ABI changes (added/changed/removed API elements), and set REVISION=0 # * increment AGE when the next release stays compatible with the previous release (only added API elements) # * set AGE=0 when the next release is incompatible with the previous release (changed/removed API elements) # For details, see . LIBANCIENT_LTVER_CURRENT=2 LIBANCIENT_LTVER_REVISION=2 LIBANCIENT_LTVER_AGE=0 # # # Dependencies (Debian/Ubuntu package names): # # * autoconf # * autoconf-archive # * automake # * libtool # * make # * pkg-config # # # Release procedure: # # ./autogen.sh # ./configure # make distcheck # make distclean # # # Verbose make output: # ./configure --disable-silent-rules # AC_PREREQ([2.68]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_FILES([Makefile libancient.pc]) AM_INIT_AUTOMAKE([1.11 -Wall -Werror foreign subdir-objects]) AM_PROG_AR LT_INIT AC_SYS_LARGEFILE PKG_PROG_PKG_CONFIG([0.24]) AM_SILENT_RULES([yes]) # C++ AC_PROG_CXX AC_PROG_INSTALL AC_SUBST([LIBANCIENT_LTVER_CURRENT]) AC_SUBST([LIBANCIENT_LTVER_REVISION]) AC_SUBST([LIBANCIENT_LTVER_AGE]) # C++ AX_CXX_COMPILE_STDCXX(17, [noext], [mandatory]) # internal (non-exposed) dependencies (pkg-config) for .pc file LIBANCIENT_PC_REQUIRES_PRIVATE= # internal (non-exposed) dependencies (plain linker) for .pc file LIBANCIENT_PC_LIBS_PRIVATE= # CFLAGS required to use the library for .pc file LIBANCIENT_PC_CFLAGS= # ANCIENT_API macro for .pc file LIBANCIENT_PC_ANCIENT_API= # C++ AC_LANG_PUSH([C++]) AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CXXFLAGS="$CXXFLAGS -fvisibility=hidden"]) AX_CHECK_COMPILE_FLAG([-fvisibility=default], [AC_DEFINE([ANCIENT_API], [__attribute__((visibility("default")))])]) AX_CHECK_COMPILE_FLAG([-fvisibility=default], [AC_SUBST([LIBANCIENT_PC_ANCIENT_API], [-DANCIENT_API_VISIBILITY_DEFAULT])]) AX_CXXFLAGS_WARN_ALL AC_LANG_POP([C++]) AC_SUBST([LIBANCIENT_PC_REQUIRES_PRIVATE]) AC_SUBST([LIBANCIENT_PC_LIBS_PRIVATE]) AC_SUBST([LIBANCIENT_PC_CFLAGS]) AC_SUBST([LIBANCIENT_PC_ANCIENT_API]) AC_OUTPUT ancient-2.2.0/extra/000077500000000000000000000000001463063262600142675ustar00rootroot00000000000000ancient-2.2.0/extra/.gitignore000066400000000000000000000000131463063262600162510ustar00rootroot00000000000000/bruteRNC1 ancient-2.2.0/extra/BruteForceRNC1Encoder.cpp000066400000000000000000000365621463063262600207730ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ // This is really quick and dirty. Works though #include #include #include #include #include #include #include #include #include #include #include #include "common/Buffer.hpp" #include "common/SubBuffer.hpp" #define FUZZY_BLOCK_CUT_THRESHOLD (256) namespace ancient::internal { class VectorBuffer : public Buffer { public: VectorBuffer(); virtual ~VectorBuffer() override final; virtual const uint8_t *data() const noexcept override final; virtual uint8_t *data() override final; virtual size_t size() const noexcept override final; virtual bool isResizable() const noexcept override final; virtual void resize(size_t newSize) override final; private: std::vector _data; }; VectorBuffer::VectorBuffer() { // nothing needed } VectorBuffer::~VectorBuffer() { // nothing needed } const uint8_t *VectorBuffer::data() const noexcept { return _data.data(); } uint8_t *VectorBuffer::data() { return _data.data(); } size_t VectorBuffer::size() const noexcept { return _data.size(); } bool VectorBuffer::isResizable() const noexcept { return true; } void VectorBuffer::resize(size_t newSize) { return _data.resize(newSize); } std::unique_ptr readFile(const std::string &fileName) { std::unique_ptr ret=std::make_unique(); std::ifstream file(fileName.c_str(),std::ios::in|std::ios::binary); bool success=false; if (file.is_open()) { file.seekg(0,std::ios::end); size_t length=size_t(file.tellg()); file.seekg(0,std::ios::beg); ret->resize(length); file.read(reinterpret_cast(ret->data()),length); success=bool(file); if (!success) ret->resize(0); file.close(); } if (!success) { fprintf(stderr,"Could not read file %s\n",fileName.c_str()); } return ret; } bool writeFile(const std::string &fileName,const Buffer &content) { bool ret=false; std::ofstream file(fileName.c_str(),std::ios::out|std::ios::binary|std::ios::trunc); if (file.is_open()) { file.write(reinterpret_cast(content.data()),content.size()); ret=bool(file); file.close(); } if (!ret) { fprintf(stderr,"Could not write file %s\n",fileName.c_str()); } return ret; } uint16_t RNCCRC(const uint8_t *buffer,size_t len) { // bit reversed 16bit CRC with 0x8005 polynomial static const uint16_t CRCTable[256]={ 0x0000,0xc0c1,0xc181,0x0140,0xc301,0x03c0,0x0280,0xc241,0xc601,0x06c0,0x0780,0xc741,0x0500,0xc5c1,0xc481,0x0440, 0xcc01,0x0cc0,0x0d80,0xcd41,0x0f00,0xcfc1,0xce81,0x0e40,0x0a00,0xcac1,0xcb81,0x0b40,0xc901,0x09c0,0x0880,0xc841, 0xd801,0x18c0,0x1980,0xd941,0x1b00,0xdbc1,0xda81,0x1a40,0x1e00,0xdec1,0xdf81,0x1f40,0xdd01,0x1dc0,0x1c80,0xdc41, 0x1400,0xd4c1,0xd581,0x1540,0xd701,0x17c0,0x1680,0xd641,0xd201,0x12c0,0x1380,0xd341,0x1100,0xd1c1,0xd081,0x1040, 0xf001,0x30c0,0x3180,0xf141,0x3300,0xf3c1,0xf281,0x3240,0x3600,0xf6c1,0xf781,0x3740,0xf501,0x35c0,0x3480,0xf441, 0x3c00,0xfcc1,0xfd81,0x3d40,0xff01,0x3fc0,0x3e80,0xfe41,0xfa01,0x3ac0,0x3b80,0xfb41,0x3900,0xf9c1,0xf881,0x3840, 0x2800,0xe8c1,0xe981,0x2940,0xeb01,0x2bc0,0x2a80,0xea41,0xee01,0x2ec0,0x2f80,0xef41,0x2d00,0xedc1,0xec81,0x2c40, 0xe401,0x24c0,0x2580,0xe541,0x2700,0xe7c1,0xe681,0x2640,0x2200,0xe2c1,0xe381,0x2340,0xe101,0x21c0,0x2080,0xe041, 0xa001,0x60c0,0x6180,0xa141,0x6300,0xa3c1,0xa281,0x6240,0x6600,0xa6c1,0xa781,0x6740,0xa501,0x65c0,0x6480,0xa441, 0x6c00,0xacc1,0xad81,0x6d40,0xaf01,0x6fc0,0x6e80,0xae41,0xaa01,0x6ac0,0x6b80,0xab41,0x6900,0xa9c1,0xa881,0x6840, 0x7800,0xb8c1,0xb981,0x7940,0xbb01,0x7bc0,0x7a80,0xba41,0xbe01,0x7ec0,0x7f80,0xbf41,0x7d00,0xbdc1,0xbc81,0x7c40, 0xb401,0x74c0,0x7580,0xb541,0x7700,0xb7c1,0xb681,0x7640,0x7200,0xb2c1,0xb381,0x7340,0xb101,0x71c0,0x7080,0xb041, 0x5000,0x90c1,0x9181,0x5140,0x9301,0x53c0,0x5280,0x9241,0x9601,0x56c0,0x5780,0x9741,0x5500,0x95c1,0x9481,0x5440, 0x9c01,0x5cc0,0x5d80,0x9d41,0x5f00,0x9fc1,0x9e81,0x5e40,0x5a00,0x9ac1,0x9b81,0x5b40,0x9901,0x59c0,0x5880,0x9841, 0x8801,0x48c0,0x4980,0x8941,0x4b00,0x8bc1,0x8a81,0x4a40,0x4e00,0x8ec1,0x8f81,0x4f40,0x8d01,0x4dc0,0x4c80,0x8c41, 0x4400,0x84c1,0x8581,0x4540,0x8701,0x47c0,0x4680,0x8641,0x8201,0x42c0,0x4380,0x8341,0x4100,0x81c1,0x8081,0x4040}; uint16_t retValue=0; for (size_t i=0;i>8)^CRCTable[(retValue&0xff)^buffer[i]]; return retValue; } // this is really really quick 'n dirty // leeway is suspicious. I can't see it from the official RNC ProPack, but seems to be present elsewhere... void packRNC(Buffer &dest,const Buffer &source,uint32_t chunkSize) { if (!chunkSize) chunkSize=32768; // seems to be a good default if (chunkSize>65536) chunkSize=65536; if (chunkSize<4096) chunkSize=4096; std::vector stream(20); stream[0]='R'; stream[1]='N'; stream[2]='C'; stream[3]=1; stream[4]=uint8_t(source.size()>>24); stream[5]=uint8_t(source.size()>>16); stream[6]=uint8_t(source.size()>>8); stream[7]=uint8_t(source.size()); stream[8]=stream[9]=stream[10]=stream[11]=0; uint16_t rawCrc=RNCCRC(source.data(),source.size()); stream[12]=uint8_t(rawCrc>>8); stream[13]=uint8_t(rawCrc); stream[14]=stream[15]=stream[16]=stream[17]=0; stream[18]=stream[19]=0; uint32_t bitStreamPosition=18; uint32_t bitAccumContent=0; uint32_t bitAccumCount=2; uint32_t offset=0; uint32_t chunkCount=0; uint32_t leeway=0; while (offset!=source.size()) { auto bitLength=[](uint32_t value)->uint32_t { uint32_t ret=0; while (value) { value>>=1; ret++; } return ret; }; // returns code, extra bits, value auto packValue=[&](uint32_t value)->std::tuple { if (value<2) return std::tuple{value,0,0}; uint32_t bits=bitLength(value)-1; value&=(1<{bits+1,bits,value}; }; // bruteforce!!! // returns length,offset auto findLongestRepeat=[&](const uint8_t *buf,uint32_t offset,uint32_t length)->std::pair { auto comparablePackedSize=[&](uint32_t value)->uint32_t { if (value<2) return 1; // fuzzy cost addition for longer codes return bitLength(value)+(bitLength(bitLength(value))<<1U); }; std::pair best{0,0}; uint32_t distance=1; while (distance<=offset && distance<=chunkSize) { uint32_t i=0; while (offset+i=2 && (comparablePackedSize(i-2)+comparablePackedSize(distance-1)+best.first*8 litFrequencies(32,0); std::vector distanceFrequencies(32,0); std::vector lengthFrequencies(32,0); // this is what makes this implementaion even more bruteforce // table index (lit=0,distance=1,length=2,byte=3,bits=4), code, extra bits, value std::vector> rawChunk; uint32_t litCountOffset=0; bool litActive=false; // sub count rawChunk.push_back(std::tuple{4,0,16,0}); auto packLit=[&]() { auto pack=packValue(std::get<3>(rawChunk[litCountOffset])); std::get<1>(rawChunk[litCountOffset])=std::get<0>(pack); std::get<2>(rawChunk[litCountOffset])=std::get<1>(pack); std::get<3>(rawChunk[litCountOffset])=std::get<2>(pack); litFrequencies[std::get<0>(pack)]+=1; }; uint32_t foundLength=1; uint32_t currentChunkSize=std::min(uint32_t(source.size())-offset,chunkSize); for (uint32_t i=offset;ibool { if (i+FUZZY_BLOCK_CUT_THRESHOLD>=currentChunkSize+offset) { currentChunkSize=i+foundLength-offset; return true; } return false; }; if (!litActive) { // literal count litCountOffset=uint32_t(rawChunk.size()); rawChunk.push_back(std::tuple{0,0,0,0}); litActive=true; std::get<3>(rawChunk[0])+=1; } auto repeat=findLongestRepeat(source.data(),i,currentChunkSize+offset); if (repeat.first) { packLit(); litActive=false; auto dist=packValue(repeat.second-1); rawChunk.push_back(std::tuple{1,std::get<0>(dist),std::get<1>(dist),std::get<2>(dist)}); distanceFrequencies[std::get<0>(dist)]+=1; auto count=packValue(repeat.first-2); rawChunk.push_back(std::tuple{2,std::get<0>(count),std::get<1>(count),std::get<2>(count)}); lengthFrequencies[std::get<0>(count)]+=1; foundLength=repeat.first; } else { rawChunk.push_back(std::tuple{3,source.data()[i],0,0}); std::get<3>(rawChunk[litCountOffset])+=1; foundLength=1; if (/*std::get<3>(rawChunk[litCountOffset])==1 &&*/ fuzzyBreak()) break; } } if (litActive) { packLit(); } else { litCountOffset=uint32_t(rawChunk.size()); rawChunk.push_back(std::tuple{0,0,0,0}); packLit(); std::get<3>(rawChunk[0])+=1; } offset+=currentChunkSize; chunkCount++; std::function writeBits=[&](uint32_t bitCount,uint32_t bits) { if (!bitCount) return; if (bitCount+bitAccumCount>16) { uint32_t bitsToWrite=16-bitAccumCount; writeBits(bitsToWrite,bits&((1<>=bitsToWrite; bitCount-=bitsToWrite; } if (!bitAccumCount) { bitStreamPosition=uint32_t(stream.size()); stream.push_back(0); stream.push_back(0); } bitAccumContent|=bits<>8; bitAccumContent=0; bitAccumCount=0; } }; auto writeByte=[&](uint8_t byte) { stream.push_back(byte); }; // also writes table to stream // result vector is code, length pairs auto createHuffmanCodeTable=[&](std::vector> &codes,const std::vector &frequencies) { std::vector> sortedList; uint32_t totalCount=0; uint32_t totalFreq=0; for (uint32_t i=0;ib.second||(a.second==b.second&&a.first>bitCount; } // use the full range while (totalUsed!=initialNorm) { uint32_t bestIndex=totalCount; uint32_t bestImprovement=0; for (uint32_t i=0;i>sortedList[i].second)<=initialNorm) { // adding cost factor here too uint32_t improvement=sortedFrequencies[i]<bestImprovement) { bestIndex=i; bestImprovement=improvement; } } } if (bestIndex==totalCount) break; totalUsed+=initialNorm>>sortedList[bestIndex].second; sortedList[bestIndex].second-=1; } // using the full range will sometimes result out-of-order indexes std::sort(sortedList.begin(),sortedList.end(),[&](const auto &a,const auto &b){return a.seconduint32_t { uint32_t ret=0; while (bitCount--) { ret<<=1; ret|=bits&1; bits>>=1; } return ret; }; uint32_t code=sortedList[i].first; codes[code].first=reverseBits(maxDepth,value); codes[code].second=sortedList[i].second; value+=1<<(maxDepth-sortedList[i].second); } for (uint32_t i=0;i> litCodes(32,std::make_pair(0,0)); std::vector> distanceCodes(32,std::make_pair(0,0)); std::vector> lengthCodes(32,std::make_pair(0,0)); uint32_t streamStart=uint32_t(stream.size()); createHuffmanCodeTable(litCodes,litFrequencies); createHuffmanCodeTable(distanceCodes,distanceFrequencies); createHuffmanCodeTable(lengthCodes,lengthFrequencies); for (uint32_t i=0;i(rawChunk[i])) { // literal case 0: writeBits(litCodes[std::get<1>(rawChunk[i])].second,litCodes[std::get<1>(rawChunk[i])].first); writeBits(std::get<2>(rawChunk[i]),std::get<3>(rawChunk[i])); break; // distance case 1: writeBits(distanceCodes[std::get<1>(rawChunk[i])].second,distanceCodes[std::get<1>(rawChunk[i])].first); writeBits(std::get<2>(rawChunk[i]),std::get<3>(rawChunk[i])); break; // length case 2: writeBits(lengthCodes[std::get<1>(rawChunk[i])].second,lengthCodes[std::get<1>(rawChunk[i])].first); writeBits(std::get<2>(rawChunk[i]),std::get<3>(rawChunk[i])); break; // bytes case 3: writeByte(std::get<1>(rawChunk[i])); break; // bits case 4: writeBits(std::get<2>(rawChunk[i]),std::get<3>(rawChunk[i])); break; default: break; } } uint32_t outputLength=uint32_t(stream.size())-streamStart; if (currentChunkSize>outputLength && outputLength-outputLength>leeway) leeway=outputLength-outputLength; } if (bitAccumCount) { stream[bitStreamPosition]=bitAccumContent; stream[bitStreamPosition+1]=bitAccumContent>>8; if (bitStreamPosition==stream.size()-2 && bitAccumCount<=8) stream.pop_back(); } uint32_t packedSize=uint32_t(stream.size()-18); stream[8]=uint8_t(packedSize>>24); stream[9]=uint8_t(packedSize>>16); stream[10]=uint8_t(packedSize>>8); stream[11]=uint8_t(packedSize); uint16_t packedCrc=RNCCRC(stream.data()+18,packedSize); stream[14]=uint8_t(packedCrc>>8); stream[15]=uint8_t(packedCrc); if (leeway>255) { fprintf(stderr,"Leeway larger than 255\n"); exit(-1); } stream[16]=leeway; stream[17]=chunkCount; dest.resize(stream.size()); std::memcpy(dest.data(),stream.data(),stream.size()); } int main(int argc,char **argv) { auto usage=[]() { fprintf(stderr,"Usage: input_raw output_packed [chunk_size]\n"); }; if (argc<3) { usage(); return -1; } auto raw{readFile(argv[1])}; VectorBuffer packed; packRNC(packed,*raw,(argc>=4)?atoi(argv[3]):0); writeFile(argv[2],packed); return 0; } } int main(int argc,char **argv) { return ancient::internal::main(argc,argv); } ancient-2.2.0/extra/Makefile000066400000000000000000000010051463063262600157230ustar00rootroot00000000000000# Copyright (C) Teemu Suutari VPATH := ../src ../src/common CC = clang CXX = clang++ COMMONFLAGS = -g -Wall -Wsign-compare -Wshorten-64-to-32 -I../src -I../api -I../api/ancient CFLAGS = $(COMMONFLAGS) CXXFLAGS = $(COMMONFLAGS) -std=c++17 -fno-rtti PROG = bruteRNC1 OBJS = Buffer.o SubBuffer.o BruteForceRNC1Encoder.o all: $(PROG) .cpp.o: $(CXX) $(CXXFLAGS) -o $@ -c $< .c.o: $(CC) $(CFLAGS) -o $@ -c $< $(PROG): $(OBJS) $(CXX) $(CFLAGS) -o $(PROG) $(OBJS) clean: rm -f $(OBJS) $(PROG) *~ src/*~ .PHONY: ancient-2.2.0/fuzzing/000077500000000000000000000000001463063262600146405ustar00rootroot00000000000000ancient-2.2.0/fuzzing/.gitignore000066400000000000000000000000061463063262600166240ustar00rootroot00000000000000/afl/ ancient-2.2.0/fuzzing/all_formats.dict000066400000000000000000000010701463063262600200060ustar00rootroot00000000000000"XPKF" "TPWM" "TDCS" "ACCA" "SASC" "SHSC" "S300" "S310" "S400" "S401" "S403" "S404" "SQSH" "SMPL" "SLZ3" "SHRI" "SHR3" "SDHC" "RNC\x01" "RNC\x02" "RLEN" "RDCN" "FRHT" "RAKE" "PP11" "PP20" "PWPK" "NUKE" "DUKE" "NONE" "ziRCONia" "MASH" "ELZX" "SLZX" "LZW5" "LZW4" "LZW2" "LZW3" "LZCB" "LZBS" "LIN2" "LIN4" "LIN1" "LIN3" "LHLB" "ATN!" "EDAM" "IMP!" "M.H." "BDPI" "CHFI" "RDC9" "Dupa" "FLT!" "PARA" "IMPL" "ILZR" "HUFF" "HFMN" "FRLE" "FBR2" "FAST" "DMS!" "TR" "DLTA" "GZIP" "CYB2" "CrM!" "CrM2" "Crm!" "Crm2" "CRM2" "CRMS" "CBR0" "CBR1" "BZP2" "BZh2" "BLZW" "ARTM" "ZENO" ancient-2.2.0/fuzzing/build.sh000077500000000000000000000002431463063262600162750ustar00rootroot00000000000000#!/usr/bin/env bash cd "${0%/*}" cd .. CXX=fuzzing/afl/afl-clang-fast++ LD=fuzzing/afl/afl-clang-fast++ MAIN_N=fuzz.o AFL_HARDEN=1 make -f Makefile.unix clean all ancient-2.2.0/fuzzing/fuzz-main.sh000077500000000000000000000006241463063262600171210ustar00rootroot00000000000000#!/usr/bin/env bash cd "${0%/*}" . ./fuzz-settings.sh # Create tmpfs for storing temporary fuzzing data mkdir $FUZZING_TEMPDIR sudo mount -t tmpfs -o size=100M none $FUZZING_TEMPDIR cp ../obj/ancient $FUZZING_TEMPDIR/ $AFL_BIN -f $FUZZING_TEMPDIR/infile01 -x all_formats.dict -t $FUZZING_TIMEOUT $FUZZING_INPUT -o $FUZZING_FINDINGS_DIR -D -M fuzzer01 $FUZZING_TEMPDIR/ancient $FUZZING_TEMPDIR/infile01 ancient-2.2.0/fuzzing/fuzz-secondary.sh000077500000000000000000000003671463063262600201700ustar00rootroot00000000000000#!/usr/bin/env bash cd "${0%/*}" . ./fuzz-settings.sh $AFL_BIN -f $FUZZING_TEMPDIR/infile02 -p exploit -x all_formats.dict -t $FUZZING_TIMEOUT $FUZZING_INPUT -o $FUZZING_FINDINGS_DIR -S fuzzer02 $FUZZING_TEMPDIR/ancient $FUZZING_TEMPDIR/infile02 ancient-2.2.0/fuzzing/fuzz-settings.sh000077500000000000000000000012151463063262600200320ustar00rootroot00000000000000#!/usr/bin/env bash # Input data for fuzzer # If you run the fuzzer for the first time, specify a directory with some input # files for the fuzzer, e.g. # FUZZING_INPUT="-i /home/foo/testcases/" # If you want to continue fuzzing using the previous findings, use: # FUZZING_INPUT=-i- FUZZING_INPUT="-i /path/to/initial" # Directory to place temporary fuzzing data into FUZZING_TEMPDIR=~/fuzzers/ancient-temp # Directory to store permanent fuzzing data (e.g. found crashes) into FUZZING_FINDINGS_DIR=~/fuzzers/ancient-fuzzing # Fuzzer timeout in ms, + = don't abort on timeout FUZZING_TIMEOUT=5000+ # Path to afl-fuzz binary AFL_BIN=afl/afl-fuzzancient-2.2.0/fuzzing/fuzz.cpp000066400000000000000000000034231463063262600163440ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include #include "common/MemoryBuffer.hpp" #include "common/SubBuffer.hpp" #include "common/StaticBuffer.hpp" #include "Decompressor.hpp" using namespace ancient::internal; std::shared_ptr readFile(const std::string &fileName) { std::shared_ptr ret=std::make_shared(0); std::ifstream file(fileName.c_str(),std::ios::in|std::ios::binary); bool success=false; if (file.is_open()) { file.seekg(0,std::ios::end); size_t length=size_t(file.tellg()); file.seekg(0,std::ios::beg); ret->resize(length); file.read(reinterpret_cast(ret->data()),length); success=bool(file); file.close(); } if (!success) { return {}; } return ret; } int main(int argc,char **argv) { (void)argc; #ifdef __AFL_HAVE_MANUAL_CONTROL __AFL_INIT(); #endif auto packed{readFile(argv[1])}; if (!packed) { return -1; } std::shared_ptr decompressor; try { decompressor=Decompressor::create(*packed,true,true); } catch (const Decompressor::InvalidFormatError&) { return -1; } catch (const Decompressor::VerificationError&) { return -1; } std::shared_ptr raw; try { raw=std::make_shared((decompressor->getRawSize())?decompressor->getRawSize():Decompressor::getMaxRawSize()); } catch (const Buffer::Error&) { return -1; } try { decompressor->decompress(*raw,true); } catch (const Decompressor::DecompressionError&) { return -1; } catch (const Decompressor::VerificationError&) { return -1; } try { raw->resize(decompressor->getRawSize()); } catch (const Buffer::Error&) { return -1; } if (decompressor->getImageOffset() || decompressor->getImageSize()) { return -1; } return 0; } ancient-2.2.0/fuzzing/get-afl.sh000077500000000000000000000006771463063262600165300ustar00rootroot00000000000000#!/usr/bin/env bash cd "${0%/*}" AFL_VERSION="$(wget --quiet -O - "https://api.github.com/repos/AFLplusplus/AFLplusplus/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')" AFL_FILENAME="$AFL_VERSION.tar.gz" AFL_URL="https://github.com/AFLplusplus/AFLplusplus/archive/$AFL_FILENAME" rm $AFL_FILENAME wget $AFL_URL || exit tar -xzvf $AFL_FILENAME rm $AFL_FILENAME cd AFLplusplus-* make source-only || exit cd .. rm -rf afl mv AFLplusplus-* aflancient-2.2.0/fuzzing/readme.md000066400000000000000000000045041463063262600164220ustar00rootroot00000000000000ancient fuzz suite ================== In this directory, you can find the necessary tools for fuzzing the ancient decoders with the American Fuzzy Lop fuzzer (afl++). Contents: * `all_formats.dict`: A dictionary containing magic bytes from all supported formats to make the life of the fuzzer a bit easier. * `fuzz-main.sh`: Script to launch the main fuzzing process. If you want to use just one fuzzer instance, run this one. * `fuzz-secondary.sh`: Script to launch the secondary fuzzing process. It is recommended to run at least two fuzzer instances, as the deterministic and random fuzz mode have been found to complement each other really well. * `fuzz-settings.sh`: Set up your preferences and afl settings here before the first run. * `fuzz.cpp`: A tiny C++ program that is used by the fuzzer to test ancient. * `get-afl.sh`: A simple script to obtain the latest version of the fuzzer. Prerequisites ============= * [afl++](https://github.com/AFLplusplus/AFLplusplus) or [afl](https://lcamtuf.coredump.cx/afl/) - the makefile expects this to be installed in `contrib/fuzzing/afl`, as it is automatically done by the `get-afl.sh` install script. * Clang with LLVM dev headers (llvm-config needs to be installed). afl also works with gcc, but our makefile has been set up to make use of afl's faster LLVM mode. How to use ========== * Run `get-afl.sh`, or manually extract afl to `contrib/fuzzing/afl`, use `make` to build afl-fuzz, `cd llvm_mode`, `make` to build afl-clang-fast. If building with either option fails because `llvm-config` cannot be found, try prepending `LLVM_CONFIG=/usr/bin/llvm-config-3.8` or similar, and read the afl manual. * Build ancient with the `build.sh` script in this directory. * Set up `fuzz-settings.sh` to your taste. Most importantly, you will have to specify the input directory for first use. The default setup mounts a tmpfs folder for all temporary files. You may change this behaviour if you do not have root privileges. * Run `fuzz-main.sh` for the first (deterministic) instance of afl-fuzz. * For a secondary instance to run on another core, run `fuzz-secondary.sh`. * If you want to make use of even more cores, make a copy of `fuzz-secondary.sh` and adjust "infile02" / "fuzzer02" to "infile03" / "fuzzer03" (they need to be unique)ancient-2.2.0/libancient.pc.in000066400000000000000000000006111463063262600162030ustar00rootroot00000000000000 prefix=@prefix@ exec_prefix=@exec_prefix@ includedir=@includedir@ libdir=@libdir@ Name: libancient Description: Ancient - Modern decompressor for old data compression formats Version: @VERSION@ Requires.private: @LIBANCIENT_PC_REQUIRES_PRIVATE@ Libs: -L${libdir} -lancient Libs.private: @LIBANCIENT_PC_LIBS_PRIVATE@ Cflags: -I${includedir} @LIBANCIENT_PC_CFLAGS@ @LIBANCIENT_PC_ANCIENT_API@ ancient-2.2.0/main.cpp000066400000000000000000000200601463063262600145720ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include #include #include #include #include #include #include #include #include #include // enables scanning. Useful for testing/debugging. Not so useful otherwise // Not good for default since creates dependency for dirent (i.e. windows blues) //#define ENABLE_SCAN 1 #ifdef ENABLE_SCAN #include #endif std::unique_ptr> readFile(const std::string &fileName) { std::unique_ptr> ret=std::make_unique>(); std::ifstream file(fileName.c_str(),std::ios::in|std::ios::binary); bool success=false; if (file.is_open()) { file.seekg(0,std::ios::end); size_t length=size_t(file.tellg()); file.seekg(0,std::ios::beg); ret->resize(length); file.read(reinterpret_cast(ret->data()),length); success=bool(file); file.close(); } if (!success) { fprintf(stderr,"Could not read file %s\n",fileName.c_str()); return std::make_unique>(); } return ret; } bool writeFile(const std::string &fileName,const uint8_t *data, size_t size) { bool ret=false; std::ofstream file(fileName.c_str(),std::ios::out|std::ios::binary|std::ios::trunc); if (file.is_open()) { file.write(reinterpret_cast(data),size); ret=bool(file); file.close(); } if (!ret) { fprintf(stderr,"Could not write file %s\n",fileName.c_str()); } return ret; } bool writeFile(const std::string &fileName,const std::vector &content) { return writeFile(fileName,content.data(),content.size()); } int main(int argc,char **argv) { auto usage=[]() { fprintf(stderr, "Ancient v2.2.0\n" "Copyright (C) Teemu Suutari\n" "\n" "Usage: ancient i[dentify] packed_input_files...\n" " - identifies compression used in a file(s)\n" "Usage: ancient v[erify] packed_input_file unpacked_comparison_file\n" " - verifies decompression against known good unpacked file\n" "Usage: ancient d[ecompress] packed_input_file output_file\n" " - decompresses single file\n"); #ifdef ENABLE_SCAN fprintf(stderr, "Usage: ancient s[can] input_dir output_dir\n" " - scans input directory recursively and stores all found\n" " - known compressed streams to separate files in output directory\n"); #endif }; if (argc<3) { usage(); return -1; } std::string cmd=argv[1]; if (cmd=="i" || cmd=="identify") { if (argc<3) { usage(); return -1; } for (int i=2;i decompressor; try { decompressor.emplace(*packed,true,true); printf("Compression of %s is %s\n",argv[i],decompressor->getName().c_str()); } catch (const ancient::InvalidFormatError&) { fprintf(stderr,"Unknown or invalid compression format in file %s\n",argv[i]); } catch (const ancient::VerificationError&) { fprintf(stderr,"Failed to validate file %s\n",argv[i]); } } return 0; } else if (cmd=="d" || cmd=="decompress" || cmd=="v" || cmd=="verify") { if (argc!=4) { usage(); return -1; } auto packed{readFile(argv[2])}; std::optional decompressor; try { decompressor.emplace(*packed,true,true); } catch (const ancient::InvalidFormatError&) { fprintf(stderr,"Unknown or invalid compression format in file %s\n",argv[2]); return -1; } catch (const ancient::VerificationError&) { fprintf(stderr,"Verify (packed) failed for %s\n",argv[2]); return -1; } std::vector raw; try { raw=decompressor->decompress(true); } catch (const ancient::DecompressionError&) { fprintf(stderr,"Decompression failed for %s\n",argv[2]); return -1; } catch (const ancient::VerificationError&) { fprintf(stderr,"Verify (raw) failed for %s\n",argv[2]); return -1; } catch (const std::bad_alloc&) { fprintf(stderr,"Out of memory\n"); return -1; } if (cmd=="d" || cmd=="decompress") { if (decompressor->getImageOffset() || decompressor->getImageSize()) { printf("File %s is disk image, decompressed stream offset is %zu, full image size is %zu, stream size is %zu\n",argv[2],decompressor->getImageOffset().value(),decompressor->getImageSize().value(),decompressor->getRawSize().value()); printf("!!! Please note !!!\n!!! The destination will be padded !!!\n\n"); } if (decompressor->getImageSize()) { std::vector pad(decompressor->getImageSize().value()); std::memcpy(&pad[decompressor->getImageOffset()?decompressor->getImageOffset().value():0],raw.data(),raw.size()); writeFile(argv[3],pad); } else { writeFile(argv[3],raw); } return 0; } else { size_t actualSize=decompressor->getImageSize()?decompressor->getImageSize().value():raw.size(); auto verify{readFile(argv[3])}; if (verify->size()!=actualSize) { fprintf(stderr,"Verify failed for %s and %s - sizes differ\n",argv[2],argv[3]); return -1; } size_t offset=decompressor->getImageOffset()?decompressor->getImageOffset().value():0; for (size_t i=0;idata()[i+offset]) { fprintf(stderr,"Verify failed for %s and %s - contents differ @ %zu\n",argv[2],argv[3],i); return -1; } } printf("Files match!\n"); return 0; } } #ifdef ENABLE_SCAN else if (cmd=="s" || cmd=="scan") { if (argc!=4) { usage(); return -1; } uint32_t fileIndex=0; std::function processDir=[&](std::string inputDir) { auto opendir=[](const char *f)->DIR* { return ::opendir(f); }; auto closedir=[](DIR *d)->int { return ::closedir(d); }; std::unique_ptr dir{opendir(inputDir.c_str()),closedir}; if (dir) { while (struct dirent *de=::readdir(dir.get())) { std::string subName(de->d_name); if (subName=="." || subName=="..") continue; std::string name=inputDir+"/"+subName; struct stat st; if (stat(name.c_str(),&st)<0) continue; if (st.st_mode&S_IFDIR) { processDir(name); } else if (st.st_mode&S_IFREG) { auto packed{readFile(name)}; for (size_t i=0;isize();) { // We will detect first, before trying the format for real if (!ancient::Decompressor::detect(packed->data()+i,packed->size()-i)) { i++; continue; } try { ancient::Decompressor decompressor{packed->data()+i,packed->size()-i,false,true}; printf("trying %s\n",decompressor.getName().c_str()); if (!decompressor.getPackedSize()) { // for formats that do not encode packed size. // we will get it from decompressor decompressor.decompress(true); } // final checks with the limited buffer and fresh decompressor const uint8_t *finalData=packed->data()+i; size_t finalSize=decompressor.getPackedSize().value(); ancient::Decompressor decompressor2{finalData,finalSize,true,true}; try { decompressor2.decompress(true); } catch (const std::bad_alloc&) { fprintf(stderr,"Out of memory\n"); i++; continue; } std::string outputName=std::string(argv[3])+"/file"+std::to_string(fileIndex++)+".pack"; printf("Found compressed stream at %zu, size %zu in file %s with type '%s', storing it into %s\n",i,decompressor2.getPackedSize().value(),name.c_str(),decompressor2.getName().c_str(),outputName.c_str()); writeFile(outputName,finalData,finalSize); i+=finalSize; continue; } catch (const ancient::Error&) { // full steam ahead (with next offset) } catch (const std::bad_alloc&) { // full steam ahead (with next offset) } i++; } } } } else { fprintf(stderr,"Could not process directory %s\n",inputDir.c_str()); } }; processDir(std::string(argv[2])); return 0; } #endif else { fprintf(stderr,"Unknown command\n"); usage(); return -1; } } ancient-2.2.0/src/000077500000000000000000000000001463063262600137335ustar00rootroot00000000000000ancient-2.2.0/src/ACCADecompressor.cpp000066400000000000000000000050171463063262600175170ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "ACCADecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" #include namespace ancient::internal { bool ACCADecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("ACCA"); } std::shared_ptr ACCADecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } ACCADecompressor::ACCADecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &ACCADecompressor::getSubName() const noexcept { static std::string name{"XPK-ACCA: Andre's code compression algorithm"}; return name; } void ACCADecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { const std::array staticBytes{ 0x00,0x01,0x02,0x03,0x04,0x08,0x10,0x20, 0x40,0x55,0x60,0x80,0xaa,0xc0,0xe0,0xff}; uint8_t tmp{readByte()}; uint32_t count{tmp&0xfU}; uint32_t code{uint32_t(tmp>>4U)}; uint32_t distance{0}; uint8_t repeatChar{0}; bool doRLE{false}; switch (code) { case 0: repeatChar=readByte(); [[fallthrough]]; case 14: count+=3; doRLE=true; break; case 1: count=(count|(uint32_t{readByte()}<<4))+19; repeatChar=readByte(); doRLE=true; break; case 2: repeatChar=staticBytes[count]; count=2; doRLE=true; break; case 15: distance=(count|(uint32_t(readByte())<<4))+3; count=uint32_t{readByte()}+14; break; default: /* 3 to 13 */ distance=(count|(uint32_t{readByte()}<<4))+3; count=code; break; } if (doRLE) { for (uint32_t i=0;i &state,bool verify); ~ACCADecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/API.cpp000066400000000000000000000075271463063262600150630ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "ancient.hpp" #include "Decompressor.hpp" #include "common/Buffer.hpp" #include "common/StaticBuffer.hpp" #include "common/WrappedVectorBuffer.hpp" #include namespace ancient { namespace internal { namespace APIv2 { class DecompressorImpl { public: ConstStaticBuffer _buffer; std::shared_ptr _decompressor; DecompressorImpl(const std::vector &packedData,bool exactSizeKnown,bool verify) : _buffer{packedData.data(), packedData.size()}, _decompressor{Decompressor::create(_buffer, exactSizeKnown, verify)} { // nothing needed } DecompressorImpl(const uint8_t *packedData,size_t packedSize,bool exactSizeKnown,bool verify) : _buffer{packedData, packedSize}, _decompressor{Decompressor::create(_buffer, exactSizeKnown, verify)} { // nothing needed } }; } } inline namespace APIv2 { Error::Error() noexcept { // nothing needed } Error::~Error() noexcept { // nothing needed } InvalidFormatError::InvalidFormatError() noexcept { // nothing needed } InvalidFormatError::~InvalidFormatError() noexcept { // nothing needed } DecompressionError::DecompressionError() noexcept { // nothing needed } DecompressionError::~DecompressionError() noexcept { // nothing needed } VerificationError::VerificationError() noexcept { // nothing needed } VerificationError::~VerificationError() noexcept { // nothing needed } // --- bool Decompressor::detect(const std::vector &packedData) noexcept { return internal::Decompressor::detect(internal::ConstStaticBuffer(packedData.data(), packedData.size())); } bool Decompressor::detect(const uint8_t *packedData, size_t packedSize) noexcept { return internal::Decompressor::detect(internal::ConstStaticBuffer(packedData, packedSize)); } Decompressor::Decompressor(const std::vector &packedData,bool exactSizeKnown,bool verify) : m_impl{std::make_unique(packedData, exactSizeKnown, verify)} { return; } Decompressor::Decompressor(const uint8_t *packedData,size_t packedSize,bool exactSizeKnown,bool verify) : m_impl{std::make_unique(packedData, packedSize, exactSizeKnown, verify)} { return; } const std::string &Decompressor::getName() const noexcept { return m_impl->_decompressor->getName(); } size_t Decompressor::getMaxPackedSize() noexcept { return internal::Decompressor::getMaxPackedSize(); } size_t Decompressor::getMaxRawSize() noexcept { return internal::Decompressor::getMaxRawSize(); } std::optional Decompressor::getPackedSize() const noexcept { size_t packedSize=m_impl->_decompressor->getPackedSize(); if (packedSize==0) { return std::nullopt; } return packedSize; } std::optional Decompressor::getRawSize() const noexcept { size_t rawSize=m_impl->_decompressor->getRawSize(); if (rawSize==0) { return std::nullopt; } return rawSize; } std::optional Decompressor::getImageSize() const noexcept { size_t imageSize=m_impl->_decompressor->getImageSize(); size_t imageOffset=m_impl->_decompressor->getImageOffset(); bool isImage=((imageSize>0)||(imageOffset>0)); if (!isImage) { return std::nullopt; } return imageSize; } std::optional Decompressor::getImageOffset() const noexcept { size_t imageSize=m_impl->_decompressor->getImageSize(); size_t imageOffset=m_impl->_decompressor->getImageOffset(); bool isImage=((imageSize>0)||(imageOffset>0)); if (!isImage) { return std::nullopt; } return imageOffset; } std::vector Decompressor::decompress(bool verify) { std::vector result((m_impl->_decompressor->getRawSize())?m_impl->_decompressor->getRawSize():0); { internal::WrappedVectorBuffer buffer(result); m_impl->_decompressor->decompress(buffer, verify); } result.shrink_to_fit(); return result; } Decompressor::~Decompressor() noexcept { // nothing needed } } } ancient-2.2.0/src/ARTMDecompressor.cpp000066400000000000000000000057541463063262600176030ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "ARTMDecompressor.hpp" #include "RangeDecoder.hpp" #include "FrequencyTree.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" #include namespace ancient::internal { bool ARTMDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("ARTM"); } std::shared_ptr ARTMDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } ARTMDecompressor::ARTMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); if (packedData.size()<2) throw Decompressor::InvalidFormatError(); } const std::string &ARTMDecompressor::getSubName() const noexcept { static std::string name{"XPK-ARTM: Arithmetic encoding compressor"}; return name; } // There really is not much to see here. // Except maybe for the fact that there is extra symbol defined but never used. // It is used in the original implementation as a terminator. In here it is considered as an error. // Logically it would decode into null-character (practically it would be instant buffer overflow) void ARTMDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { class BitReader : public RangeDecoder::BitReader { public: BitReader(ForwardInputStream &stream) noexcept: _reader{stream} { // nothing needed } ~BitReader() noexcept=default; uint32_t readBit() final { return _reader.readBits8(1); } uint32_t readBits(uint32_t count) { return _reader.readBits8(count); } private: LSBBitReader _reader; }; ForwardInputStream inputStream{_packedData,0,_packedData.size(),3U}; ForwardOutputStream outputStream{rawData,0,rawData.size()}; BitReader bitReader{inputStream}; RangeDecoder decoder{bitReader,uint16_t(rotateBits(bitReader.readBits(16U),16U))}; // first one will never be used, but doing it this way saves us on some nasty arith // on every place later FrequencyTree tree; std::array characters; for (uint32_t i=0;i<257;i++) { tree.add(i,1); characters[i]=256U-i; } while (!outputStream.eof()) { uint16_t value=decoder.decode(tree.getTotal()); uint16_t low,freq; uint16_t symbol=tree.decode(value,low,freq); if (!symbol) throw Decompressor::DecompressionError(); decoder.scale(low,low+freq,tree.getTotal()); outputStream.writeByte(characters[symbol]); if (tree.getTotal()==0x3fffU) { for (uint32_t i=1;i<=256;i++) tree.set(i,(tree[i]+1)>>1); } uint16_t i; for (i=symbol;i<256&&tree[i+1]==tree[i];i++); if (i!=symbol) std::swap(characters[symbol],characters[i]); tree.add(i,1); } } } ancient-2.2.0/src/ARTMDecompressor.hpp000066400000000000000000000014551463063262600176020ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef ARTMDECOMPRESSOR_HPP #define ARTMDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class ARTMDecompressor : public XPKDecompressor { public: ARTMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~ARTMDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/BLZWDecompressor.cpp000066400000000000000000000044701463063262600176100ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "BLZWDecompressor.hpp" #include "LZWDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool BLZWDecompressor::detectHeaderXPK(uint32_t hdr) { return hdr==FourCC("BLZW"); } std::shared_ptr BLZWDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } BLZWDecompressor::BLZWDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); _maxBits=_packedData.readBE16(0); if (_maxBits<9 || _maxBits>20) throw Decompressor::InvalidFormatError();; _stackLength=uint32_t(_packedData.readBE16(2))+5; } const std::string &BLZWDecompressor::getSubName() const noexcept { static std::string name{"XPK-BLZW: LZW-compressor"}; return name; } void BLZWDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,4,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; auto writeByte=[&](uint8_t value) { outputStream.writeByte(value); }; uint32_t codeBits{9U}; auto readCode=[&]()->uint32_t { return readBits(codeBits); }; uint32_t firstCode=readCode(); LZWDecoder decoder{1U<<_maxBits,259U,_stackLength,firstCode}; decoder.write(firstCode,false,writeByte); while (!outputStream.eof()) { switch (uint32_t code{readBits(codeBits)};code) { case 256: throw Decompressor::DecompressionError(); break; case 257: codeBits=9U; firstCode=readCode(); decoder.reset(firstCode); decoder.write(firstCode,false,writeByte); break; case 258: if (codeBits>=24) throw Decompressor::DecompressionError(); codeBits++; break; default: decoder.write(code,!decoder.isLiteral(code),writeByte); decoder.add(code); break; } } } } ancient-2.2.0/src/BLZWDecompressor.hpp000066400000000000000000000015301463063262600176070ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef BLZWDECOMPRESSOR_HPP #define BLZWDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class BLZWDecompressor : public XPKDecompressor { public: BLZWDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~BLZWDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr); static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _maxBits{0}; uint32_t _stackLength{0}; }; } #endif ancient-2.2.0/src/BZIP2Decompressor.cpp000066400000000000000000000270711463063262600176620ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "BZIP2Decompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/MemoryBuffer.hpp" #include "common/CRC32.hpp" #include "common/Common.hpp" #include namespace ancient::internal { bool BZIP2Decompressor::detectHeader(uint32_t hdr) noexcept { return ((hdr&0xffff'ff00U)==FourCC("BZh\0") && (hdr&0xffU)>='1' && (hdr&0xffU)<='9'); } bool BZIP2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return (hdr==FourCC("BZP2")); } std::shared_ptr BZIP2Decompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,exactSizeKnown,verify); } std::shared_ptr BZIP2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } BZIP2Decompressor::BZIP2Decompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData{packedData}, _packedSize{0} { uint32_t hdr=packedData.readBE32(0); if (!detectHeader(hdr)) throw Decompressor::InvalidFormatError();; _blockSize=((hdr&0xffU)-'0')*100'000; } BZIP2Decompressor::BZIP2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData}, _packedSize{_packedData.size()} { uint32_t blockHdr=packedData.readBE32(0); if (!detectHeader(blockHdr)) throw Decompressor::InvalidFormatError();; _blockSize=((blockHdr&0xffU)-'0')*100'000; } const std::string &BZIP2Decompressor::getName() const noexcept { static std::string name{"bz2: bzip2"}; return name; } const std::string &BZIP2Decompressor::getSubName() const noexcept { static std::string name{"XPK-BZP2: bzip2"}; return name; } size_t BZIP2Decompressor::getPackedSize() const noexcept { // no way to know before decompressing return _packedSize; } size_t BZIP2Decompressor::getRawSize() const noexcept { // same thing, decompression needed first return _rawSize; } void BZIP2Decompressor::decompressImpl(Buffer &rawData,bool verify) { size_t packedSize=_packedSize?_packedSize:_packedData.size(); ForwardInputStream inputStream{_packedData,4,packedSize}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; AutoExpandingForwardOutputStream outputStream{rawData}; // stream verification // // there is so much wrong in bzip2 CRC-calculation :( // 1. The bit ordering is opposite what everyone else does with CRC32 // 2. The block CRCs are calculated separately, no way of calculating a complete // CRC without knowing the block layout // 3. The CRC is the end of the stream and the stream is bit aligned. You // can't read CRC without decompressing the stream. uint32_t crc=0; auto calculateBlockCRC=[&](size_t blockPos,size_t blockSize) { crc=(crc<<1)|(crc>>31); crc^=CRC32Rev(rawData,blockPos,blockSize,0); }; HuffmanDecoder selectorDecoder { // incomplete Huffman table. errors possible HuffmanCode{1,0b000000,uint8_t{0}}, HuffmanCode{2,0b000010,uint8_t{1}}, HuffmanCode{3,0b000110,uint8_t{2}}, HuffmanCode{4,0b001110,uint8_t{3}}, HuffmanCode{5,0b011110,uint8_t{4}}, HuffmanCode{6,0b111110,uint8_t{5}} }; HuffmanDecoder deltaDecoder { HuffmanCode{1,0b00,0}, HuffmanCode{2,0b10,1}, HuffmanCode{2,0b11,-1} }; MemoryBuffer tmpBuffer{_blockSize}; // This is the dark, ancient secret of bzip2. // versions before 0.9.5 had a data randomization for "too regular" // data problematic for the bwt-implementation at that time. // although it is never utilized anymore, the support is still there // And this is exactly the kind of ancient stuff we want to support :) // // On this specific part (since it is a table of magic numbers) // we have no way other than copying it from the original reference // Table has a separate copyright, lets have it as a separate file as well #include "BZIP2Table.hpp" for (;;) { uint32_t blockHdrHigh{readBits(32)}; uint32_t blockHdrLow{readBits(16)}; if (blockHdrHigh==0x31415926U && blockHdrLow==0x5359U) { // a block // this is rather spaghetti... readBits(32); // block crc, not interested bool randomized{!!readBit()}; // basically the random inserted is one LSB after n-th bytes // per defined in the table. uint32_t randomPos{1}; uint32_t randomCounter{randomTable[0]-1U}; auto randomBit=[&]()->bool { // Beauty is in the eye of the beholder: this is smallest form to hide the ugliness return (!randomCounter--)?randomCounter=randomTable[randomPos++&511]:false; }; uint32_t currentPtr{readBits(24)}; uint32_t currentBlockSize{0}; { uint32_t numHuffmanItems{2}; std::array huffmanValues; { // this is just a little bit inefficient but still we reading bit by bit since // reference does it. (bitsream format details do not spill over) std::array usedMap; for (uint32_t i=0;i<16;i++) usedMap[i]=readBit(); std::array huffmanMap; for (uint32_t i=0;i<16;i++) { for (uint32_t j=0;j<16;j++) huffmanMap[i*16+j]=(usedMap[i])?readBit():false; } for (uint32_t i=0;i<256;i++) if (huffmanMap[i]) numHuffmanItems++; if (numHuffmanItems==2) throw DecompressionError(); for (uint32_t currentValue=0,i=0;i<256;i++) if (huffmanMap[i]) huffmanValues[currentValue++]=i; } uint32_t huffmanGroups{readBits(3)}; if (huffmanGroups<2 || huffmanGroups>6) throw DecompressionError(); uint32_t selectorsUsed{readBits(15)}; if (!selectorsUsed) throw DecompressionError(); std::vector huffmanSelectorList(selectorsUsed); auto unMTF=[](uint8_t value,auto &map)->uint8_t { uint8_t ret{map[value]}; if (value) { uint8_t tmp=map[value]; for (uint32_t i=value;i;i--) map[i]=map[i-1]; map[0]=tmp; } return ret; }; // create Huffman selectors std::array selectorMTFMap{0,1,2,3,4,5}; for (uint32_t i=0;i=huffmanGroups) throw DecompressionError(); huffmanSelectorList[i]=item; } typedef HuffmanDecoder BZIP2Decoder; std::vector dataDecoders{huffmanGroups}; // Create all tables for (uint32_t i=0;i bitLengths; uint32_t currentBits{readBits(5)}; for (uint32_t j=0;j20) throw DecompressionError(); bitLengths[j]=currentBits; } dataDecoders[i].createOrderlyHuffmanTable(bitLengths,numHuffmanItems); } // Huffman decode + unRLE + unMTF BZIP2Decoder *currentHuffmanDecoder{nullptr}; uint32_t currentHuffmanIndex{0}; std::array dataMTFMap; for (uint32_t i=0;i_blockSize) throw DecompressionError(); for (uint32_t i=0;i=selectorsUsed) throw DecompressionError(); currentHuffmanDecoder=&dataDecoders[huffmanSelectorList[currentHuffmanIndex++]]; } uint32_t symbolMTF{currentHuffmanDecoder->decode(readBit)}; // stop marker is referenced only once, and it is the last one // This means we do no have to un-MTF it for detection if (symbolMTF==numHuffmanItems-1) break; if (currentBlockSize>=_blockSize) throw DecompressionError(); if (symbolMTF<2) { currentRunLength+=currentRLEWeight<=_blockSize) throw DecompressionError(); tmpBuffer[currentBlockSize++]=huffmanValues[symbol]; } } decodeRLE(); if (currentPtr>=currentBlockSize) throw DecompressionError(); } // inverse BWT + final RLE decoding. // there are a few dark corners here as well // 1. Can the stream end at 4 literals without count? I assume it is a valid optimization (and that this does not spillover to next block) // 2. Can the RLE-step include counts 252 to 255 even if reference does not do them? I assume yes here as here as well // 3. Can the stream be empty? We do not take issue here about that (that should be culled out earlier already) std::array sums; for (uint32_t i=0;i<256;i++) sums[i]=0; for (uint32_t i=0;i rank; for (uint32_t tot=0,i=0;i<256;i++) { rank[i]=tot; tot+=sums[i]; } // not at all happy about the memory consumption, but it simplifies the implementation a lot // and by sacrificing 4*size (size as in actual block size) we do not have to have slow search nor another temporary buffer // since by calculating forward table we can do forward decoding of the data on the same pass as iBWT // // also, because I'm lazy std::vector forwardIndex(currentBlockSize); for (uint32_t i=0;i &state,bool verify); ~BZIP2Decompressor() noexcept=default; size_t getRawSize() const noexcept final; size_t getPackedSize() const noexcept final; const std::string &getName() const noexcept final; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; size_t _blockSize{0}; size_t _packedSize{0}; size_t _rawSize{0}; }; } #endif ancient-2.2.0/src/BZIP2Table.hpp000066400000000000000000000121121463063262600162370ustar00rootroot00000000000000/* -------------------------------------------------------------------------- This program, "bzip2", the associated library "libbzip2", and all documentation, are copyright (C) 1996-2010 Julian R Seward. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 4. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Julian Seward, jseward@bzip.org bzip2/libbzip2 version 1.0.6 of 6 September 2010 -------------------------------------------------------------------------- */ // content formatted, numbers -1 (i.e. this is altered version by clause 3) static const uint16_t randomTable[512]={ 619-1,720-1,127-1,481-1,931-1,816-1,813-1,233-1,566-1,247-1,985-1,724-1,205-1,454-1,863-1,491-1, 741-1,242-1,949-1,214-1,733-1,859-1,335-1,708-1,621-1,574-1, 73-1,654-1,730-1,472-1,419-1,436-1, 278-1,496-1,867-1,210-1,399-1,680-1,480-1, 51-1,878-1,465-1,811-1,169-1,869-1,675-1,611-1,697-1, 867-1,561-1,862-1,687-1,507-1,283-1,482-1,129-1,807-1,591-1,733-1,623-1,150-1,238-1, 59-1,379-1, 684-1,877-1,625-1,169-1,643-1,105-1,170-1,607-1,520-1,932-1,727-1,476-1,693-1,425-1,174-1,647-1, 73-1,122-1,335-1,530-1,442-1,853-1,695-1,249-1,445-1,515-1,909-1,545-1,703-1,919-1,874-1,474-1, 882-1,500-1,594-1,612-1,641-1,801-1,220-1,162-1,819-1,984-1,589-1,513-1,495-1,799-1,161-1,604-1, 958-1,533-1,221-1,400-1,386-1,867-1,600-1,782-1,382-1,596-1,414-1,171-1,516-1,375-1,682-1,485-1, 911-1,276-1, 98-1,553-1,163-1,354-1,666-1,933-1,424-1,341-1,533-1,870-1,227-1,730-1,475-1,186-1, 263-1,647-1,537-1,686-1,600-1,224-1,469-1, 68-1,770-1,919-1,190-1,373-1,294-1,822-1,808-1,206-1, 184-1,943-1,795-1,384-1,383-1,461-1,404-1,758-1,839-1,887-1,715-1, 67-1,618-1,276-1,204-1,918-1, 873-1,777-1,604-1,560-1,951-1,160-1,578-1,722-1, 79-1,804-1, 96-1,409-1,713-1,940-1,652-1,934-1, 970-1,447-1,318-1,353-1,859-1,672-1,112-1,785-1,645-1,863-1,803-1,350-1,139-1, 93-1,354-1, 99-1, 820-1,908-1,609-1,772-1,154-1,274-1,580-1,184-1, 79-1,626-1,630-1,742-1,653-1,282-1,762-1,623-1, 680-1, 81-1,927-1,626-1,789-1,125-1,411-1,521-1,938-1,300-1,821-1, 78-1,343-1,175-1,128-1,250-1, 170-1,774-1,972-1,275-1,999-1,639-1,495-1, 78-1,352-1,126-1,857-1,956-1,358-1,619-1,580-1,124-1, 737-1,594-1,701-1,612-1,669-1,112-1,134-1,694-1,363-1,992-1,809-1,743-1,168-1,974-1,944-1,375-1, 748-1, 52-1,600-1,747-1,642-1,182-1,862-1, 81-1,344-1,805-1,988-1,739-1,511-1,655-1,814-1,334-1, 249-1,515-1,897-1,955-1,664-1,981-1,649-1,113-1,974-1,459-1,893-1,228-1,433-1,837-1,553-1,268-1, 926-1,240-1,102-1,654-1,459-1, 51-1,686-1,754-1,806-1,760-1,493-1,403-1,415-1,394-1,687-1,700-1, 946-1,670-1,656-1,610-1,738-1,392-1,760-1,799-1,887-1,653-1,978-1,321-1,576-1,617-1,626-1,502-1, 894-1,679-1,243-1,440-1,680-1,879-1,194-1,572-1,640-1,724-1,926-1, 56-1,204-1,700-1,707-1,151-1, 457-1,449-1,797-1,195-1,791-1,558-1,945-1,679-1,297-1, 59-1, 87-1,824-1,713-1,663-1,412-1,693-1, 342-1,606-1,134-1,108-1,571-1,364-1,631-1,212-1,174-1,643-1,304-1,329-1,343-1, 97-1,430-1,751-1, 497-1,314-1,983-1,374-1,822-1,928-1,140-1,206-1, 73-1,263-1,980-1,736-1,876-1,478-1,430-1,305-1, 170-1,514-1,364-1,692-1,829-1, 82-1,855-1,953-1,676-1,246-1,369-1,970-1,294-1,750-1,807-1,827-1, 150-1,790-1,288-1,923-1,804-1,378-1,215-1,828-1,592-1,281-1,565-1,555-1,710-1, 82-1,896-1,831-1, 547-1,261-1,524-1,462-1,293-1,465-1,502-1, 56-1,661-1,821-1,976-1,991-1,658-1,869-1,905-1,758-1, 745-1,193-1,768-1,550-1,608-1,933-1,378-1,286-1,215-1,979-1,792-1,961-1, 61-1,688-1,793-1,644-1, 986-1,403-1,106-1,366-1,905-1,644-1,372-1,567-1,466-1,434-1,645-1,210-1,389-1,550-1,919-1,135-1, 780-1,773-1,635-1,389-1,707-1,100-1,626-1,958-1,165-1,504-1,920-1,176-1,193-1,713-1,857-1,265-1, 203-1, 50-1,668-1,108-1,645-1,990-1,626-1,197-1,510-1,357-1,358-1,850-1,858-1,364-1,936-1,638-1}; ancient-2.2.0/src/CBR0Decompressor.cpp000066400000000000000000000034361463063262600175210ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "CBR0Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool CBR0Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { // CBR1 is practical joke: it is the same as CBR0 but with ID changed return hdr==FourCC("CBR0") || hdr==FourCC("CBR1"); } std::shared_ptr CBR0Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } CBR0Decompressor::CBR0Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData}, _isCBR0{hdr==FourCC("CBR0")} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &CBR0Decompressor::getSubName() const noexcept { static std::string nameCBR0{"XPK-CBR0: RLE-compressor"}; static std::string nameCBR1{"XPK-CBR1: RLE-compressor"}; return (_isCBR0)?nameCBR0:nameCBR1; } void CBR0Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; ForwardOutputStream outputStream{rawData,0,rawData.size()}; // barely different than RLEN, however the count is well defined here. while (!outputStream.eof()) { if (uint32_t count{inputStream.readByte()};count<128) { count++; for (uint32_t i=0;i &state,bool verify); ~CBR0Decompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; bool _isCBR0; }; } #endif ancient-2.2.0/src/CRMDecompressor.cpp000066400000000000000000000160121463063262600174460ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "CRMDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "DLTADecode.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "VariableLengthCodeDecoder.hpp" #include "common/OverflowCheck.hpp" #include "common/Common.hpp" namespace ancient::internal { bool CRMDecompressor::detectHeader(uint32_t hdr) noexcept { switch (hdr) { case FourCC("CrM!"): [[fallthrough]]; case FourCC("CrM2"): [[fallthrough]]; case FourCC("Crm!"): [[fallthrough]]; case FourCC("Crm2"): [[fallthrough]]; case 0x1805'1973U: // Fears [[fallthrough]]; case FourCC("CD\xb3\xb9"): // BiFi 2 [[fallthrough]]; case FourCC("DCS!"): // Sonic Attack/DualCrew-Shining [[fallthrough]]; case FourCC("Iron"): // Sun / TRSI [[fallthrough]]; case FourCC("MSS!"): // Infection / Mystic [[fallthrough]]; case FourCC("mss!"): return true; default: return false; } } bool CRMDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("CRM2") || hdr==FourCC("CRMS"); } std::shared_ptr CRMDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,0,verify); } std::shared_ptr CRMDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } CRMDecompressor::CRMDecompressor(const Buffer &packedData,uint32_t recursionLevel,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { uint32_t hdr{packedData.readBE32(0)}; if (!detectHeader(hdr) || packedData.size()<20) throw Decompressor::InvalidFormatError(); if (hdr==0x1805'1973U || hdr==FourCC("CD\xb3\xb9") || hdr==FourCC("Iron") || hdr==FourCC("MSS!")) hdr=FourCC("CrM2"); if (hdr==FourCC("mss!")) hdr=FourCC("Crm2"); if (hdr==FourCC("DCS!")) hdr=FourCC("CrM!"); _rawSize=packedData.readBE32(6); _packedSize=packedData.readBE32(10); if (!_rawSize || !_packedSize || _rawSize>getMaxRawSize() || _packedSize>getMaxPackedSize() || OverflowCheck::sum(_packedSize,14U)>packedData.size()) throw Decompressor::InvalidFormatError(); if (((hdr>>8)&0xff)=='m') _isSampled=true; if ((hdr&0xff)=='2') _isLZH=true; } CRMDecompressor::CRMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : CRMDecompressor{packedData,recursionLevel,verify} { _isXPKDelta=(hdr==FourCC("CRMS")); } const std::string &CRMDecompressor::getName() const noexcept { static std::string names[4]={ "CrM!: Crunch-Mania standard-mode", "Crm!: Crunch-Mania standard-mode, sampled", "CrM2: Crunch-Mania LZH-mode", "Crm2: Crunch-Mania LZH-mode, sampled"}; return names[(_isLZH?2:0)+(_isSampled?1:0)]; } const std::string &CRMDecompressor::getSubName() const noexcept { // the XPK-id is not used in decompressing process, // but there is a real id inside the stream // This means we can have frankenstein configurations, // although in practice we don't static std::string names[2]={ "XPK-CRM2: Crunch-Mania LZH-mode", "XPK-CRMS: Crunch-Mania LZH-mode, sampled"}; return names[(_isXPKDelta?1:0)]; } size_t CRMDecompressor::getPackedSize() const noexcept { return _packedSize+14; } size_t CRMDecompressor::getRawSize() const noexcept { return _rawSize; } void CRMDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw Decompressor::DecompressionError(); BackwardInputStream inputStream{_packedData,14,_packedSize+14-6}; LSBBitReader bitReader{inputStream}; { // There are empty bits?!? at the start of the stream. take them out size_t bufOffset{_packedSize+14-6}; uint32_t originalBitsContent{_packedData.readBE32(bufOffset)}; uint16_t originalShift{_packedData.readBE16(bufOffset+4)}; uint8_t bufBitsLength{uint8_t(originalShift+16)}; uint32_t bufBitsContent{originalBitsContent>>(16-originalShift)}; bitReader.reset(bufBitsContent,bufBitsLength); } auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; BackwardOutputStream outputStream{rawData,0,_rawSize}; if (_isLZH) { typedef HuffmanDecoder CRMHuffmanDecoder; auto readHuffmanTable=[&](CRMHuffmanDecoder &dec,uint32_t codeLength) { uint32_t maxDepth{readBits(4)}; if (!maxDepth) throw Decompressor::DecompressionError(); std::array lengthTable; for (uint32_t i=0;i>(maxDepth-depth),value}); code+=1<<(maxDepth-depth); } } }; do { CRMHuffmanDecoder lengthDecoder,distanceDecoder; readHuffmanTable(lengthDecoder,9); readHuffmanTable(distanceDecoder,4); uint32_t items{readBits(16)+1}; for (uint32_t i=0;i lengthDecoder { HuffmanCode{1,0b000,uint8_t{0}}, HuffmanCode{2,0b010,uint8_t{1}}, HuffmanCode{3,0b110,uint8_t{2}}, HuffmanCode{3,0b111,uint8_t{3}} }; HuffmanDecoder distanceDecoder { HuffmanCode{1,0b00,uint8_t{1}}, HuffmanCode{2,0b10,uint8_t{0}}, HuffmanCode{2,0b11,uint8_t{2}} }; VariableLengthCodeDecoder lengthVLC{1,2,4,8}; VariableLengthCodeDecoder distanceVLC{5,9,14}; while (!outputStream.eof()) { if (readBit()) { outputStream.writeByte(readBits(8)); } else { uint8_t lengthIndex=lengthDecoder.decode(readBit); if (uint32_t count{lengthVLC.decode(readBits,lengthIndex)+2U};count==23) { if (readBit()) { count=readBits(5)+15; } else { count=readBits(14)+15; } for (uint32_t i=0;i23) count--; uint8_t distanceIndex{distanceDecoder.decode(readBit)}; uint32_t distance{distanceVLC.decode(readBits,distanceIndex)}; outputStream.copy(distance,count); } } } } if (!outputStream.eof()) throw Decompressor::DecompressionError(); if (_isSampled) DLTADecode::decode(rawData,rawData,0,_rawSize); } void CRMDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); return decompressImpl(rawData,verify); } } ancient-2.2.0/src/CRMDecompressor.hpp000066400000000000000000000030001463063262600174440ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef CRMDECOMPRESSOR_HPP #define CRMDECOMPRESSOR_HPP #include "Decompressor.hpp" #include "XPKDecompressor.hpp" namespace ancient::internal { class CRMDecompressor : public Decompressor, public XPKDecompressor { public: CRMDecompressor(const Buffer &packedData,uint32_t recursionLevel,bool verify); CRMDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~CRMDecompressor() noexcept=default; const std::string &getName() const noexcept final; const std::string &getSubName() const noexcept final; size_t getPackedSize() const noexcept final; size_t getRawSize() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _packedSize{0}; uint32_t _rawSize{0}; bool _isLZH{false}; // "normal" compression or LZH compression bool _isSampled{false}; // normal or "sampled" i.e. obsfuscated bool _isXPKDelta{false}; // If delta encoding defined in XPK }; } #endif ancient-2.2.0/src/CYB2Decoder.cpp000066400000000000000000000034121463063262600164240ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "common/SubBuffer.hpp" #include "CYB2Decoder.hpp" #include "XPKMain.hpp" #include "common/Common.hpp" namespace ancient::internal { bool CYB2Decoder::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("CYB2"); } std::shared_ptr CYB2Decoder::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } CYB2Decoder::CYB2Decoder(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor(recursionLevel), _packedData(packedData) { if (!detectHeaderXPK(hdr) || _packedData.size()<=10) throw Decompressor::InvalidFormatError(); _blockHeader=_packedData.readBE32(0); // after the block header, the next 6 bytes seem to be // 00 64 00 00 00 00 // Those bytes do not seem to be terribly important though... if (verify) { // trigger child checks... ConstSubBuffer blockData{_packedData,10,_packedData.size()-10}; std::shared_ptr subState; auto sub=XPKMain::createDecompressor(_blockHeader,_recursionLevel+1,blockData,subState,true); } } const std::string &CYB2Decoder::getSubName() const noexcept { static std::string name{"XPK-CYB2: xpkCybPrefs container"}; return name; } void CYB2Decoder::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ConstSubBuffer blockData{_packedData,10,_packedData.size()-10}; std::shared_ptr state; auto sub{XPKMain::createDecompressor(_blockHeader,_recursionLevel+1,blockData,state,verify)}; sub->decompressImpl(rawData,previousData,verify); } } ancient-2.2.0/src/CYB2Decoder.hpp000066400000000000000000000014551463063262600164360ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef CYB2DECODER_HPP #define CYB2DECODER_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class CYB2Decoder : public XPKDecompressor { public: CYB2Decoder(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~CYB2Decoder() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _blockHeader; }; } #endif ancient-2.2.0/src/CompactDecompressor.cpp000066400000000000000000000054041463063262600204160ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "CompactDecompressor.hpp" #include "DynamicHuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" #include namespace ancient::internal { bool CompactDecompressor::detectHeader(uint32_t hdr) noexcept { // If a compact file has signature of 0x1fff it means the its endianess is wrongly configured // (This is easy since it only knows of vax and pdp-11) // Files with a wrong endianess are almost salvageable - they just wrongly encode first and last byte // Thus supporting them would be in vain return (hdr>>16)==0xff1fU; } std::shared_ptr CompactDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,exactSizeKnown,verify); } CompactDecompressor::CompactDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData{packedData}, _exactSizeKnown{exactSizeKnown} { if (_packedData.size()<2U) throw InvalidFormatError(); uint32_t hdr{_packedData.readBE16(0)}; if (!detectHeader(hdr<<16)) throw InvalidFormatError(); if (exactSizeKnown) _packedSize=packedData.size(); } const std::string &CompactDecompressor::getName() const noexcept { static std::string name{"C: Compact"}; return name; } size_t CompactDecompressor::getPackedSize() const noexcept { // no way to know before decompressing return _packedSize; } size_t CompactDecompressor::getRawSize() const noexcept { // same thing, decompression needed first return _rawSize; } void CompactDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream{_packedData,2U,_packedSize?_packedSize:_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { // left is right and right is left return bitReader.readBits8(1U)^1U; }; AutoExpandingForwardOutputStream outputStream{rawData}; DynamicHuffmanDecoder<258U> decoder{3U}; uint32_t codeCount{0}; std::array mapper; // Magic! mapper[codeCount++]=256U; mapper[codeCount++]=257U; mapper[codeCount++]=readBits(8U); outputStream.writeByte(uint8_t(mapper[2])); for(;;) { uint32_t code{decoder.decode(readBit)}; decoder.update(code); code=mapper[code]; if (code==257U) { if (codeCount==258U) throw DecompressionError(); decoder.addCode(); decoder.update(codeCount); code=mapper[codeCount++]=readBits(8U); } else if (code==256U) break; outputStream.writeByte(code); } _rawSize=outputStream.getOffset(); if (_exactSizeKnown && inputStream.getOffset()!=_packedSize) throw DecompressionError(); _packedSize=inputStream.getOffset(); } } ancient-2.2.0/src/CompactDecompressor.hpp000066400000000000000000000014661463063262600204270ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef COMPACTDECOMPRESSOR_HPP #define COMPACTDECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class CompactDecompressor : public Decompressor { public: CompactDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); ~CompactDecompressor() noexcept=default; size_t getRawSize() const noexcept final; size_t getPackedSize() const noexcept final; const std::string &getName() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: const Buffer &_packedData; size_t _packedSize{0}; size_t _rawSize{0}; bool _exactSizeKnown; }; } #endif ancient-2.2.0/src/CompressDecompressor.cpp000066400000000000000000000073221463063262600206240ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "CompressDecompressor.hpp" #include "LZWDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool CompressDecompressor::detectHeader(uint32_t hdr) noexcept { return ((hdr>>16)==0x1f9dU); } std::shared_ptr CompressDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,exactSizeKnown,verify); } CompressDecompressor::CompressDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData{packedData} { // Can't do anything with undefined size stream if (!exactSizeKnown) throw InvalidFormatError(); if (_packedData.size()<3U) throw InvalidFormatError(); uint32_t hdr=_packedData.readBE16(0); if (!detectHeader(hdr<<16)) throw InvalidFormatError(); uint8_t tmp{_packedData.read8(2)}; _hasBlocks=tmp&0x80U; tmp&=0x7fU; if (tmp<9U||tmp>16U) throw InvalidFormatError(); _maxBits=tmp; } const std::string &CompressDecompressor::getName() const noexcept { static std::string names[2]={ "Z: Compress (old)", "Z: Compress (new)"}; return names[_hasBlocks?1:0]; } size_t CompressDecompressor::getPackedSize() const noexcept { // only from input return _packedData.size(); } size_t CompressDecompressor::getRawSize() const noexcept { // same thing, decompression needed first return _rawSize; } void CompressDecompressor::decompressImpl(Buffer &rawData,bool verify) { // Special case for empty file if (_packedData.size()==3U) { _rawSize=0U; return; } ForwardInputStream inputStream{_packedData,3,_packedData.size()}; LSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; AutoExpandingForwardOutputStream outputStream{rawData}; auto writeByte=[&](uint8_t value) { outputStream.writeByte(value); }; uint32_t codeBits{9U}; size_t prevCodePos=inputStream.getOffset(); uint32_t firstCode{readBits(codeBits)}; LZWDecoder decoder{1U<<_maxBits,_hasBlocks?257U:256U,8192U,firstCode}; decoder.write(firstCode,false,writeByte); // This is actually surprising for a compressor // that was popular at time: There are silly bugs // and wastage of bytes since codes are read in blocks // and thrown away in reset and code size change // the worst thing is setting max bits to 9 which turns instead // unbounded bit size compressor (here we throw in case we can more than 16 bits) // gzip seems to limit the minimum max bits to 12ish, which is more sane // (debugging that was entertaining though) uint32_t codeCounter{1}; // previous read for first byte included auto reset=[&]() { bitReader.reset(0,0); prevCodePos+=codeBits; inputStream.setOffset(prevCodePos); codeCounter=0; }; bool maxReached{false}; auto readCode=[&]()->uint32_t { if (!maxReached && decoder.getCurrentIndex()>=(1U<16U) throw DecompressionError(); } if (!codeCounter) prevCodePos=inputStream.getOffset(); if (++codeCounter==8U) codeCounter=0; return readBits(codeBits); }; // since the codes are larger than 8 bits, we can do this while (!inputStream.eof()) { if (uint32_t code{readCode()};_hasBlocks && code==256U) { reset(); if (inputStream.eof()) break; codeBits=9U; maxReached=false; firstCode=readBits(codeBits); codeCounter++; decoder.reset(firstCode); decoder.write(firstCode,false,writeByte); } else { decoder.write(code,!decoder.isLiteral(code),writeByte); decoder.add(code); } } _rawSize=outputStream.getOffset(); } } ancient-2.2.0/src/CompressDecompressor.hpp000066400000000000000000000014611463063262600206270ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef COMPRESSDECOMPRESSOR_HPP #define COMPRESSDECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class CompressDecompressor : public Decompressor { public: CompressDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); ~CompressDecompressor() noexcept=default; size_t getRawSize() const noexcept final; size_t getPackedSize() const noexcept final; const std::string &getName() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: const Buffer &_packedData; size_t _rawSize{0}; bool _hasBlocks; uint32_t _maxBits; }; } #endif ancient-2.2.0/src/DEFLATEDecompressor.cpp000066400000000000000000000242311463063262600200730ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "DEFLATEDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "VariableLengthCodeDecoder.hpp" #include "common/CRC32.hpp" #include "common/OverflowCheck.hpp" #include "common/Common.hpp" #include namespace ancient::internal { static uint32_t Adler32(const Buffer &buffer,size_t offset,size_t len) { uint32_t s1=1,s2=0; for (size_t i=0;i=65521) s1-=65521; s2+=s1; if (s2>=65521) s2-=65521; } return (s2<<16)|s1; } bool DEFLATEDecompressor::detectHeader(uint32_t hdr) noexcept { return ((hdr>>16U)==0x1f8bU)||((hdr>>16U)==0x1fa1U); } bool DEFLATEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return (hdr==FourCC("GZIP")); } std::shared_ptr DEFLATEDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,exactSizeKnown,verify); } std::shared_ptr DEFLATEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } bool DEFLATEDecompressor::detectZLib() { if (_packedData.size()<6) return false; // no knowledge about rawSize, before decompression // packedSize told by decompressor _packedSize=uint32_t(_packedData.size()); _packedOffset=2; uint8_t cm{_packedData.read8(0)}; if ((cm&0xf)!=8) return false; if ((cm&0xf0)>0x70) return false; uint8_t flags{_packedData.read8(1)}; if (flags&0x20) { if (_packedSize<8) return false; _packedOffset+=4; } if (((uint16_t(cm)<<8)|uint16_t(flags))%31) return false; _type=Type::ZLib; return true; } DEFLATEDecompressor::DEFLATEDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData{packedData}, _exactSizeKnown{exactSizeKnown} { if (_packedData.size()<2U) throw InvalidFormatError(); uint32_t hdr{_packedData.readBE16(0)}; if (!detectHeader(hdr<<16U)) throw InvalidFormatError(); if (hdr==0x1f8bU) { // Standard Gzip if (_packedData.size()<18U) throw InvalidFormatError(); uint8_t cm{_packedData.read8(2)}; if (cm!=8) throw InvalidFormatError();; uint8_t flags{_packedData.read8(3)}; if (flags&0xe0) throw InvalidFormatError();; uint32_t currentOffset{10}; if (flags&4) { uint16_t xlen{_packedData.readLE16(currentOffset)}; currentOffset+=uint32_t(xlen)+2; } auto skipString=[&]() { uint8_t ch; do { ch=_packedData.read8(currentOffset); currentOffset++; } while (ch); }; if (flags&8) skipString(); // FNAME if (flags&16) skipString(); // FCOMMENT if (flags&2) currentOffset+=2; // FHCRC, not using that since it is only for header _packedOffset=currentOffset; if (OverflowCheck::sum(currentOffset,8U)>_packedData.size()) throw InvalidFormatError(); _type=Type::GZIP; } else { // Quasijarus if (_packedData.size()<10U) throw InvalidFormatError(); _packedOffset=2U; _type=Type::Quasijarus; } if (_exactSizeKnown) { _packedSize=_packedData.size(); _rawSize=_packedData.readLE32(_packedData.size()-4); if (!_rawSize || _rawSize>getMaxRawSize()) throw InvalidFormatError(); } } DEFLATEDecompressor::DEFLATEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectZLib()) { _packedSize=packedData.size(); _packedOffset=0; _type=Type::Raw; } } DEFLATEDecompressor::DEFLATEDecompressor(const Buffer &packedData,size_t packedSize,size_t rawSize,bool isZlib,bool verify,bool deflate64) : _packedData{packedData}, _deflate64{deflate64} { _packedSize=packedSize; if (_packedSize>_packedData.size()) throw InvalidFormatError(); if (isZlib) { // if it is not real zlib-stream fail. if (!detectZLib()) throw InvalidFormatError(); } else { // raw stream _packedOffset=0; _rawSize=rawSize; _type=Type::Raw; } } const std::string &DEFLATEDecompressor::getName() const noexcept { static std::string names[4]={ "gzip: Deflate", "Z: Quasijarus Strong Compression", "zlib: Deflate", "raw: Deflate/Deflate64"}; return names[static_cast(_type)]; } const std::string &DEFLATEDecompressor::getSubName() const noexcept { static std::string name{"XPK-GZIP: Deflate"}; return name; } size_t DEFLATEDecompressor::getPackedSize() const noexcept { // no way to know before decompressing return _packedSize; } size_t DEFLATEDecompressor::getRawSize() const noexcept { // same thing, decompression needed first return _rawSize; } void DEFLATEDecompressor::decompressImpl(Buffer &rawData,bool verify) { size_t packedSize{_packedSize?_packedSize:_packedData.size()}; ForwardInputStream inputStream{_packedData,_packedOffset,packedSize}; LSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; AutoExpandingForwardOutputStream outputStream{rawData}; VariableLengthCodeDecoder lengthVLC{ 0,0,0,0,0,0,0,0, 1,1,1,1,2,2,2,2, 3,3,3,3,4,4,4,4, 5,5,5,5,0}; VariableLengthCodeDecoder distanceVLC{ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9,10,10, 11,11,12,12,13,13}; VariableLengthCodeDecoder distanceVLC64{ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9,10,10, 11,11,12,12,13,13,14,14}; bool final; do { final=readBit(); uint32_t blockType{readBits(2)}; if (!blockType) { bitReader.reset(); uint16_t len=inputStream.readByte(); len|=uint16_t(inputStream.readByte())<<8; uint16_t nlen=inputStream.readByte(); nlen|=uint16_t(inputStream.readByte())<<8; if (len!=(nlen^0xffffU)) throw DecompressionError(); outputStream.produce(*inputStream.consume(len)); } else if (blockType==1 || blockType==2) { typedef HuffmanDecoder DEFLATEDecoder; DEFLATEDecoder llDecoder; DEFLATEDecoder distanceDecoder; if (blockType==1) { for (uint32_t i=0;i<24;i++) llDecoder.insert(HuffmanCode{7,i,i+256}); for (uint32_t i=0;i<144;i++) llDecoder.insert(HuffmanCode{8,i+0x30,i}); for (uint32_t i=0;i<8;i++) llDecoder.insert(HuffmanCode{8,i+0xc0,i+280}); for (uint32_t i=0;i<112;i++) llDecoder.insert(HuffmanCode{9,i+0x190,i+144}); for (uint32_t i=0;i<32;i++) distanceDecoder.insert(HuffmanCode{5,i,i}); } else { uint32_t hlit=readBits(5)+257; // lets just error here, it is simpler if (hlit>286) throw DecompressionError(); uint32_t hdist{readBits(5)+1}; uint32_t hclen{readBits(4)+4}; std::array lengthTable; for (uint32_t i=0;i<19;i++) lengthTable[i]=0; const std::array lengthTableOrder={ 16,17,18, 0, 8, 7, 9, 6, 10, 5,11, 4,12, 3,13, 2, 14, 1,15}; for (uint32_t i=0;i llTableBits; std::array distanceTableBits; uint8_t prevValue{0}; uint32_t i{0}; while (i=hlit+hdist) throw DecompressionError(); if (i>=hlit) distanceTableBits[i-hlit]=value; else llTableBits[i]=value; prevValue=value; i++; }; if (uint32_t code{bitLengthDecoder.decode(readBit)};code<16U) { insert(code); } else switch (code) { case 16U: if (i) { uint32_t count{readBits(2U)+3U}; for (uint32_t j=0;jpackedSize) throw DecompressionError(); _packedSize=inputStream.getOffset()+8; } else if (_type==Type::ZLib) { if (OverflowCheck::sum(inputStream.getOffset(),4U)>packedSize) throw DecompressionError(); _packedSize=inputStream.getOffset()+4; } else { _packedSize=inputStream.getOffset(); } if (verify) { if (_type==Type::GZIP || _type==Type::Quasijarus) { uint32_t crc{_packedData.readLE32(inputStream.getOffset())}; if (CRC32(rawData,0,_rawSize,0)!=crc) throw VerificationError(); } else if (_type==Type::ZLib) { uint32_t adler{_packedData.readBE32(inputStream.getOffset())}; if (Adler32(rawData,0,_rawSize)!=adler) throw VerificationError(); } } } void DEFLATEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { decompressImpl(rawData,verify); } } ancient-2.2.0/src/DEFLATEDecompressor.hpp000066400000000000000000000032361463063262600201020ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef DEFLATEDECOMPRESSOR_HPP #define DEFLATEDECOMPRESSOR_HPP #include "Decompressor.hpp" #include "XPKDecompressor.hpp" namespace ancient::internal { class DEFLATEDecompressor : public Decompressor, public XPKDecompressor { public: DEFLATEDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); DEFLATEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); DEFLATEDecompressor(const Buffer &packedData,size_t packedSize,size_t rawSize,bool isZlib,bool verify,bool deflate64); // zlib or completely raw stream ~DEFLATEDecompressor() noexcept=default; size_t getRawSize() const noexcept final; size_t getPackedSize() const noexcept final; const std::string &getName() const noexcept final; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: bool detectZLib(); enum class Type { GZIP=0, Quasijarus, ZLib, Raw }; const Buffer &_packedData; size_t _packedSize{0}; size_t _packedOffset{0}; size_t _rawSize{0}; Type _type; bool _exactSizeKnown{true}; bool _deflate64{false}; }; } #endif ancient-2.2.0/src/DLTADecode.cpp000066400000000000000000000025241463063262600162720ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "DLTADecode.hpp" #include "common/OverflowCheck.hpp" #include "common/Common.hpp" namespace ancient::internal { bool DLTADecode::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("DLTA"); } std::shared_ptr DLTADecode::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } DLTADecode::DLTADecode(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &DLTADecode::getSubName() const noexcept { static std::string name{"XPK-DLTA: Delta encoding"}; return name; } void DLTADecode::decode(Buffer &bufferDest,const Buffer &bufferSrc,size_t offset,size_t size) { uint8_t ctr{0}; for (size_t i=0;i &state,bool verify); ~DLTADecode() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); // static method for easy external usage. Buffers can be the same for in-place replacement static void decode(Buffer &bufferDest,const Buffer &bufferSrc,size_t offset,size_t size); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/DMSDecompressor.cpp000066400000000000000000000442371463063262600174620ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "DMSDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "DynamicHuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "VariableLengthCodeDecoder.hpp" #include "common/MemoryBuffer.hpp" #include "common/CRC16.hpp" #include "common/OverflowCheck.hpp" #include "common/Common.hpp" namespace ancient::internal { bool DMSDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("DMS!"); } std::shared_ptr DMSDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,verify); } DMSDecompressor::DMSDecompressor(const Buffer &packedData,bool verify) : _packedData{packedData} { uint32_t hdr{packedData.readBE32(0)}; if (!detectHeader(hdr) || packedData.size()<56) throw InvalidFormatError(); if (verify && CRC16(packedData,4,50,0)!=packedData.readBE16(54)) throw VerificationError(); uint16_t info{packedData.readBE16(10)}; _isObsfuscated=info&2; // using 16 bit key is not encryption, it is obsfuscation _isHD=info&16; if (info&32) throw InvalidFormatError(); // MS-DOS disk // packed data in header is useless, as is rawsize and track numbers // they are not always filled if (packedData.readBE16(50)>6) throw InvalidFormatError(); // either FMS or unknown const std::array contextSizes{0,0,256,16384,16384,4096,8192}; // now calculate the real packed size, including headers uint32_t offset{56}; uint32_t accountedSize{0}; uint32_t lastTrackSize{0}; uint32_t numTracks{0}; uint32_t minTrack{80}; uint32_t prevTrack{0}; while (offset+206) throw InvalidFormatError(); _contextBufferSize=std::max(_contextBufferSize,contextSizes[mode]); uint8_t flags{_packedData.read8(offset+12)}; if ((mode>=2 && mode<=4) || (mode>=5 && (flags&4))) _tmpBufferSize=std::max(_tmpBufferSize,uint32_t(_packedData.readBE16(offset+8))); uint32_t packedChunkLength{packedData.readBE16(offset+6)}; if (OverflowCheck::sum(offset,20U,packedChunkLength)>packedData.size()) throw InvalidFormatError(); if (verify && CRC16(packedData,offset+20,packedChunkLength,0)!=packedData.readBE16(offset+16)) throw VerificationError(); if (trackNo<80) { if (trackNo>=numTracks) lastTrackSize=_packedData.readBE16(offset+10); minTrack=std::min(minTrack,trackNo); numTracks=std::max(numTracks,trackNo); prevTrack=trackNo; } offset+=packedChunkLength+20; accountedSize+=packedChunkLength; // this is the real exit critea, unfortunately if (trackNo>=79 && trackNo<0x8000U) break; } uint32_t trackSize{(_isHD)?22528U:11264U}; _rawOffset=minTrack*trackSize; if (minTrack>=numTracks) throw InvalidFormatError(); _minTrack=minTrack; _rawSize=(numTracks-minTrack)*trackSize+lastTrackSize; _imageSize=trackSize*80; _packedSize=offset; if (_packedSize>getMaxPackedSize()) throw InvalidFormatError(); } const std::string &DMSDecompressor::getName() const noexcept { static std::string name{"DMS: Disk Masher System"}; return name; } size_t DMSDecompressor::getPackedSize() const noexcept { return _packedSize; } size_t DMSDecompressor::getRawSize() const noexcept { return _rawSize; } size_t DMSDecompressor::getImageSize() const noexcept { return _imageSize; } size_t DMSDecompressor::getImageOffset() const noexcept { return _rawOffset; } void DMSDecompressor::decompressImpl(Buffer &rawData,bool verify) { uint32_t restartPosition{0}; if (!_isObsfuscated) { decompressImpl(rawData,verify,restartPosition); } else { while (restartPosition<0x20000U) { // more than single run here is really rare. It means that first track CRC succeeds // but later something else fails try { decompressImpl(rawData,verify,restartPosition); return; } catch (const Buffer::Error &) { // just continue } catch (const Decompressor::Error &) { // just continue } restartPosition++; } throw DecompressionError(); } } // TODO: Too much state for a single method. too convoluted // needs to be split void DMSDecompressor::decompressImpl(Buffer &rawData,bool verify,uint32_t &restartPosition) { if (rawData.size()<_rawSize) throw DecompressionError(); MemoryBuffer contextBuffer{_contextBufferSize}; MemoryBuffer tmpBuffer{_tmpBufferSize}; uint32_t limitedDecompress{~0U}; class ObsfuscatedInputStream { public: ObsfuscatedInputStream(const Buffer &data,uint32_t start,uint32_t length,bool obsfuscate) : _inputStream{data,start,OverflowCheck::sum(start,length)}, _obsfuscate{obsfuscate} { // nothing needed } ~ObsfuscatedInputStream() noexcept=default; void reset(size_t startOffset,size_t endOffset) { _inputStream.reset(startOffset,endOffset); } uint8_t readByte() { uint8_t ch{_inputStream.readByte()}; if (!_obsfuscate) { return ch; } else { uint8_t ret{uint8_t(ch^_passAccumulator)}; _passAccumulator=(_passAccumulator>>1)+uint16_t(ch); return ret; } } void setCode(uint16_t passAccumulator) { _passAccumulator=passAccumulator; } bool eof() const { return _inputStream.getOffset()==_inputStream.getEndOffset(); } private: ForwardInputStream _inputStream; bool _obsfuscate; uint16_t _passAccumulator{0}; }; ObsfuscatedInputStream inputStream{_packedData,0,0,_isObsfuscated}; MSBBitReader bitReader{inputStream}; auto initInputStream=[&](uint32_t start,uint32_t length) { inputStream.reset(start,OverflowCheck::sum(start,length)); bitReader.reset(); }; auto finishStream=[&]() { if (_isObsfuscated && limitedDecompress==~0U) while (!inputStream.eof()) inputStream.readByte(); }; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream{rawData,0,0}; auto initOutputStream=[&](uint32_t start,uint32_t length) { outputStream.reset(start,OverflowCheck::sum(start,length)); }; // fill unused tracks with zeros std::memset(rawData.data(),0,_rawSize); bool doInitContext{true}; // quick: context used is 256 bytes // medium & deep: context used is 16384 bytes // heavy: context used is 4096/8192 bytes uint32_t contextLocation; std::unique_ptr> deepDecoder; auto initContext=[&]() { if (doInitContext) { if (_contextBufferSize) std::memset(contextBuffer.data(),0,_contextBufferSize); contextLocation=0; deepDecoder.reset(); doInitContext=false; } }; auto checksum=[](const Buffer &src,uint32_t offset,uint32_t srcLength)->uint16_t { uint16_t ret{0}; for (uint32_t i=0;i=limitedDecompress) return; uint8_t ch{input.readByte()}; uint32_t count{1}; if (ch==0x90U) { count=input.readByte(); if (!count) count=1; else ch=input.readByte(); if (count==0xffU) { count=uint32_t(input.readByte())<<8; count|=input.readByte(); } } for (uint32_t i=0;i=limitedDecompress) return; if (readBits(1)) { output.writeByte(contextBuffer[contextLocation++]=readBits(8)); contextLocation&=0xffU; } else { uint32_t count{readBits(2)+2}; uint8_t offset{uint8_t(contextLocation-readBits(8)-1)}; for (uint32_t i=0;i=limitedDecompress) return; if (readBits(1)) { output.writeByte(contextBuffer[contextLocation++]=readBits(8)); contextLocation&=0x3fffU; } else { uint32_t count{lengthDecoder.decode(readBits,readBits(4U))}; uint32_t rawDist{lengthDecoder.decode([&](uint32_t c) { return (uint32_t(count&0xfU)<<(c-4U))|readBits(c-4U); },((count>>4U)&0xfU))}; count=(count>>8U)+3U; uint32_t offset{contextLocation-rawDist-1U}; for (uint32_t i=0;i>(); while (!output.eof()) { if (output.getOffset()>=limitedDecompress) return; uint32_t symbol=deepDecoder->decode(readBit); if (deepDecoder->getMaxFrequency()==0x8000U) deepDecoder->halve(); deepDecoder->update(symbol); if (symbol<256) { output.writeByte(contextBuffer[contextLocation++]=symbol); contextLocation&=0x3fffU; } else { uint32_t count{symbol-253}; // minimum repeat is 3 uint32_t offset{contextLocation-lengthDecoder.decode(readBits,readBits(4U))-1U}; for (uint32_t i=0;i> symbolDecoder,offsetDecoder; bool heavyLastInitialized{false}; // this is part of initContext on some implementations. screwy!!! uint32_t heavyLastOffset; auto unpackHeavy=[&](auto &output,bool initTables,bool use8kDict) { initContext(); // well, this works. Why this works? dunno if (!heavyLastInitialized) { heavyLastOffset=use8kDict?0U:~0U; heavyLastInitialized=true; } auto readTable=[&](std::unique_ptr> &decoder,uint32_t countBits,uint32_t valueBits) { decoder=std::make_unique>(); uint32_t count{readBits(countBits)}; if (count) { std::array lengthBuffer; // in order to speed up the deObsfuscation, do not send the hopeless // data into slow CreateOrderlyHuffmanTable uint64_t sum{0}; for (uint32_t i=0;i(uint64_t(1U)<<32)) throw DecompressionError(); } lengthBuffer[i]=bits; } decoder->createOrderlyHuffmanTable(lengthBuffer,count); } else { uint32_t index{readBits(countBits)}; decoder->setEmpty(index); } }; if (initTables) { readTable(symbolDecoder,9,5); readTable(offsetDecoder,5,4); } uint32_t mask{use8kDict?0x1fffU:0xfffU}; uint32_t bitLength{use8kDict?14U:13U}; while (!output.eof()) { if (output.getOffset()>=limitedDecompress) return; uint32_t symbol=symbolDecoder->decode(readBit); if (symbol<256) { output.writeByte(contextBuffer[contextLocation++]=symbol); contextLocation&=mask; } else { uint32_t count{symbol-253}; // minimum repeat is 3 uint32_t offsetLength{offsetDecoder->decode(readBit)}; uint32_t rawOffset{heavyLastOffset}; if (offsetLength!=bitLength) { if (offsetLength) rawOffset=(1<<(offsetLength-1))|readBits(offsetLength-1); else rawOffset=0; heavyLastOffset=rawOffset; } uint32_t offset{contextLocation-rawOffset-1}; for (uint32_t i=0;i=0x8000U) { initInputStream(packedOffset+20,packedChunkLength); finishStream(); continue; } if (rawChunkLength>trackLength) throw DecompressionError(); if (trackNo>80) throw DecompressionError(); // should not happen, already excluded uint32_t dataOffset{trackNo*trackLength}; if (_rawOffset>dataOffset) throw DecompressionError(); auto handleTrackSize=[&]() { if (limitedDecompress!=~0U) return; if (!outputStream.eof() && outputStream.getOffset()&0x3ffU) throw DecompressionError(); }; // this is screwy, but it is, what it is auto processBlock=[&](bool doRLE,auto func,auto&&... params) { auto applyFix=[&]() { if (mode>=5U && !_isObsfuscated) { uint32_t missingNo{uint32_t(outputStream.getEndOffset()-outputStream.getOffset())}; uint16_t protoSum{checksum(rawData,dataOffset-_rawOffset,rawChunkLength-missingNo)}; uint16_t fileSum{_packedData.readBE16(packedOffset+14)}; // too many ways to interpret the data if (missingNo>1) throw DecompressionError(); for (uint32_t i=0;i=0x100U) throw DecompressionError(); *outputStream.history(1)=uint8_t(fixByte); } } else handleTrackSize(); }; initInputStream(packedOffset+20,packedChunkLength); if (doRLE) { ForwardOutputStream tmpOutputStream{tmpBuffer,0,tmpChunkLength}; try { func(tmpOutputStream,params...); } catch (const DecompressionError &) { // nothing needed } finishStream(); uint32_t missingNo{static_cast(tmpOutputStream.getEndOffset()-tmpOutputStream.getOffset())}; ForwardInputStream tmpInputStream{tmpBuffer,0,tmpChunkLength-missingNo}; initOutputStream(dataOffset-_rawOffset,rawChunkLength); try { unRLE(outputStream,tmpInputStream); } catch (const DecompressionError &) { // nothing needed } applyFix(); } else { initOutputStream(dataOffset-_rawOffset,rawChunkLength); try { func(outputStream,params...); } catch (const DecompressionError &) { // nothing needed } applyFix(); } finishStream(); }; auto processBlockCode=[&](bool doRLE,auto func,auto&&... params) { if (!_isObsfuscated || trackNo!=_minTrack || _codeFixed) return processBlock(doRLE,func,params...); // fast try if (!trackNo && restartPosition<0x10000U) for (;restartPosition<0x10000U;restartPosition++) { try { doInitContext=true; inputStream.setCode(restartPosition); limitedDecompress=8; processBlock(doRLE,func,params...); if ((rawData.readBE32(0)&0xffff'ff00U)!=FourCC("DOS\0")) continue; // now see if the candidate is any good doInitContext=true; inputStream.setCode(restartPosition); limitedDecompress=~0U; processBlock(doRLE,func,params...); if (checksum(rawData,dataOffset-_rawOffset,rawChunkLength)!=_packedData.readBE16(packedOffset+14)) continue; _codeFixed=true; return; } catch (const Buffer::Error &) { // just continue } catch (const Decompressor::Error &) { // just continue } } // slow round limitedDecompress=~0U; for (;restartPosition<0x20000U;restartPosition++) { try { doInitContext=true; inputStream.setCode(restartPosition); processBlock(doRLE,func,params...); if (checksum(rawData,dataOffset-_rawOffset,rawChunkLength)!=_packedData.readBE16(packedOffset+14)) continue; _codeFixed=true; return; } catch (const Buffer::Error &) { // just continue } catch (const Decompressor::Error &) { // just continue } } throw DecompressionError(); }; switch (mode) { case 0: processBlockCode(false,unpackNone); rawChunkLength=packedChunkLength; break; case 1: processBlockCode(false,[&](auto &output) { unRLE(output,inputStream); handleTrackSize(); }); break; case 2: processBlockCode(true,unpackQuick); break; case 3: processBlockCode(true,unpackMedium); break; case 4: processBlockCode(true,unpackDeep); break; // heavy flags: // 2: (re-)initialize/read tables // 4: do RLE // heavy1 uses 4k dictionary (mode 5), whereas heavy2 uses 8k dictionary case 5: [[fallthrough]]; case 6: processBlockCode(flags&4,unpackHeavy,flags&2,mode==6); break; default: throw DecompressionError(); } if (!(flags&1)) doInitContext=true; if (verify && checksum(rawData,dataOffset-_rawOffset,rawChunkLength)!=_packedData.readBE16(packedOffset+14)) throw VerificationError(); } } } ancient-2.2.0/src/DMSDecompressor.hpp000066400000000000000000000021051463063262600174530ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef DMSDECOMPRESSOR_HPP #define DMSDECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class DMSDecompressor : public Decompressor { public: DMSDecompressor(const Buffer &packedData,bool verify); ~DMSDecompressor() noexcept=default; const std::string &getName() const noexcept final; size_t getPackedSize() const noexcept final; size_t getRawSize() const noexcept final; size_t getImageSize() const noexcept final; size_t getImageOffset() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: void decompressImpl(Buffer &rawData,bool verify,uint32_t &restartPosition); const Buffer &_packedData; uint32_t _packedSize{0}; uint32_t _rawSize{0}; uint32_t _contextBufferSize{0}; uint32_t _tmpBufferSize{0}; uint32_t _imageSize; uint32_t _rawOffset; uint32_t _minTrack; bool _isHD; bool _isObsfuscated; }; } #endif ancient-2.2.0/src/Decompressor.cpp000066400000000000000000000067111463063262600171110ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "Decompressor.hpp" #include #include #include "BZIP2Decompressor.hpp" #include "CompactDecompressor.hpp" #include "CompressDecompressor.hpp" #include "CRMDecompressor.hpp" #include "DEFLATEDecompressor.hpp" #include "DMSDecompressor.hpp" #include "FreezeDecompressor.hpp" #include "IMPDecompressor.hpp" #include "LOBDecompressor.hpp" #include "MMCMPDecompressor.hpp" #include "PackDecompressor.hpp" #include "PPDecompressor.hpp" #include "RNCDecompressor.hpp" #include "SCOCompressDecompressor.hpp" #include "StoneCrackerDecompressor.hpp" #include "TPWMDecompressor.hpp" #include "VicXDecompressor.hpp" #include "XPKMain.hpp" namespace ancient::internal { // --- static std::vector(*)(const Buffer&,bool,bool)>> decompressors={ {BZIP2Decompressor::detectHeader,BZIP2Decompressor::create}, {CompactDecompressor::detectHeader,CompactDecompressor::create}, {CompressDecompressor::detectHeader,CompressDecompressor::create}, {CRMDecompressor::detectHeader,CRMDecompressor::create}, {DEFLATEDecompressor::detectHeader,DEFLATEDecompressor::create}, {DMSDecompressor::detectHeader,DMSDecompressor::create}, {FreezeDecompressor::detectHeader,FreezeDecompressor::create}, {IMPDecompressor::detectHeader,IMPDecompressor::create}, {LOBDecompressor::detectHeader,LOBDecompressor::create}, {MMCMPDecompressor::detectHeader,MMCMPDecompressor::create}, {PackDecompressor::detectHeader,PackDecompressor::create}, {PPDecompressor::detectHeader,PPDecompressor::create}, {RNCDecompressor::detectHeader,RNCDecompressor::create}, {SCOCompressDecompressor::detectHeader,SCOCompressDecompressor::create}, {TPWMDecompressor::detectHeader,TPWMDecompressor::create}, {VicXDecompressor::detectHeader,VicXDecompressor::create}, {XPKMain::detectHeader,XPKMain::create}, // Putting StoneCracker last since detection can be accidentally be detected instead of correct format {StoneCrackerDecompressor::detectHeader,StoneCrackerDecompressor::create} }; std::shared_ptr Decompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { try { uint32_t hdr{(packedData.size()>=4)?packedData.readBE32(0):(uint32_t(packedData.readBE16(0))<<16)}; for (auto &it : decompressors) { if (it.first(hdr)) return it.second(packedData,exactSizeKnown,verify); } throw InvalidFormatError(); } catch (const Buffer::Error&) { throw InvalidFormatError(); } } bool Decompressor::detect(const Buffer &packedData) noexcept { if (packedData.size()<2) return false; try { uint32_t hdr{(packedData.size()>=4)?packedData.readBE32(0):(uint32_t(packedData.readBE16(0))<<16)}; for (auto &it : decompressors) if (it.first(hdr)) return true; return false; } catch (const Buffer::Error&) { return false; } } void Decompressor::decompress(Buffer &rawData,bool verify) { // Simplifying the implementation of sub-decompressors. Just let the buffer-exception pass here, // and that will get translated into Decompressor exceptions try { decompressImpl(rawData,verify); } catch (const Buffer::Error&) { throw DecompressionError(); } } size_t Decompressor::getImageSize() const noexcept { return 0; } size_t Decompressor::getImageOffset() const noexcept { return 0; } // 1G should be enough for everyone (this is retro!) size_t Decompressor::getMaxPackedSize() noexcept { return 0x4000'0000U; } size_t Decompressor::getMaxRawSize() noexcept { return 0x4000'0000U; } } ancient-2.2.0/src/Decompressor.hpp000066400000000000000000000062351463063262600171170ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef DECOMPRESSOR_HPP #define DECOMPRESSOR_HPP #include #include #include #include #include "common/Buffer.hpp" #include "ancient.hpp" namespace ancient::internal { class Decompressor { protected: Decompressor() noexcept=default; public: using Error = ancient::Error; using InvalidFormatError = ancient::InvalidFormatError; using DecompressionError = ancient::DecompressionError; using VerificationError = ancient::VerificationError; Decompressor(const Decompressor&)=delete; Decompressor& operator=(const Decompressor&)=delete; virtual ~Decompressor() noexcept=default; // Name returned is human readable long name virtual const std::string &getName() const noexcept=0; // PackedSize or RawSize are taken from the stream if available, 0 otherwise. // for those compressors having 0 sizes, running decompression will update // the values. (make sure to allocate big-enough buffer for decompression) // There are exceptions: Some decompressors need to exact size of the packed data // in order to decompress. For those providing a indefinitely size packed stream // will not work // use the "exactSizeKnown" flag for create to tell whether you know the size or not virtual size_t getPackedSize() const noexcept=0; virtual size_t getRawSize() const noexcept=0; // Actual decompression. // verify checksum if verify==true // can throw DecompressionError if stream cant be unpacked // can throw VerificationError if verify enabled and checksum does not match void decompress(Buffer &rawData,bool verify); // in case of disk image based formats the data does not necessarily start // from logical beginnig of the image but it is offsetted inside the logical image // (f.e. DMS). getDataOffset will return the offset (or 0 if not relevant or if offset does not exist) // getImageSize will return the size of the the logical image, or 0 if not image-based format virtual size_t getImageSize() const noexcept; virtual size_t getImageOffset() const noexcept; // the functions are there to protect against "accidental" large files when parsing headers // a.k.a. 16M should be enough for everybody (sizes do not have to accurate i.e. // compressors can exclude header content for simplification) // This entirely ok for the context of "old computers" and their files, // for other usages these need to be tuned up static size_t getMaxPackedSize() noexcept; static size_t getMaxRawSize() noexcept; // Main entrypoint // if verify=true then check the packedData for errors: CRC or other checksum if available // check exactSizeKnown from size documentation // can throw InvalidFormatError if stream is not recognized or it is invalid // can throw VerificationError if verify enabled and checksum does not match static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); // Detect signature whether it matches to any known compressor // This does not guarantee the data is decompressable though, only signature is read static bool detect(const Buffer &packedData) noexcept; protected: virtual void decompressImpl(Buffer &rawData,bool verify)=0; }; } #endif ancient-2.2.0/src/DynamicHuffmanDecoder.hpp000066400000000000000000000131051463063262600206230ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef DYNAMICHUFFMANDECODER_HPP #define DYNAMICHUFFMANDECODER_HPP #include #include #include // For exception #include "Decompressor.hpp" namespace ancient::internal { template class DynamicHuffmanDecoder { public: DynamicHuffmanDecoder(uint32_t initialCount=maxCount) : _initialCount{initialCount} { if (_initialCount>maxCount) throw Decompressor::DecompressionError(); reset(); } ~DynamicHuffmanDecoder() noexcept=default; void reset() { _count=_initialCount; if (!_count) return; for (uint32_t i=0;i<_count;i++) { _nodes[i].frequency=1; _nodes[i].index=i+(maxCount-_count)*2; _nodes[i].parent=maxCount*2-_count+(i>>1); _nodes[i].leftLeaf=0; _nodes[i].rightLeaf=0; _codeMap[i+(maxCount-_count)*2]=i; } for (uint32_t i=maxCount*2-_count,j=0;i=_count)?j+(maxCount-_count)*2:j}; uint32_t r{(j+1>=_count)?j+1+(maxCount-_count)*2:(j+1)}; _nodes[i].frequency=_nodes[l].frequency+_nodes[r].frequency; _nodes[i].index=i; _nodes[i].parent=maxCount+(i>>1); _nodes[i].leftLeaf=l; _nodes[i].rightLeaf=r; _codeMap[i]=i; } } template uint32_t decode(F bitReader) const { if (!_count) throw Decompressor::DecompressionError(); if (_count==1) return 0; uint32_t code{maxCount*2-2}; while (code>=maxCount) code=bitReader()?_nodes[code].rightLeaf:_nodes[code].leftLeaf; return code; } void update(uint32_t code) { if (code>=_count) throw Decompressor::DecompressionError(); // this is a bug in LH2. Nobody else uses this codepath, so we can let it be... if (_count==1) { _nodes[0].frequency=1; return; } while (code!=maxCount*2-2) { _nodes[code].frequency++; uint32_t index{_nodes[code].index}; uint32_t destIndex{index}; uint32_t freq{_nodes[code].frequency}; while (destIndex!=maxCount*2-2 && freq>_nodes[_codeMap[destIndex+1]].frequency) destIndex++; if (index!=destIndex) { auto getParentLeaf=[&](uint32_t currentCode)->uint32_t& { Node &parent{_nodes[_nodes[currentCode].parent]}; return parent.leftLeaf==currentCode?parent.leftLeaf:parent.rightLeaf; }; uint32_t destCode{_codeMap[destIndex]}; std::swap(_nodes[code].index,_nodes[destCode].index); std::swap(_codeMap[index],_codeMap[destIndex]); std::swap(getParentLeaf(code),getParentLeaf(destCode)); std::swap(_nodes[code].parent,_nodes[destCode].parent); } code=_nodes[code].parent; } _nodes[code].frequency++; } // halve the frequencies rounding upwards void halve() { if (!_count) return; else if (_count==1) { _nodes[0].frequency=(_nodes[0].frequency+1)>>1; return; } for (uint32_t i=(maxCount-_count)*2,j=(maxCount-_count)*2;i>1; _nodes[i].parent=maxCount+(_nodes[i].index>>1); _codeMap[_nodes[i].index]=i; } for (uint32_t i=maxCount*2-_count,j=(maxCount-_count)*2;i>1); _nodes[i].leftLeaf=l; _nodes[i].rightLeaf=r; _codeMap[i]=i; for (uint32_t k=i;freq<_nodes[_codeMap[k-1]].frequency;k--) { uint32_t &code{_codeMap[k]}; uint32_t &destCode{_codeMap[k-1]}; std::swap(_nodes[code].index,_nodes[destCode].index); std::swap(_nodes[code].parent,_nodes[destCode].parent); std::swap(code,destCode); } } } // Defined as in LH2 void addCode() { if (_count>=maxCount) throw Decompressor::DecompressionError(); uint32_t newIndex{(maxCount-_count-1)*2}; if (!_count) { _nodes[0].frequency=0; _nodes[0].index=newIndex-1; _nodes[0].parent=maxCount*2-2; _nodes[0].leftLeaf=0; _nodes[0].rightLeaf=0; _codeMap[newIndex-1]=0; _count++; } else { _nodes[_count].frequency=0; _nodes[_count].index=newIndex; _nodes[_count].parent=maxCount*2-_count-1; _nodes[_count].leftLeaf=0; _nodes[_count].rightLeaf=0; _codeMap[newIndex]=_count; uint32_t insertIndex{newIndex+2}; uint32_t repNode; uint32_t parentNode; uint32_t insertNode{maxCount*2-_count-1}; if (_count>1) { _codeMap[insertIndex-1]=_codeMap[insertIndex]; _nodes[_codeMap[insertIndex-1]].index--; repNode=_codeMap[(maxCount-_count)*2]; parentNode=_nodes[repNode].parent; (_nodes[parentNode].leftLeaf==repNode?_nodes[parentNode].leftLeaf:_nodes[parentNode].rightLeaf)=insertNode; _nodes[repNode].parent=insertNode; } else { repNode=0; parentNode=maxCount*2-1; } _nodes[insertNode].frequency=_nodes[repNode].frequency; _nodes[insertNode].index=insertIndex; _nodes[insertNode].parent=parentNode; _nodes[insertNode].leftLeaf=_count; _nodes[insertNode].rightLeaf=repNode; _codeMap[insertIndex]=insertNode; Node &parent{_nodes[parentNode]}; if (_count>1 && _nodes[parent.leftLeaf].index>_nodes[parent.rightLeaf].index) std::swap(parent.leftLeaf,parent.rightLeaf); _count++; } } uint32_t getMaxFrequency() const noexcept { return _nodes[maxCount*2-2].frequency; } private: struct Node { uint32_t frequency; uint32_t index; uint32_t parent; uint32_t leftLeaf; uint32_t rightLeaf; }; uint32_t _initialCount; uint32_t _count; std::array _nodes; std::array _codeMap; }; } #endif ancient-2.2.0/src/FASTDecompressor.cpp000066400000000000000000000037451463063262600175730ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "FASTDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool FASTDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("FAST"); } std::shared_ptr FASTDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } FASTDecompressor::FASTDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &FASTDecompressor::getSubName() const noexcept { static std::string name{"XPK-FAST: Fast LZ77 compressor"}; return name; } void FASTDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream forwardInputStream{_packedData,0,_packedData.size()}; BackwardInputStream backwardInputStream{_packedData,0,_packedData.size()}; forwardInputStream.link(backwardInputStream); backwardInputStream.link(forwardInputStream); MSBBitReader bitReader{backwardInputStream}; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1); }; auto readByte=[&]()->uint8_t { return forwardInputStream.readByte(); }; auto readShort=[&]()->uint16_t { return backwardInputStream.readBE16(); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint16_t ld{readShort()}; uint32_t count{std::min(18U-(ld&0xf),uint32_t(outputStream.getEndOffset()-outputStream.getOffset()))}; uint32_t distance{uint32_t(ld>>4U)}; outputStream.copy(distance,count); } } } } ancient-2.2.0/src/FASTDecompressor.hpp000066400000000000000000000014551463063262600175740ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef FASTDECOMPRESSOR_HPP #define FASTDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class FASTDecompressor : public XPKDecompressor { public: FASTDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~FASTDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/FBR2Decompressor.cpp000066400000000000000000000043421463063262600175230ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "FBR2Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool FBR2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("FBR2"); } std::shared_ptr FBR2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } FBR2Decompressor::FBR2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError();; } const std::string &FBR2Decompressor::getSubName() const noexcept { static std::string name{"XPK-FBR2: FBR2 CyberYAFA compressor"}; return name; } void FBR2Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; ForwardOutputStream outputStream{rawData,0,rawData.size()}; uint8_t mode=inputStream.readByte(); while (!outputStream.eof()) { bool doCopy{false}; uint32_t count{0}; switch (mode) { case 33: count=uint32_t(inputStream.readByte())<<24; count|=uint32_t(inputStream.readByte())<<16; count|=uint32_t(inputStream.readByte())<<8; count|=uint32_t(inputStream.readByte()); if (count>=0x8000'0000) { doCopy=true; count=0-count; } break; case 67: count=uint32_t(inputStream.readByte())<<8; count|=uint32_t(inputStream.readByte()); if (count>=0x8000) { doCopy=true; count=0x10000-count; } break; case 100: count=uint32_t(inputStream.readByte()); if (count>=0x80) { doCopy=true; count=0x100-count; } break; default: throw Decompressor::DecompressionError(); } count++; if (doCopy) { for (uint32_t i=0;i &state,bool verify); ~FBR2Decompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/FRLEDecompressor.cpp000066400000000000000000000031751463063262600175630ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "FRLEDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool FRLEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("FRLE"); } std::shared_ptr FRLEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } FRLEDecompressor::FRLEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &FRLEDecompressor::getSubName() const noexcept { static std::string name{"XPK-FRLE: RLE-compressor"}; return name; } void FRLEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; ForwardOutputStream outputStream{rawData,0,rawData.size()}; while (!outputStream.eof()) { auto countMod=[](uint32_t count)->uint32_t { return (32-(count&0x1f))+(count&0x60); }; if (uint32_t count{inputStream.readByte()};count<128) { count=countMod(count); for (uint32_t i=0;i &state,bool verify); ~FRLEDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool override) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/FreezeDecompressor.cpp000066400000000000000000000074311463063262600202520ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "FreezeDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "DynamicHuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool FreezeDecompressor::detectHeader(uint32_t hdr) noexcept { return ((hdr>>16)==0x1f9eU||(hdr>>16)==0x1f9fU); } std::shared_ptr FreezeDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,exactSizeKnown,verify); } FreezeDecompressor::FreezeDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData{packedData}, _exactSizeKnown{exactSizeKnown} { if (_packedData.size()<2U) throw InvalidFormatError(); uint32_t hdr{_packedData.readBE16(0)}; if (!detectHeader(hdr<<16)) throw InvalidFormatError(); _isOldVersion=hdr==0x1f9eU; if (_isOldVersion) { _hufTable={0,0,1,3,8,12,24,16}; } else { if (_packedData.size()<5U) throw InvalidFormatError(); uint16_t tmp{_packedData.readLE16(2U)}; if (tmp&0x8000U) throw InvalidFormatError(); uint8_t tmp2{_packedData.read8(4U)}; if (tmp2&0xc0U) throw InvalidFormatError(); _hufTable={uint8_t(tmp&1U),uint8_t((tmp>>1)&3U),uint8_t((tmp>>3)&7U),uint8_t((tmp>>6)&0xfU),uint8_t(tmp>>10),tmp2}; uint32_t count{62}; uint32_t weights{256}; for (uint32_t i=0;i<6U;i++) count-=_hufTable[i]; for (uint32_t i=0,j=7U;i<6U;i++,j--) weights-=uint32_t(_hufTable[i])< bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; AutoExpandingForwardOutputStream outputStream{rawData}; DynamicHuffmanDecoder<511U> decoder{_isOldVersion?315U:511U}; HuffmanDecoder distanceDecoder; { std::array distanceHighBits; uint32_t j{0}; for (uint32_t i=0;i<8U;i++) { if (j+_hufTable[i]>64U) throw DecompressionError(); for (uint32_t k=0;k<_hufTable[i];k++) distanceHighBits[j++]=i+1U; } distanceDecoder.createOrderlyHuffmanTable(distanceHighBits,j); } uint32_t distanceBits{_isOldVersion?6U:7U}; for(;;) { uint32_t code{decoder.decode(readBit)}; if (decoder.getMaxFrequency()==0x8000U) decoder.halve(); decoder.update(code); if (code==256U) break; if (code<256U) { outputStream.writeByte(code); } else { uint32_t distance{uint32_t(distanceDecoder.decode(readBit)< namespace ancient::internal { class FreezeDecompressor : public Decompressor { public: FreezeDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); ~FreezeDecompressor() noexcept=default; size_t getRawSize() const noexcept final; size_t getPackedSize() const noexcept final; const std::string &getName() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: const Buffer &_packedData; size_t _packedSize{0}; size_t _rawSize{0}; bool _isOldVersion; bool _exactSizeKnown; std::array _hufTable; }; } #endif ancient-2.2.0/src/FrequencyTree.hpp000066400000000000000000000076051463063262600172350ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef FREQUENCYTREE_HPP #define FREQUENCYTREE_HPP #include #include #include #include // For exception #include "Decompressor.hpp" namespace ancient::internal { template class FrequencyTree { public: FrequencyTree() noexcept { for (uint32_t i=0;i<_size;i++) _tree[i]=0; } ~FrequencyTree() noexcept=default; U decode(T value,T &low,T &freq) const { if (value>=_tree[_size-1]) throw Decompressor::DecompressionError(); U symbol{0}; low=0; for (uint32_t i=_levels-2;;i--) { T tmp{_tree[_levelOffsets[i]+symbol]}; if (uint32_t(symbol+1)<_levelSizes[i] && value>=tmp) { symbol++; low+=tmp; value-=tmp; } if (!i) break; symbol<<=1; } freq=_tree[symbol]; return symbol; } template void onNotZero(F func) { uint32_t step{1U<<(_levels-1)}; uint32_t level{_levels-1}; for (uint32_t symbol=0;symbol>level)]) { if (level) { level--; step>>=1; } else { func(symbol); break; } } symbol+=step; if (!(symbol&step)) { level++; step<<=1; } } } // only works with tree of zeros and ones (and I don't want to specialize this on bool. that would also be ugly) template void onNotOne(F func) { uint32_t step{1U<<(_levels-1)}; uint32_t level{_levels-1}; for (uint32_t symbol=0;symbol>level)]!=std::min(step,uint32_t(V)-symbol)) { if (level) { level--; step>>=1; } else { func(symbol); break; } } symbol+=step; if (!(symbol&step)) { level++; step<<=1; } } } T operator[](U symbol) const { if (symbol>=V) throw Decompressor::DecompressionError(); return _tree[symbol]; } void add(U symbol,typename std::make_signed::type freq) { if (symbol>=V) throw Decompressor::DecompressionError(); if (!freq) return; for (uint32_t i=0;i<_levels;i++) { _tree[_levelOffsets[i]+symbol]+=freq; symbol>>=1; } } void set(U symbol,T freq) { if (symbol>=V) throw Decompressor::DecompressionError(); // TODO: check behavior on large numbers typename std::make_signed::type delta=freq-_tree[symbol]; add(symbol,delta); } T getTotal() const noexcept { return _tree[_size-1]; } private: static constexpr uint32_t levelSize(uint32_t level) { uint32_t ret=V; for (uint32_t i=0;i>1; return ret; } static constexpr uint32_t levels() { uint32_t ret{0}; while (levelSize(ret)!=1) ret++; return ret+1; } static constexpr uint32_t size() { uint32_t ret{0}; for (uint32_t i=0;i static constexpr auto makeLevelOffsetSequence(std::integer_sequence) { return std::integer_sequence{}; } template static constexpr auto makeLevelSizeSequence(std::integer_sequence) { return std::integer_sequence{}; } template static constexpr std::array makeArray(std::integer_sequence) { return std::array{{I...}}; } static constexpr uint32_t _size{size()}; static constexpr uint32_t _levels{levels()}; static constexpr std::array _levelOffsets{makeArray(makeLevelOffsetSequence(std::make_integer_sequence{}))}; static constexpr std::array _levelSizes{makeArray(makeLevelSizeSequence(std::make_integer_sequence{}))}; std::array _tree; }; } #endif ancient-2.2.0/src/HFMNDecompressor.cpp000066400000000000000000000052051463063262600175570ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "HFMNDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/OverflowCheck.hpp" #include "common/Common.hpp" namespace ancient::internal { bool HFMNDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("HFMN"); } std::shared_ptr HFMNDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } HFMNDecompressor::HFMNDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr) || packedData.size()<4) throw Decompressor::InvalidFormatError(); uint16_t tmp{packedData.readBE16(0)}; if (tmp&3U) throw Decompressor::InvalidFormatError(); // header is being written in 4 byte chunks _headerSize=tmp&0x1ffU; // the top 7 bits are flags. No definition what they are and they are ignored in decoder... if (OverflowCheck::sum(_headerSize,4U)>packedData.size()) throw Decompressor::InvalidFormatError(); _rawSize=packedData.readBE16(_headerSize+2U); if (!_rawSize) throw Decompressor::InvalidFormatError(); _headerSize+=4; } const std::string &HFMNDecompressor::getSubName() const noexcept { static std::string name{"XPK-HFMN: Huffman compressor"}; return name; } void HFMNDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardOutputStream outputStream{rawData,0,rawData.size()}; HuffmanDecoder decoder; if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); { ForwardInputStream inputStream{_packedData,2,_headerSize}; MSBBitReader bitReader{inputStream}; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; uint32_t code{1}; uint32_t codeBits{1}; for (;;) { if (!readBit()) { uint32_t lit=rotateBits(bitReader.readBits8(8),8); decoder.insert(HuffmanCode{codeBits,code,lit}); while (!(code&1) && codeBits) { codeBits--; code>>=1; } if (!codeBits) break; code--; } else { code=(code<<1)+1; codeBits++; } } } ForwardInputStream inputStream{_packedData,_headerSize,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; while (!outputStream.eof()) outputStream.writeByte(decoder.decode(readBit)); } } ancient-2.2.0/src/HFMNDecompressor.hpp000066400000000000000000000015271463063262600175670ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef HFMNDECOMPRESSOR_HPP #define HFMNDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class HFMNDecompressor : public XPKDecompressor { public: HFMNDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~HFMNDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; size_t _headerSize; size_t _rawSize; }; } #endif ancient-2.2.0/src/HUFFDecompressor.cpp000066400000000000000000000043211463063262600175550ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "HUFFDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool HUFFDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("HUFF"); } std::shared_ptr HUFFDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } HUFFDecompressor::HUFFDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr) || packedData.size()<6) throw Decompressor::InvalidFormatError(); // version: only 0 is defined uint16_t ver{packedData.readBE16(0)}; if (ver) throw Decompressor::InvalidFormatError(); // password: we do not support it... uint32_t pwd{packedData.readBE32(2)}; if (pwd!=0xabadcafe) throw Decompressor::InvalidFormatError(); } const std::string &HUFFDecompressor::getSubName() const noexcept { static std::string name{"XPK-HUFF: Huffman compressor"}; return name; } void HUFFDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,6,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; HuffmanDecoder decoder; for (uint32_t i=0;i<256;i++) { uint8_t codeBits{uint8_t(readByte()+1)}; if (!codeBits) continue; if (codeBits>32) throw Decompressor::DecompressionError(); uint32_t code{0}; int32_t shift{-codeBits}; for (uint32_t j=0;j>shift)&((1< &state,bool verify); ~HUFFDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/HuffmanDecoder.hpp000066400000000000000000000112571463063262600173240ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef HUFFMANDECODER_HPP #define HUFFMANDECODER_HPP #include #include #include #include #include // For exception #include "Decompressor.hpp" namespace ancient::internal { template struct HuffmanCode { uint32_t length; uint32_t code; T value; HuffmanCode(uint32_t _length,uint32_t _code,T _value) noexcept : length{_length}, code{_code}, value{_value} { // nothing needed } ~HuffmanCode() noexcept=default; }; template class OptionalHuffmanDecoder; template class HuffmanDecoder { friend class OptionalHuffmanDecoder; private: struct Node { uint32_t left,right; T value; Node(uint32_t _left,uint32_t _right,T _value) : left{_left}, right{_right}, value{_value} { // nothing needed } Node(Node &&source) : left{source.left}, right{source.right}, value{source.value} { // nothing needed } Node& operator=(Node &&source) { if (this!=&source) { left=source.left; right=source.right; value=source.value; } return *this; } }; public: HuffmanDecoder() noexcept=default; template HuffmanDecoder(const Args&& ...args) : HuffmanDecoder() { const HuffmanCode list[sizeof...(args)]={args...}; for (auto &item : list) insert(item); } ~HuffmanDecoder() noexcept=default; void reset() { _table.clear(); } template const T &decode(F bitReader) const { if (!_table.size()) throw Decompressor::DecompressionError(); uint32_t i{0}; while (_table[i].left || _table[i].right) { i=bitReader()?_table[i].right:_table[i].left; if (!i) throw Decompressor::DecompressionError(); } return _table[i].value; } void insert(const HuffmanCode &code) { uint32_t i{0}; uint32_t length={uint32_t(_table.size())}; for (int32_t currentBit=code.length;currentBit>=0;currentBit--) { uint32_t codeBit={(currentBit && ((code.code>>(currentBit-1U))&1U))?1U:0}; if (i!=length) { if (!currentBit || (!_table[i].left && !_table[i].right)) throw Decompressor::DecompressionError(); uint32_t &tmp{codeBit?_table[i].right:_table[i].left}; if (!tmp) tmp=i=length; else i=tmp; } else { _table.emplace_back((currentBit&&!codeBit)?length+1:0,(currentBit&&codeBit)?length+1:0,currentBit?T():code.value); length++; i++; } } } // create orderly Huffman table, as used by Deflate and Bzip2 (and many others) template void createOrderlyHuffmanTable(const std::array &bitLengths,uint32_t bitTableLength) { if (bitTableLength>N) throw Decompressor::DecompressionError(); uint8_t minDepth{32}; uint8_t maxDepth{0}; // some optimization: more tables std::array firstIndex; std::array lastIndex; std::vector nextIndex(bitTableLength); for (uint32_t i=1;i<33;i++) firstIndex[i]=0xffffU; uint32_t realItems{0}; for (uint32_t i=0;i32) throw Decompressor::DecompressionError(); if (length) { if (lengthmaxDepth) maxDepth=length; if (firstIndex[length]==0xffffU) { firstIndex[length]=i; lastIndex[length]=i; } else { nextIndex[lastIndex[length]]=i; lastIndex[length]=i; } realItems++; } } if (!maxDepth) throw Decompressor::DecompressionError(); // optimization, the multiple depends how sparse the tree really is. (minimum is *2) // usually it is sparse. _table.reserve(realItems*3); uint32_t code{0}; for (uint32_t depth=minDepth;depth<=maxDepth;depth++) { if (firstIndex[depth]!=0xffffU) nextIndex[lastIndex[depth]]=bitTableLength; for (uint32_t i=firstIndex[depth];i{depth,code>>(maxDepth-depth),T(i)}); code+=1<<(maxDepth-depth); } } } private: std::vector _table; }; template class OptionalHuffmanDecoder { public: OptionalHuffmanDecoder() noexcept=default; ~OptionalHuffmanDecoder() noexcept=default; void reset() { _base.reset(); } void setEmpty(T value) { reset(); _emptyValue=value; } template T decode(F bitReader) const { if (!_base._table.size()) return _emptyValue; else return _base.decode(bitReader); } void insert(const HuffmanCode &code) { _base.insert(code); } template void createOrderlyHuffmanTable(const std::array &bitLengths,uint32_t bitTableLength) { _base.createOrderlyHuffmanTable(bitLengths,bitTableLength); } private: HuffmanDecoder _base; T _emptyValue{0}; }; } #endif ancient-2.2.0/src/ILZRDecompressor.cpp000066400000000000000000000037151463063262600176130ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "ILZRDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool ILZRDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("ILZR"); } std::shared_ptr ILZRDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } ILZRDecompressor::ILZRDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr) || packedData.size()<2) throw Decompressor::InvalidFormatError(); _rawSize=_packedData.readBE16(0); if (!_rawSize) throw Decompressor::InvalidFormatError(); } const std::string &ILZRDecompressor::getSubName() const noexcept { static std::string name{"XPK-ILZR: Incremental Lempel-Ziv-Renau compressor"}; return name; } void ILZRDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); ForwardInputStream inputStream{_packedData,2,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; uint32_t bits{8}; while (!outputStream.eof()) { if (readBits(1)) { outputStream.writeByte(readBits(8)); } else { while (outputStream.getOffset()>(1ULL<=outputStream.getOffset()) throw Decompressor::DecompressionError(); outputStream.copy(outputStream.getOffset()-position,count); } } } } ancient-2.2.0/src/ILZRDecompressor.hpp000066400000000000000000000015041463063262600176120ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef ILZRDECOMPRESSOR_HPP #define ILZRDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class ILZRDecompressor : public XPKDecompressor { public: ILZRDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~ILZRDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; size_t _rawSize{0}; }; } #endif ancient-2.2.0/src/IMPDecompressor.cpp000066400000000000000000000166471463063262600174700ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "IMPDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" #include "common/OverflowCheck.hpp" #include namespace ancient::internal { static bool readIMPHeader(uint32_t hdr,uint32_t &addition) noexcept { switch (hdr) { case FourCC("ATN!"): // Team 17 games [[fallthrough]]; case FourCC("EDAM"): // Indy Heat [[fallthrough]]; case FourCC("IMP!"): // Original!! [[fallthrough]]; case FourCC("M.H."): // Georg Glaxo addition=7; return true; case FourCC("BDPI"): // Dizzy's excellent adventures addition=0x6e8; return true; case FourCC("CHFI"): // Dizzy's excellent adventures, Bubble and Squeak addition=0xfe4; return true; case FourCC("RDC9"): // Telekommando 2 (Files do not contain checksum) [[fallthrough]]; // I haven't got these files to be sure what is the addition case FourCC("Dupa"): [[fallthrough]]; case FourCC("FLT!"): [[fallthrough]]; case FourCC("PARA"): addition=0; // disable checksum for now return true; default: return false; } } bool IMPDecompressor::detectHeader(uint32_t hdr) noexcept { uint32_t dummy; return readIMPHeader(hdr,dummy); } bool IMPDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("IMPL"); } std::shared_ptr IMPDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,verify); } std::shared_ptr IMPDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } IMPDecompressor::IMPDecompressor(const Buffer &packedData,bool verify) : _packedData{packedData} { uint32_t hdr{packedData.readBE32(0)}; uint32_t checksumAddition; if (!readIMPHeader(hdr,checksumAddition) || packedData.size()<0x32) throw InvalidFormatError(); _rawSize=packedData.readBE32(4); _endOffset=packedData.readBE32(8); if ((_endOffset&1) || _endOffset<0xc || _endOffset+0x32>packedData.size() || !_rawSize || !_endOffset || _rawSize>getMaxRawSize() || _endOffset>getMaxPackedSize()) throw InvalidFormatError(); uint32_t checksum{packedData.readBE32(_endOffset+0x2e)}; if (verify && checksumAddition) { // size is divisible by 2 uint32_t sum{checksumAddition}; for (uint32_t i=0;i<_endOffset+0x2e;i+=2) sum+=uint32_t(_packedData.readBE16(i)); if (checksum!=sum) throw InvalidFormatError(); } } IMPDecompressor::IMPDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr) || packedData.size()<0x2e) throw InvalidFormatError(); _rawSize=packedData.readBE32(4); _endOffset=packedData.readBE32(8); if ((_endOffset&1) || _endOffset<0xc || OverflowCheck::sum(_endOffset,0x2eU)>packedData.size()) throw InvalidFormatError(); _isXPK=true; } const std::string &IMPDecompressor::getName() const noexcept { static std::string name{"IMP: File Imploder"}; return name; } const std::string &IMPDecompressor::getSubName() const noexcept { static std::string name{"XPK-IMPL: File Imploder"}; return name; } size_t IMPDecompressor::getPackedSize() const noexcept { return _endOffset+0x32; } size_t IMPDecompressor::getRawSize() const noexcept { return _rawSize; } void IMPDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); class IMPInputStream { public: IMPInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset) : _buffer{buffer}, _currentOffset{endOffset}, _endOffset{startOffset}, _refOffset{endOffset} { if (_currentOffset<_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); uint8_t markerByte{buffer.read8(_currentOffset+16)}; if (!(markerByte&0x80)) { if (_currentOffset==_endOffset) throw Decompressor::DecompressionError(); _currentOffset--; } } ~IMPInputStream() noexcept=default; uint8_t readByte() { // streamreader with funny ordering auto sourceOffset=[&](size_t i)->size_t { if (i>=12) { return i; } else { if (i<4) { return i+_refOffset+8; } else if (i<8) { return i+_refOffset; } else { return i+_refOffset-8; } } }; if (_currentOffset<=_endOffset) throw Decompressor::DecompressionError(); return _buffer[sourceOffset(--_currentOffset)]; } bool eof() const { return _currentOffset==_endOffset; } private: const Buffer &_buffer; size_t _currentOffset; size_t _endOffset; size_t _refOffset; }; IMPInputStream inputStream{_packedData,0,_endOffset}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; // the anchor-bit does not seem always to be at the correct place { uint8_t halfByte{_packedData.read8(_endOffset+17)}; for (uint32_t i=0;i<7;i++) if (halfByte&(1<>(i+1),7-i); break; } } BackwardOutputStream outputStream{rawData,0,_rawSize}; // tables std::array,2> distanceValues; for (uint32_t i=0;i<8;i++) distanceValues[i>>2][i&3]=_packedData.readBE16(_endOffset+18+i*2); std::array,3> distanceBits; for (uint32_t i=0;i<12;i++) distanceBits[i>>2][i&3]=_packedData.read8(_endOffset+34+i); // length, distance & literal counts are all intertwined HuffmanDecoder lldDecoder { HuffmanCode{1,0b00000,uint8_t{0}}, HuffmanCode{2,0b00010,uint8_t{1}}, HuffmanCode{3,0b00110,uint8_t{2}}, HuffmanCode{4,0b01110,uint8_t{3}}, HuffmanCode{5,0b11110,uint8_t{4}}, HuffmanCode{5,0b11111,uint8_t{5}} }; HuffmanDecoder lldDecoder2 { HuffmanCode{1,0b00,uint8_t{0}}, HuffmanCode{2,0b10,uint8_t{1}}, HuffmanCode{2,0b11,uint8_t{2}} }; const std::array literalLengths{6,10,10,18}; const std::array,3> literalBits{{ {1,1,1,1}, {2,3,3,4}, {4,5,7,14}}}; uint32_t litLength{_packedData.readBE32(_endOffset+12)}; for (;;) { for (uint32_t i=0;i &state,bool verify); ~IMPDecompressor() noexcept=default; const std::string &getName() const noexcept final; const std::string &getSubName() const noexcept final; size_t getPackedSize() const noexcept final; size_t getRawSize() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _rawSize{0}; uint32_t _endOffset{0}; bool _isXPK{false}; }; } #endif ancient-2.2.0/src/InputStream.cpp000066400000000000000000000073261463063262600167220ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "InputStream.hpp" #include "common/OverflowCheck.hpp" #include "common/SubBuffer.hpp" namespace ancient::internal { ForwardInputStream::ForwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset,size_t overrunAllowance) : _buffer{buffer}, _currentOffset{startOffset}, _endOffset{endOffset}, _overrunAllowance{overrunAllowance} { if (_currentOffset>_endOffset || _currentOffset>_buffer.size() || _endOffset>_buffer.size()) throw Decompressor::DecompressionError(); } void ForwardInputStream::reset(size_t startOffset,size_t endOffset) { _currentOffset=startOffset; _endOffset=endOffset; if (_currentOffset>_endOffset || _currentOffset>_buffer.size() || _endOffset>_buffer.size()) throw Decompressor::DecompressionError(); if (_linkedInputStream) _linkedInputStream->setEndOffset(_currentOffset); } uint8_t ForwardInputStream::readByte() { if (_currentOffset>=_endOffset) { if (_overrunAllowance) { _overrunAllowance--; return 0; } throw Decompressor::DecompressionError(); } uint8_t ret{_buffer[_currentOffset++]}; if (_linkedInputStream) _linkedInputStream->setEndOffset(_currentOffset); return ret; } uint16_t ForwardInputStream::readBE16() { uint16_t b0{readByte()}; uint16_t b1{readByte()}; return (b0<<8)|b1; } uint32_t ForwardInputStream::readBE32() { uint32_t b0{readByte()}; uint32_t b1{readByte()}; uint32_t b2{readByte()}; uint32_t b3{readByte()}; return (b0<<24)|(b1<<16)|(b2<<8)|b3; } uint16_t ForwardInputStream::readLE16() { uint16_t b0{readByte()}; uint16_t b1{readByte()}; return (b1<<8)|b0; } uint32_t ForwardInputStream::readLE32() { uint32_t b0{readByte()}; uint32_t b1{readByte()}; uint32_t b2{readByte()}; uint32_t b3{readByte()}; return (b3<<24)|(b2<<16)|(b1<<8)|b0; } std::shared_ptr ForwardInputStream::consume(size_t bytes) { if (OverflowCheck::sum(_currentOffset,bytes)>_endOffset) throw Decompressor::DecompressionError(); auto ret{std::make_shared(_buffer,_currentOffset,bytes)}; _currentOffset+=bytes; if (_linkedInputStream) _linkedInputStream->setEndOffset(_currentOffset); return ret; } void ForwardInputStream::setOffset(size_t offset) { if (offset>_endOffset) throw Decompressor::DecompressionError(); _currentOffset=offset; if (_linkedInputStream) _linkedInputStream->setEndOffset(_currentOffset); } BackwardInputStream::BackwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset) : _buffer{buffer}, _currentOffset{endOffset}, _endOffset{startOffset} { if (_currentOffset<_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); } uint8_t BackwardInputStream::readByte() { if (_currentOffset<=_endOffset) throw Decompressor::DecompressionError(); uint8_t ret=_buffer[--_currentOffset]; if (_linkedInputStream) _linkedInputStream->setEndOffset(_currentOffset); return ret; } uint16_t BackwardInputStream::readBE16() { uint16_t b0{readByte()}; uint16_t b1{readByte()}; return (b1<<8)|b0; } uint32_t BackwardInputStream::readBE32() { uint32_t b0{readByte()}; uint32_t b1{readByte()}; uint32_t b2{readByte()}; uint32_t b3{readByte()}; return (b3<<24)|(b2<<16)|(b1<<8)|b0; } uint16_t BackwardInputStream::readLE16() { uint16_t b0{readByte()}; uint16_t b1{readByte()}; return (b0<<8)|b1; } uint32_t BackwardInputStream::readLE32() { uint32_t b0{readByte()}; uint32_t b1{readByte()}; uint32_t b2{readByte()}; uint32_t b3{readByte()}; return (b0<<24)|(b1<<16)|(b2<<8)|b3; } void BackwardInputStream::setOffset(size_t offset) { if (offset<_endOffset) throw Decompressor::DecompressionError(); _currentOffset=offset; if (_linkedInputStream) _linkedInputStream->setEndOffset(_currentOffset); } } ancient-2.2.0/src/InputStream.hpp000066400000000000000000000121241463063262600167170ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef INPUTSTREAM_HPP #define INPUTSTREAM_HPP #include #include #include #include #include "common/Buffer.hpp" // for exceptions #include "Decompressor.hpp" namespace ancient::internal { class BackwardInputStream; class ForwardInputStream { friend class BackwardInputStream; public: ForwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset,size_t overrunAllowance=0); ~ForwardInputStream() noexcept=default; void reset(size_t startOffset,size_t endOffset); uint8_t readByte(); uint16_t readBE16(); uint32_t readBE32(); uint16_t readLE16(); uint32_t readLE32(); std::shared_ptr consume(size_t bytes); bool eof() const noexcept { return _currentOffset==_endOffset; } size_t getOffset() const noexcept { return _currentOffset; } size_t getEndOffset() const noexcept { return _endOffset; } void link(BackwardInputStream &stream) noexcept { _linkedInputStream=&stream; } void setOffset(size_t offset); private: void setEndOffset(size_t offset) noexcept { _endOffset=offset; } const Buffer &_buffer; size_t _currentOffset; size_t _endOffset; size_t _overrunAllowance; BackwardInputStream *_linkedInputStream{nullptr}; }; class BackwardInputStream { friend class ForwardInputStream; public: BackwardInputStream(const Buffer &buffer,size_t startOffset,size_t endOffset); ~BackwardInputStream() noexcept=default; uint8_t readByte(); uint16_t readBE16(); uint32_t readBE32(); uint16_t readLE16(); uint32_t readLE32(); bool eof() const noexcept { return _currentOffset==_endOffset; } size_t getOffset() const noexcept { return _currentOffset; } void link(ForwardInputStream &stream) noexcept { _linkedInputStream=&stream; } void setOffset(size_t offset); private: void setEndOffset(size_t offset) noexcept { _endOffset=offset; } const Buffer &_buffer; size_t _currentOffset; size_t _endOffset; ForwardInputStream *_linkedInputStream{nullptr}; }; template class LSBBitReader { public: LSBBitReader(T &inputStream) noexcept : _inputStream{inputStream} { // nothing needed } ~LSBBitReader() noexcept=default; uint32_t readBits8(uint32_t count) { return readBitsGeneric(count,[&](){ return std::make_pair(_inputStream.readByte(),uint8_t(8)); }); } uint32_t readBitsBE16(uint32_t count) { return readBitsGeneric(count,[&](){ return std::make_pair(_inputStream.readBE16(),uint8_t(16)); }); } uint32_t readBitsBE32(uint32_t count) { return readBitsGeneric(count,[&](){ return std::make_pair(_inputStream.readBE32(),uint8_t(32)); }); } uint32_t readBitsLE16(uint32_t count) { return readBitsGeneric(count,[&](){ return std::make_pair(_inputStream.readLE16(),uint8_t(16)); }); } uint32_t getBufContent() const noexcept { return _bufContent; } uint8_t getBufLength() const noexcept { return _bufLength; } void reset(uint32_t bufContent=0,uint8_t bufLength=0) noexcept { _bufContent=bufContent; _bufLength=bufLength; } template uint32_t readBitsGeneric(uint32_t count,F readWord) { uint32_t ret{0}; uint32_t pos{0}; if (count>32) throw Decompressor::DecompressionError(); while (count) { if (!_bufLength) std::tie(_bufContent,_bufLength)=readWord(); uint8_t maxCount{std::min(uint8_t(count),_bufLength)}; ret|=(_bufContent&((1<>=maxCount; _bufLength-=maxCount; count-=maxCount; pos+=maxCount; } return ret; } private: T &_inputStream; uint32_t _bufContent{0}; uint8_t _bufLength{0}; }; template class MSBBitReader { public: MSBBitReader(T &inputStream) noexcept : _inputStream{inputStream} { // nothing needed } ~MSBBitReader() noexcept=default; uint32_t readBits8(uint32_t count) { return readBitsGeneric(count,[&](){ return std::make_pair(_inputStream.readByte(),uint8_t(8)); }); } uint32_t readBitsBE16(uint32_t count) { return readBitsGeneric(count,[&](){ return std::make_pair(_inputStream.readBE16(),uint8_t(16)); }); } uint32_t readBitsBE32(uint32_t count) { return readBitsGeneric(count,[&](){ return std::make_pair(_inputStream.readBE32(),uint8_t(32)); }); } uint32_t readBitsLE16(uint32_t count) { return readBitsGeneric(count,[&](){ return std::make_pair(_inputStream.readLE16(),uint8_t(16)); }); } uint32_t getBufContent() const noexcept { return _bufContent; } uint8_t getBufLength() const noexcept { return _bufLength; } void reset(uint32_t bufContent=0,uint8_t bufLength=0) noexcept { _bufContent=bufContent; _bufLength=bufLength; } template uint32_t readBitsGeneric(uint32_t count,F readWord) { uint32_t ret{0}; if (count>32) throw Decompressor::DecompressionError(); while (count) { if (!_bufLength) std::tie(_bufContent,_bufLength)=readWord(); uint8_t maxCount{std::min(uint8_t(count),_bufLength)}; _bufLength-=maxCount; ret=(ret<>_bufLength)&((1< LHLBDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } LHLBDecompressor::LHLBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &LHLBDecompressor::getSubName() const noexcept { static std::string name{"XPK-LHLB: LZRW-compressor"}; return name; } void LHLBDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1U); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); // Same logic as in Choloks pascal implementation // Differences to LH1: // - LHLB does not halve probabilities at 32k // - 314 vs. 317 sized huffman entry // - no end code // - different distance/count logic DynamicHuffmanDecoder<317> decoder; VariableLengthCodeDecoder vlcDecoder{5,5,6,6,6,7,7,7,7,8,8,8,9,9,9,10}; while (!outputStream.eof()) { uint32_t code=decoder.decode(readBit); if (code==316U) break; if (decoder.getMaxFrequency()<0x8000U) decoder.update(code); if (code<256U) { outputStream.writeByte(code); } else { uint32_t distance{vlcDecoder.decode(readBits,readBits(4U))}; uint32_t count{code-255U}; outputStream.copy(distance,count); } } } } ancient-2.2.0/src/LHLBDecompressor.hpp000066400000000000000000000014551463063262600175600ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LHLBDECOMPRESSOR_HPP #define LHLBDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class LHLBDecompressor : public XPKDecompressor { public: LHLBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~LHLBDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/LIN1Decompressor.cpp000066400000000000000000000055711463063262600175400ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LIN1Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool LIN1Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LIN1") || hdr==FourCC("LIN3"); } std::shared_ptr LIN1Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } LIN1Decompressor::LIN1Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); _ver=(hdr==FourCC("LIN1"))?1:3; if (packedData.size()<5) throw Decompressor::InvalidFormatError(); uint32_t tmp{packedData.readBE32(0)}; if (tmp) throw Decompressor::InvalidFormatError(); // password set } const std::string &LIN1Decompressor::getSubName() const noexcept { static std::string name1{"XPK-LIN1: LIN1 LINO packer"}; static std::string name3{"XPK-LIN3: LIN3 LINO packer"}; return (_ver==1)?name1:name3; } void LIN1Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,5U,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; size_t rawSize{rawData.size()}; ForwardOutputStream outputStream{rawData,0,rawSize}; while (!outputStream.eof()) { if (!readBits(1U)) { outputStream.writeByte(readByte()^0x55U); } else { uint32_t count{3U}; if (readBits(1U)) { count=readBits(2U); if (count==3U) { count=readBits(3U); if (count==7U) { count=readBits(4U); if (count==15U) { count=readByte(); if (count==0xffU) throw Decompressor::DecompressionError(); count+=3U; } else count+=14U; } else count+=7U; } else count+=4U; } uint32_t distance{0}; switch (readBits(2)) { case 0: distance=readByte()+1; break; case 1: distance=uint32_t(readBits(2U))<<8U; distance|=readByte(); distance+=0x101U; break; case 2: distance=uint32_t(readBits(4U))<<8U; distance|=readByte(); distance+=0x501U; break; case 3: distance=uint32_t(readBits(6U))<<8U; distance|=readByte(); distance+=0x1501U; break; } // buggy compressors count=std::min(count,uint32_t(rawSize-outputStream.getOffset())); if (!count) throw Decompressor::DecompressionError(); outputStream.copy(distance,count); } } } } ancient-2.2.0/src/LIN1Decompressor.hpp000066400000000000000000000015011463063262600175320ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LIN1DECOMPRESSOR_HPP #define LIN1DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class LIN1Decompressor : public XPKDecompressor { public: LIN1Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~LIN1Decompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _ver{0}; }; } #endif ancient-2.2.0/src/LIN2Decompressor.cpp000066400000000000000000000152531463063262600175370ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LIN2Decompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" #include "common/OverflowCheck.hpp" #include "common/SubBuffer.hpp" namespace ancient::internal { bool LIN2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LIN2") || hdr==FourCC("LIN4"); } std::shared_ptr LIN2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } LIN2Decompressor::LIN2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); _ver=(hdr==FourCC("LIN2"))?2U:4U; if (packedData.size()<10U) throw Decompressor::InvalidFormatError(); uint32_t tmp{packedData.readBE32(0)}; if (tmp) throw Decompressor::InvalidFormatError(); // password set // LIN4 is very similar to LIN2 - it only has 5 bit literals instead of 4 bit literals // (and thus larger table at the end of the stream) // Also, the huffman decoder for length is different _endStreamOffset=packedData.size()-1U; while (_endStreamOffset && _packedData[--_endStreamOffset]!=0xffU); // end stream // 1 byte, byte before 0xff // 0x10 bytes/0x20 for table if (_endStreamOffset<0x11U+0xaU) throw Decompressor::InvalidFormatError(); _endStreamOffset-=(_ver==2U)?0x11U:0x21U; size_t midStreamOffset{_ver==2?0x16U:0x26U}; // midstream // from endstream without // add 0x10/0x20 byte back to point after table // add 6 bytes to point to correct place tmp=packedData.readBE32(4U); if (OverflowCheck::sum(_endStreamOffset,midStreamOffset) bitsReader{forwardInputStream}; MSBBitReader bitReader{middleInputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitsReader.readBits8(count); }; { uint8_t tmp{middleInputStream.readByte()}; if (tmp>8U) throw Decompressor::DecompressionError(); bitReader.reset(middleInputStream.readByte()>>tmp,8U-tmp); } auto readBit=[&]()->uint8_t { return bitReader.readBits8(1U); }; bool buf4Incomplete{false}; uint8_t nibbleContent{0}; { uint8_t tmp{_packedData.read8(9U)}; buf4Incomplete=!!tmp; if (buf4Incomplete) nibbleContent=backwardInputStream.readByte(); } // this is a rather strange thing... auto read4Bits=[&](bool fullByte)->uint8_t { if (!fullByte) { buf4Incomplete=!buf4Incomplete; if (!buf4Incomplete) { return nibbleContent&0xfU; } else { nibbleContent=backwardInputStream.readByte(); return nibbleContent>>4U; } } else { if (buf4Incomplete) { uint8_t ret{uint8_t(nibbleContent&0xfU)}; nibbleContent=backwardInputStream.readByte(); ret|=nibbleContent&0xf0U; return ret; } else { return backwardInputStream.readByte(); } } }; const ConstSubBuffer literalTable{_packedData,_endStreamOffset,_packedData.size()-_endStreamOffset}; size_t rawSize=rawData.size(); ForwardOutputStream outputStream{rawData,0,rawSize}; // little meh to initialize both (intentionally deleted copy/assign) HuffmanDecoder lengthDecoder2 { HuffmanCode{1,0b000000,uint8_t{3}}, HuffmanCode{3,0b000100,uint8_t{4}}, HuffmanCode{3,0b000101,uint8_t{5}}, HuffmanCode{3,0b000110,uint8_t{6}}, HuffmanCode{6,0b111000,uint8_t{7}}, HuffmanCode{6,0b111001,uint8_t{8}}, HuffmanCode{6,0b111010,uint8_t{9}}, HuffmanCode{6,0b111011,uint8_t{10}}, HuffmanCode{6,0b111100,uint8_t{11}}, HuffmanCode{6,0b111101,uint8_t{12}}, HuffmanCode{6,0b111110,uint8_t{13}}, HuffmanCode{6,0b111111,uint8_t{0}} }; HuffmanDecoder lengthDecoder4 { HuffmanCode{2,0b0000000,uint8_t{3}}, HuffmanCode{2,0b0000001,uint8_t{4}}, HuffmanCode{2,0b0000010,uint8_t{5}}, HuffmanCode{4,0b0001100,uint8_t{6}}, HuffmanCode{4,0b0001101,uint8_t{7}}, HuffmanCode{4,0b0001110,uint8_t{8}}, HuffmanCode{7,0b1111000,uint8_t{9}}, HuffmanCode{7,0b1111001,uint8_t{10}}, HuffmanCode{7,0b1111010,uint8_t{11}}, HuffmanCode{7,0b1111011,uint8_t{12}}, HuffmanCode{7,0b1111100,uint8_t{13}}, HuffmanCode{7,0b1111101,uint8_t{14}}, HuffmanCode{7,0b1111110,uint8_t{15}}, HuffmanCode{7,0b1111111,uint8_t{0}} }; auto &lengthDecoder=_ver==2?lengthDecoder2:lengthDecoder4; uint32_t minBits{1}; while (!outputStream.eof()) { if (!readBits(1U)) { if (readBit()) { outputStream.writeByte(read4Bits(true)); } else { if (_ver==4U) { outputStream.writeByte(literalTable[(read4Bits(false)<<1U)+readBit()]); } else outputStream.writeByte(literalTable[read4Bits(false)]); } } else { uint32_t count=lengthDecoder.decode([&](){return readBits(1);}); if (!count) { count=readBits(4U); if (count==0xfU) { count=readBits(8U); if (count!=0xffU) count+=3U; else throw Decompressor::DecompressionError(); } else count+=(_ver==2U)?14U:16U; } uint32_t distance; bool isMax{false}; do { uint32_t bits{readBits(3U)+minBits}; distance=readBits(bits); isMax=(distance==((1U< &state,bool verify); ~LIN2Decompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _ver{0}; size_t _endStreamOffset{0}; size_t _midStreamOffset{0}; }; } #endif ancient-2.2.0/src/LOBDecompressor.cpp000066400000000000000000000214431463063262600174450ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LOBDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "HuffmanDecoder.hpp" #include "LZWDecoder.hpp" #include "common/Common.hpp" #include "common/SubBuffer.hpp" #include "common/MemoryBuffer.hpp" #include "common/OverflowCheck.hpp" namespace ancient::internal { bool LOBDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("\001LOB")||hdr==FourCC("\002LOB")||hdr==FourCC("\003LOB"); } std::shared_ptr LOBDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,verify); } LOBDecompressor::LOBDecompressor(const Buffer &packedData,bool verify) : _packedData{packedData} { uint32_t hdr{packedData.readBE32(0)}; if (!detectHeader(hdr) || packedData.size()<12U) throw InvalidFormatError(); _methodCount=hdr>>24U; uint8_t method{packedData.read8(4U)}; if (method<1U || method>6U) throw InvalidFormatError(); if (_methodCount==1U) { _rawSize=packedData.readBE32(4U)&0xff'ffffU; if (!_rawSize || _rawSize>getMaxRawSize()) throw InvalidFormatError(); } _packedSize=OverflowCheck::sum(packedData.readBE32(8U),12U); // now parse the huffman tables to get the correct size if (method==2U) { uint32_t count{2U}; for (uint32_t i=0;ipackedData.size()) throw InvalidFormatError(); uint32_t t1{_packedData.read8(_packedSize+i)}; uint32_t t2{_packedData.read8(_packedSize+i+1U)}; if (t1!=t2) { t1=std::max(t1,t2); count=std::max(t1+i+4U,count); if (count>1024U) throw InvalidFormatError(); } } _packedSize+=count; } if (_packedSize>packedData.size()) throw InvalidFormatError(); } const std::string &LOBDecompressor::getName() const noexcept { static std::string names[3]={ "LOB: LOB's File Compressor", "LOB: LOB's File Compressor (double compressed)", "LOB: LOB's File Compressor (triple compressed)"}; return names[_methodCount-1U]; } size_t LOBDecompressor::getPackedSize() const noexcept { return _packedSize; } size_t LOBDecompressor::getRawSize() const noexcept { // not known for recursive compressions return _rawSize; } void LOBDecompressor::decompressRound(Buffer &rawData,const Buffer &packedData) { uint8_t method{packedData.read8(0)}; if (method<1U || method>6U) throw DecompressionError(); uint32_t rawSize{packedData.readBE32(0)&0xff'ffffU}; uint32_t packedSize{OverflowCheck::sum(packedData.readBE32(4U),8U)}; ForwardInputStream inputStream{packedData,8U,packedSize}; MSBBitReader bitReader{inputStream}; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1U); }; auto readBits=[&](uint32_t bits)->uint32_t { return bitReader.readBits8(bits); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,rawSize}; auto writeByte=[&](uint8_t value) { outputStream.writeByte(value); }; switch (method) { // BMC (RLE compressor) case 1U: while (!outputStream.eof()) { uint32_t count{readByte()}; if (count<0x80U) { count++; for (uint32_t i=0;i decoder; std::array tree; uint32_t count=2U; for (uint32_t i=0;i1024U) throw DecompressionError(); } else { tree[i]=0; tree[i+1]=t1; } } auto branch=[&](uint32_t node,uint32_t length,uint32_t bits,auto branch)->void { if (node>=count) throw DecompressionError(); if (tree[node]) { length++; bits<<=1U; if (length>=32U) throw DecompressionError(); branch(node+tree[node],length,bits,branch); if (node+1>=count) throw DecompressionError(); branch(node+tree[node+1],length,bits|1U,branch); } else { if (!length) throw DecompressionError(); decoder.insert(HuffmanCode{length,bits,uint8_t(tree[node+1])}); } }; branch(0,0,0,branch); while (!outputStream.eof()) outputStream.writeByte(decoder.decode(readBit)); } break; // LZW (12-bit fixed code LZW) case 3U: { uint32_t firstCode{readBits(12U)}; LZWDecoder decoder{4096U,256U,65536U,firstCode}; decoder.write(firstCode,false,writeByte); while (!outputStream.eof()) { uint32_t code{readBits(12U)}; if (code==0xfffU) { firstCode=readBits(12U); decoder.reset(firstCode); decoder.write(firstCode,false,writeByte); } else { decoder.write(code,!decoder.isLiteral(code),writeByte); decoder.add(code); } } } break; // LZB (9 to 12-bit LZW) case 4U: { uint32_t codeBits{9U}; uint32_t firstCode{readBits(codeBits)}; if (!firstCode--) throw DecompressionError(); LZWDecoder decoder{4096U,256U,65536U,firstCode}; decoder.write(firstCode,false,writeByte); while (!outputStream.eof()) { if (codeBits!=12U && decoder.getCurrentIndex()+1U>=(1U< decoder { HuffmanCode{2,0b000,uint8_t{0}}, HuffmanCode{2,0b001,uint8_t{1}}, HuffmanCode{3,0b100,uint8_t{2}}, HuffmanCode{3,0b101,uint8_t{3}}, HuffmanCode{3,0b110,uint8_t{4}}, HuffmanCode{3,0b111,uint8_t{5}} }; // will fail if size<2 outputStream.writeByte(readByte()); outputStream.writeByte(readByte()); while (!outputStream.eof()) { uint32_t count,distance; switch (uint32_t value=decoder.decode(readBit)) { case 0: count=readBits(3U)+1U; for (uint32_t i=0;igetMaxRawSize()) throw DecompressionError(); MemoryBuffer tmpBuffer{rawSize}; decompressRound(tmpBuffer,ConstSubBuffer(_packedData,4U,_packedData.size()-4U)); rawSize=tmpBuffer.readBE32(0U)&0xff'ffffU; if (!rawSize || rawSize>getMaxRawSize()) throw DecompressionError(); _rawSize=rawSize; if (rawData.size()<_rawSize) rawData.resize(rawSize); decompressRound(rawData,tmpBuffer); } break; case 3U: { uint32_t rawSize{_packedData.readBE32(4U)&0xff'ffffU}; if (!rawSize || rawSize>getMaxRawSize()) throw DecompressionError(); MemoryBuffer tmpBuffer{rawSize}; decompressRound(tmpBuffer,ConstSubBuffer(_packedData,4U,_packedData.size()-4U)); rawSize=tmpBuffer.readBE32(0U)&0xff'ffffU; if (!rawSize || rawSize>getMaxRawSize()) throw DecompressionError(); MemoryBuffer tmpBuffer2{rawSize}; decompressRound(tmpBuffer2,tmpBuffer); rawSize=tmpBuffer2.readBE32(0U)&0xff'ffffU; if (!rawSize || rawSize>getMaxRawSize()) throw DecompressionError(); _rawSize=rawSize; if (rawData.size()<_rawSize) rawData.resize(rawSize); decompressRound(rawData,tmpBuffer2); } break; default: throw DecompressionError(); break; } } } ancient-2.2.0/src/LOBDecompressor.hpp000066400000000000000000000015271463063262600174530ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LOBDECOMPRESSOR_HPP #define LOBDECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class LOBDecompressor : public Decompressor { public: LOBDecompressor(const Buffer &packedData,bool verify); ~LOBDecompressor() noexcept=default; const std::string &getName() const noexcept final; size_t getPackedSize() const noexcept final; size_t getRawSize() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: static void decompressRound(Buffer &rawData,const Buffer &packedData); const Buffer &_packedData; uint32_t _rawSize{0}; uint32_t _packedSize{0}; uint32_t _methodCount; }; } #endif ancient-2.2.0/src/LZBSDecompressor.cpp000066400000000000000000000036671463063262600176130ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZBSDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool LZBSDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LZBS"); } std::shared_ptr LZBSDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } LZBSDecompressor::LZBSDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr) || _packedData.size()<1) throw Decompressor::InvalidFormatError(); } const std::string &LZBSDecompressor::getSubName() const noexcept { static std::string name{"XPK-LZBS: LZBS CyberYAFA compressor"}; return name; } void LZBSDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,1,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return rotateBits(bitReader.readBits8(count),count); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; uint32_t bits{0}; uint32_t maxBits{_packedData[0]}; while (!outputStream.eof()) { if (!readBits(1)) { outputStream.writeByte(readBits(8)); } else { uint32_t count{readBits(8)+2}; if (count==2) { count=readBits(12); if (!count) throw Decompressor::DecompressionError(); for (uint32_t i=0;i=(1ULL< &state,bool verify); ~LZBSDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/LZCBDecompressor.cpp000066400000000000000000000107261463063262600175650ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZCBDecompressor.hpp" #include "RangeDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "FrequencyTree.hpp" #include "common/Common.hpp" namespace ancient::internal { template class FrequencyDecoder { public: FrequencyDecoder(RangeDecoder &decoder) : _decoder{decoder} { // nothing needed } ~FrequencyDecoder() noexcept=default; template uint16_t decode(F readFunc) { uint16_t freq{0}; uint16_t symbol; uint16_t value{_decoder.decode(_threshold+_tree.getTotal())}; if (value>=_threshold) { uint16_t low; symbol=_tree.decode(value-_threshold,low,freq); _decoder.scale(_threshold+low,_threshold+low+freq,_threshold+_tree.getTotal()); if (freq==1 && _threshold>1) _threshold--; } else { _decoder.scale(0,_threshold,_threshold+_tree.getTotal()); symbol=readFunc(); // A bug in the encoder if (!symbol && _tree[symbol]) symbol=T; _threshold++; } _tree.add(symbol,1); if (_threshold+_tree.getTotal()>=0x3ffdU) { for (uint32_t i=0;i>1); _threshold=(_threshold>>1)+1; } return symbol; } private: RangeDecoder &_decoder; FrequencyTree _tree; uint16_t _threshold{1}; }; bool LZCBDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LZCB"); } std::shared_ptr LZCBDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } LZCBDecompressor::LZCBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (packedData.size()<2) throw Decompressor::InvalidFormatError(); } const std::string &LZCBDecompressor::getSubName() const noexcept { static std::string name{"XPK-LZCB: LZ-compressor"}; return name; } void LZCBDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { class BitReader : public RangeDecoder::BitReader { public: BitReader(ForwardInputStream &stream) : _reader(stream) { // nothing needed } ~BitReader() noexcept=default; uint32_t readBit() final { return _reader.readBitsBE32(1); } uint32_t readBits(uint32_t bitCount) { return _reader.readBitsBE32(bitCount); } private: MSBBitReader _reader; }; ForwardInputStream inputStream{_packedData,0,_packedData.size(),7U}; ForwardOutputStream outputStream{rawData,0,rawData.size()}; BitReader bitReader{inputStream}; RangeDecoder rangeDecoder{bitReader,uint16_t(bitReader.readBits(16))}; // Ugly duplicates auto readByte=[&]()->uint16_t { uint16_t ret=rangeDecoder.decode(0x100U); rangeDecoder.scale(ret,ret+1,0x100U); return ret; }; auto readCount=[&]()->uint16_t { uint16_t ret=rangeDecoder.decode(0x101U); rangeDecoder.scale(ret,ret+1,0x101U); return ret; }; FrequencyDecoder<256> baseLiteralDecoder{rangeDecoder}; FrequencyDecoder<257> repeatCountDecoder{rangeDecoder}; FrequencyDecoder<257> literalCountDecoder{rangeDecoder}; FrequencyDecoder<256> distanceDecoder{rangeDecoder}; std::array>,256> literalDecoders; uint8_t ch{uint8_t(baseLiteralDecoder.decode(readByte))}; outputStream.writeByte(ch); bool lastIsLiteral{true}; while (!outputStream.eof()) { uint32_t count{repeatCountDecoder.decode(readCount)}; if (count) { if (count==0x100U) { uint32_t tmp; do { tmp=readByte(); count+=tmp; } while (tmp==0xffU); } count+=lastIsLiteral?5:4; uint32_t distance{uint32_t(distanceDecoder.decode(readByte)<<8U)}; distance|=readByte(); ch=outputStream.copy(distance,count); lastIsLiteral=false; } else { uint16_t literalCount; do { literalCount=literalCountDecoder.decode(readCount); if (!literalCount) throw Decompressor::DecompressionError(); for (uint32_t i=0;i>(rangeDecoder); ch=uint8_t(literalDecoder->decode([&]() { return baseLiteralDecoder.decode(readByte); })); outputStream.writeByte(ch); } } while (literalCount==0x100U); lastIsLiteral=true; } } } } ancient-2.2.0/src/LZCBDecompressor.hpp000066400000000000000000000014551463063262600175710ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZCBDECOMPRESSOR_HPP #define LZCBDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class LZCBDecompressor : public XPKDecompressor { public: LZCBDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~LZCBDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/LZW2Decompressor.cpp000066400000000000000000000036201463063262600175640ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZW2Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool LZW2Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LZW2") || hdr==FourCC("LZW3"); } std::shared_ptr LZW2Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } LZW2Decompressor::LZW2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); _ver=(hdr==FourCC("LZW2"))?2:3; } const std::string &LZW2Decompressor::getSubName() const noexcept { static std::string name2{"XPK-LZW2: LZW2 CyberYAFA compressor"}; static std::string name3{"XPK-LZW3: LZW3 CyberYAFA compressor"}; return (_ver==2)?name2:name3; } void LZW2Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; LSBBitReader bitReader{inputStream}; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint32_t distance{uint32_t(readByte())<<8}; distance|=uint32_t(readByte()); if (!distance) throw Decompressor::DecompressionError(); distance=65536-distance; uint32_t count{uint32_t(readByte())+4}; outputStream.copy(distance,count); } } } } ancient-2.2.0/src/LZW2Decompressor.hpp000066400000000000000000000015011463063262600175650ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZW2DECOMPRESSOR_HPP #define LZW2DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class LZW2Decompressor : public XPKDecompressor { public: LZW2Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~LZW2Decompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _ver{0}; }; } #endif ancient-2.2.0/src/LZW4Decompressor.cpp000066400000000000000000000033751463063262600175750ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZW4Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool LZW4Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LZW4"); } std::shared_ptr LZW4Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } LZW4Decompressor::LZW4Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &LZW4Decompressor::getSubName() const noexcept { static std::string name{"XPK-LZW4: LZW4 CyberYAFA compressor"}; return name; } void LZW4Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1U); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint32_t distance={uint32_t(readByte())<<8U}; distance|=readByte(); if (!distance) throw Decompressor::DecompressionError(); distance=65536U-distance; uint32_t count=uint32_t(readByte())+3; outputStream.copy(distance,count); } } } } ancient-2.2.0/src/LZW4Decompressor.hpp000066400000000000000000000014551463063262600175770ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZW4DECOMPRESSOR_HPP #define LZW4DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class LZW4Decompressor : public XPKDecompressor { public: LZW4Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~LZW4Decompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/LZW5Decompressor.cpp000066400000000000000000000043321463063262600175700ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZW5Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool LZW5Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("LZW5"); } std::shared_ptr LZW5Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } LZW5Decompressor::LZW5Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &LZW5Decompressor::getSubName() const noexcept { static std::string name{"XPK-LZW5: LZW5 CyberYAFA compressor"}; return name; } void LZW5Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto read2Bits=[&]()->uint32_t { return bitReader.readBitsBE32(2U); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; while (!outputStream.eof()) { uint32_t distance,count; auto readld=[&]()->uint32_t { uint32_t ret{uint32_t(readByte())<<8}; ret|=readByte(); if (!ret) throw Decompressor::DecompressionError(); return ret; }; switch (read2Bits()) { case 0: outputStream.writeByte(readByte()); break; case 1U: distance=readld(); count=(distance&3U)+2U; distance=0x4000-(distance>>2U); outputStream.copy(distance,count); break; case 2U: distance=readld(); count=(distance&15U)+2U; distance=0x1000U-(distance>>4U); outputStream.copy(distance,count); break; case 3U: distance=readld(); count=uint32_t(readByte())+3U; distance=0x10000U-distance; outputStream.copy(distance,count); break; default: throw Decompressor::DecompressionError(); } } } } ancient-2.2.0/src/LZW5Decompressor.hpp000066400000000000000000000014551463063262600176000ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZW5DECOMPRESSOR_HPP #define LZW5DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class LZW5Decompressor : public XPKDecompressor { public: LZW5Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~LZW5Decompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/LZWDecoder.cpp000066400000000000000000000015231463063262600164020ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZWDecoder.hpp" #include "common/Common.hpp" namespace ancient::internal { LZWDecoder::LZWDecoder(uint32_t maxCode,uint32_t literalCodes,uint32_t stackLength,uint32_t firstCode) : _maxCode{maxCode}, _literalCodes{literalCodes}, _stackLength{stackLength}, _freeIndex{literalCodes}, _prevCode{firstCode}, _prefix{std::make_unique(maxCode-literalCodes)}, _suffix{std::make_unique(maxCode-literalCodes)}, _stack{std::make_unique(stackLength)} { // nothing needed } void LZWDecoder::reset(uint32_t firstCode) { _freeIndex=_literalCodes; _prevCode=firstCode; } void LZWDecoder::add(uint32_t code) { if (_freeIndex<_maxCode) { _suffix[_freeIndex-_literalCodes]=_newCode; _prefix[_freeIndex-_literalCodes]=_prevCode; _freeIndex++; } _prevCode=code; } } ancient-2.2.0/src/LZWDecoder.hpp000066400000000000000000000030061463063262600164050ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZWDECODER_HPP #define LZWDECODER_HPP #include #include "Decompressor.hpp" #include "OutputStream.hpp" namespace ancient::internal { class LZWDecoder { public: LZWDecoder(uint32_t maxCode,uint32_t literalCodes,uint32_t stackLength,uint32_t firstCode); ~LZWDecoder() noexcept=default; void reset(uint32_t firstCode); void add(uint32_t code); template void write(uint32_t code,bool addNew,F func) { auto suffixLookup=[&](uint32_t value)->uint32_t { if (value>=_freeIndex) throw Decompressor::DecompressionError(); return (value<_literalCodes)?value:_suffix[value-_literalCodes]; }; uint32_t stackPos{0}; uint32_t tmp{_newCode}; if (addNew) code=_prevCode; _newCode=suffixLookup(code); while (code>=_literalCodes) { if (stackPos+1>=_stackLength) throw Decompressor::DecompressionError(); _stack[stackPos++]=_newCode; code=_prefix[code-_literalCodes]; _newCode=suffixLookup(code); } _stack[stackPos++]=_newCode; while (stackPos) func(_stack[--stackPos]); if (addNew) func(tmp); } bool isLiteral(uint32_t code) { return code<_freeIndex; } bool isFull() { return _freeIndex==_maxCode; } uint32_t getCurrentIndex() { return _freeIndex; } private: uint32_t _maxCode; uint32_t _literalCodes; uint32_t _stackLength; uint32_t _freeIndex; uint32_t _prevCode; uint32_t _newCode{0}; std::unique_ptr _prefix; std::unique_ptr _suffix; std::unique_ptr _stack; }; } #endif ancient-2.2.0/src/LZXDecompressor.cpp000066400000000000000000000152571463063262600175140ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "LZXDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "DLTADecode.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "VariableLengthCodeDecoder.hpp" #include "common/CRC32.hpp" #include "common/Common.hpp" #include "common/OverflowCheck.hpp" namespace ancient::internal { bool LZXDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("ELZX") || hdr==FourCC("SLZX"); } std::shared_ptr LZXDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } LZXDecompressor::LZXDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); if (hdr==FourCC("SLZX")) _isSampled=true; // There is no good spec on the LZX header content -> lots of unknowns here if (_packedData.size()<41) throw Decompressor::InvalidFormatError(); // XPK LZX compression is embedded single file of LZX -> read first file. Ignore rest // this will include flags, which need to be zero anyway uint32_t streamHdr=_packedData.readBE32(0); if (streamHdr!=FourCC("LZX\0")) throw Decompressor::InvalidFormatError(); _rawSize=_packedData.readLE32(12); _packedSize=_packedData.readLE32(16); _rawCRC=_packedData.readLE32(32); uint32_t headerCRC{_packedData.readLE32(36)}; uint8_t tmp{_packedData.read8(21)}; if (tmp && tmp!=2) throw Decompressor::InvalidFormatError(); if (tmp==2) _isCompressed=true; _packedOffset=41U+uint32_t(_packedData.read8(40U)); _packedOffset+=_packedData.read8(24U); _packedSize+=_packedOffset; if (_packedSize>_packedData.size()) throw Decompressor::InvalidFormatError(); if (verify) { uint32_t crc{CRC32(_packedData,10,26,0)}; for (uint32_t i=0;i<4;i++) crc=CRC32Byte(0,crc); crc=CRC32(_packedData,40,_packedOffset-40,crc); if (crc!=headerCRC) throw Decompressor::VerificationError(); } } const std::string &LZXDecompressor::getSubName() const noexcept { static std::string nameE{"XPK-ELZX: LZX-compressor"}; static std::string nameS{"XPK-SLZX: LZX-compressor with delta encoding"}; return (_isSampled)?nameS:nameE; } void LZXDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); if (!_isCompressed) { if (_packedSize!=_rawSize) throw Decompressor::DecompressionError(); std::memcpy(rawData.data(),_packedData.data()+_packedOffset,_rawSize); return; } ForwardInputStream inputStream{_packedData,_packedOffset,_packedSize}; LSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBitsBE16(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1U); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; typedef HuffmanDecoder LZXDecoder; // possibly padded/reused later if multiple blocks std::array literalTable; for (uint32_t i=0;i<768U;i++) literalTable[i]=0; LZXDecoder literalDecoder; uint32_t previousDistance{1}; VariableLengthCodeDecoder vlcDecoder{ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,10, 11,11,12,12,13,13,14,14}; while (!outputStream.eof()) { auto createHuffmanTable=[&](LZXDecoder &dec,const auto &bitLengths,uint32_t bitTableLength) { uint8_t minDepth{16}; uint8_t maxDepth{0}; for (uint32_t i=0;imaxDepth) maxDepth=bitLengths[i]; } if (!maxDepth) return; dec.createOrderlyHuffmanTable(bitLengths,bitTableLength); }; uint32_t method{readBits(3U)}; if (method<1U || method>3U) throw Decompressor::DecompressionError(); LZXDecoder distanceDecoder; if (method==3U) { std::array bitLengths; for (uint32_t i=0;i<8U;i++) bitLengths[i]=readBits(3U); createHuffmanTable(distanceDecoder,bitLengths,8U); } size_t blockLength{readBits(8)<<16}; blockLength|=readBits(8)<<8; blockLength|=readBits(8); if (OverflowCheck::sum(blockLength,outputStream.getOffset())>_rawSize) throw Decompressor::DecompressionError(); if (method!=1U) { literalDecoder.reset(); for (uint32_t pos=0,block=0;block<2;block++) { uint32_t adjust{(block)?0:1U}; uint32_t maxPos{(block)?768U:256U}; LZXDecoder bitLengthDecoder; { std::array lengthTable; for (uint32_t i=0;i<20U;i++) lengthTable[i]=readBits(4U); createHuffmanTable(bitLengthDecoder,lengthTable,20U); } while (posmaxPos-pos) count=maxPos-pos; while (count--) literalTable[pos++]=value; }; auto symDecode=[&](uint32_t value)->uint32_t { return (literalTable[pos]+17U-value)%17U; }; switch (symbol) { case 17U: doRepeat(readBits(4U)+3U+adjust,0); break; case 18U: doRepeat(readBits(6U-adjust)+19U+adjust,0); break; case 19U: { uint32_t count{readBit()+3U+adjust}; doRepeat(count,symDecode(bitLengthDecoder.decode(readBit))); } break; default: literalTable[pos++]=symDecode(symbol); break; } } } createHuffmanTable(literalDecoder,literalTable,768U); } while (blockLength) { uint32_t symbol{literalDecoder.decode(readBit)}; if (symbol<256U) { outputStream.writeByte(symbol); blockLength--; } else { symbol-=256U; uint32_t distance; if ((symbol&0x1fU)>=8U && method==3U) { distance=vlcDecoder.decode([&](uint32_t count) { uint32_t tmp={readBits(count-3U)<<3U}; return tmp|distanceDecoder.decode(readBit); },symbol&0x1fU); } else { distance=vlcDecoder.decode(readBits,symbol&0x1fU); if (!distance) distance=previousDistance; } previousDistance=distance; uint32_t count{vlcDecoder.decode(readBits,symbol>>5U)+3U}; if (count>blockLength) throw Decompressor::DecompressionError(); outputStream.copy(distance,count); blockLength-=count; } } } if (verify) { uint32_t crc{CRC32(rawData,0,_rawSize,0)}; if (crc!=_rawCRC) throw Decompressor::VerificationError(); } if (_isSampled) DLTADecode::decode(rawData,rawData,0,_rawSize); } } ancient-2.2.0/src/LZXDecompressor.hpp000066400000000000000000000017001463063262600175050ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZXDECOMPRESSOR_HPP #define LZXDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class LZXDecompressor : public XPKDecompressor { public: LZXDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~LZXDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; bool _isSampled{false}; bool _isCompressed{false}; size_t _packedSize{0}; size_t _packedOffset{0}; size_t _rawSize{0}; uint32_t _rawCRC{0}; }; } #endif ancient-2.2.0/src/MASHDecompressor.cpp000066400000000000000000000060361463063262600175620ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "MASHDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "VariableLengthCodeDecoder.hpp" #include "common/Common.hpp" namespace ancient::internal { bool MASHDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("MASH"); } std::shared_ptr MASHDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } MASHDecompressor::MASHDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &MASHDecompressor::getSubName() const noexcept { static std::string name{"XPK-MASH: LZRW-compressor"}; return name; } void MASHDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1U); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; size_t rawSize{rawData.size()}; ForwardOutputStream outputStream{rawData,0,rawSize}; HuffmanDecoder litDecoder { HuffmanCode{1,0b000000,0U}, HuffmanCode{2,0b000010,1U}, HuffmanCode{3,0b000110,2U}, HuffmanCode{4,0b001110,3U}, HuffmanCode{5,0b011110,4U}, HuffmanCode{6,0b111110,5U}, HuffmanCode{6,0b111111,6U} }; VariableLengthCodeDecoder vlcDecoder{5,7,9,10,11,12,13,14}; while (!outputStream.eof()) { uint32_t litLength{litDecoder.decode(readBit)}; if (litLength==6) { uint32_t litBits; for (litBits=1;litBits<=17;litBits++) if (!readBit()) break; if (litBits==17) throw Decompressor::DecompressionError(); litLength=readBits(litBits)+(1< &state,bool verify); ~MASHDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/MMCMPDecompressor.cpp000066400000000000000000000163651463063262600177110ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "MMCMPDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/OverflowCheck.hpp" #include "common/Common.hpp" #include namespace ancient::internal { bool MMCMPDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("ziRC"); } std::shared_ptr MMCMPDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,exactSizeKnown,verify); } MMCMPDecompressor::MMCMPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData{packedData} { if (!detectHeader(packedData.readBE32(0)) || packedData.readBE32(4U)!=FourCC("ONia") || packedData.readLE16(8U)!=14U || packedData.size()<24U) throw InvalidFormatError(); _version=packedData.readLE16(10U); _blocks=packedData.readLE16(12U); _blocksOffset=packedData.readLE32(18U); _rawSize=packedData.readLE32(14U); if (_rawSize>getMaxRawSize()) throw InvalidFormatError(); if (OverflowCheck::sum(_blocksOffset,uint32_t(_blocks)*4U)>packedData.size()) throw InvalidFormatError(); _packedSize=0; for (uint32_t i=0;i<_blocks;i++) { uint32_t blockAddr{packedData.readLE32(OverflowCheck::sum(_blocksOffset,i*4U))}; if (OverflowCheck::sum(blockAddr,20U)>=packedData.size()) throw InvalidFormatError(); uint32_t blockSize{OverflowCheck::sum(packedData.readLE32(blockAddr+4U),uint32_t(packedData.readLE16(blockAddr+12U))*8U+20U)}; _packedSize=std::max(_packedSize,OverflowCheck::sum(blockAddr,blockSize)); } if (_packedSize>packedData.size()) throw InvalidFormatError(); } const std::string &MMCMPDecompressor::getName() const noexcept { static std::string name="MMCMP: Music Module Compressor"; return name; } size_t MMCMPDecompressor::getPackedSize() const noexcept { return _packedSize; } size_t MMCMPDecompressor::getRawSize() const noexcept { return _rawSize; } void MMCMPDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); // MMCMP allows gaps in data. Although not used in practice still we memset before decompressing to be sure std::memset(rawData.data(),0,rawData.size()); const std::array extraBits8{3,3,3,3, 2,1,0,0}; const std::array extraBits16{4,4,4,4, 3,2,1,0, 0,0,0,0, 0,0,0,0}; for (uint32_t i=0;i<_blocks;i++) { uint32_t blockAddr{_packedData.readLE32(_blocksOffset+i*4U)}; uint32_t unpackedBlockSize{_packedData.readLE32(blockAddr)}; uint32_t packedBlockSize{_packedData.readLE32(blockAddr+4U)}; uint32_t fileChecksum{_packedData.readLE32(blockAddr+8U)}; uint32_t subBlocks{_packedData.readLE16(blockAddr+12U)}; uint16_t flags{_packedData.readLE16(blockAddr+14U)}; uint32_t packTableSize{_packedData.readLE16(blockAddr+16U)}; if (packTableSize>packedBlockSize) throw DecompressionError(); uint32_t bitCount{uint32_t{_packedData.readLE16(blockAddr+18U)}+1U}; ForwardInputStream inputStream{_packedData,OverflowCheck::sum(blockAddr,subBlocks*8U,20U,packTableSize),OverflowCheck::sum(blockAddr,subBlocks*8U,20U,packedBlockSize)}; LSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; uint32_t currentSubBlock{0}; uint32_t outputOffset{0}; uint32_t outputSize{0}; auto readNextSubBlock=[&]() { if (currentSubBlock>=subBlocks) throw DecompressionError(); outputOffset=_packedData.readLE32(blockAddr+currentSubBlock*8U+20U); outputSize=_packedData.readLE32(blockAddr+currentSubBlock*8U+24U); if (OverflowCheck::sum(outputOffset,outputSize)>_rawSize) throw DecompressionError(); currentSubBlock++; }; uint32_t checksum{0}; uint32_t checksumPartial{0}; uint32_t checksumRot{0}; auto writeByte=[&](uint8_t value,bool allowOverrun=false) { while (!outputSize) { if (allowOverrun && currentSubBlock>=subBlocks) return; readNextSubBlock(); } outputSize--; rawData[outputOffset++]=value; if (verify) { if (_version>=0x1310U) { checksum^=value; checksum=(checksum<<1U)|(checksum>>31U); } else { checksumPartial|=((uint32_t)value)<8) throw DecompressionError(); std::array oldValue{0,0}; uint32_t chIndex{0}; uint32_t tableOffset{blockAddr+subBlocks*8U+20U}; for (uint32_t j=0;j=threshold) { if (uint32_t newBitCount{(readBits(extraBitCount)|((value-threshold)<=packTableSize) throw DecompressionError(); value=_packedData[tableOffset+value]; if (flags&0x2U) { // delta value+=oldValue[chIndex]; oldValue[chIndex]=value; if (flags&0x100U) chIndex^=1U; // stereo } writeByte(value); } } else { // 16 bit compression // shameless copy-paste from 8-bit variant, with minor changes if (bitCount>16) throw DecompressionError(); std::array oldValue{0,0}; uint32_t chIndex{0}; for (uint32_t j=0;j=threshold) { if (uint32_t newBitCount{(readBits(extraBitCount)|((value-threshold)<>=1; if (flags&0x2U) { // delta sValue+=oldValue[chIndex]; oldValue[chIndex]=sValue; if (flags&0x100U) chIndex^=1U; // stereo } else if (!(flags&0x200U)) sValue^=0x8000; // abs16 if (flags&0x400U) { // big ending writeByte(sValue>>8); writeByte(sValue,true); } else { // little endian writeByte(sValue); writeByte(sValue>>8,true); } } } if (verify && checksum!=fileChecksum) throw VerificationError(); } } } ancient-2.2.0/src/MMCMPDecompressor.hpp000066400000000000000000000015341463063262600177060ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef MMCMPDECOMPRESSOR_HPP #define MMCMPDECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class MMCMPDecompressor : public Decompressor { public: MMCMPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); ~MMCMPDecompressor() noexcept=default; const std::string &getName() const noexcept final; size_t getPackedSize() const noexcept final; size_t getRawSize() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: const Buffer &_packedData; uint32_t _packedSize{0}; uint32_t _rawSize{0}; uint32_t _blocksOffset{0}; uint32_t _blocks{0}; uint16_t _version{0}; }; } #endif ancient-2.2.0/src/NONEDecompressor.cpp000066400000000000000000000022671463063262600175730ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "NONEDecompressor.hpp" #include "common/Common.hpp" namespace ancient::internal { bool NONEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("NONE"); } std::shared_ptr NONEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } NONEDecompressor::NONEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &NONEDecompressor::getSubName() const noexcept { static std::string name{"XPK-NONE: Null compressor"}; return name; } void NONEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_packedData.size()) throw Decompressor::DecompressionError(); std::memcpy(rawData.data(),_packedData.data(),_packedData.size()); } } ancient-2.2.0/src/NONEDecompressor.hpp000066400000000000000000000014551463063262600175760ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef NONEDECOMPRESSOR_HPP #define NONEDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class NONEDecompressor : public XPKDecompressor { public: NONEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~NONEDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/NUKEDecompressor.cpp000066400000000000000000000065121463063262600175730ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "NUKEDecompressor.hpp" #include "DLTADecode.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "VariableLengthCodeDecoder.hpp" #include "common/Common.hpp" namespace ancient::internal { bool NUKEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("NUKE") || hdr==FourCC("DUKE"); } std::shared_ptr NUKEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } NUKEDecompressor::NUKEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); if (hdr==FourCC("DUKE")) _isDUKE=true; } const std::string &NUKEDecompressor::getSubName() const noexcept { static std::string nameN{"XPK-NUKE: LZ77-compressor"}; static std::string nameD{"XPK-DUKE: LZ77-compressor with delta encoding"}; return (_isDUKE)?nameD:nameN; } void NUKEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { // there are 2 streams, reverse stream for bytes and // normal stream for bits, the bit stream is divided // into single bit, 2 bit, 4 bit and random accumulator ForwardInputStream forwardInputStream{_packedData,0,_packedData.size()}; BackwardInputStream backwardInputStream{_packedData,0,_packedData.size()}; forwardInputStream.link(backwardInputStream); backwardInputStream.link(forwardInputStream); MSBBitReader bit1Reader{forwardInputStream}; MSBBitReader bit2Reader{forwardInputStream}; LSBBitReader bit4Reader{forwardInputStream}; MSBBitReader bitXReader{forwardInputStream}; auto readBit=[&]()->uint32_t { return bit1Reader.readBitsBE16(1); }; auto read2Bits=[&]()->uint32_t { return bit2Reader.readBitsBE16(2); }; auto read4Bits=[&]()->uint32_t { return bit4Reader.readBitsBE32(4); }; auto readBits=[&](uint32_t count)->uint32_t { return bitXReader.readBitsBE16(count); }; auto readByte=[&]()->uint8_t { return backwardInputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; VariableLengthCodeDecoder vlcDecoder{ 4, 6, 8, 9, -4, 7, 9,11,13,14, -5, 7, 9,11,13,14}; for (;;) { if (!readBit()) { uint32_t count{0}; if (readBit()) { count=1; } else { uint32_t tmp; do { tmp=read2Bits(); if (tmp) count+=5-tmp; else count+=3; } while (!tmp); } for (uint32_t i=0;i &state,bool verify); ~NUKEDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; bool _isDUKE{false}; }; } #endif ancient-2.2.0/src/OutputStream.cpp000066400000000000000000000121061463063262600171130ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include "OutputStream.hpp" // for exceptions #include "Decompressor.hpp" #include "common/Common.hpp" #include "common/OverflowCheck.hpp" namespace ancient::internal { ForwardOutputStreamBase::ForwardOutputStreamBase(Buffer &buffer,size_t startOffset) : _buffer{buffer}, _startOffset{startOffset}, _currentOffset{startOffset} { // should call ensureSize but can't } void ForwardOutputStreamBase::writeByte(uint8_t value) { ensureSize(_currentOffset+1); _buffer[_currentOffset++]=value; } uint8_t ForwardOutputStreamBase::copy(size_t distance,size_t count) { ensureSize(OverflowCheck::sum(_currentOffset,count)); if (!distance || OverflowCheck::sum(_startOffset,distance)>_currentOffset) throw Decompressor::DecompressionError(); uint8_t ret{0}; for (size_t i=0;i_currentOffset) { size_t prevSize{prevBuffer.size()}; if (_startOffset+distance>_currentOffset+prevSize) throw Decompressor::DecompressionError(); size_t prevDist{_startOffset+distance-_currentOffset}; prevCount=std::min(count,prevDist); const uint8_t *prev{&prevBuffer[prevSize-prevDist]}; for (size_t i=0;i_currentOffset) { prevCount=std::min(count,_startOffset+distance-_currentOffset); for (size_t i=0;i_currentOffset) throw Decompressor::DecompressionError(); return &_buffer[_currentOffset-distance]; } uint8_t *ForwardOutputStreamBase::history(size_t distance) { if (OverflowCheck::sum(distance,_startOffset)>_currentOffset) throw Decompressor::DecompressionError(); return &_buffer[_currentOffset-distance]; } void ForwardOutputStreamBase::produce(const Buffer &src) { ensureSize(OverflowCheck::sum(_currentOffset,src.size())); std::memcpy(&_buffer[_currentOffset],src.data(),src.size()); _currentOffset+=src.size(); } // --- ForwardOutputStream::ForwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset) : ForwardOutputStreamBase{buffer,startOffset}, _endOffset{endOffset} { if (_startOffset>_endOffset || _endOffset>_buffer.size()) throw Decompressor::DecompressionError(); } void ForwardOutputStream::reset(size_t startOffset,size_t endOffset) { _currentOffset=_startOffset=startOffset; _endOffset=endOffset; if (_startOffset>_endOffset || _endOffset>_buffer.size()) throw Decompressor::DecompressionError(); } void ForwardOutputStream::ensureSize(size_t offset) { if (offset>_endOffset) throw Decompressor::DecompressionError(); } // --- AutoExpandingForwardOutputStream::AutoExpandingForwardOutputStream(Buffer &buffer) : ForwardOutputStreamBase{buffer,0} { // nothing needed } AutoExpandingForwardOutputStream::~AutoExpandingForwardOutputStream() noexcept { // trim if (_hasExpanded && _currentOffset!=_buffer.size()) _buffer.resize(_currentOffset); } void AutoExpandingForwardOutputStream::ensureSize(size_t offset) { if (offset>Decompressor::getMaxRawSize()) throw Decompressor::DecompressionError(); if (offset>_buffer.size()) { _buffer.resize(offset+_advance); _hasExpanded=true; } } // --- BackwardOutputStream::BackwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset) : _buffer{buffer}, _startOffset{startOffset}, _currentOffset{endOffset}, _endOffset{endOffset} { if (_startOffset>_endOffset || _currentOffset>buffer.size() || _endOffset>buffer.size()) throw Decompressor::DecompressionError(); } void BackwardOutputStream::writeByte(uint8_t value) { if (_currentOffset<=_startOffset) throw Decompressor::DecompressionError(); _buffer[--_currentOffset]=value; } uint8_t BackwardOutputStream::copy(size_t distance,size_t count) { if (!distance || OverflowCheck::sum(_startOffset,count)>_currentOffset || OverflowCheck::sum(_currentOffset,distance)>_endOffset) throw Decompressor::DecompressionError(); uint8_t ret{0}; for (size_t i=0;i #include #include "common/Buffer.hpp" namespace ancient::internal { class ForwardOutputStreamBase { public: ForwardOutputStreamBase(Buffer &buffer,size_t startOffset); virtual ~ForwardOutputStreamBase() noexcept=default; void writeByte(uint8_t value); uint8_t copy(size_t distance,size_t count); uint8_t copy(size_t distance,size_t count,const Buffer &prevBuffer); uint8_t copy(size_t distance,size_t count,uint8_t defaultChar); uint8_t *history(size_t distance); const uint8_t *history(size_t distance) const; void produce(const Buffer &src); size_t getOffset() const { return _currentOffset; } protected: virtual void ensureSize(size_t offset)=0; Buffer &_buffer; size_t _startOffset; size_t _currentOffset; }; class ForwardOutputStream : public ForwardOutputStreamBase { public: ForwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset); ~ForwardOutputStream() noexcept=default; void reset(size_t startOffset,size_t endOffset); bool eof() const { return _currentOffset==_endOffset; } size_t getEndOffset() const { return _endOffset; } protected: void ensureSize(size_t offset) final; private: size_t _endOffset; }; class AutoExpandingForwardOutputStream : public ForwardOutputStreamBase { public: AutoExpandingForwardOutputStream(Buffer &buffer); ~AutoExpandingForwardOutputStream() noexcept; protected: void ensureSize(size_t offset) final; private: static constexpr size_t _advance{65536U}; bool _hasExpanded=false; }; class BackwardOutputStream { public: BackwardOutputStream(Buffer &buffer,size_t startOffset,size_t endOffset); ~BackwardOutputStream() noexcept=default; void writeByte(uint8_t value); uint8_t copy(size_t distance,size_t count); bool eof() const { return _currentOffset==_startOffset; } size_t getOffset() const { return _currentOffset; } private: Buffer &_buffer; size_t _startOffset; size_t _currentOffset; size_t _endOffset; }; } #endif ancient-2.2.0/src/PPDecompressor.cpp000066400000000000000000000335011463063262600173460ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "PPDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { PPDecompressor::PPState::PPState(uint32_t mode) noexcept : _cachedMode{mode} { // nothing needed } bool PPDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("PP11") || hdr==FourCC("PP20") || hdr==FourCC("PX20") || hdr==FourCC("CHFC") // Sky High Stuntman || hdr==FourCC("DEN!") // Jewels - Crossroads || hdr==FourCC("DXS9") // Hopp oder Top, Punkt Punkt Punkt || hdr==FourCC("H.D.") // F1 Challenge || hdr==FourCC("RVV!"); // Hoi AGA Remix } bool PPDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("PWPK"); } std::shared_ptr PPDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,exactSizeKnown,verify); } std::shared_ptr PPDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } PPDecompressor::PPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData{packedData} { if (!exactSizeKnown || packedData.size()<16U) throw InvalidFormatError(); // no scanning support _dataStart=_packedData.size()-4; uint32_t hdr{packedData.readBE32(0)}; if (!detectHeader(hdr)) throw InvalidFormatError(); if (hdr==FourCC("PX20")) { if (packedData.size()<18U) throw InvalidFormatError(); _isObsfuscated=true; } uint32_t mode{packedData.readBE32(_isObsfuscated?6U:4U)}; if (mode!=0x9090909 && mode!=0x90a0a0a && mode!=0x90a0b0b && mode!=0x90a0c0c && mode!=0x90a0c0d) throw InvalidFormatError(); for (uint32_t i=0;i<4;i++) { _modeTable[i]=mode>>24U; mode<<=8; } uint32_t tmp{packedData.readBE32(_dataStart)}; _rawSize=tmp>>8U; _startShift=tmp&0xffU; if (!_rawSize || _startShift>=0x20U || _rawSize>getMaxRawSize()) throw InvalidFormatError(); } PPDecompressor::PPDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr) || packedData.size()<4U) throw InvalidFormatError(); _dataStart=_packedData.size()-4; uint32_t mode; if (state.get()) { mode=static_cast(state.get())->_cachedMode; } else { mode=packedData.readBE32(_dataStart); if (mode>4) throw InvalidFormatError(); state.reset(new PPState(mode)); if (_dataStart<4U) throw InvalidFormatError(); _dataStart-=4; } const std::array modeMap{0x9090909,0x90a0a0a,0x90a0b0b,0x90a0c0c,0x90a0c0d}; mode=modeMap[mode]; for (uint32_t i=0;i<4;i++) { _modeTable[i]=mode>>24; mode<<=8; } uint32_t tmp{packedData.readBE32(_dataStart)}; _rawSize=tmp>>8U; _startShift=tmp&0xffU; if (!_rawSize || _startShift>=0x20U || _rawSize>getMaxRawSize()) throw InvalidFormatError(); _isXPK=true; } const std::string &PPDecompressor::getName() const noexcept { static std::string name{"PP: PowerPacker"}; return name; } const std::string &PPDecompressor::getSubName() const noexcept { static std::string name{"XPK-PWPK: PowerPacker"}; return name; } size_t PPDecompressor::getPackedSize() const noexcept { return 0; } size_t PPDecompressor::getRawSize() const noexcept { return _rawSize; } /* Here comes the unobsfuscation (It would be stretch to call it as encryption cracking) The PowerPacker uses following method for obsfuscating files. First string of maximum of 16 characters is taken as a password (ISO8859-1 assumed, although anything with null termination goes. In fact the 16 characters limit is UI thing) Then both a 32-bit key and 16-bit checksum are calculated from the string. the algorithm for the key is as follows uint32_t ret=0; while (uint8_t ch=*(str++)) { ret<<<1U; ret+=ch; swap(ret); } return ret; and the algorithm for the checksum (stored in header at offset 4) is as follows uint16_t ret=0; while (uint8_t ch=*(str++)) { ret>>>=ch&0xfU; ret+=ch; } return ret; When the official decompressor tries to decrypt, it first checks the checksum of the key. (This is the only checksum PowerPacker has) After that the 32-bit is used to xor every long word read from the stream. Now, there are many ways you could skin this cat. One simple thing would be to attack the password. By limiting to printable characters and having dictionary of usual word would greatly help here. But this would add complexity to the code and by adding any dictionary would just be more ballast. Since the actual decompression is 32-bit xor, it would also be beneficial to do just a histogram with 4 buckets. However, the input files are pretty short and PP algorithm is very good at mangling bytes and bits so there is not neccessarily enough data to get a good idea of the key. Going through the 32-bit keyspace is not complicated in itself either. The only complication is that since there are not checksum for the output, the only way to validate the key is to check that input forms a correct bitstream. This is what ppcrack does. It takes minutes / hours or maybe even a day to go through a single file depending the size of the file, possible key and whether you are lucky. It obviously could be optimized by parallelization (or even dumping the whole set to GPU). Although this would bring results faster, it would make this library system dependent and also probably would raise a few eyebrows (am I accidentally running mining operation?!?) so the best option is actually to do it smarter. And smarter here is really a stretch but we try Due to the nature of the format, the fail paths happen early in the process, and if you know that some tree path does not yield results, it can be completely ignored. Thus making a tree of the possible choices taken wrt. to key bits is beneficial. In naive complexity calculation this would divide the effort into half. But in practice the algorithm fails quickly i.e. when some input does not fit it is usually the end of the possibilities.. And this will bring the time into sub-second to go through the keyspace. This would in fact be enough, but we are not done yet. We can leverage the weaknesses in the implementation. First of all, first bit of the input needs to be zero (literal character). This reduces the keyspace to a maximum of 31-bits. But this is not all. There are filler bits at the end. Last long word of the input is something where possible incomplete word is stored (ranging from 1 to 31 bits of data). It would have been prudent to clear these bits after the xor-bits but they are not. Instead they are filled pre-xor with (depending on version) 1. address of buffer when encoded + size. And since mostly Amiga allocates buffers with 8 byte alignment this means 3 free bits of key 2. stack pointer. Due to the memory allocation alignment and predictable path we can guess that last three bits are 110 (This is not applicable when PP library is used) If these approaches fail the "slower" path is chosen. This makes the algorithm fast in general, and even faster when we can leverage the filler bits. I believe you could run this on unaccelerated Amiga 500 and get the key in no time. :D For the actual user, the obsfuscation is now nothing but a minor speed bump. Little bit on the code style. I started this with co-routines in my mind. But I do not want to go with c++20. At least not yet. Also it would be questionable whether it would perform. The next thing that I did I removed the exception throwing on the failure case for finding a key. Unfortunately it doubles the running speed compared to the c-style flagged variable break. I think the code went from readable to whacky in this process. Sorry about that In any case this should serve as a warning to anyone trying to create their own crypto. */ void PPDecompressor::findKeyRound(BackwardInputStream &inputStream,LSBBitReader &bitReader,uint32_t keyBits,uint32_t keyMask,uint32_t outputPosition,uint32_t &iterCount) { uint32_t inputOffset; uint32_t bufContent; uint8_t bufLength; uint32_t savedOutputPosition; bool failed{false}; auto readBit=[&]()->uint32_t { if (failed) return 0; uint32_t bitPos{32U-bitReader.getBufLength()}; if (bitPos==32U) bitPos=0; if (!bitPos && inputStream.getOffset()<=10U) { failed=true; return 0; } uint32_t bit{bitReader.readBitsBE32(1U)}; if ((keyMask>>bitPos)&1U) return bit^((keyBits>>bitPos)&1U); // meh uint32_t tmpInputOffset{uint32_t(inputStream.getOffset())}; uint32_t tmpBufContent{bitReader.getBufContent()}; uint8_t tmpBufLength{bitReader.getBufLength()}; // try 0 inputStream.setOffset(inputOffset); bitReader.reset(bufContent,bufLength); findKeyRound(inputStream,bitReader,keyBits,keyMask|(1U<uint32_t { if (failed) return 0; uint32_t ret{0}; while (count--) { ret=(ret<<1U)|readBit(); if (failed) return 0; } return ret; }; auto consumeBits=[&](uint32_t count) { if (failed) return; uint8_t bufLength{bitReader.getBufLength()}; uint32_t bits{std::min(uint32_t(bufLength),count)}; bitReader.readBitsBE32(bits); count-=bits; if (!count) return; uint32_t bytes{(count>>3U)&~3U}; uint32_t offset{uint32_t(inputStream.getOffset())}; if (offsetoutputPosition) failed=true; if (failed) break; consumeBits(count*8U); outputPosition-=count; } if (!outputPosition || failed) break; uint32_t modeIndex{readBits(2)}; if (failed) break; uint32_t count,distance; if (modeIndex==3) { distance=readBits(readBit()?_modeTable[modeIndex]:7)+1; if (failed) break; count=5; for (;;) { uint32_t tmp{readBits(3)}; if (failed) break; count+=tmp; if (tmp<7) break; } } else { count=modeIndex+2; distance=readBits(_modeTable[modeIndex])+1; } if (outputPosition+distance>_rawSize || count>outputPosition) failed=true; if (failed) break; outputPosition-=count; iterCount++; } if (failed) return; // If not all bits are resolved, that is bad if (keyMask==0xffff'ffffU) throw DoneException(keyBits); } void PPDecompressor::findKey(uint32_t keyBits,uint32_t keyMask) { BackwardInputStream inputStream{_packedData,10,_dataStart}; LSBBitReader bitReader{inputStream}; bitReader.readBitsBE32(_startShift); uint32_t iterCount=0; findKeyRound(inputStream,bitReader,keyBits,keyMask,uint32_t(_rawSize),iterCount); } void PPDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); uint32_t key=0; if (_isObsfuscated) { uint32_t fillerData{_packedData.readBE32(_dataStart-4U)}; // although this not help too much since we always start from zero, it will // make the stack frame shorter uint32_t keyBits{fillerData}; uint32_t keyMask{1U<<_startShift}; keyBits&=keyMask; // now the fuzzy try { if (_startShift) { // SP uint32_t bitCount{std::min(uint32_t(_startShift),3U)}; uint32_t bitMask{((1U< bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return rotateBits(bitReader.readBitsGeneric(count,[&](){ return std::make_pair(inputStream.readBE32()^key,32U); }),count); }; auto readBit=[&]()->uint32_t { return readBits(1); }; readBits(_startShift); BackwardOutputStream outputStream{rawData,0,_rawSize}; for (;;) { if (!readBit()) { uint32_t count{1}; // This does not make much sense I know. But it is what it is... for (;;) { uint32_t tmp{readBits(2)}; count+=tmp; if (tmp<3) break; } for (uint32_t i=0;i namespace ancient::internal { class PPDecompressor : public Decompressor, public XPKDecompressor { private: class PPState : public XPKDecompressor::State { public: PPState(uint32_t mode) noexcept; ~PPState() noexcept=default; uint32_t _cachedMode; }; public: PPDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); PPDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~PPDecompressor() noexcept=default; const std::string &getName() const noexcept final; const std::string &getSubName() const noexcept final; size_t getPackedSize() const noexcept final; size_t getRawSize() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: class DoneException : public std::exception { public: DoneException(uint32_t key) noexcept : _key(key) {} ~DoneException() noexcept=default; uint32_t getKey() const noexcept { return _key; } private: uint32_t _key; }; void findKeyRound(BackwardInputStream &inputStream,LSBBitReader &bitReader,uint32_t keyBits,uint32_t keyMask,uint32_t outputPosition,uint32_t &iterCount); void findKey(uint32_t keyBits,uint32_t keyMask); const Buffer &_packedData; size_t _dataStart{0}; size_t _rawSize{0}; uint8_t _startShift{0}; std::array _modeTable; bool _isObsfuscated{false}; bool _isXPK{false}; }; } #endif ancient-2.2.0/src/PPMQDecompressor.cpp000066400000000000000000000375741463063262600176220ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "common/Common.hpp" #include "PPMQDecompressor.hpp" #include "RangeDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "RangeDecoder.hpp" #include "FrequencyTree.hpp" #include #include #include namespace ancient::internal { bool PPMQDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("PPMQ"); } std::shared_ptr PPMQDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } PPMQDecompressor::PPMQDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &PPMQDecompressor::getSubName() const noexcept { static std::string name{"XPK-PPMQ: PPM compressor"}; return name; } void PPMQDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { class BitReader : public RangeDecoder::BitReader { public: BitReader(ForwardInputStream &stream) noexcept : _reader{stream} { // nothing needed } ~BitReader() noexcept=default; uint32_t readBit() final { return _reader.readBitsBE32(1); } uint32_t readBits(uint32_t bitCount) { return _reader.readBitsBE32(bitCount); } private: MSBBitReader _reader; }; ForwardInputStream inputStream{_packedData,0,_packedData.size(),16U}; ForwardOutputStream outputStream{rawData,0,rawData.size()}; BitReader bitReader{inputStream}; uint32_t history{0}; uint8_t history5{0}; auto addToHistory=[&](uint8_t ch) { history5=history>>24U; history=(history<<8U)|ch; }; for (uint32_t i=0;i<5U;i++) { // files shorter than 5 bytes are not supported by the encoder. // In practice this most probably just means padding if (outputStream.eof()) return; uint8_t ch{uint8_t(bitReader.readBits(8U))}; outputStream.writeByte(ch); addToHistory(ch); } RangeDecoder decoder{bitReader,uint16_t(bitReader.readBits(16))}; class InclusionList { public: class InclusionCallback { public: InclusionCallback(InclusionList &parent) noexcept { parent.registerCallback(this); } ~InclusionCallback() noexcept=default; virtual void symbolIncluded(uint8_t symbol) noexcept=0; virtual void symbolExcluded(uint8_t symbol) noexcept=0; }; InclusionList() noexcept { for (uint32_t i=0;i<256U;i++) _tree.set(i,1); } ~InclusionList() noexcept=default; void reset() noexcept { _tree.onNotOne([&](uint32_t i) { _tree.set(i,1); for (auto callback : _callbacks) callback->symbolIncluded(i); }); } void exclude(uint8_t symbol) noexcept { if (_tree[symbol]) { _tree.set(symbol,0); for (auto callback : _callbacks) callback->symbolExcluded(symbol); } } bool isIncluded(uint8_t symbol) const noexcept { return _tree[symbol]; } uint32_t getTotal() const noexcept { return _tree.getTotal(); } uint8_t decode(uint16_t value,uint16_t &low,uint16_t &freq) const { return _tree.decode(value,low,freq); } private: void registerCallback(InclusionCallback *callback) noexcept { _callbacks.push_back(callback); } FrequencyTree _tree; std::vector _callbacks; }; class ShadedFrequencyTree : public InclusionList::InclusionCallback { public: ShadedFrequencyTree(InclusionList &inclusionList) noexcept : InclusionCallback{inclusionList}, _inclusionList{inclusionList} { for (uint32_t i=0;i<256U;i++) _charCounts[i]=0; } ~ShadedFrequencyTree() noexcept=default; uint16_t operator[](uint8_t symbol) const { return _tree[symbol]; } void add(uint8_t symbol,int16_t freq) { if (_inclusionList.isIncluded(symbol)) _tree.add(symbol,freq); _charCounts[symbol]+=freq; } void set(uint8_t symbol,uint16_t freq) { if (_inclusionList.isIncluded(symbol)) _tree.set(symbol,freq); _charCounts[symbol]=freq; } uint16_t getTotal() const noexcept { return _tree.getTotal(); } void excludeAll() noexcept { _tree.onNotZero([&](uint32_t i) { _inclusionList.exclude(i); }); } uint8_t decode(uint16_t value,uint16_t &low,uint16_t &freq) const { return _tree.decode(value,low,freq); } private: void symbolIncluded(uint8_t symbol) noexcept final { _tree.set(uint16_t(symbol),_charCounts[symbol]); } void symbolExcluded(uint8_t symbol) noexcept final { _tree.set(uint16_t(symbol),0); } InclusionList &_inclusionList; FrequencyTree _tree; std::array _charCounts; }; // Sparse frequency map with MTF, no good data type for this one. // Note that there is no sense to do MTF here. It is just something // the original implementation did (maybe to optimize linked list traversing) // Worse yet, this embeds to the exclusion logic in the original. // By using just tree and separate MFT, the mem-usage would be in gigabytes -> lets not do that // No matter how I rotate this rubics cube, the star's wont align to do something nice // so lets do something ugly class ShadedSparseMTFFrequencyList { public: struct Node { uint16_t freq; uint8_t symbol; }; ShadedSparseMTFFrequencyList(InclusionList &inclusionList) noexcept : _inclusionList{inclusionList} { // nothing needed } ~ShadedSparseMTFFrequencyList() noexcept=default; uint16_t getTotal() const noexcept { uint16_t ret{0}; for (auto &node : _nodes) { if (_inclusionList.isIncluded(node.symbol)) ret+=node.freq; } return ret; } size_t size() const noexcept { return _nodes.size(); } void excludeAll() noexcept { for (auto &node : _nodes) _inclusionList.exclude(node.symbol); } void scale() noexcept { for (auto it=_nodes.begin();it!=_nodes.end();) { auto next=it++; it->freq>>=1U; if (!it->freq) _nodes.erase(it); it=next; } } Node &decode(uint16_t value,uint16_t &low,uint16_t &freq) { uint16_t tmp{0}; for (auto it=_nodes.begin();it!=_nodes.end();it++) { auto &node{*it}; if (_inclusionList.isIncluded(node.symbol)) { if (value _nodes; InclusionList &_inclusionList; }; // After some eye-bleach we are ready to continue class Model { public: Model(RangeDecoder &decoder,InclusionList &inclusionList) noexcept : _decoder{decoder}, _inclusionList{inclusionList} { // nothing needed } virtual ~Model() noexcept=default; virtual bool decode(uint32_t history,uint8_t history5,uint8_t &ch)=0; virtual void mark(uint8_t ch) noexcept=0; protected: RangeDecoder &_decoder; InclusionList &_inclusionList; }; class Model2 : public Model { public: using contextFunc=std::tuple(*)(uint32_t,uint8_t) noexcept; Model2(RangeDecoder &decoder,InclusionList &inclusionList,contextFunc cf) noexcept : Model{decoder,inclusionList}, _cf{cf} { for (uint32_t i=0;i<32;i++) for (uint32_t j=0;j<18;j++) { _freqs[i][j]=1U; _totals[i][j]=j?(j<<2)+1U:2U; } } ~Model2() noexcept=default; bool decode(uint32_t history,uint8_t history5,uint8_t &ch) final { auto context{_cf(history,history5)}; auto scale=[&](Context &ctx,uint16_t total) { if (total+ctx.escapeFreq==0x4000U) { ctx.escapeFreq=(ctx.escapeFreq>>1U)+1U; ctx.nodes.scale(); } }; if (auto it=_contexts.find(context);it!=_contexts.end()) { Context &ctx{it->second}; if (ctx.nodes.size()==1U) { auto &node{ctx.nodes.front()}; uint16_t count{std::min(node.freq,uint16_t{17U})}; uint32_t index{std::get<0>(context)&0x1fU}; if (_totals[index][count]>16300U) { _freqs[index][count]>>=1U; _totals[index][count]>>=1U; if (!_freqs[index][count]) { _freqs[index][count]++; _totals[index][count]+=20U; } } if (node.freq>16300U) node.freq>>=1U; if (_inclusionList.isIncluded(node.symbol)) { uint16_t freq{_freqs[index][count]}; uint16_t total{_totals[index][count]}; uint16_t value{_decoder.decode(total)}; if (value1U) ctx.escapeFreq--; node.freq++; ch=node.symbol; scale(ctx,total+1U); return true; } } _delayedContext=context; _addNewContext=true; return false; } void mark(uint8_t ch) noexcept final { if (_addNewContext) { if (auto it=_contexts.find(_delayedContext);it!=_contexts.end()) { it->second.nodes.addNew(ch); } else { _contexts.emplace(_delayedContext,Context{_inclusionList,ch}); } _addNewContext=false; } } private: struct Context { Context(InclusionList &inclusionList,uint8_t ch) noexcept : escapeFreq{1U}, nodes{inclusionList} { nodes.addNew(ch); }; ~Context() noexcept=default; uint16_t escapeFreq; ShadedSparseMTFFrequencyList nodes; }; contextFunc _cf; bool _addNewContext=false; std::tuple _delayedContext; std::map,Context> _contexts; std::array,32> _freqs; std::array,32> _totals; }; class Model1 : public Model { public: using contextFunc=std::pair(*)(uint32_t) noexcept; Model1(RangeDecoder &decoder,InclusionList &inclusionList,contextFunc cf) noexcept : Model{decoder,inclusionList}, _cf{cf} { // nothing needed } ~Model1() noexcept=default; bool decode(uint32_t history,uint8_t history5,uint8_t &ch) final { auto context{_cf(history)}; auto scale=[&](Context &ctx,uint16_t total) { if (total+ctx.escapeFreq==0x4000U) { ctx.escapeFreq=(ctx.escapeFreq>>1U)+1U; ctx.nodes.scale(); } }; if (auto it=_contexts.find(context);it!=_contexts.end()) { Context &ctx{it->second}; uint16_t total{ctx.nodes.getTotal()}; uint16_t value{_decoder.decode(total+ctx.escapeFreq)}; if (value1U) ctx.escapeFreq--; node.freq++; ch=node.symbol; scale(ctx,total+1U); return true; } } _delayedContext=context; _addNewContext=true; return false; } void mark(uint8_t ch) noexcept final { if (_addNewContext) { if (auto it=_contexts.find(_delayedContext);it!=_contexts.end()) { it->second.nodes.addNew(ch); } else { _contexts.emplace(_delayedContext,Context{_inclusionList,ch}); } _addNewContext=false; } } private: struct Context { Context(InclusionList &inclusionList,uint8_t ch) noexcept : escapeFreq{1U}, nodes{inclusionList} { nodes.addNew(ch); }; ~Context() noexcept=default; uint16_t escapeFreq; ShadedSparseMTFFrequencyList nodes; }; contextFunc _cf; bool _addNewContext=false; std::pair _delayedContext; std::map,Context> _contexts; }; // A simple arithmetic encoder (but with sparse array) // Fallback built in, since they have unhealthy dependency to this model class Model0 : public Model { public: Model0(RangeDecoder &decoder,InclusionList &inclusionList) noexcept : Model{decoder,inclusionList}, _tree{inclusionList} { // nothing needed } ~Model0() noexcept=default; bool decode(uint32_t history,uint8_t history5,uint8_t &ch) final { uint16_t value{_decoder.decode(_tree.getTotal()+_escapeFreq)}; if (value<_escapeFreq) { _decoder.scale(0,_escapeFreq,_tree.getTotal()+_escapeFreq); // since we could not decode it, it is no symbol we know and // we can exclude them all _tree.excludeAll(); decodeFallback(ch); } else { uint16_t low,freq; uint8_t symbol{_tree.decode(value-_escapeFreq,low,freq)}; _decoder.scale(_escapeFreq+low,_escapeFreq+low+freq,_tree.getTotal()+_escapeFreq); switch (_tree[symbol]) { case 0: _escapeFreq++; break; case 1: if (_escapeFreq>1U) _escapeFreq--; break; default: // nothing needed break; } _tree.add(symbol,1); ch=symbol; } return true; } void mark(uint8_t ch) noexcept final { // nothing needed } private: void decodeFallback(uint8_t &ch) { uint16_t value{_decoder.decode(_inclusionList.getTotal())}; uint16_t low,freq; ch=_inclusionList.decode(value,low,freq); _decoder.scale(low,low+freq,_inclusionList.getTotal()); _tree.set(uint16_t(ch),1U); _escapeFreq++; } uint16_t _escapeFreq{1U}; ShadedFrequencyTree _tree; }; InclusionList inclusionList; // Different sources put different names on the models: PPM, PPMI etc. with different contexts // Also, nothing seems to be exactly same as some public sources. The xpk library is "special" // -> I just call them with numbers and letters so I don't get confused Model2 model2A{decoder,inclusionList,[](uint32_t c,uint8_t c5)noexcept{return std::make_tuple(c,uint16_t(c^(c>>15U)),c5);}}; Model2 model2B{decoder,inclusionList,[](uint32_t c,uint8_t c5)noexcept{return std::make_tuple(c,uint16_t(c^(c>>15U)),uint8_t{0});}}; Model1 model1A{decoder,inclusionList,[](uint32_t c)noexcept{return std::make_pair(c&0xff'ffffU,uint16_t(c^(c>>7U)));}}; Model1 model1B{decoder,inclusionList,[](uint32_t c)noexcept{return std::make_pair(c&0xffffU,uint16_t(c&0xffffU));}}; Model1 model1C{decoder,inclusionList,[](uint32_t c)noexcept{return std::make_pair(c&0xffU,uint16_t(c&0xffU));}}; Model0 model0{decoder,inclusionList}; const std::array models{&model2A,&model2B,&model1A,&model1B,&model1C,&model0}; while (!outputStream.eof()) { inclusionList.reset(); for (auto model : models) { uint8_t ch; if (model->decode(history,history5,ch)) { for (auto backModel : models) { if (backModel==model) break; backModel->mark(ch); } addToHistory(ch); outputStream.writeByte(ch); break; } } } } } ancient-2.2.0/src/PPMQDecompressor.hpp000066400000000000000000000014551463063262600176140ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef PPMQDECOMPRESSOR_HPP #define PPMQDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class PPMQDecompressor : public XPKDecompressor { public: PPMQDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~PPMQDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/PackDecompressor.cpp000066400000000000000000000102201463063262600176760ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include "PackDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "DynamicHuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool PackDecompressor::detectHeader(uint32_t hdr) noexcept { return ((hdr>>16)==0x1f1eU||(hdr>>16)==0x1f1fU); } std::shared_ptr PackDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,exactSizeKnown,verify); } PackDecompressor::PackDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData{packedData} { if (_packedData.size()<6U) throw InvalidFormatError(); uint32_t hdr{_packedData.readBE16(0)}; if (!detectHeader(hdr<<16U)) throw InvalidFormatError(); _isOldVersion=hdr==0x1f1fU; if (exactSizeKnown) _packedSize=packedData.size(); if (_isOldVersion) { // PDP endian!!! _rawSize=(uint32_t(_packedData.readLE16(2U))<<16U)|_packedData.readLE16(4); } else { _rawSize=_packedData.readBE32(2U); } if (_rawSize>getMaxRawSize() || (_isOldVersion && !_rawSize)) throw InvalidFormatError(); } const std::string &PackDecompressor::getName() const noexcept { static std::string names[2]{ "z: Pack (Old)", "z: Pack"}; return names[_isOldVersion?0:1U]; } size_t PackDecompressor::getPackedSize() const noexcept { // no way to know before decompressing return _packedSize; } size_t PackDecompressor::getRawSize() const noexcept { return _rawSize; } void PackDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream{_packedData,6,_packedSize?_packedSize:_packedData.size()}; ForwardOutputStream outputStream{rawData,0,rawData.size()}; MSBBitReader bitReader{inputStream}; if (_isOldVersion) { HuffmanDecoder decoder; { std::array tree; uint32_t count=inputStream.readLE16(); if (count>=1024U) throw DecompressionError(); for (uint32_t i=0;ivoid { if (node>=count) throw DecompressionError(); if (tree[node]) { length++; bits<<=1U; if (length>24U) throw DecompressionError(); branch(node+tree[node],length,bits,branch); if (node+1>=count) throw DecompressionError(); branch(node+tree[node+1],length,bits|1U,branch); } else { if (!length) throw DecompressionError(); decoder.insert(HuffmanCode{length,bits,uint8_t(tree[node+1])}); } }; branch(0,0,0,branch); } auto readBit=[&]()->uint32_t { return bitReader.readBitsLE16(1); }; while (outputStream.getOffset()!=_rawSize) outputStream.writeByte(decoder.decode(readBit)); } else { HuffmanDecoder decoder; // interesting ordering... { uint32_t maxLevel{inputStream.readByte()}; if (!maxLevel || maxLevel>24U) throw DecompressionError(); std::array levelCounts; for (uint32_t i=0;i>(23U-i),symbol}); code+=1U<<(23U-i); } code-=levelCounts[i]<<(23U-i); } } auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; while (outputStream.getOffset()!=_rawSize) { uint16_t code{decoder.decode(readBit)}; if (code==0x100U) { if (outputStream.getOffset()!=_rawSize) throw DecompressionError(); break; } outputStream.writeByte(uint8_t(code)); } } // we do not verify the exact packed length here since official encoder // tends to add few bytes at the end _packedSize=inputStream.getOffset(); } } ancient-2.2.0/src/PackDecompressor.hpp000066400000000000000000000014441463063262600177130ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef PACKDECOMPRESSOR_HPP #define PACKDECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class PackDecompressor : public Decompressor { public: PackDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); ~PackDecompressor() noexcept=default; size_t getRawSize() const noexcept final; size_t getPackedSize() const noexcept final; const std::string &getName() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: const Buffer &_packedData; size_t _packedSize{0}; size_t _rawSize{0}; bool _isOldVersion; }; } #endif ancient-2.2.0/src/RAKEDecompressor.cpp000066400000000000000000000132111463063262600175450ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "RAKEDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" #include namespace ancient::internal { bool RAKEDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return (hdr==FourCC("FRHT") || hdr==FourCC("RAKE")); } std::shared_ptr RAKEDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } RAKEDecompressor::RAKEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData}, _isRAKE{hdr==FourCC("RAKE")} { if (!detectHeaderXPK(hdr) || packedData.size()<4) throw Decompressor::InvalidFormatError(); _midStreamOffset=packedData.readBE16(2); if (_midStreamOffset>=packedData.size()) throw Decompressor::InvalidFormatError(); } const std::string &RAKEDecompressor::getSubName() const noexcept { static std::string nameFRHT{"XPK-FRHT: LZ77-compressor"}; static std::string nameRAKE{"XPK-RAKE: LZ77-compressor"}; return (_isRAKE)?nameRAKE:nameFRHT; } void RAKEDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { // 2 streams // 1st: bit stream starting from _midStreamOffset(+1) going to packedSize // 2nd: byte stream starting from _midStreamOffset going backwards to 4 ForwardInputStream forwardInputStream{_packedData,_midStreamOffset+(_midStreamOffset&1),_packedData.size()}; BackwardInputStream backwardInputStream{_packedData,4,_midStreamOffset}; MSBBitReader bitReader{forwardInputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBitsBE32(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1); }; auto readByte=[&]()->uint8_t { return backwardInputStream.readByte(); }; { uint16_t tmp{_packedData.readBE16(0)}; if (tmp>32) throw Decompressor::DecompressionError(); uint32_t content{forwardInputStream.readBE32()}; bitReader.reset(content>>tmp,32-tmp); } BackwardOutputStream outputStream{rawData,0,rawData.size()}; HuffmanDecoder lengthDecoder; // is there some logic into this? const std::array,255> decTable{{ { 1,0x01},{ 3,0x03},{ 5,0x05},{ 6,0x09},{ 7,0x0c},{ 9,0x13},{12,0x34},{18,0xc0}, {18,0xc2},{18,0xc3},{18,0xc6},{16,0x79},{18,0xc7},{18,0xd6},{18,0xd7},{18,0xd8}, {17,0xa8},{17,0x92},{17,0x8a},{17,0x82},{16,0x6c},{17,0x94},{18,0xda},{18,0xca}, {16,0x7b},{13,0x36},{13,0x39},{13,0x48},{14,0x49},{14,0x50},{15,0x62},{15,0x5e}, {16,0x6f},{17,0x83},{17,0x87},{15,0x56},{11,0x21},{12,0x31},{13,0x38},{13,0x3d}, { 8,0x0f},{ 4,0x04},{ 6,0x08},{10,0x1c},{12,0x27},{13,0x42},{13,0x3a},{12,0x30}, {12,0x32},{ 9,0x16},{ 8,0x11},{ 7,0x0b},{ 5,0x06},{10,0x19},{10,0x1a},{10,0x18}, {11,0x26},{17,0x98},{17,0x99},{17,0x9b},{17,0x9e},{17,0x9f},{17,0xa6},{16,0x73}, {17,0x7f},{17,0x81},{17,0x84},{17,0x85},{15,0x5d},{14,0x4d},{14,0x4f},{13,0x45}, {13,0x3c},{ 9,0x17},{10,0x1d},{12,0xff},{13,0x41},{17,0x8c},{18,0xaa},{19,0xdb}, {19,0xdc},{16,0x77},{15,0x63},{16,0x7c},{16,0x76},{16,0x71},{16,0x7d},{12,0x2c}, {13,0x3b},{16,0x7a},{16,0x75},{15,0x55},{15,0x60},{16,0x74},{17,0xa4},{18,0xab}, {18,0xac},{ 7,0x0a},{ 6,0x07},{ 9,0x15},{11,0x20},{11,0x24},{10,0x1b},{ 8,0x10}, { 9,0x12},{12,0x33},{14,0x4b},{15,0x53},{19,0xdd},{19,0xde},{18,0xad},{19,0xdf}, {19,0xe0},{18,0xae},{17,0x88},{18,0xaf},{19,0xe1},{19,0xe2},{13,0x37},{12,0x2e}, {18,0xb0},{18,0xb1},{19,0xe3},{19,0xe4},{18,0xb2},{18,0xb3},{19,0xe5},{19,0xe6}, {19,0xe7},{19,0xe8},{18,0xb4},{17,0x9a},{18,0xb5},{18,0xb6},{18,0xb7},{19,0xe9}, {19,0xea},{18,0xb8},{19,0xeb},{19,0xec},{19,0xed},{19,0xee},{18,0xb9},{19,0xef}, {19,0xf0},{18,0xbb},{18,0xbc},{19,0xf1},{19,0xf2},{18,0xbd},{18,0xbe},{19,0xf3}, {19,0xf4},{18,0xbf},{18,0xc1},{19,0xf5},{19,0xf6},{18,0xc4},{18,0xc5},{17,0x95}, {18,0xc8},{18,0xc9},{19,0xf7},{19,0xf8},{18,0xcb},{18,0xcc},{19,0xf9},{19,0xfa}, {18,0xcd},{18,0xce},{17,0x96},{18,0xcf},{18,0xd0},{19,0xfb},{19,0xfc},{18,0xd1}, {18,0xd2},{18,0xd3},{17,0x9c},{17,0x9d},{18,0xd4},{18,0xd5},{17,0xa0},{17,0xa1}, {17,0xa2},{17,0xa3},{17,0xa5},{19,0xfd},{19,0xfe},{18,0xd9},{17,0xa7},{16,0x66}, {15,0x54},{15,0x57},{16,0x6b},{16,0x68},{14,0x4c},{14,0x4e},{12,0x28},{11,0x23}, { 8,0x0e},{ 7,0x0d},{10,0x1f},{13,0x47},{15,0x64},{15,0x58},{15,0x59},{15,0x5a}, {12,0x29},{13,0x3e},{15,0x5f},{17,0x8e},{18,0xba},{18,0xa9},{16,0x70},{14,0x4a}, {12,0x2a},{ 9,0x14},{11,0x22},{12,0x2f},{16,0x7e},{16,0x67},{16,0x69},{16,0x65}, {15,0x51},{16,0x78},{16,0x6a},{13,0x46},{11,0x25},{16,0x72},{16,0x6e},{15,0x5b}, {15,0x61},{15,0x52},{13,0x40},{13,0x43},{13,0x44},{13,0x3f},{15,0x5c},{17,0x93}, {17,0x80},{17,0x8d},{17,0x8b},{17,0x86},{17,0x89},{17,0x97},{17,0x8f},{17,0x90}, {17,0x91},{16,0x6d},{12,0x2b},{12,0x2d},{12,0x35},{10,0x1e},{ 3,0x02}}}; uint32_t hufCode{0}; for (auto &it: decTable) { lengthDecoder.insert(HuffmanCode{it[0],hufCode>>(32-it[0]),it[1]}); hufCode+=1<<(32-it[0]); } while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint32_t count{lengthDecoder.decode(readBit)}; count+=2; uint32_t distance; if (!readBit()) { distance=uint32_t(readByte())+1; } else { if (!readBit()) { distance=((readBits(3)<<8)|uint32_t(readByte()))+0x101; } else { distance=((readBits(6)<<8)|uint32_t(readByte()))+0x901; } } outputStream.copy(distance,count); } } } } ancient-2.2.0/src/RAKEDecompressor.hpp000066400000000000000000000015341463063262600175570ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef RAKEDECOMPRESSOR_HPP #define RAKEDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class RAKEDecompressor : public XPKDecompressor { public: RAKEDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~RAKEDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; bool _isRAKE; size_t _midStreamOffset{0}; }; } #endif ancient-2.2.0/src/RDCNDecompressor.cpp000066400000000000000000000043641463063262600175620ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "RDCNDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool RDCNDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("RDCN"); } std::shared_ptr RDCNDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } RDCNDecompressor::RDCNDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &RDCNDecompressor::getSubName() const noexcept { static std::string name{"XPK-RDCN: Ross data compression"}; return name; } void RDCNDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint8_t tmp{readByte()}; uint32_t count{tmp&0xfU}; uint32_t code{uint32_t(tmp>>4U)}; uint32_t distance{0}; uint8_t repeatChar{0}; bool doRLE{false}; switch (code) { case 0: repeatChar=readByte(); count+=3; doRLE=true; break; case 1: count=(count|(uint32_t(readByte())<<4U))+19U; repeatChar=readByte(); doRLE=true; break; case 2: distance=(count|(uint32_t(readByte())<<4U))+3U; count=uint32_t(readByte())+16U; break; default: /* 3 to 15 */ distance=(count|(uint32_t(readByte())<<4U))+3U; count=code; break; } if (doRLE) { for (uint32_t i=0;i &state,bool verify); ~RDCNDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/RLENDecompressor.cpp000066400000000000000000000033341463063262600175700ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "RLENDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool RLENDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("RLEN"); } std::shared_ptr RLENDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } RLENDecompressor::RLENDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &RLENDecompressor::getSubName() const noexcept { static std::string name{"XPK-RLEN: RLE-compressor"}; return name; } void RLENDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; ForwardOutputStream outputStream{rawData,0,rawData.size()}; while (!outputStream.eof()) { uint32_t count{inputStream.readByte()}; if (count<128) { if (!count) throw Decompressor::DecompressionError(); // lets have this as error... for (uint32_t i=0;i &state,bool verify); ~RLENDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/RNCDecompressor.cpp000066400000000000000000000304051463063262600174510ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "RNCDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/CRC16.hpp" #include "common/OverflowCheck.hpp" #include "common/Common.hpp" #include "VariableLengthCodeDecoder.hpp" #include // This allows decompression of pc compressed files from unonfficial (and unpatched) compressor // PC games do not need chunk count, and are happy to read these files. // Official tools put it and amiga decompressors require it #define ALLOW_MISSING_CHUNKS 1 namespace ancient::internal { bool RNCDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("RNC\001") || hdr==FourCC("RNC\002") || hdr==FourCC("...\001"); // Total Carnage } std::shared_ptr RNCDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,verify); } RNCDecompressor::RNCDecompressor(const Buffer &packedData,bool verify) : _packedData{packedData} { uint32_t hdr{packedData.readBE32(0)}; _rawSize=packedData.readBE32(4); _packedSize=packedData.readBE32(8); if (!_rawSize || !_packedSize || _rawSize>getMaxRawSize() || _packedSize>getMaxPackedSize()) throw InvalidFormatError(); bool verified{false}; if (hdr==FourCC("RNC\001")) { // now detect between old and new version // since the old and the new version share the same id, there is no foolproof way // to tell them apart. It is easier to prove that it is not something by finding // specific invalid bitstream content. // well, this is silly though but lets assume someone has made old format RNC1 with total size less than 19 if (packedData.size()<=18U) { _ver=Version::RNC1Old; } else { uint8_t newStreamStart{packedData.read8(18U)}; uint8_t oldStreamStart{packedData.read8(_packedSize+11U)}; // Check that stream starts with a literal(s) if (!(oldStreamStart&0x80U)) _ver=Version::RNC1New; // New stream have two bits in start as a filler on new stream. Those are always 0 // (although this is not strictly mandated) // + // Even though it is possible to make new RNC1 stream which starts with zero literal table size, // it is extremely unlikely else if ((newStreamStart&3U) || !(newStreamStart&0x7cU)) _ver=Version::RNC1Old; // now the last resort: check CRC. else if (_packedData.size()>=OverflowCheck::sum(_packedSize,18U) && CRC16(_packedData,18U,_packedSize,0)==packedData.readBE16(14U)) { _ver=Version::RNC1New; verified=true; } else _ver=Version::RNC1Old; } } else if (hdr==FourCC("RNC\002")) { // ...and detect between the new and old format of RNC2 if (packedData.size()<=18U) { _ver=Version::RNC2Old; } else { // RNC2Old is very similar to RNC1Old, RNC2 has padding at start which makes things more complex uint8_t newStreamStart{packedData.read8(18U)}; uint8_t oldStreamStart{packedData.read8(_packedSize+10U)}; // Check that stream starts with a literal(s) if (!(oldStreamStart&0x80U)) _ver=Version::RNC2New; // First command needs to be LIT/MOV! (after 2 ballast bits) else if ((newStreamStart&0x30U)==0x30U) _ver=Version::RNC2Old; // now the last resort: check CRC. else if (_packedData.size()>=OverflowCheck::sum(_packedSize,18U) && CRC16(_packedData,18U,_packedSize,0)==packedData.readBE16(14U)) { _ver=Version::RNC2New; verified=true; } else _ver=Version::RNC2Old; } } else if (hdr==FourCC("...\001")) { _ver=Version::RNC1New; } else throw InvalidFormatError(); uint32_t hdrSize{(_ver==Version::RNC1Old || _ver==Version::RNC2Old)?12U:18U}; if (OverflowCheck::sum(_packedSize,hdrSize)>packedData.size()) throw InvalidFormatError(); if (_ver!=Version::RNC1Old && _ver!=Version::RNC2Old) { _rawCRC=packedData.readBE16(12U); _chunks=packedData.read8(17U); if (verify && !verified) { if (CRC16(_packedData,18,_packedSize,0)!=packedData.readBE16(14)) throw VerificationError(); } } } const std::string &RNCDecompressor::getName() const noexcept { static std::string names[4]={ {"RNC1: Rob Northen RNC1 Compressor (old)"}, {"RNC1: Rob Northen RNC1 Compressor"}, {"RNC2: Rob Northen RNC2 Compressor (old)"}, {"RNC2: Rob Northen RNC2 Compressor"}}; return names[static_cast(_ver)]; } size_t RNCDecompressor::getPackedSize() const noexcept { if (_ver==Version::RNC1Old || _ver==Version::RNC2Old) return _packedSize+12U; else return _packedSize+18U; } size_t RNCDecompressor::getRawSize() const noexcept { return _rawSize; } void RNCDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); switch (_ver) { case Version::RNC1Old: return RNCDecompressOld(rawData,verify,false); case Version::RNC1New: return RNC1DecompressNew(rawData,verify); case Version::RNC2Old: return RNCDecompressOld(rawData,verify,true); case Version::RNC2New: return RNC2DecompressNew(rawData,verify); default: throw DecompressionError(); } } void RNCDecompressor::RNCDecompressOld(Buffer &rawData,bool verify,bool rnc2) { BackwardInputStream inputStream{_packedData,12U,_packedSize+12U}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; uint32_t lastDistanceBits{12}; uint32_t lastLengthBits{10}; if (rnc2) { uint32_t tmp=readByte()+1U; lastDistanceBits=tmp&0xfU; lastLengthBits=(tmp>>4U)+1U; } // the anchor-bit does not seem always to be at the correct place { uint8_t halfByte{readByte()}; for (uint32_t i=0;i<7;i++) if (halfByte&(1<>(i+1),7-i); break; } } BackwardOutputStream outputStream{rawData,0,_rawSize}; HuffmanDecoder lengthDecoder { HuffmanCode{1,0b0000,uint8_t{0}}, HuffmanCode{2,0b0010,uint8_t{1}}, HuffmanCode{3,0b0110,uint8_t{2}}, HuffmanCode{4,0b1110,uint8_t{3}}, HuffmanCode{4,0b1111,uint8_t{4}} }; HuffmanDecoder distanceDecoder { HuffmanCode{1,0b00,uint8_t{1}}, HuffmanCode{2,0b10,uint8_t{0}}, HuffmanCode{2,0b11,uint8_t{2}} }; VariableLengthCodeDecoder litVlcDecoder1{1,1,2,2,3,10}; VariableLengthCodeDecoder litVlcDecoder2{1,1,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}; VariableLengthCodeDecoder lengthVlcDecoder{0,0,1,2,lastLengthBits}; VariableLengthCodeDecoder distanceVlcDecoder{5,8,lastDistanceBits}; for (;;) { uint32_t litLength{rnc2?litVlcDecoder2.decodeCascade(readBits):litVlcDecoder1.decodeCascade(readBits)}; for (uint32_t i=0;i bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBitsLE16(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsLE16(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,_rawSize}; typedef HuffmanDecoder RNC1HuffmanDecoder; // helpers auto readHuffmanTable=[&](RNC1HuffmanDecoder &dec) { uint32_t length{readBits(5)}; if (!length) return; std::array lengthTable; for (uint32_t i=0;iuint32_t { // this is kind of non-specced uint32_t ret{dec.decode(readBit)}; if (ret>=2U) ret=(1U<<(ret-1U))|readBits(ret-1U); return ret; }; auto processLiterals=[&](const RNC1HuffmanDecoder &dec) { uint32_t litLength{huffmanDecode(dec)}; for (uint32_t i=0;i bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,_rawSize}; // Huffman decoding enum class Cmd { LIT=0, // 0, Literal MOV, // 10, Move bytes + length + distance, Get bytes if length=9 + 4bits MV2, // 110, Move 2 bytes MV3, // 1110, Move 3 bytes CND // 1111, Conditional copy, or EOF }; HuffmanDecoder cmdDecoder { HuffmanCode{1,0b0000,Cmd::LIT}, HuffmanCode{2,0b0010,Cmd::MOV}, HuffmanCode{3,0b0110,Cmd::MV2}, HuffmanCode{4,0b1110,Cmd::MV3}, HuffmanCode{4,0b1111,Cmd::CND} }; /* length of 9 is a marker for literals */ HuffmanDecoder lengthDecoder { HuffmanCode{2,0b000,uint8_t{4}}, HuffmanCode{2,0b010,uint8_t{5}}, HuffmanCode{3,0b010,uint8_t{6}}, HuffmanCode{3,0b011,uint8_t{7}}, HuffmanCode{3,0b110,uint8_t{8}}, HuffmanCode{3,0b111,uint8_t{9}} }; HuffmanDecoder distanceDecoder { HuffmanCode{1,0b000000,uint8_t{0}}, HuffmanCode{3,0b000110,uint8_t{1}}, HuffmanCode{4,0b001000,uint8_t{2}}, HuffmanCode{4,0b001001,uint8_t{3}}, HuffmanCode{5,0b010101,uint8_t{4}}, HuffmanCode{5,0b010111,uint8_t{5}}, HuffmanCode{5,0b011101,uint8_t{6}}, HuffmanCode{5,0b011111,uint8_t{7}}, HuffmanCode{6,0b101000,uint8_t{8}}, HuffmanCode{6,0b101001,uint8_t{9}}, HuffmanCode{6,0b101100,uint8_t{10}}, HuffmanCode{6,0b101101,uint8_t{11}}, HuffmanCode{6,0b111000,uint8_t{12}}, HuffmanCode{6,0b111001,uint8_t{13}}, HuffmanCode{6,0b111100,uint8_t{14}}, HuffmanCode{6,0b111101,uint8_t{15}} }; // helpers auto readDistance=[&]()->uint32_t { uint8_t distMult{distanceDecoder.decode(readBit)}; uint8_t distByte{readByte()}; return (uint32_t(distByte)|(uint32_t(distMult)<<8))+1; }; auto moveBytes=[&](uint32_t distance,uint32_t count)->void { if (!count) throw DecompressionError(); outputStream.copy(distance,count); }; readBit(); readBit(); uint8_t foundChunks{0}; bool done{false}; while (!done && foundChunks<_chunks) { Cmd cmd{cmdDecoder.decode(readBit)}; switch (cmd) { case Cmd::LIT: outputStream.writeByte(readByte()); break; case Cmd::MOV: { uint8_t count{lengthDecoder.decode(readBit)}; if (count!=9) moveBytes(readDistance(),count); else { uint32_t rep{(readBits(4U)+3U)*4U}; for (uint32_t i=0;i create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: enum class Version { RNC1Old=0, RNC1New, RNC2Old, RNC2New }; void RNCDecompressOld(Buffer &rawData,bool verify,bool rnc2); void RNC1DecompressNew(Buffer &rawData,bool verify); void RNC2DecompressNew(Buffer &rawData,bool verify); const Buffer &_packedData; uint32_t _rawSize{0}; uint32_t _packedSize{0}; uint16_t _rawCRC{0}; uint8_t _chunks{0}; Version _ver; }; } #endif ancient-2.2.0/src/RangeDecoder.cpp000066400000000000000000000016711463063262600167660ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "RangeDecoder.hpp" namespace ancient::internal { RangeDecoder::RangeDecoder(BitReader &bitReader,uint16_t initialValue) : _bitReader{bitReader}, _stream{initialValue} { // nothing needed } uint16_t RangeDecoder::decode(uint16_t length) { return ((uint32_t(_stream-_low)+1)*length-1)/(uint32_t(_high-_low)+1); } void RangeDecoder::scale(uint16_t newLow,uint16_t newHigh,uint16_t newRange) { uint32_t range{uint32_t(_high-_low)+1U}; _high=(range*newHigh)/newRange+_low-1U; _low=(range*newLow)/newRange+_low; auto doubleContext=[&](uint16_t decr) { _low-=decr; _high-=decr; _stream-=decr; _low<<=1; _high=(_high<<1)|1U; _stream=(_stream<<1)|_bitReader.readBit(); }; for (;;) { if (_high<0x8000U) { doubleContext(0U); } else if (_low>=0x8000U) { doubleContext(0x8000U); } else if (_low>=0x4000U && _high<0xc000U) { doubleContext(0x4000U); } else break; } } } ancient-2.2.0/src/RangeDecoder.hpp000066400000000000000000000012231463063262600167640ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef RANGEDECODER_HPP #define RANGEDECODER_HPP #include namespace ancient::internal { // used by too many compressors... class RangeDecoder { public: class BitReader { public: BitReader() noexcept=default; virtual ~BitReader() noexcept=default; virtual uint32_t readBit()=0; }; RangeDecoder(BitReader &bitReader,uint16_t initialValue); ~RangeDecoder() noexcept=default; uint16_t decode(uint16_t length); void scale(uint16_t newLow,uint16_t newHigh,uint16_t newRange); private: BitReader &_bitReader; uint16_t _low{0}; uint16_t _high{0xffffU}; uint16_t _stream; }; } #endif ancient-2.2.0/src/SCOCompressDecompressor.cpp000066400000000000000000000103741463063262600211720ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SCOCompressDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" #include namespace ancient::internal { bool SCOCompressDecompressor::detectHeader(uint32_t hdr) noexcept { return ((hdr>>16)==0x1fa0U); } std::shared_ptr SCOCompressDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,exactSizeKnown,verify); } SCOCompressDecompressor::SCOCompressDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData{packedData}, _exactSizeKnown{exactSizeKnown} { if (_packedData.size()<4U) throw InvalidFormatError(); uint32_t hdr{_packedData.readBE16(0)}; if (!detectHeader(hdr<<16)) throw InvalidFormatError(); if (exactSizeKnown) _packedSize=packedData.size(); } const std::string &SCOCompressDecompressor::getName() const noexcept { static std::string name{"SCO Compress LZH"}; return name; } size_t SCOCompressDecompressor::getPackedSize() const noexcept { // no way to know before decompressing return _packedSize; } size_t SCOCompressDecompressor::getRawSize() const noexcept { // same thing, decompression needed first return _rawSize; } void SCOCompressDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream{_packedData,2U,_packedSize?_packedSize:_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1U); }; AutoExpandingForwardOutputStream outputStream{rawData}; OptionalHuffmanDecoder decoder; OptionalHuffmanDecoder distanceDecoder; // Almost straight steal from LHX uint32_t blockRemaining{0}; for(;;) { if (!blockRemaining) { blockRemaining=readBits(16); // 0 is break in SCO if (!blockRemaining) break; auto createTable=[&](OptionalHuffmanDecoder &dest,uint32_t bits,bool enableHole) { std::array symbolBits; uint32_t length{readBits(bits)}; if (!length) { dest.setEmpty(readBits(bits)); } else { for (uint32_t i=0;ilength) throw DecompressionError(); for (uint32_t j=0;j tmpDecoder; createTable(tmpDecoder,5,true); decoder.reset(); std::array symbolBits; uint32_t length=readBits(9); if (!length) { decoder.setEmpty(readBits(9)); } else { for (uint32_t i=0;ilength) throw DecompressionError(); for (uint32_t j=0;j create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: const Buffer &_packedData; size_t _packedSize{0}; size_t _rawSize{0}; bool _exactSizeKnown; }; } #endif ancient-2.2.0/src/SDHCDecompressor.cpp000066400000000000000000000053011463063262600175450ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "common/SubBuffer.hpp" #include "SDHCDecompressor.hpp" #include "XPKMain.hpp" #include "DLTADecode.hpp" #include "common/Common.hpp" namespace ancient::internal { bool SDHCDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SDHC"); } std::shared_ptr SDHCDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } SDHCDecompressor::SDHCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr) || _packedData.size()<2) throw Decompressor::InvalidFormatError(); _mode=_packedData.readBE16(0); if (verify && (_mode&0x8000U)) { ConstSubBuffer src{_packedData,2U,_packedData.size()-2U}; XPKMain::createDecompressor(_recursionLevel+1,src,true); } } const std::string &SDHCDecompressor::getSubName() const noexcept { static std::string name{"XPK-SDHC: Sample delta huffman compressor"}; return name; } void SDHCDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ConstSubBuffer src{_packedData,2U,_packedData.size()-2U}; if (_mode&0x8000U) { auto main=XPKMain::createDecompressor(_recursionLevel+1,src,verify); main->decompress(rawData,verify); } else { if (src.size()!=rawData.size()) throw Decompressor::DecompressionError(); std::memcpy(rawData.data(),src.data(),src.size()); } size_t length{rawData.size()&~3U}; auto deltaDecodeMono=[&]() { uint16_t ctr{0}; for (size_t i=0;i>8U; rawData[i+1U]=ctr&0xffU; } }; auto deltaDecodeStereo=[&]() { uint16_t ctr1{0}; uint16_t ctr2{0}; for (size_t i=0;i>8U; rawData[i+1U]=ctr1&0xffU; rawData[i+2U]=ctr2>>8U; rawData[i+3U]=ctr2&0xffU; } }; switch (_mode&15) { case 1: DLTADecode::decode(rawData,rawData,0,length); [[fallthrough]]; case 0: DLTADecode::decode(rawData,rawData,0,length); break; case 3: deltaDecodeMono(); [[fallthrough]]; case 2: deltaDecodeMono(); break; case 11: deltaDecodeStereo(); [[fallthrough]]; case 10: deltaDecodeStereo(); break; default: throw Decompressor::DecompressionError(); } } } ancient-2.2.0/src/SDHCDecompressor.hpp000066400000000000000000000015021463063262600175510ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SDHCDECOMPRESSOR_HPP #define SDHCDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class SDHCDecompressor : public XPKDecompressor { public: SDHCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~SDHCDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; uint16_t _mode{0}; }; } #endif ancient-2.2.0/src/SHRXDecompressor.cpp000066400000000000000000000173551463063262600176240ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SHRXDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { SHRXDecompressor::SHRXState::SHRXState() noexcept { for (uint32_t i=0;i<999;i++) ar[i]=0; } bool SHRXDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SHRI")||hdr==FourCC("SHR3"); } std::shared_ptr SHRXDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } SHRXDecompressor::SHRXDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData}, _state{state} { if (!detectHeaderXPK(hdr) || _packedData.size()<6) throw Decompressor::InvalidFormatError(); _ver=_packedData.read8(0); if (!_ver || _ver>2) throw Decompressor::InvalidFormatError(); _isSHR3=hdr==FourCC("SHR3"); if (!_isSHR3) { // second byte defines something that does not seem to be terribly important... uint8_t tmp{_packedData.read8(2)}; if (tmp&0x80U) { _rawSize=~_packedData.readBE32(2)+1U; _startOffset=6; } else { _rawSize=_packedData.readBE16(2); _startOffset=4; } } else _startOffset=1; if (!_state) { if (_ver==2) throw Decompressor::InvalidFormatError(); _state.reset(new SHRXState()); } } const std::string &SHRXDecompressor::getSubName() const noexcept { static std::string name3{"XPK-SHR3: LZ-compressor with arithmetic encoding"}; static std::string nameI{"XPK-SHRI: LZ-compressor with arithmetic encoding"}; return _isSHR3?name3:nameI; } void SHRXDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (!_isSHR3 && rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); ForwardInputStream inputStream{_packedData,_startOffset,_packedData.size()}; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); // This follows quite closely Choloks pascal reference std::array ar; auto resum=[&]() { for (uint32_t i=498;i;i--) ar[i]=ar[i*2]+ar[i*2+1]; }; auto init=[&]() { for (uint32_t i=0;i<499;i++) ar[i]=0; for (uint32_t i=0;i<256;i++) ar[i+499]=(i<32||i>126)?1:3; for (uint32_t i=256+499;i<999;i++) ar[i]=0; resum(); }; auto update=[&](uint32_t updateIndex,uint32_t increment) { if (updateIndex>=499) return; updateIndex+=499; while (updateIndex) { ar[updateIndex]+=increment; updateIndex>>=1; } if (ar[1]>=0x2000) { for (uint32_t i=499;i<998;i++) if (ar[i]) ar[i]=(ar[i]>>1)+1; resum(); } }; auto scale=[&](uint32_t a,uint32_t b,uint32_t mult)->uint32_t { if (!b) throw Decompressor::DecompressionError(); uint32_t tmp,tmp2; if (_isSHR3) { tmp=(0x10000U/b); tmp2=((0x10000U%b)<<16)/b; } else { tmp=(a<<16)/b; tmp2=(((a<<16)%b)<<16)/b; } return ((mult&0xffffU)*tmp>>16)+((mult>>16)*tmp2>>16)+(mult>>16)*tmp; }; uint32_t vlen{0}; uint32_t vnext{0}; const std::array updates1{358,359,386,387,414,415}; const std::array updates2{442,456,470,484}; auto upgrade=[&]() { if (vnext>=65532) { vnext=~0U; } else if (!vlen) { vnext=1; } else { uint32_t vvalue{vnext-1U}; if (vvalue<48U) update(vvalue+256U,1U); uint32_t bits{0}; uint32_t compare{4}; while (vvalue>=compare) { vvalue-=compare; compare<<=1; bits++; } if (bits>=14) { vnext=~0U; } else { if (!vvalue) { if (bits<7) { for (uint32_t i=304;i<=307;i++) update((bits<<2)+i,1); } if (bits<13) { for (uint32_t i=332;i<=333;i++) update((bits<<1)+i,1); } for (auto it : updates1) update((bits<<1)+it,1); for (auto it : updates2) update(bits+it,1); } if (vnext<49) { vnext++; } else if (vnext==49) { vnext=61; } else { vnext=(vnext<<1)+3; } } } }; uint32_t stream{0}; uint32_t shift{0}; auto refillStream=[&]() { while (shift<0x100'0000) { stream=(stream<<8)|uint32_t(readByte()); shift<<=8; } }; auto getSymbol=[&]()->uint32_t { if (!(shift>>16)) throw Decompressor::DecompressionError(); uint32_t vvalue{(stream/(shift>>16U))&0xffffU}; uint32_t threshold{(ar[1]*vvalue)>>16U}; uint32_t arIndex{1}; uint32_t result{0}; do { arIndex<<=1; uint32_t tmp=ar[arIndex]+result; if (threshold>=tmp) { result=tmp; arIndex++; } } while (arIndex<499); uint32_t rawValue; uint32_t newValue; if (_isSHR3) { rawValue=scale(0,ar[1],shift); newValue=rawValue*result; } else { rawValue=0; // not used newValue=scale(result,ar[1],shift); } if (newValue>stream) { while (newValue>stream) { if (--arIndex<499) arIndex+=499; result-=ar[arIndex]; newValue=_isSHR3?rawValue*result:scale(result,ar[1],shift); } } else { result+=ar[arIndex]; while (result=998) arIndex-=499; result+=ar[arIndex]; newValue=compare; } } stream-=newValue; shift=_isSHR3?rawValue*ar[arIndex]:scale(ar[arIndex],ar[1U],shift); uint32_t addition{(ar[1U]>>10U)+3U}; arIndex-=499U; update(arIndex,addition); refillStream(); return arIndex; }; auto getCode=[&](uint32_t size)->uint32_t { uint32_t ret{0}; while (size--) { ret<<=1; shift>>=1; if (stream>=shift) { ret++; stream-=shift; } refillStream(); } return ret; }; if (_ver==1) { init(); update(498,1); shift=0x8000'0000U; } else { SHRXState *state{static_cast(_state.get())}; vlen=state->vlen; vnext=state->vnext; shift=state->shift; for (uint32_t i=0;i<999;i++) ar[i]=state->ar[i]; } stream=inputStream.readBE32(); while (!outputStream.eof()) { while (vlen>=vnext) upgrade(); uint32_t code{getSymbol()}; if (code<256) { outputStream.writeByte(code); vlen++; } else { auto distanceAddition=[](uint32_t i)->uint32_t { return ((1<<(i+2))-1)&~0x3U; }; uint32_t count,distance; if (code<304) { count=2; distance=code-255; } else if (code<332) { uint32_t tmp=code-304; uint32_t extra=getCode(tmp>>2); distance=((extra<<2)|(tmp&3))+distanceAddition(tmp>>2)+1; count=3; } else if (code<358) { uint32_t tmp=code-332; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=4; } else if (code<386) { uint32_t tmp=code-358; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=5; } else if (code<414) { uint32_t tmp=code-386; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=6; } else if (code<442) { uint32_t tmp=code-414; uint32_t extra=getCode((tmp>>1)+1); distance=((extra<<1)|(tmp&1))+distanceAddition(tmp>>1)+1; count=7; } else if (code<498) { uint32_t tmp=code-442; uint32_t d=tmp/14; uint32_t m=tmp%14; count=getCode(d+2)+distanceAddition(d)+8; distance=getCode(m+2)+distanceAddition(m)+1; } else { count=getCode(16); distance=getCode(16); } vlen+=count; if (!count) throw Decompressor::DecompressionError(); outputStream.copy(distance,count,previousData); } } SHRXState *state{static_cast(_state.get())}; state->vlen=vlen; state->vnext=vnext; state->shift=shift; for (uint32_t i=0;i<999;i++) state->ar[i]=ar[i]; } } ancient-2.2.0/src/SHRXDecompressor.hpp000066400000000000000000000023261463063262600176210ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SHRXDECOMPRESSOR_HPP #define SHRXDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" #include namespace ancient::internal { class SHRXDecompressor : public XPKDecompressor { private: class SHRXState : public XPKDecompressor::State { public: SHRXState() noexcept; virtual ~SHRXState() noexcept=default; uint32_t vlen{0}; uint32_t vnext{0}; uint32_t shift{0}; std::array ar; }; public: SHRXDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~SHRXDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _ver{0}; size_t _startOffset{0}; size_t _rawSize{0}; bool _isSHR3{false}; std::shared_ptr &_state; // reference!!! }; } #endif ancient-2.2.0/src/SLZ3Decompressor.cpp000066400000000000000000000033611463063262600175630ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SLZ3Decompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool SLZ3Decompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SLZ3"); } std::shared_ptr SLZ3Decompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } SLZ3Decompressor::SLZ3Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &SLZ3Decompressor::getSubName() const noexcept { static std::string name{"XPK-SLZ3: SLZ3 CyberYAFA compressor"}; return name; } void SLZ3Decompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { if (!readBit()) { outputStream.writeByte(readByte()); } else { uint8_t tmp{readByte()}; if (!tmp) throw Decompressor::DecompressionError(); uint32_t distance={uint32_t(tmp&0xf0U)<<4U}; distance|=readByte(); uint32_t count{uint32_t(tmp&0xfU)+2U}; outputStream.copy(distance,count); } } } } ancient-2.2.0/src/SLZ3Decompressor.hpp000066400000000000000000000014551463063262600175720ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SLZ3DECOMPRESSOR_HPP #define SLZ3DECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class SLZ3Decompressor : public XPKDecompressor { public: SLZ3Decompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~SLZ3Decompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/SMPLDecompressor.cpp000066400000000000000000000037351463063262600176100ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SMPLDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "HuffmanDecoder.hpp" #include "common/Common.hpp" namespace ancient::internal { bool SMPLDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SMPL"); } std::shared_ptr SMPLDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } SMPLDecompressor::SMPLDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr) || packedData.size()<2) throw Decompressor::InvalidFormatError(); if (packedData.readBE16(0)!=1U) throw Decompressor::InvalidFormatError(); } const std::string &SMPLDecompressor::getSubName() const noexcept { static std::string name{"XPK-SMPL: Huffman compressor with delta encoding"}; return name; } void SMPLDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,2U,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; HuffmanDecoder decoder; for (uint32_t i=0;i<256;i++) { uint32_t codeLength{readBits(4U)}; if (!codeLength) continue; if (codeLength==15) codeLength=readBits(4U)+15U; uint32_t code{readBits(codeLength)}; decoder.insert(HuffmanCode{codeLength,code,i}); } uint8_t accum{0}; while (!outputStream.eof()) { uint32_t code{decoder.decode(readBit)}; accum+=code; outputStream.writeByte(accum); } } } ancient-2.2.0/src/SMPLDecompressor.hpp000066400000000000000000000014551463063262600176120ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SMPLDECOMPRESSOR_HPP #define SMPLDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class SMPLDecompressor : public XPKDecompressor { public: SMPLDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~SMPLDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/SQSHDecompressor.cpp000066400000000000000000000124331463063262600176060ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SQSHDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "HuffmanDecoder.hpp" #include "VariableLengthCodeDecoder.hpp" #include "common/Common.hpp" #include namespace ancient::internal { bool SQSHDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SQSH"); } std::shared_ptr SQSHDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } SQSHDecompressor::SQSHDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr) || packedData.size()<3) throw Decompressor::InvalidFormatError(); _rawSize=packedData.readBE16(0); if (!_rawSize) throw Decompressor::InvalidFormatError(); } const std::string &SQSHDecompressor::getSubName() const noexcept { static std::string name{"XPK-SQSH: Compressor for sampled sounds"}; return name; } void SQSHDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { if (rawData.size()!=_rawSize) throw Decompressor::DecompressionError(); ForwardInputStream inputStream{_packedData,2,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readSignedBits=[&](uint8_t bits)->int32_t { int32_t ret{int32_t(readBits(bits))}; if (ret&(1<<(bits-1))) ret|=~0U<uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,_rawSize}; HuffmanDecoder modDecoder { HuffmanCode{1,0b0001,uint8_t{0}}, HuffmanCode{2,0b0000,uint8_t{1}}, HuffmanCode{3,0b0010,uint8_t{2}}, HuffmanCode{4,0b0110,uint8_t{3}}, HuffmanCode{4,0b0111,uint8_t{4}} }; HuffmanDecoder lengthDecoder { HuffmanCode{1,0b0000,uint8_t{0}}, HuffmanCode{2,0b0010,uint8_t{1}}, HuffmanCode{3,0b0110,uint8_t{2}}, HuffmanCode{4,0b1110,uint8_t{3}}, HuffmanCode{4,0b1111,uint8_t{4}} }; HuffmanDecoder distanceDecoder { HuffmanCode{1,0b01,uint8_t{1}}, HuffmanCode{2,0b00,uint8_t{0}}, HuffmanCode{2,0b01,uint8_t{2}} }; // first byte is special uint8_t currentSample{readByte()}; outputStream.writeByte(currentSample); uint32_t accum1{0}; uint32_t accum2{0}; uint32_t prevBits{0}; const std::array,7> bitLengthTable{{ {2,3,4,5,6,7,8,0}, {3,2,4,5,6,7,8,0}, {4,3,5,2,6,7,8,0}, {5,4,6,2,3,7,8,0}, {6,5,7,2,3,4,8,0}, {7,6,8,2,3,4,5,0}, {8,7,6,2,3,4,5,0}}}; VariableLengthCodeDecoder lengthVlcDecoder{1,1,1,3,5}; VariableLengthCodeDecoder distanceVlcDecoder{8,12,14}; while (!outputStream.eof()) { uint8_t bits{0}; uint32_t count{0}; bool doRepeat{false}; if (accum1>=8U) { auto handleCondCase=[&]() { if (bits==8U) { if (accum2<20U) { count=1U; } else { count=2U; accum2+=8U; } } else { count=5U; accum2+=8U; } }; auto handleTable=[&](uint32_t newBits) { if (prevBits<2U || !newBits) throw Decompressor::DecompressionError(); bits=bitLengthTable[prevBits-2U][newBits-1U]; if (!bits) throw Decompressor::DecompressionError(); handleCondCase(); }; switch (modDecoder.decode(readBit)) { case 0: if (prevBits==8U) { bits=8U; handleCondCase(); } else { bits=prevBits; count=5U; accum2+=8U; } break; case 1U: doRepeat=true; break; case 2U: handleTable(2U); break; case 3U: handleTable(3U); break; case 4U: handleTable(readBits(2U)+4U); break; default: throw Decompressor::DecompressionError(); } } else { if (readBit()) { doRepeat=true; } else { count=1U; bits=8U; } } if (doRepeat) { /* uint32_t lengthIndex=lengthDecoder.decode(readBit); static const uint8_t lengthBits[5]={1,1,1,3,5}; static const uint32_t lengthAdditions[5]={2,4,6,8,16}; count=readBits(lengthBits[lengthIndex])+lengthAdditions[lengthIndex];*/ count=lengthVlcDecoder.decode(readBits,lengthDecoder.decode(readBit))+2U; if (count>=3) { if (accum1) accum1--; if (count>3 && accum1) accum1--; } /* uint32_t distanceIndex=distanceDecoder.decode(readBit); static const uint8_t distanceBits[3]={12,8,14}; static const uint32_t distanceAdditions[3]={0x101,1,0x1101}; uint32_t distance=readBits(distanceBits[distanceIndex])+distanceAdditions[distanceIndex];*/ uint32_t distance{distanceVlcDecoder.decode(readBits,distanceDecoder.decode(readBit))+1U}; count=std::min(count,uint32_t(_rawSize-outputStream.getOffset())); currentSample=outputStream.copy(distance,count); } else { count=std::min(count,uint32_t(_rawSize-outputStream.getOffset())); for (uint32_t i=0;i>3U; } } } ancient-2.2.0/src/SQSHDecompressor.hpp000066400000000000000000000015051463063262600176110ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SQSHDECOMPRESSOR_HPP #define SQSHDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class SQSHDecompressor : public XPKDecompressor { public: SQSHDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~SQSHDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _rawSize{0}; }; } #endif ancient-2.2.0/src/SXSCDecompressor.cpp000066400000000000000000000542031463063262600176110ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SXSCDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "DLTADecode.hpp" #include "common/MemoryBuffer.hpp" #include "common/Common.hpp" #include namespace ancient::internal { SXSCDecompressor::SXSCReader::SXSCReader(ForwardInputStream &stream) : _reader(stream) { // nothing needed } uint32_t SXSCDecompressor::SXSCReader::readBit() { return _reader.readBits8(1); } // --- bool SXSCDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("SASC")||hdr==FourCC("SHSC"); } std::shared_ptr SXSCDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } SXSCDecompressor::SXSCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData}, _isHSC{hdr==FourCC("SHSC")} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &SXSCDecompressor::getSubName() const noexcept { static std::string nameASC{"XPK-SASC: LZ-compressor with arithmetic and delta encoding"}; static std::string nameHSC{"XPK-SHSC: Context modeling compressor"}; return _isHSC?nameHSC:nameASC; } void SXSCDecompressor::decompressASC(Buffer &rawData,ForwardInputStream &inputStream) { ForwardOutputStream outputStream{rawData,0,rawData.size()}; uint16_t bitReaderInitialValue; bitReaderInitialValue=inputStream.readByte()<<8; bitReaderInitialValue|=inputStream.readByte(); SXSCReader bitReader{inputStream}; RangeDecoder arithDecoder(bitReader,bitReaderInitialValue); // decoder for literal, copy, end decision // two thresholds -> 3 symbols, last symbol is break with size of 1 std::array bitThreshold1{40,40,40,40}; std::array bitThreshold2{40,40,40,40}; uint32_t bitPos{0}; // generics for the other decoder auto tableElements=[](auto &table)->uint16_t { return uint16_t((table.size()+1U)>>1U); }; auto initTable=[&](auto &table,uint16_t initialValue) { uint32_t length{tableElements(table)}; for (uint32_t i=0;i>1)+length) table[i]+=value; if (table[length*2-2]>=max) { for (uint32_t i=0;i1) table[i]>>=1; for (uint32_t j=0,i=length;iuint16_t { return table[table.size()-1U]; }; auto decodeSymbol=[&](auto &table,uint16_t &value)->uint16_t { uint32_t length{tableElements(table)}; uint32_t threshold{0}; uint32_t i{length*2U-4U}; while (i>=length) { uint32_t child=(i-length)<<1; if (value-threshold>=table[i]) { threshold+=table[i]; child+=2; } i=child; } if (value-threshold>=table[i]) { threshold+=table[i]; i++; } value=threshold; return i; }; // literal decoder std::array litInitial; std::array litDynamic; uint16_t litThreshold{1}; initTable(litInitial,1U); initTable(litDynamic,0); // distance / length decoder std::array distanceCodes; std::array countInitial; std::array countDynamic; uint16_t countThreshold{8}; initTable(distanceCodes,0); initTable(countInitial,1U); initTable(countDynamic,0); updateTable(distanceCodes,6000U,0,24U); uint32_t distanceIndex=0; auto twoStepArithDecoder=[&](auto &initialTable,auto &dynamicTable,uint16_t &threshold,uint16_t max,uint16_t step,uint16_t updateRange)->uint16_t { uint16_t value{arithDecoder.decode(tableSize(dynamicTable)+threshold)}; uint16_t ret; if (valueupdateRange?ret-updateRange:0;istep?threshold-step:1U; return ret; }; for (;;) { uint16_t bitSize{uint16_t(bitThreshold1[bitPos]+bitThreshold2[bitPos])}; uint16_t bitValue{arithDecoder.decode(bitSize+1U)}; if (bitValue==bitSize) break; bool bit{bitValue=6000U) { if (!(bitThreshold1[bitPos]>>=1U)) bitThreshold1[bitPos]=1U; if (!(bitThreshold2[bitPos]>>=1U)) bitThreshold2[bitPos]=1U; } bitPos=(bitPos<<1U&2U)|(bit?0:1U); if (bit) { // literal outputStream.writeByte(uint8_t(twoStepArithDecoder(litInitial,litDynamic,litThreshold,1000U,1U,8U))); } else { // copy while (outputStream.getOffset()>(1ULL<=2U) { uint16_t minRange{uint16_t(1U<<(distanceBits-1U))}; uint16_t range{distanceIndex==distanceBits?uint16_t(std::min(outputStream.getOffset(),size_t(31200U))-minRange):minRange}; distance=arithDecoder.decode(range); arithDecoder.scale(distance,distance+1,range); distance+=minRange; } distance++; uint32_t count{twoStepArithDecoder(countInitial,countDynamic,countThreshold,6000,8,4)}; if (count==15U) { count=783U; } else if (count>=16U) { uint16_t value=arithDecoder.decode(16U); arithDecoder.scale(value,value+1U,16U); count=((count-16U)<<4U)+value+15U; } count+=3U; outputStream.copy(distance,count); } } if (!outputStream.eof()) throw Decompressor::DecompressionError(); } template class CheckedArray { public: CheckedArray() : _buffer(N) { // nothing needed } ~CheckedArray() noexcept=default; T &operator[](size_t i) { if (i>=N) throw Decompressor::DecompressionError(); return _buffer[i]; } const T &operator[](size_t i) const { if (i>=N) throw Decompressor::DecompressionError(); return _buffer[i]; } private: std::vector _buffer; }; // The horror. It needs to follow exactly the original logic, even if that logic is not very good. void SXSCDecompressor::decompressHSC(Buffer &rawData,ForwardInputStream &inputStream) { struct Model { std::array context; uint16_t hashPointer; uint16_t expiryPrevious; uint16_t expiryNext; uint16_t frequencyTotal; uint16_t escapeFrequency; uint8_t contextLength; uint8_t characterCount; uint8_t refreshCounter; }; struct Frequency { uint16_t frequency; uint16_t next; uint8_t character; }; struct HashItem { uint16_t data; uint16_t random; }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; uint16_t bitReaderInitialValue; bitReaderInitialValue=inputStream.readByte()<<8; bitReaderInitialValue|=inputStream.readByte(); SXSCReader bitReader{inputStream}; RangeDecoder arithDecoder{bitReader,bitReaderInitialValue}; uint8_t maxContextLength{4}; int16_t dropCount{2500}; int16_t contextSearchLength{0}; CheckedArray models{}; for (uint32_t i=0;i<10000U;i++) { auto &m{models[i]}; for (uint32_t j=0;j<4U;j++) m.context[j]=0; m.hashPointer=0; m.expiryPrevious=i-1U; m.expiryNext=i+1U; m.frequencyTotal=0; m.escapeFrequency=0; m.contextLength=0xffU; m.characterCount=0; m.refreshCounter=0; } std::array currentContext; for (uint32_t i=0;i<4U;i++) currentContext[i]=0; uint16_t firstExpiry{0}; uint16_t lastExpiry{9999}; uint16_t freeBlockPointer{10000}; uint16_t releaseBlock{0}; CheckedArray frequencies{}; for (uint32_t i=0;i<32760U;i++) { auto &f{frequencies[i]}; f.frequency=0; f.next=(i>=10000&&i<32759)?i+1:0xffffU; f.character=0; } CheckedArray hashes{}; for (uint32_t i=0,j=10U;i<0x4000U;i++) { auto &h{hashes[i]}; h.data=0xffffU; // constants used are 2147483647 % / 16807 int32_t integerPart=j/127773U; int32_t fractionalPart=j%127773U; int32_t tmp=16807*fractionalPart-2836*integerPart; j=tmp<0?tmp+0x7fff'ffff:tmp; h.random=j&0x3fffU; } std::array hashStack; for (uint32_t i=0;i<5U;i++) hashStack[i]=0; std::array characterMask; std::array characterMaskStack; for (uint32_t i=0;i<256U;i++) { characterMask[i]=false; characterMaskStack[i]=0; } int16_t characterMaskStackPointer{0}; std::array initialEscapeChar; for (uint32_t i=0;i<5U;i++) initialEscapeChar[i]=i?15U:16U; uint8_t escapeCharacterCounter{0}; uint16_t stackPointer{0}; std::array contextPointer; std::array frequencyArrayIndex; for (uint32_t i=0;i<5U;i++) { contextPointer[i]=0; frequencyArrayIndex[i]=0; } auto loopBreaker=[](uint16_t i) { if (i>=0x8000U) throw Decompressor::DecompressionError(); }; auto findNext=[&]()->uint16_t { for (int32_t i=contextSearchLength-1;i>=0;i--) { for (uint32_t lb=0,j=hashes[hashStack[i]].data;j!=0xffffU;j=models[j].hashPointer,loopBreaker(lb++)) { if (i==models[j].contextLength) { if ([&]()->bool { for (int32_t k=0;k=value) break; i++; } arithDecoder.scale(i,i+1,size); break; } // madness!!! auto getEscFrequency=[&](uint16_t value,uint16_t i)->uint16_t { auto &model{models[i]}; if (model.frequencyTotal==1) return initialEscapeChar[model.contextLength]>=16?2:1; if (model.characterCount==0xffU) return 1; uint16_t tmp{uint16_t(uint16_t(model.characterCount)*2U+2U)}; if (model.characterCount && tmp>=model.frequencyTotal) { value=int32_t(value)*tmp/model.frequencyTotal; if (model.characterCount+1==model.frequencyTotal) value+=tmp>>2; } if (!value) value++; return value; }; int16_t freq{0}; int16_t escapeFreq{0}; int16_t currentFrequency{0}; int16_t frequencyTotal{0}; auto decodeCf=[&](uint16_t shift,bool cmCondition)->uint16_t { freq<<=shift; uint16_t i; uint16_t value{uint16_t(arithDecoder.decode(freq+escapeFreq)>>shift)}; uint32_t lb{0}; for (i=index;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) { auto &frequency{frequencies[i]}; if (cmCondition||!characterMask[frequency.character]) { if (frequencyTotal+frequency.frequency<=value) { frequencyTotal+=frequency.frequency; } else { currentFrequency=frequency.frequency<bool { if (chPos==0xffffU) { arithDecoder.scale(freq,freq+escapeFreq,freq+escapeFreq); if (models[index].frequencyTotal==1 && initialEscapeChar[models[index].contextLength]<32) initialEscapeChar[models[index].contextLength]++; uint16_t prevI{0}; for (uint16_t lb=0,i=index;i!=0xffffU;prevI=i,i=frequencies[i].next,loopBreaker(lb++)) { auto &frequency{frequencies[i]}; if (cmCondition||!characterMask[frequency.character]) { if (characterMaskStackPointer==256) throw Decompressor::DecompressionError(); characterMaskStack[characterMaskStackPointer++]=frequency.character; characterMask[frequency.character]=true; } } contextPointer[insertPos]=index|0x8000U; frequencyArrayIndex[insertPos]=prevI; ch=256; return true; } else { arithDecoder.scale(frequencyTotal,frequencyTotal+currentFrequency,freq+escapeFreq); if (models[index].frequencyTotal==1 && initialEscapeChar[models[index].contextLength]) initialEscapeChar[models[index].contextLength]--; contextPointer[insertPos]=index; frequencyArrayIndex[insertPos]=chPos; ch=frequencies[chPos].character; return false; } }; if (characterMaskStackPointer) { for (uint16_t lb=0,i=index;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) { auto &frequency{frequencies[i]}; if (!characterMask[frequency.character]) { freq+=frequency.frequency; if (frequency.frequency<3) escapeFreq++; } } escapeFreq=getEscFrequency(escapeFreq,index); uint16_t chPos{decodeCf(0,false)}; if (stackPointer==5U) throw Decompressor::DecompressionError(); if (!decodeCh(chPos,stackPointer,false)) { if (escapeCharacterCounter==10U) throw Decompressor::DecompressionError(); escapeCharacterCounter++; } stackPointer++; } else { freq=models[index].frequencyTotal; escapeFreq=getEscFrequency(models[index].escapeFrequency,index); uint16_t chPos{decodeCf((escapeCharacterCounter>=5)?(freq<5 && escapeCharacterCounter==10)?2:1:0,true)}; stackPointer=1; if (decodeCh(chPos,0,true)) { escapeCharacterCounter=0; } else { if (escapeCharacterCounter<10) escapeCharacterCounter++; } } if (ch!=256) { if (index!=firstExpiry) { auto &model{models[index]}; if (index==lastExpiry) { lastExpiry=model.expiryPrevious; } else { models[model.expiryNext].expiryPrevious=model.expiryPrevious; models[model.expiryPrevious].expiryNext=model.expiryNext; } models[firstExpiry].expiryPrevious=index; model.expiryNext=firstExpiry; firstExpiry=index; } break; } } if (ch==256) break; while (stackPointer) { uint16_t freqIndex{frequencyArrayIndex[--stackPointer]}; uint16_t pointer{contextPointer[stackPointer]}; auto &model{models[pointer&0x7fffU]}; if (pointer&0x8000U) { if (freeBlockPointer==0xffffU) { for (uint16_t i=0;i<=stackPointer;) { // yuck uint32_t lb{0}; do { releaseBlock=(releaseBlock!=9999U)?releaseBlock+1:0; loopBreaker(lb++); } while (frequencies[releaseBlock].next==0xffffU); for (i=0;i<=stackPointer;i++) if ((contextPointer[i]&0x7fffU)==releaseBlock) break; } auto &frequencyRB{frequencies[releaseBlock]}; auto &modelRB{models[releaseBlock]}; uint16_t f{frequencyRB.frequency}; for (uint16_t lb=0,i=frequencyRB.next;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) if (frequencies[i].frequencyfrequencies[freqIndex].frequency*2) { model.refreshCounter--; } else if (model.refreshCounter<4U) { model.refreshCounter++; } // one ugly scaler if (!model.refreshCounter||model.frequencyTotal>=8000U) { model.refreshCounter++; model.escapeFrequency=0; model.frequencyTotal=0; for (uint16_t lb=0,i=pointer&0x7fffU;i!=0xffffU;i=frequencies[i].next,loopBreaker(lb++)) { if (frequencies[i].frequency>1) { uint16_t tmp{frequencies[i].frequency>>=1}; model.frequencyTotal+=tmp; if (tmp<3) model.escapeFrequency++; } else { model.escapeFrequency++; model.frequencyTotal++; } } } } uint8_t maxLength=maxContextLength+1; while (maxLength-->minLength) { auto hash=[&](const std::array &block,uint32_t length)->uint16_t { uint16_t ret{0}; for (uint32_t i=0;i=2); std::unique_ptr tmpBuffer; if (needsTmpBuffer) tmpBuffer=std::make_unique(rawData.size()); if (_isHSC) decompressHSC(needsTmpBuffer?*tmpBuffer:rawData,inputStream); else decompressASC(needsTmpBuffer?*tmpBuffer:rawData,inputStream); // Mono, High byte only // also includes de-interleaving auto deltaDecode16BE=[&]() { size_t length=rawData.size(); uint8_t ctr{0}; for (size_t i=0,j=0;j>1)+i]; } if (length&1) rawData[length-1]=(*tmpBuffer)[length-1]; }; // Stereo, High byte only // also includes de-interleaving auto deltaDecode16LE=[&]() { size_t length=rawData.size(); uint8_t ctr{0}; for (size_t i=0,j=0;j>1)+i]; ctr+=(*tmpBuffer)[i]; rawData[j+1]=ctr; } if (length&1) rawData[length-1]=(*tmpBuffer)[length-1]; }; switch (mode) { case 0: // no delta break; case 1: DLTADecode::decode(rawData,rawData,0,rawData.size()); break; case 2: deltaDecode16BE(); break; case 3: deltaDecode16LE(); break; default: throw Decompressor::DecompressionError(); } } } ancient-2.2.0/src/SXSCDecompressor.hpp000066400000000000000000000024001463063262600176060ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SXSCDECOMPRESSOR_HPP #define SXSCDECOMPRESSOR_HPP #include #include "XPKDecompressor.hpp" #include "InputStream.hpp" #include "RangeDecoder.hpp" namespace ancient::internal { class SXSCDecompressor : public XPKDecompressor { public: SXSCDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~SXSCDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: class SXSCReader : public RangeDecoder::BitReader { public: SXSCReader(ForwardInputStream &stream); ~SXSCReader() noexcept=default; uint32_t readBit() final; private: MSBBitReader _reader; }; void decompressASC(Buffer &rawData,ForwardInputStream &inputStream); void decompressHSC(Buffer &rawData,ForwardInputStream &inputStream); const Buffer &_packedData; bool _isHSC; }; } #endif ancient-2.2.0/src/StoneCrackerDecompressor.cpp000066400000000000000000000400751463063262600214160ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "StoneCrackerDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "HuffmanDecoder.hpp" #include "VariableLengthCodeDecoder.hpp" #include "common/Common.hpp" #include "common/MemoryBuffer.hpp" namespace ancient::internal { bool StoneCrackerDecompressor::detectHeaderAndGeneration(uint32_t hdr,uint32_t &generation) noexcept { // Stonecracker 2.71 (and others from 2.69 - 2.8 series) do not have any sensible identification value // first 3 bytes are constants for RLE-encoder (byte values for RLE-encoder, at least they are unique) // last byte is bit length. // // 2.92 and 2.99 do not have either any proper identification word either, however its // bit lengts for decompressor are stored in the first 4 bytes, which forms identifiable // value. // // Thus for detecting 2.71 and friends, we are creating lots of false positives here. // At least we can rule those out later when detecting the actual header content // Final complication is that values for 2.71/2.9X overlap, this we need to handle // later as well if (hdr>=0x08090a08U && hdr<=0x08090a0eU && hdr!=0x08090a09U) { // can be generation 1 as well. needs to be determined later generation=2; return true; } if ((hdr&0xffU)>=0x08U&&(hdr&0xffU)<=0x0eU) { uint8_t byte0{uint8_t(hdr>>24U)}; uint8_t byte1{uint8_t(hdr>>16U)}; uint8_t byte2{uint8_t(hdr>>8U)}; // only limiter I can think of apart from the last byte is that the rle-bytes are unique if (byte0!=byte1 && byte0!=byte2 && byte1!=byte2) { generation=1; return true; } } // Specials switch (hdr&0xffff'ff00U) { case FourCC("1AM\000"): // Reunion generation=3; return true; case FourCC("2AM\000"): // Reunion generation=6; return true; default: break; } // From 3.00 and onwards we can be certain of the format switch (hdr) { case FourCC("S300"): generation=3; return true; case FourCC("S310"): generation=4; return true; case FourCC("S400"): generation=5; return true; case FourCC("S401"): generation=6; return true; case FourCC("S403"): [[fallthrough]]; case FourCC("Z&G!"): // Switchback / Rebels [[fallthrough]]; case FourCC("ZULU"): // Whammer Slammer / Rebels generation=7; return true; case FourCC("S404"): [[fallthrough]]; case FourCC("AYS!"): // High Anxiety / Abyss generation=8; return true; default: return false; } } bool StoneCrackerDecompressor::detectHeader(uint32_t hdr) noexcept { uint32_t dummy; return detectHeaderAndGeneration(hdr,dummy); } std::shared_ptr StoneCrackerDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,exactSizeKnown,verify); } void StoneCrackerDecompressor::initialize(const Buffer &packedData,uint32_t hdr) { auto readModes=[&](uint32_t value) { for (uint32_t i=0;i<4U;i++) { _modes[i]=value>>24U; if (_modes[i]<8U || _modes[i]>14U) throw InvalidFormatError(); value<<=8U; } }; switch (_generation) { case 1: _dataOffset=18U; _rle[0]=hdr>>24U; _rle[1]=hdr>>16U; _rle[2]=hdr>>8U; _modes[0]=hdr; if (packedData.size()<_dataOffset) throw InvalidFormatError(); for (uint32_t i=1;i<3U;i++) { _modes[i]=packedData.read8(i+15U); if (_modes[i]<4U || _modes[i]>7U) throw InvalidFormatError(); } _rleSize=packedData.readBE32(4U); _rawSize=packedData.readBE32(8U); _packedSize=packedData.readBE32(12U); break; case 2: readModes(hdr); [[fallthrough]]; case 4: [[fallthrough]]; case 5: [[fallthrough]]; case 6: _dataOffset=12U; if (packedData.size()<_dataOffset) throw InvalidFormatError(); _rawSize=packedData.readBE32(4U); _packedSize=packedData.readBE32(8U); break; case 3: _dataOffset=16U; if (packedData.size()<_dataOffset) throw InvalidFormatError(); readModes(packedData.readBE32(4)); _rawSize=packedData.readBE32(8U); _packedSize=packedData.readBE32(12U); break; case 7: case 8: _dataOffset=16U; if (packedData.size()<_dataOffset+2U) throw InvalidFormatError(); _rawSize=packedData.readBE32(8U); _packedSize=packedData.readBE32(12U)+2U; break; default: throw InvalidFormatError(); break; } if (_packedSize>getMaxPackedSize() || _rawSize>getMaxRawSize()) throw InvalidFormatError(); // Final sanity checks on old formats, especially on 2.71 which can still be false positive // For a sanity check we assume the compressor is actually compressing, which is not a exactly wrong thing to do // (of course there could be expanding files, but it is doubtful someone would actually used them) // Also, the compressor seem to crash on large files. Lets cap the filesize to 1M if (_generation==1 && (_rleSize>_rawSize || _packedSize>_rleSize || _rawSize>1048'576U)) throw InvalidFormatError(); _packedSize+=_dataOffset; if (_packedSize>packedData.size()) throw InvalidFormatError(); } StoneCrackerDecompressor::StoneCrackerDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData{packedData} { uint32_t hdr{packedData.readBE32(0)}; if (!detectHeaderAndGeneration(hdr,_generation)) throw InvalidFormatError(); bool initialized{false}; // this is the fallback if we have accidentally identified the wrong version if (_generation==2) { try { initialize(packedData,hdr); initialized=true; } catch (const Error &) { _generation=1; } } if (!initialized) initialize(packedData,hdr); } const std::string &StoneCrackerDecompressor::getName() const noexcept { switch (_generation) { case 1: { static std::string name{"SC: StoneCracker v2.69 - v2.81"}; return name; } case 2: { static std::string name{"SC: StoneCracker v2.92, v2.99"}; return name; } case 3: { static std::string name{"S300: StoneCracker v3.00"}; return name; } case 4: { static std::string name{"S310: StoneCracker v3.10, v3.11b"}; return name; } case 5: { static std::string name{"S400: StoneCracker pre v4.00"}; return name; } case 6: { static std::string name{"S401: StoneCracker v4.01"}; return name; } case 7: { static std::string name{"S403: StoneCracker v4.02a"}; return name; } case 8: { static std::string name{"S404: StoneCracker v4.10"}; return name; } } static std::string dummy=""; return dummy; } size_t StoneCrackerDecompressor::getPackedSize() const noexcept { return _packedSize; } size_t StoneCrackerDecompressor::getRawSize() const noexcept { return _rawSize; } // v2.71 void StoneCrackerDecompressor::decompressGen1(Buffer &rawData) { BackwardInputStream inputStream{_packedData,_dataOffset,_packedSize}; MSBBitReader bitReader{inputStream}; MemoryBuffer tmpBuffer(_rleSize); BackwardOutputStream outputStream{tmpBuffer,0,_rleSize}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBitsBE32(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1); }; // anchor-bit handling { uint32_t value{inputStream.readBE32()}; uint32_t tmp{value}; uint32_t count{0}; while (tmp) { tmp<<=1; count++; } if (count) count--; if (count) bitReader.reset(value>>(32U-count),count); } VariableLengthCodeDecoder countVlcDecoder{3,_modes[2]+1U,-1,0,0,0,_modes[1]+1U}; std::array distanceBits{0,0,0,8,9,10,uint8_t(_modes[0]+1U)}; HuffmanDecoder scDecoder { HuffmanCode{2,0b000,uint8_t{0x80}}, HuffmanCode{3,0b010,uint8_t{0x81}}, HuffmanCode{3,0b011,uint8_t{6}}, HuffmanCode{2,0b010,uint8_t{3}}, HuffmanCode{3,0b110,uint8_t{4}}, HuffmanCode{3,0b111,uint8_t{5}} }; while (!outputStream.eof()) { uint8_t index{scDecoder.decode(readBit)}; uint32_t count{countVlcDecoder.decode(readBits,index&0x7fU)}; if (index&0x80U) { if (!count) count=8U; // why?!? for (uint32_t i=0;i bitReader{inputStream}; BackwardOutputStream outputStream{rawData,0,_rawSize}; // anchor-bit handling { uint32_t value{inputStream.readBE32()}; if (value) for (uint32_t i=31U;i>0;i--) { if (value&(1U<uint32_t { return bitReader.readBitsBE32(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1); }; auto readCount=[&](uint32_t threshold,uint32_t bits)->uint32_t { uint32_t ret{0}; uint32_t tmp; do { tmp=rotateBits(readBits(bits),bits); ret+=tmp; } while (tmp==threshold); return ret; }; bool gen3{_generation>=3U}; while (!outputStream.eof()) { if (readBit()) { uint32_t count{readCount(7U,3U)}; if (gen3) count++; // for 2.92 zero count could meant count of 65536 // for 2.99 it would be 4G // nevertheless it is an error // 3.00 fixes this by +1 if (!count) throw DecompressionError(); for (uint32_t i=0;i bitReader{inputStream}; ForwardOutputStream outputStream{rawData,0,_rawSize}; auto readBits=[&](uint32_t count)->uint32_t { if (_generation==4U) return bitReader.readBitsBE32(count); else return bitReader.readBitsBE16(count); }; auto readBit=[&]()->uint32_t { return readBits(1); }; auto readCount=[&](uint32_t threshold,uint32_t bits)->uint32_t { uint32_t ret{0}; uint32_t tmp; do { tmp=readBits(bits); ret+=tmp; } while (tmp==threshold); return ret; }; const std::array modes{10U,11U,12U,8U}; while (!outputStream.eof()) { if (readBit()) { uint32_t mode{readBits(2U)}; uint32_t distance{readBits(modes[mode])}; // will obviously throw if distance is 0 uint32_t count; if (mode>=2U) { if (_generation==4) count=readCount(15U,4U)+3U; else count=readCount(7U,3U)+3U; } else { count=mode+3U; } // yet another bug if (count>_rawSize-uint32_t(outputStream.getOffset())) count=_rawSize-uint32_t(outputStream.getOffset()); outputStream.copy(distance,count); } else { uint32_t count=readCount(7U,3U); // A regression in 3.10 that is not fixed until 4.01 if (_generation==6U) count++; if (!count) throw DecompressionError(); for (uint32_t i=0;i bitReader{inputStream}; BackwardOutputStream outputStream{rawData,0,_rawSize}; // incomplete value at start { uint16_t bitCount{_packedData.readBE16(_packedSize-2U)}; if (bitCount>16U) throw DecompressionError(); uint16_t value{inputStream.readBE16()}; bitReader.reset(value,bitCount&0xff); } auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBitsBE16(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1); }; VariableLengthCodeDecoder vlcDecoder{5,8,10,12}; while (!outputStream.eof()) { if (readBit()) { uint32_t distance{vlcDecoder.decode(readBits,readBits(2U))+1U}; // christmas tree! uint32_t count{2U}; if (!readBit()) { count++; if (!readBit()) { count++; if (!readBit()) { count++; uint32_t tmp; do { tmp=readBits(3U); count+=tmp; } while (tmp==7U); } } } outputStream.copy(distance,count); } else { outputStream.writeByte(readBits(8)); } } } // v4.10 void StoneCrackerDecompressor::decompressGen8(Buffer &rawData) { BackwardInputStream inputStream{_packedData,_dataOffset,_packedSize-2U}; MSBBitReader bitReader{inputStream}; BackwardOutputStream outputStream{rawData,0,_rawSize}; // incomplete value at start uint16_t modeBits{0}; { uint16_t bitCount{uint16_t(_packedData.readBE16(_packedSize-2U)&15U)}; uint16_t value{inputStream.readBE16()}; bitReader.reset(value>>(16U-bitCount),bitCount&0xff); modeBits=inputStream.readBE16(); if (modeBits<10U || modeBits>14U) throw DecompressionError(); } auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBitsBE16(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBitsBE16(1); }; VariableLengthCodeDecoder countVlcDecoder{0,1,2,3,2,1,0,8,-3,2,0,5}; VariableLengthCodeDecoder distanceVlcDecoder{5,9,modeBits}; HuffmanDecoder countDecoder { HuffmanCode{1,0b00000000,uint8_t{0x80}}, HuffmanCode{2,0b00000011,uint8_t{1}}, HuffmanCode{3,0b00000101,uint8_t{2}}, HuffmanCode{4,0b00001000,uint8_t{7}}, HuffmanCode{5,0b00010010,uint8_t{3}}, HuffmanCode{6,0b00100110,uint8_t{4}}, HuffmanCode{7,0b01001110,uint8_t{5}}, HuffmanCode{8,0b10011110,uint8_t{6}}, HuffmanCode{8,0b10011111,uint8_t{0x8bU}} }; HuffmanDecoder distanceDecoder { HuffmanCode{1,0b01,uint8_t{2}}, HuffmanCode{2,0b00,uint8_t{1}}, HuffmanCode{2,0b01,uint8_t{0}} }; while (!outputStream.eof()) { uint8_t countIndex{countDecoder.decode(readBit)}; uint32_t count{countVlcDecoder.decode(readBits,countIndex&0x7fU)+1U}; uint32_t tmp; if (count==278U) do { tmp=readBits(8U); count+=tmp; } while (tmp==0xffU); if (countIndex&0x80U) { for (uint32_t i=0;i namespace ancient::internal { class StoneCrackerDecompressor : public Decompressor { public: StoneCrackerDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify); ~StoneCrackerDecompressor() noexcept=default; const std::string &getName() const noexcept final; size_t getPackedSize() const noexcept final; size_t getRawSize() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: static bool detectHeaderAndGeneration(uint32_t hdr,uint32_t &generation) noexcept; void initialize(const Buffer &packedData,uint32_t hdr); void decompressGen1(Buffer &rawData); void decompressGen23(Buffer &rawData); void decompressGen456(Buffer &rawData); void decompressGen7(Buffer &rawData); void decompressGen8(Buffer &rawData); const Buffer &_packedData; uint32_t _rawSize{0}; uint32_t _packedSize{0}; uint32_t _rleSize{0}; std::array _modes; std::array _rle; uint32_t _generation; uint32_t _dataOffset; }; } #endif ancient-2.2.0/src/TDCSDecompressor.cpp000066400000000000000000000043071463063262600175660ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "TDCSDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool TDCSDecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("TDCS"); } std::shared_ptr TDCSDecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } TDCSDecompressor::TDCSDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); } const std::string &TDCSDecompressor::getSubName() const noexcept { static std::string name{"XPK-TDCS: LZ77-compressor"}; return name; } void TDCSDecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,0,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto read2Bits=[&]()->uint32_t { return bitReader.readBitsBE32(2); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; while (!outputStream.eof()) { uint32_t distance{0}; uint32_t count{0}; uint32_t tmp; switch (read2Bits()) { case 0: outputStream.writeByte(readByte()); break; case 1: tmp=uint32_t(readByte())<<8U; tmp|=uint32_t(readByte()); count=(tmp&3U)+3U; distance=((tmp>>2U)^0x3fffU)+1U; break; case 2: tmp=uint32_t(readByte())<<8U; tmp|=uint32_t(readByte()); count=(tmp&0xfU)+3U; distance=((tmp>>4U)^0xfffU)+1U; break; case 3: distance=uint32_t(readByte())<<8U; distance|=uint32_t(readByte()); count=uint32_t(readByte())+3U; if (!distance) throw Decompressor::DecompressionError(); distance=(distance^0xffffU)+1U; break; default: throw Decompressor::DecompressionError(); } if (count && distance) outputStream.copy(distance,count); } } } ancient-2.2.0/src/TDCSDecompressor.hpp000066400000000000000000000014551463063262600175740ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef TDCSDECOMPRESSOR_HPP #define TDCSDECOMPRESSOR_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class TDCSDecompressor : public XPKDecompressor { public: TDCSDecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~TDCSDecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/TPWMDecompressor.cpp000066400000000000000000000040451463063262600176170ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "TPWMDecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool TPWMDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("TPWM"); } std::shared_ptr TPWMDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,verify); } TPWMDecompressor::TPWMDecompressor(const Buffer &packedData,bool verify) : _packedData{packedData} { uint32_t hdr{packedData.readBE32(0)}; if (!detectHeader(hdr) || packedData.size()<12) throw InvalidFormatError(); _rawSize=packedData.readBE32(4); if (!_rawSize || _rawSize>getMaxRawSize()) throw InvalidFormatError(); } const std::string &TPWMDecompressor::getName() const noexcept { static std::string name="TPWM: Turbo Packer"; return name; } size_t TPWMDecompressor::getPackedSize() const noexcept { // No packed size in the stream :( // After decompression, we can tell how many bytes were actually used return _decompressedPackedSize; } size_t TPWMDecompressor::getRawSize() const noexcept { return _rawSize; } void TPWMDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); ForwardInputStream inputStream{_packedData,8U,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1U); }; auto readByte=[&]()->uint8_t { return inputStream.readByte(); }; ForwardOutputStream outputStream{rawData,0,_rawSize}; while (!outputStream.eof()) { if (readBit()) { uint8_t byte1{readByte()}; uint8_t byte2{readByte()}; uint32_t distance{(uint32_t(byte1&0xf0U)<<4U)|byte2}; uint32_t count{uint32_t(byte1&0xfU)+3U}; count=std::min(count,uint32_t(_rawSize-outputStream.getOffset())); outputStream.copy(distance,count); } else { outputStream.writeByte(readByte()); } } _decompressedPackedSize=inputStream.getOffset(); } } ancient-2.2.0/src/TPWMDecompressor.hpp000066400000000000000000000014051463063262600176210ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef TPWMDECOMPRESSOR_HPP #define TPWMDECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class TPWMDecompressor : public Decompressor { public: TPWMDecompressor(const Buffer &packedData,bool verify); ~TPWMDecompressor() noexcept=default; const std::string &getName() const noexcept final; size_t getPackedSize() const noexcept final; size_t getRawSize() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: const Buffer &_packedData; uint32_t _rawSize{0}; size_t _decompressedPackedSize{0}; }; } #endif ancient-2.2.0/src/VariableLengthCodeDecoder.hpp000066400000000000000000000036351463063262600214230ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef VARIABLELENGTHCODEDECODER_HPP #define VARIABLELENGTHCODEDECODER_HPP #include #include #include // For exception #include "Decompressor.hpp" namespace ancient::internal { template class VariableLengthCodeDecoder { public: // negative lengths can be used to reset the offset template VariableLengthCodeDecoder(Args ...args) noexcept : _bitLengths{createBitLength(args)...} { // Probably this could be someway done as a nice constexpr initializer list, but I can't // see an easy way. Let it be for now uint32_t length{0}; uint32_t i{0}; auto foldOffsets=[&](auto value) noexcept { if constexpr (std::is_signed_v) if (value<0) { _offsets[i]=0; length=1U<<-value; i++; return; } _offsets[i]=length; length+=1U< uint32_t decode(F bitReader,uint32_t base) const { if (base>=N) throw Decompressor::DecompressionError(); return _offsets[base]+bitReader(_bitLengths[base]); } template uint32_t decodeCascade(F bitReader) const { for (uint32_t i=0;i static constexpr uint8_t createBitLength(T value) noexcept { if constexpr (std::is_signed_v) return uint8_t(value>=0?value:-value); return uint8_t(value); } const std::array _bitLengths; std::array _offsets; }; template VariableLengthCodeDecoder(Args...args)->VariableLengthCodeDecoder; } #endif ancient-2.2.0/src/VicXDecompressor.cpp000066400000000000000000000072161463063262600177040ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include "VicXDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "DynamicHuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" #include "common/OverflowCheck.hpp" namespace ancient::internal { bool VicXDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("Vice") || hdr==FourCC("Vic2"); } std::shared_ptr VicXDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,exactSizeKnown,verify); } VicXDecompressor::VicXDecompressor(const Buffer &packedData,bool exactSizeKnown,bool verify) : _packedData{packedData} { uint32_t hdr{_packedData.readBE32(0)}; if (!detectHeader(hdr)) throw InvalidFormatError(); _isVic2=hdr==FourCC("Vic2"); if (_isVic2) { _rawSize=_packedData.readBE32(4U); _packedSize=OverflowCheck::sum(_packedData.readBE32(8U),12U); if (!_rawSize || _packedSize<1036U || _packedSize>packedData.size() || _rawSize>getMaxRawSize() || _packedSize>getMaxPackedSize()) throw InvalidFormatError(); } } const std::string &VicXDecompressor::getName() const noexcept { static std::string names[2]{ "Vice: Huffman compressor with RLE", "Vic2: Huffman compressor with RLE"}; return names[_isVic2?1:0U]; } size_t VicXDecompressor::getPackedSize() const noexcept { // no way to know before decompressing for Vice return _packedSize; } size_t VicXDecompressor::getRawSize() const noexcept { // no way to know before decompressing for Vice return _rawSize; } void VicXDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream{_packedData,_isVic2?12U:4U,_packedSize?_packedSize:_packedData.size()}; LSBBitReader bitReader{inputStream}; HuffmanDecoder decoder; { std::array tree; for (uint32_t i=0;i<511U;i++) tree[i]=inputStream.readBE16(); inputStream.readBE16(); // ballast auto branch=[&](uint32_t node,uint32_t length,uint32_t bits,auto branch)->void { uint32_t tmp=tree[node]; if (tmp&0x8000U) { if (!length) throw DecompressionError(); decoder.insert(HuffmanCode{length,bits,uint8_t(tmp)}); } else { if ((tmp&1U) || tmp>=0x1feU) throw DecompressionError(); length++; if (length>24U) throw DecompressionError(); bits<<=1U; tmp>>=1U; branch(tmp,length,bits,branch); branch(tmp+256U,length,bits|1U,branch); } }; branch(255U,0,0,branch); } auto readBit=[&]()->uint32_t { return bitReader.readBitsBE32(1U); }; if (_isVic2) { ForwardOutputStream outputStream{rawData,0,_rawSize}; uint8_t rleMarker=decoder.decode(readBit); while (outputStream.getOffset()!=_rawSize) { uint8_t code{decoder.decode(readBit)}; if (code==rleMarker) { uint32_t count{decoder.decode(readBit)}; if (count) { count+=3U; code=decoder.decode(readBit); for (uint32_t i=0;i create(const Buffer &packedData,bool exactSizeKnown,bool verify); private: const Buffer &_packedData; size_t _packedSize{0}; size_t _rawSize{0}; bool _isVic2; }; } #endif ancient-2.2.0/src/XPKDecompressor.cpp000066400000000000000000000003601463063262600174660ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "XPKDecompressor.hpp" #include "XPKMain.hpp" namespace ancient::internal { XPKDecompressor::XPKDecompressor(uint32_t recursionLevel) : _recursionLevel{recursionLevel} { // nothing needed } } ancient-2.2.0/src/XPKDecompressor.hpp000066400000000000000000000015051463063262600174750ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef XPKDECOMPRESSOR_HPP #define XPKDECOMPRESSOR_HPP #include #include #include #include "Decompressor.hpp" namespace ancient::internal { class XPKDecompressor { public: class State { public: State(const State&)=delete; State& operator=(const State&)=delete; State() noexcept=default; virtual ~State() noexcept=default; }; XPKDecompressor(const XPKDecompressor&)=delete; XPKDecompressor& operator=(const XPKDecompressor&)=delete; XPKDecompressor(uint32_t recursionLevel=0); virtual ~XPKDecompressor() noexcept=default; virtual const std::string &getSubName() const noexcept=0; // Actual decompression virtual void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify)=0; protected: uint32_t _recursionLevel; }; } #endif ancient-2.2.0/src/XPKMain.cpp000066400000000000000000000254561463063262600157220ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include "common/SubBuffer.hpp" #include "common/OverflowCheck.hpp" #include "common/Common.hpp" #include "XPKMain.hpp" #include "XPKDecompressor.hpp" #include "ACCADecompressor.hpp" #include "ARTMDecompressor.hpp" #include "BLZWDecompressor.hpp" #include "BZIP2Decompressor.hpp" #include "CBR0Decompressor.hpp" #include "CRMDecompressor.hpp" #include "CYB2Decoder.hpp" #include "DEFLATEDecompressor.hpp" #include "DLTADecode.hpp" #include "FASTDecompressor.hpp" #include "FBR2Decompressor.hpp" #include "FRLEDecompressor.hpp" #include "HFMNDecompressor.hpp" #include "HUFFDecompressor.hpp" #include "ILZRDecompressor.hpp" #include "IMPDecompressor.hpp" #include "LHLBDecompressor.hpp" #include "LIN1Decompressor.hpp" #include "LIN2Decompressor.hpp" #include "LZBSDecompressor.hpp" #include "LZCBDecompressor.hpp" #include "LZW2Decompressor.hpp" #include "LZW4Decompressor.hpp" #include "LZW5Decompressor.hpp" #include "LZXDecompressor.hpp" #include "MASHDecompressor.hpp" #include "NONEDecompressor.hpp" #include "NUKEDecompressor.hpp" #include "PPDecompressor.hpp" #include "PPMQDecompressor.hpp" #include "RAKEDecompressor.hpp" #include "RDCNDecompressor.hpp" #include "RLENDecompressor.hpp" #include "SDHCDecompressor.hpp" #include "SHRXDecompressor.hpp" #include "SLZ3Decompressor.hpp" #include "SMPLDecompressor.hpp" #include "SQSHDecompressor.hpp" #include "SXSCDecompressor.hpp" #include "TDCSDecompressor.hpp" #include "ZENODecompressor.hpp" #include "XPKUnimplemented.hpp" #include namespace ancient::internal { bool XPKMain::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("XPKF"); } std::shared_ptr XPKMain::create(const Buffer &packedData,bool verify,bool exactSizeKnown) { return std::shared_ptr{new XPKMain{packedData,verify,0}}; } static std::vector(*)(uint32_t,uint32_t,const Buffer&,std::shared_ptr&,bool)>> XPKDecompressors={ {ACCADecompressor::detectHeaderXPK,ACCADecompressor::create}, {ARTMDecompressor::detectHeaderXPK,ARTMDecompressor::create}, {BLZWDecompressor::detectHeaderXPK,BLZWDecompressor::create}, {BZIP2Decompressor::detectHeaderXPK,BZIP2Decompressor::create}, {CBR0Decompressor::detectHeaderXPK,CBR0Decompressor::create}, {CRMDecompressor::detectHeaderXPK,CRMDecompressor::create}, {CYB2Decoder::detectHeaderXPK,CYB2Decoder::create}, {DEFLATEDecompressor::detectHeaderXPK,DEFLATEDecompressor::create}, {DLTADecode::detectHeaderXPK,DLTADecode::create}, {FASTDecompressor::detectHeaderXPK,FASTDecompressor::create}, {FBR2Decompressor::detectHeaderXPK,FBR2Decompressor::create}, {FRLEDecompressor::detectHeaderXPK,FRLEDecompressor::create}, {HFMNDecompressor::detectHeaderXPK,HFMNDecompressor::create}, {HUFFDecompressor::detectHeaderXPK,HUFFDecompressor::create}, {ILZRDecompressor::detectHeaderXPK,ILZRDecompressor::create}, {IMPDecompressor::detectHeaderXPK,IMPDecompressor::create}, {LHLBDecompressor::detectHeaderXPK,LHLBDecompressor::create}, {LIN1Decompressor::detectHeaderXPK,LIN1Decompressor::create}, {LIN2Decompressor::detectHeaderXPK,LIN2Decompressor::create}, {LZBSDecompressor::detectHeaderXPK,LZBSDecompressor::create}, {LZCBDecompressor::detectHeaderXPK,LZCBDecompressor::create}, {LZW2Decompressor::detectHeaderXPK,LZW2Decompressor::create}, {LZW4Decompressor::detectHeaderXPK,LZW4Decompressor::create}, {LZW5Decompressor::detectHeaderXPK,LZW5Decompressor::create}, {LZXDecompressor::detectHeaderXPK,LZXDecompressor::create}, {MASHDecompressor::detectHeaderXPK,MASHDecompressor::create}, {NONEDecompressor::detectHeaderXPK,NONEDecompressor::create}, {NUKEDecompressor::detectHeaderXPK,NUKEDecompressor::create}, {PPDecompressor::detectHeaderXPK,PPDecompressor::create}, {PPMQDecompressor::detectHeaderXPK,PPMQDecompressor::create}, {RAKEDecompressor::detectHeaderXPK,RAKEDecompressor::create}, {RDCNDecompressor::detectHeaderXPK,RDCNDecompressor::create}, {RLENDecompressor::detectHeaderXPK,RLENDecompressor::create}, {SDHCDecompressor::detectHeaderXPK,SDHCDecompressor::create}, {SHRXDecompressor::detectHeaderXPK,SHRXDecompressor::create}, {SLZ3Decompressor::detectHeaderXPK,SLZ3Decompressor::create}, {SMPLDecompressor::detectHeaderXPK,SMPLDecompressor::create}, {SQSHDecompressor::detectHeaderXPK,SQSHDecompressor::create}, {SXSCDecompressor::detectHeaderXPK,SXSCDecompressor::create}, {TDCSDecompressor::detectHeaderXPK,TDCSDecompressor::create}, {ZENODecompressor::detectHeaderXPK,ZENODecompressor::create}, {XPKUnimplemented::detectHeaderXPK,XPKUnimplemented::create}}; XPKMain::XPKMain(const Buffer &packedData,bool verify,uint32_t recursionLevel) : _packedData{packedData} { if (recursionLevel>=getMaxRecursionLevel()) throw InvalidFormatError(); if (packedData.size()<44U) throw InvalidFormatError(); uint32_t hdr{packedData.readBE32(0)}; if (!detectHeader(hdr)) throw InvalidFormatError(); _packedSize=packedData.readBE32(4U); _type=packedData.readBE32(8U); _rawSize=packedData.readBE32(12U); if (!_rawSize || !_packedSize) throw InvalidFormatError(); if (_rawSize>getMaxRawSize() || _packedSize>getMaxPackedSize()) throw InvalidFormatError(); uint8_t flags{packedData.read8(32U)}; _longHeaders=(flags&1U)?true:false; if (flags&2U) _hasPassword=true; // Late failure so we can identify format if (flags&4U) // extra header { _headerSize=38U+uint32_t(packedData.readBE16(36U)); } else { _headerSize=36U; } if (OverflowCheck::sum(_packedSize,8U)>packedData.size()) throw InvalidFormatError(); bool found=false; for (auto &it : XPKDecompressors) { if (it.first(_type)) { found=true; break; } } if (!found) throw InvalidFormatError(); auto headerChecksum=[](const Buffer &buffer,size_t offset,size_t len)->bool { if (!len || OverflowCheck::sum(offset,len)>buffer.size()) return false; uint8_t tmp{0}; for (size_t i=0;ibool { if (!len || OverflowCheck::sum(offset,len)>buffer.size()) return false; std::array tmp{0,0}; for (size_t i=0;i>8) && tmp[1]==(checkValue&0xff); }; if (verify) { if (!headerChecksum(_packedData,0,36U)) throw VerificationError(); std::shared_ptr state; forEachChunk([&](const Buffer &header,const Buffer &chunk,uint32_t rawChunkSize,uint8_t chunkType)->bool { if (!headerChecksum(header,0,header.size())) throw VerificationError(); uint16_t hdrCheck{header.readBE16(2U)}; if (chunk.size() && !chunkChecksum(chunk,0,chunk.size(),hdrCheck)) throw VerificationError(); if (chunkType==1U) { auto sub=createDecompressor(_type,_recursionLevel,chunk,state,true); } else if (chunkType!=0 && chunkType!=15U) throw InvalidFormatError(); return true; }); } } const std::string &XPKMain::getName() const noexcept { std::shared_ptr sub; std::shared_ptr state; try { forEachChunk([&](const Buffer &header,const Buffer &chunk,uint32_t rawChunkSize,uint8_t chunkType)->bool { try { sub=createDecompressor(_type,_recursionLevel,chunk,state,false); } catch (const Error&) { // should not happen since the code is already tried out, // however, lets handle the case gracefully } return false; }); } catch (const Buffer::Error&) { // ditto } static std::string invName{""}; return (sub)?sub->getSubName():invName; } size_t XPKMain::getPackedSize() const noexcept { return _packedSize+8U; } size_t XPKMain::getRawSize() const noexcept { return _rawSize; } void XPKMain::decompressImpl(Buffer &rawData,bool verify) { if (rawData.size()<_rawSize) throw DecompressionError(); if (_hasPassword) throw DecompressionError(); uint32_t destOffset{0}; std::shared_ptr state; forEachChunk([&](const Buffer &header,const Buffer &chunk,uint32_t rawChunkSize,uint8_t chunkType)->bool { if (OverflowCheck::sum(destOffset,rawChunkSize)>rawData.size()) throw DecompressionError(); if (!rawChunkSize) return true; ConstSubBuffer previousBuffer{rawData,0,destOffset}; SubBuffer DestBuffer{rawData,destOffset,rawChunkSize}; switch (chunkType) { case 0: if (rawChunkSize!=chunk.size()) throw DecompressionError();; std::memcpy(DestBuffer.data(),chunk.data(),rawChunkSize); break; case 1: { try { auto sub{createDecompressor(_type,_recursionLevel,chunk,state,false)}; sub->decompressImpl(DestBuffer,previousBuffer,verify); } catch (const InvalidFormatError&) { // we should throw a correct error throw DecompressionError(); } } break; case 15U: break; default: return false; } destOffset+=rawChunkSize; return true; }); if (destOffset!=_rawSize) throw DecompressionError(); if (verify) { if (std::memcmp(_packedData.data()+16U,rawData.data(),std::min(_rawSize,16U))) throw DecompressionError(); } } std::shared_ptr XPKMain::createDecompressor(uint32_t recursionLevel,const Buffer &buffer,bool verify) { return std::shared_ptr{new XPKMain{buffer,verify,recursionLevel+1U}}; } std::shared_ptr XPKMain::createDecompressor(uint32_t type,uint32_t recursionLevel,const Buffer &buffer,std::shared_ptr &state,bool verify) { for (auto &it : XPKDecompressors) { if (it.first(type)) return it.second(type,recursionLevel,buffer,state,verify); } throw InvalidFormatError(); } template void XPKMain::forEachChunk(F func) const { uint32_t currentOffset{0}; uint32_t rawSize,packedSize; bool isLast{false}; while (currentOffset<_packedSize+8U && !isLast) { auto readDualValue=[&](uint32_t offsetShort,uint32_t offsetLong,uint32_t &value) { if (_longHeaders) { value=_packedData.readBE32(currentOffset+offsetLong); } else { value=uint32_t(_packedData.readBE16(currentOffset+offsetShort)); } }; uint32_t chunkHeaderLen{_longHeaders?12U:8U}; if (!currentOffset) { // return first; currentOffset=_headerSize; } else { uint32_t tmp; readDualValue(4U,4U,tmp); tmp=((tmp+3U)&~3U); if (OverflowCheck::sum(tmp,currentOffset,chunkHeaderLen)>_packedSize) throw InvalidFormatError(); currentOffset+=chunkHeaderLen+tmp; } readDualValue(4U,4U,packedSize); readDualValue(6U,8U,rawSize); ConstSubBuffer hdr{_packedData,currentOffset,chunkHeaderLen}; ConstSubBuffer chunk{_packedData,currentOffset+chunkHeaderLen,packedSize}; uint8_t type{_packedData.read8(currentOffset)}; if (!func(hdr,chunk,rawSize,type)) return; if (type==15U) isLast=true; } if (!isLast) throw InvalidFormatError(); } } ancient-2.2.0/src/XPKMain.hpp000066400000000000000000000031631463063262600157160ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef XPKMAIN_HPP #define XPKMAIN_HPP #include "Decompressor.hpp" #include "XPKDecompressor.hpp" #include namespace ancient::internal { class XPKMain : public Decompressor { private: XPKMain(const Buffer &packedData,bool verify,uint32_t recursionLevel); public: ~XPKMain() noexcept=default; const std::string &getName() const noexcept final; size_t getPackedSize() const noexcept final; size_t getRawSize() const noexcept final; void decompressImpl(Buffer &rawData,bool verify) final; static bool detectHeader(uint32_t hdr) noexcept; static std::shared_ptr create(const Buffer &packedData,bool exactSizeKnown,bool verify); // Can be used for direct recursion static std::shared_ptr createDecompressor(uint32_t recursionLevel,const Buffer &buffer,bool verify); // Can be used to create directly decoder for chunk (needed by CYB2) static std::shared_ptr createDecompressor(uint32_t type,uint32_t recursionLevel,const Buffer &buffer,std::shared_ptr &state,bool verify); private: static void registerDecompressor(bool(*detect)(uint32_t),std::shared_ptr(*create)(uint32_t,uint32_t,const Buffer&,std::shared_ptr&,bool)); static constexpr uint32_t getMaxRecursionLevel() noexcept { return 4; } template void forEachChunk(F func) const; const Buffer &_packedData; uint32_t _packedSize{0}; uint32_t _rawSize{0}; uint32_t _headerSize{0}; uint32_t _type{0}; bool _longHeaders{false}; uint32_t _recursionLevel{0}; bool _hasPassword{false}; }; } #endif ancient-2.2.0/src/XPKUnimplemented.cpp000066400000000000000000000067741463063262600176460ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "XPKUnimplemented.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" namespace ancient::internal { bool XPKUnimplemented::detectHeaderXPK(uint32_t hdr) noexcept { auto &modes{getModes()}; for (auto &mode : modes) if (mode.fourcc==hdr) return true; return false; } std::shared_ptr XPKUnimplemented::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } XPKUnimplemented::XPKUnimplemented(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _modeIndex{0} { if (!detectHeaderXPK(hdr)) throw Decompressor::InvalidFormatError(); auto &modes{getModes()}; for (uint32_t i=0;i &XPKUnimplemented::getModes() { // So here are the remaining XPK-libraries info. All there is, all there ever was. // (And I'm sure after writing that someone points me to 7 more formats) // Unimplemented reasons are as follows: // 1. Missing - there is no compressor available anywhere // 2. PowerPC only - Amiga OS 4 (or 3.5+) libraries that are nothing but modern compression methods wrapped in AmigaOS // 3. Floating point based formats - Fragile formats that require exact 68881/68882 semantics // 4. Encryption formats - Encryption formats requiring a proper passphrase static std::vector modes={ Mode{FourCC("BLFH"),"XPK-BLFH: Blowfish encryption (unimplemented)"}, // Encryption format Mode{FourCC("BZIP"),"XPK-BZIP: Bzip v1 (unimplemented)"}, // PowerPC only Mode{FourCC("CAST"),"XPK-CAST: CAST encryption (unimplemented)"}, // Encryption format Mode{FourCC("ENCO"),"XPK-ENCO: Unsafe encryption (unimplemented)"}, // Encryption format Mode{FourCC("DHUF"),"XPK-DHUF: Huffman compressor (unimplemented)"}, // Missing (All the libraries that exist are broken) Mode{FourCC("DMCB"),"XPK-DMCB: Arithmetic compressor (unimplemented)"}, // Floating point based format Mode{FourCC("DMCD"),"XPK-DMCD: Arithmetic compressor (unimplemented)"}, // Floating point based format Mode{FourCC("DMCI"),"XPK-DMCI: Arithmetic compressor (unimplemented)"}, // Missing Mode{FourCC("DMCU"),"XPK-DMCU: Arithmetic compressor (unimplemented)"}, // Floating point based format Mode{FourCC("FEAL"),"XPK-FEAL: FEAL-N encryption (unimplemented)"}, // Encryption format Mode{FourCC("IDEA"),"XPK-IDEA: IDEA encryption (unimplemented)"}, // Encryption format Mode{FourCC("L2XZ"),"XPK-L2XZ: LZMA2 compressor (unimplemented)"}, // PowerPC only Mode{FourCC("LZ40"),"XPK-LZ40: LZ4 compressor (unimplemented)"}, // PowerPC only Mode{FourCC("LZMA"),"XPK-LZMA: LZMA2 compressor (unimplemented)"}, // PowerPC only Mode{FourCC("NUID"),"XPK-NUID: IDEA encryption + NUKE (unimplemented)"},// Encryption format Mode{FourCC("SHID"),"XPK-SHID: IDEA encryption + SHRI (unimplemented)"},// Encryption format Mode{FourCC("TLTA"),"XPK-TLTA: TLTA encoder (unimplemented)"}}; // Missing, no idea what this is return modes; } } ancient-2.2.0/src/XPKUnimplemented.hpp000066400000000000000000000016141463063262600176370ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef XPKUNIMPLEMENTED_HPP #define XPKUNIMPLEMENTED_HPP #include "XPKDecompressor.hpp" namespace ancient::internal { class XPKUnimplemented : public XPKDecompressor { public: XPKUnimplemented(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); ~XPKUnimplemented() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: struct Mode { uint32_t fourcc; std::string name; }; static std::vector &getModes(); uint32_t _modeIndex; }; } #endif ancient-2.2.0/src/ZENODecompressor.cpp000066400000000000000000000063651463063262600176120ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "ZENODecompressor.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/Common.hpp" #include namespace ancient::internal { bool ZENODecompressor::detectHeaderXPK(uint32_t hdr) noexcept { return hdr==FourCC("ZENO"); } std::shared_ptr ZENODecompressor::create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) { return std::make_shared(hdr,recursionLevel,packedData,state,verify); } ZENODecompressor::ZENODecompressor(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify) : XPKDecompressor{recursionLevel}, _packedData{packedData} { if (!detectHeaderXPK(hdr) || _packedData.size()<6) throw Decompressor::InvalidFormatError(); // first 4 bytes is checksum for password. It needs to be zero if (_packedData.readBE32(0)) throw Decompressor::InvalidFormatError(); _maxBits=_packedData.read8(4U); if (_maxBits<9U || _maxBits>20U) throw Decompressor::InvalidFormatError(); _startOffset=uint32_t(_packedData.read8(5U))+6U; if (_startOffset>=_packedData.size()) throw Decompressor::InvalidFormatError(); } const std::string &ZENODecompressor::getSubName() const noexcept { static std::string name{"XPK-ZENO: LZW-compressor"}; return name; } void ZENODecompressor::decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) { ForwardInputStream inputStream{_packedData,_startOffset,_packedData.size()}; MSBBitReader bitReader{inputStream}; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; ForwardOutputStream outputStream{rawData,0,rawData.size()}; uint32_t maxCode{1U<<_maxBits}; uint32_t stackLength{5000U}; // magic constant std::vector prefix(maxCode-258U); std::vector suffix(maxCode-258U); std::vector stack(stackLength); uint32_t freeIndex,codeBits; auto init=[&]() { codeBits=9U; freeIndex=258U; }; init(); uint32_t prevCode{readBits(9U)}; uint32_t newCode{prevCode}; suffix[freeIndex-258]=0; prefix[freeIndex-258]=0; freeIndex++; outputStream.writeByte(newCode); while (!outputStream.eof()) { if (freeIndex+3U>=(1U<=258U) { do { if (stackPos+1>=stackLength || tmp>=freeIndex) throw Decompressor::DecompressionError(); stack[stackPos++]=suffix[tmp-258U]; tmp=prefix[tmp-258U]; } while (tmp>=258U); stack[stackPos++]=newCode=tmp; while (stackPos) outputStream.writeByte(stack[--stackPos]); } else { newCode=tmp; outputStream.writeByte(tmp); if (stackPos) outputStream.writeByte(stack[0]); } } if (freeIndex &state,bool verify); ~ZENODecompressor() noexcept=default; const std::string &getSubName() const noexcept final; void decompressImpl(Buffer &rawData,const Buffer &previousData,bool verify) final; static bool detectHeaderXPK(uint32_t hdr) noexcept; static std::shared_ptr create(uint32_t hdr,uint32_t recursionLevel,const Buffer &packedData,std::shared_ptr &state,bool verify); private: const Buffer &_packedData; uint32_t _maxBits{0}; size_t _startOffset{0}; }; } #endif ancient-2.2.0/src/attic/000077500000000000000000000000001463063262600150375ustar00rootroot00000000000000ancient-2.2.0/src/attic/Lzh/000077500000000000000000000000001463063262600155745ustar00rootroot00000000000000ancient-2.2.0/src/attic/Lzh/LH1Decompressor.cpp000066400000000000000000000034631463063262600212600ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LH1Decompressor.hpp" #include "../HuffmanDecoder.hpp" #include "../DynamicHuffmanDecoder.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" namespace ancient::internal { LH1Decompressor::LH1Decompressor(const Buffer &packedData) : _packedData(packedData) { // nothing needed } LH1Decompressor::~LH1Decompressor() { // nothing needed } size_t LH1Decompressor::getRawSize() const noexcept { // N/A return 0; } size_t LH1Decompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &LH1Decompressor::getName() const noexcept { static std::string name="LHA: LH1"; return name; } void LH1Decompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); DynamicHuffmanDecoder<314> decoder; static const uint8_t distanceHighBits[64]={ 3,4,4,4,5,5,5,5, 5,5,5,5,6,6,6,6, 6,6,6,6,6,6,6,6, 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8 }; HuffmanDecoder distanceDecoder; distanceDecoder.createOrderlyHuffmanTable(distanceHighBits,64); while (!outputStream.eof()) { uint32_t code=decoder.decode(readBit); if (decoder.getMaxFrequency()==0x8000U) decoder.halve(); decoder.update(code); if (code<256) { outputStream.writeByte(code); } else { uint32_t distance=distanceDecoder.decode(readBit); distance=(distance<<6)|readBits(6); distance++; uint32_t count=code-253; outputStream.copy(distance,count,0x20); } } } } ancient-2.2.0/src/attic/Lzh/LH1Decompressor.hpp000066400000000000000000000011301463063262600212520ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LH1DECOMPRESSOR_HPP #define LH1DECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class LH1Decompressor : public Decompressor { public: LH1Decompressor(const Buffer &packedData); virtual ~LH1Decompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/attic/Lzh/LH2Decompressor.cpp000066400000000000000000000060671463063262600212640ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LH2Decompressor.hpp" #include "../HuffmanDecoder.hpp" #include "../DynamicHuffmanDecoder.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" namespace ancient::internal { LH2Decompressor::LH2Decompressor(const Buffer &packedData) : _packedData(packedData) { // nothing needed } LH2Decompressor::~LH2Decompressor() { // nothing needed } size_t LH2Decompressor::getRawSize() const noexcept { // N/A return 0; } size_t LH2Decompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &LH2Decompressor::getName() const noexcept { static std::string name="LHA: LH2"; return name; } // This is probably fishiest of the fishy formats I've worked with // Basically it uses Dynamic Huffman decoder but instead of static // size, it is dynamically growing. However, that part is not well // defined. Thus // a. It is very hard to use LH2 using generic implementation // instead of the specific one used by LHA // b. There are bugs in encoder which need to be baked in the decoder // as well (Probably there is lots of unneccesary stuff in addCode, // but better safe than sorry) // c. LH 1.9x and UNLHA32 refuse to use LH2 beyond 8k files. Thus // we can only guess if the wraparound is correct, since nothing // should use it // jLHA in theory supports LH2 without limitations, but it produces // broken bitstream. // // So far this works with the files I've tried from "official" LHA void LH2Decompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); DynamicHuffmanDecoder<286> valueDecoder; DynamicHuffmanDecoder<128> distanceDecoder(0); uint32_t distancePosition=0; uint32_t distCount=0; while (!outputStream.eof()) { uint32_t code=valueDecoder.decode(readBit); if (valueDecoder.getMaxFrequency()==0x8000U) valueDecoder.halve(); valueDecoder.update(code); if (code<256) { outputStream.writeByte(code); } else { if (code==285) code+=readBits(8); uint32_t count=code-253; // Bug in LH2 where count does not match frequency. // Makes things more complicated auto updateDist=[&](uint32_t code) { if (distCount==0x8000U) { distanceDecoder.halve(); distCount=distanceDecoder.getMaxFrequency(); } distanceDecoder.update(code); distCount++; }; uint32_t maxDist=std::min(uint32_t((outputStream.getOffset()+63)>>6),128U); if (distancePosition!=maxDist) { for (uint32_t i=distancePosition;i bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); OptionalHuffmanDecoder decoder; static const uint8_t distanceHighBits[128]={ 2,4,4,5,5,5,6,6, 6,6,6,6,6,7,7,7, 7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8, 8,8,8,8,8,8,9,9, 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9 }; OptionalHuffmanDecoder distanceDecoder; uint32_t blockRemaining=0; while (!outputStream.eof()) { if (!blockRemaining) { blockRemaining=readBits(16); if (!blockRemaining) blockRemaining=0x10000; // not strictly needed as a lambda, but cleaner this way auto createDecoderTable=[&]() { decoder.reset(); uint8_t symbolBits[286]; uint32_t oneCount=0; for (uint32_t i=0;i<286;i++) { if (readBit()) symbolBits[i]=readBits(4)+1; else symbolBits[i]=0; if (symbolBits[i]==1) oneCount++; if (i==2 && oneCount==3) { decoder.setEmpty(readBits(9)); return; } } decoder.createOrderlyHuffmanTable(symbolBits,286); }; // ditto auto createDistanceDecoderTable=[&]() { distanceDecoder.reset(); if (readBit()) { uint8_t symbolBits[128]; uint32_t oneCount=0; for (uint32_t i=0;i<128;i++) { symbolBits[i]=readBits(4); if (symbolBits[i]==1) oneCount++; if (i==2 && oneCount==3) { // 7 bits would be fine, but whatever distanceDecoder.setEmpty(readBits(9)); return; } } distanceDecoder.createOrderlyHuffmanTable(symbolBits,128); } else distanceDecoder.createOrderlyHuffmanTable(distanceHighBits,128); }; createDecoderTable(); createDistanceDecoderTable(); } blockRemaining--; uint32_t code=decoder.decode(readBit); if (code<256) { outputStream.writeByte(code); } else { if (code==285) code+=readBits(8); uint32_t distance=distanceDecoder.decode(readBit); distance=(distance<<6)|readBits(6); distance++; uint32_t count=code-253; outputStream.copy(distance,count,0x20); } } } } ancient-2.2.0/src/attic/Lzh/LH3Decompressor.hpp000066400000000000000000000011301463063262600212540ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LH3DECOMPRESSOR_HPP #define LH3DECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class LH3Decompressor : public Decompressor { public: LH3Decompressor(const Buffer &packedData); virtual ~LH3Decompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/attic/Lzh/LHXDecompressor.cpp000066400000000000000000000106171463063262600213260ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LHXDecompressor.hpp" #include "../HuffmanDecoder.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" namespace ancient::internal { LHXDecompressor::LHXDecompressor(const Buffer &packedData) : _packedData(packedData), _method(0) { // nothing needed } LHXDecompressor::LHXDecompressor(const Buffer &packedData,uint32_t method) : _packedData(packedData), _method(method-3) { if (method<4 || method>8) throw InvalidFormatError(); } LHXDecompressor::~LHXDecompressor() { // nothing needed } size_t LHXDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t LHXDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &LHXDecompressor::getName() const noexcept { static std::string name="LHA: LH4, LH5, LH6, LH7, LH8, LHX"; return name; } void LHXDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); OptionalHuffmanDecoder decoder; OptionalHuffmanDecoder distanceDecoder; static const struct { uint32_t mask; uint32_t distanceTableSize; uint32_t distanceBits; } methodTable[6] = { {0x7ffffU,20,5}, // LHX {0xfffU,14,4}, // LH4 {0x1fffU,14,4}, // LH5 {0x7fffU,16,5}, // LH6 {0xffffU,17,5}, // LH7 {0xffffU,17,5} // LH8 // LH8 is the only Joe Jared method seen. // It exists in early versions of LH7-tool // In those versions LH8 is just synonym for LH7 // However, LH9+ is something I have not seen at // all (i.e. not in DLH7021Q/WLH7021Q.) // There would be one more version (0.21R) but // that seems lost. Most likely minor bump from Q->R // do not enable those formats either and the LH9+ // formats are nothing but hopes for "future work" // that never materialized (until I'm proven wrong ofc) }; uint32_t blockRemaining=0; while (!outputStream.eof()) { if (!blockRemaining) { blockRemaining=readBits(16); if (!blockRemaining) blockRemaining=0x10000; auto createTable=[&](OptionalHuffmanDecoder &dest,uint32_t count,uint32_t bits,bool enableHole) { uint8_t symbolBits[20]; uint32_t length=readBits(bits); if (!length) { dest.setEmpty(readBits(bits)); } else if (length<=count) { for (uint32_t i=0;i32) throw DecompressionError(); symbolBits[i++]=value; if (i==3 && enableHole) { uint32_t zeros=readBits(2); if (i+zeros>length) throw DecompressionError(); for (uint32_t j=0;j tmpDecoder; createTable(tmpDecoder,19,5,true); decoder.reset(); uint8_t symbolBits[511]; uint32_t length=readBits(9); if (!length) { decoder.setEmpty(readBits(9)); } else { for (uint32_t i=0;ilength) throw DecompressionError(); for (uint32_t j=0;j bitReader(inputStream); auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; auto readByte=[&]()->uint32_t { return inputStream.readByte(); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); StaticBuffer<4096> prevBuffer; { uint8_t *bufPtr=prevBuffer.data(); for (uint32_t i=0;i<18;i++) *(bufPtr++)=0; for (uint32_t i=0;i<256;i++) for (uint32_t j=0;j<13;j++) *(bufPtr++)=i; for (uint32_t i=0;i<256;i++) *(bufPtr++)=i; for (uint32_t i=0;i<256;i++) *(bufPtr++)=255-i; for (uint32_t i=0;i<128;i++) *(bufPtr++)=0; for (uint32_t i=0;i<110;i++) *(bufPtr++)=' '; } while (!outputStream.eof()) { if (readBit()) { outputStream.writeByte(readByte()); } else { uint32_t byte1=readByte(); uint32_t byte2=readByte(); uint32_t distance=((outputStream.getOffset()-byte1-((byte2&0xf0U)<<4)-19)&0xfffU)+1; uint32_t count=(byte2&0xfU)+3; outputStream.copy(distance,count,prevBuffer); } } } } ancient-2.2.0/src/attic/Lzh/LZ5Decompressor.hpp000066400000000000000000000011301463063262600213000ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZ5DECOMPRESSOR_HPP #define LZ5DECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class LZ5Decompressor : public Decompressor { public: LZ5Decompressor(const Buffer &packedData); virtual ~LZ5Decompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/attic/Lzh/LZHDecompressor.cpp000066400000000000000000000060511463063262600213250ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include "LZHDecompressor.hpp" #include "LH1Decompressor.hpp" #include "LH2Decompressor.hpp" #include "LH3Decompressor.hpp" #include "LHXDecompressor.hpp" #include "LZ5Decompressor.hpp" #include "LZSDecompressor.hpp" #include "PMDecompressor.hpp" namespace ancient::internal { LZHDecompressor::LZHDecompressor(const Buffer &packedData,const std::string &method) : _packedData(packedData), _method(method) { // nothing needed } LZHDecompressor::~LZHDecompressor() { // nothing needed } size_t LZHDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t LZHDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &LZHDecompressor::getName() const noexcept { static std::string name="Lzh"; return name; } void LZHDecompressor::decompressImpl(Buffer &rawData,bool verify) { enum class Compressor { LH0=0, LH1, LH2, LH3, LH4, LH5, LH6, LH7, LH8, LHX, LZ4, LZ5, LZS, PM0, PM1, PM2 }; static std::map compressorMap{ {"-lh0-",Compressor::LH0}, {"-lh1-",Compressor::LH1}, {"-lh2-",Compressor::LH2}, {"-lh3-",Compressor::LH3}, {"-lh4-",Compressor::LH4}, {"-lh5-",Compressor::LH5}, {"-lh6-",Compressor::LH6}, {"-lh7-",Compressor::LH7}, {"-lh8-",Compressor::LH8}, {"-lhx-",Compressor::LHX}, {"-lz4-",Compressor::LZ4}, {"-lz5-",Compressor::LZ5}, {"-lzs-",Compressor::LZS}, {"-pm0-",Compressor::PM0}, {"-pm1-",Compressor::PM1}, {"-pm2-",Compressor::PM2} }; auto it=compressorMap.find(_method); if (it==compressorMap.end()) throw DecompressionError(); switch (it->second) { case Compressor::LH0: case Compressor::LZ4: case Compressor::PM0: if (rawData.size()!=_packedData.size()) throw DecompressionError(); std::memcpy(rawData.data(),_packedData.data(),rawData.size()); break; case Compressor::LH1: { LH1Decompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::LH2: { LH2Decompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::LH3: { LH3Decompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::LH4: case Compressor::LH5: case Compressor::LH6: case Compressor::LH7: case Compressor::LH8: { LHXDecompressor dec(_packedData,static_cast(it->second)-static_cast(Compressor::LH4)+4); dec.decompress(rawData,verify); } break; case Compressor::LHX: { LHXDecompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::LZ5: { LZ5Decompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::LZS: { LZSDecompressor dec(_packedData); dec.decompress(rawData,verify); } break; case Compressor::PM1: case Compressor::PM2: { PMDecompressor dec(_packedData,static_cast(it->second)-static_cast(Compressor::PM1)+1); dec.decompress(rawData,verify); } break; } } } ancient-2.2.0/src/attic/Lzh/LZHDecompressor.hpp000066400000000000000000000012111463063262600213230ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZHDECOMPRESSOR_HPP #define LZHDECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class LZHDecompressor : public Decompressor { public: LZHDecompressor(const Buffer &packedData,const std::string &method); virtual ~LZHDecompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; std::string _method; }; } #endif ancient-2.2.0/src/attic/Lzh/LZSDecompressor.cpp000066400000000000000000000024071463063262600213410ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "LZSDecompressor.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" namespace ancient::internal { LZSDecompressor::LZSDecompressor(const Buffer &packedData) : _packedData(packedData) { // nothing needed } LZSDecompressor::~LZSDecompressor() { // nothing needed } size_t LZSDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t LZSDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &LZSDecompressor::getName() const noexcept { static std::string name="LHA: LZS"; return name; } void LZSDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); while (!outputStream.eof()) { if (readBit()) { outputStream.writeByte(readBits(8)); } else { uint32_t distance=((outputStream.getOffset()-readBits(11)-18)&0x7ffU)+1; uint32_t count=readBits(4)+2; outputStream.copy(distance,count,0x20); } } } } ancient-2.2.0/src/attic/Lzh/LZSDecompressor.hpp000066400000000000000000000011301463063262600213360ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef LZSDECOMPRESSOR_HPP #define LZSDECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class LZSDecompressor : public Decompressor { public: LZSDecompressor(const Buffer &packedData); virtual ~LZSDecompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; }; } #endif ancient-2.2.0/src/attic/Lzh/PMDecompressor.cpp000066400000000000000000000264171463063262600212140ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "PMDecompressor.hpp" #include "../HuffmanDecoder.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" namespace ancient::internal { PMDecompressor::PMDecompressor(const Buffer &packedData,uint32_t version) : _packedData(packedData), _version(version) { if (version!=1 && version!=2) throw InvalidFormatError(); } PMDecompressor::~PMDecompressor() { // nothing needed } size_t PMDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t PMDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &PMDecompressor::getName() const noexcept { static std::string name="LHA: PM1, PM2"; return name; } uint8_t PMDecompressor::decodeMTF(uint8_t value,uint8_t map[]) { return map[value]; } void PMDecompressor::updateMTF(uint8_t value,uint8_t map[]) { for (uint32_t i=0;;i++) { if (map[i]==value) { value=i; break; } } if (value) { uint8_t tmp=map[value]; for (uint32_t i=value;i;i--) map[i]=map[i-1]; map[0]=tmp; } } void PMDecompressor::createMTFMap(uint8_t map[]) { for (uint32_t i=0,j=0x20;j<0x80;i++,j++) map[i]=j; for (uint32_t i=0x60,j=0;j<0x20;i++,j++) map[i]=j; for (uint32_t i=0x80,j=0xa0;j<0xe0;i++,j++) map[i]=j; for (uint32_t i=0xc0,j=0x80;j<0xa0;i++,j++) map[i]=j; for (uint32_t i=0xe0,j=0xe0;j<0x100;i++,j++) map[i]=j; } void PMDecompressor::decompressImpl(Buffer &rawData,bool verify) { if (_version==1) decompressImplPM1(rawData,verify); else decompressImplPM2(rawData,verify); } void PMDecompressor::decompressImplPM1(Buffer &rawData,bool verify) { static const struct { uint8_t length; uint8_t code; } treeDefinitions[32][6] { // This is madness, I had to write a special program to decode this. // Also, in the original there is a bug @ index 17, as identified by // lhasa (we fix it when creating the decoder) {{4, 0b0000},{4, 0b0001},{3, 0b001},{2, 0b01},{2, 0b10},{2, 0b11}}, {{3, 0b000},{3, 0b001},{3, 0b010},{2, 0b10},{2, 0b11},{3, 0b011}}, {{3, 0b000},{3, 0b001},{2, 0b01},{2, 0b10},{3, 0b110},{3, 0b111}}, {{2, 0b00},{3, 0b010},{3, 0b011},{2, 0b10},{3, 0b110},{3, 0b111}}, {{2, 0b00},{3, 0b010},{2, 0b10},{3, 0b011},{3, 0b110},{3, 0b111}}, {{2, 0b00},{3, 0b010},{2, 0b10},{2, 0b11},{4, 0b0110},{4, 0b0111}}, {{2, 0b00},{2, 0b01},{3, 0b100},{3, 0b101},{3, 0b110},{3, 0b111}}, {{2, 0b00},{2, 0b01},{3, 0b100},{2, 0b11},{4, 0b1010},{4, 0b1011}}, {{2, 0b00},{2, 0b01},{2, 0b10},{3, 0b110},{4, 0b1110},{4, 0b1111}}, {{1, 0b0},{4, 0b1000},{3, 0b101},{3, 0b110},{3, 0b111},{4, 0b1001}}, {{1, 0b0},{4, 0b1000},{3, 0b101},{2, 0b11},{5,0b10010},{5,0b10011}}, {{1, 0b0},{4, 0b1000},{4, 0b1001},{3, 0b101},{3, 0b110},{3, 0b111}}, {{1, 0b0},{3, 0b100},{4, 0b1010},{3, 0b110},{3, 0b111},{4, 0b1011}}, {{1, 0b0},{3, 0b100},{3, 0b101},{3, 0b110},{4, 0b1110},{4, 0b1111}}, {{1, 0b0},{3, 0b100},{2, 0b11},{4, 0b1010},{5,0b10110},{5,0b10111}}, {{1, 0b0},{2, 0b10},{4, 0b1100},{4, 0b1101},{4, 0b1110},{4, 0b1111}}, {{1, 0b0},{2, 0b10},{3, 0b110},{4, 0b1110},{5,0b11110},{5,0b11111}}, {{3, 0b000},{3, 0b001},{2, 0b01},{2, 0b10},{2, 0b11},{0, 0b0}}, {{2, 0b00},{3, 0b010},{2, 0b10},{2, 0b11},{3, 0b011},{0, 0b0}}, {{2, 0b00},{2, 0b01},{2, 0b10},{3, 0b110},{3, 0b111},{0, 0b0}}, {{1, 0b0},{4, 0b1000},{3, 0b101},{2, 0b11},{4, 0b1001},{0, 0b0}}, {{1, 0b0},{3, 0b100},{3, 0b101},{3, 0b110},{3, 0b111},{0, 0b0}}, {{1, 0b0},{3, 0b100},{2, 0b11},{4, 0b1010},{4, 0b1011},{0, 0b0}}, {{1, 0b0},{2, 0b10},{3, 0b110},{4, 0b1110},{4, 0b1111},{0, 0b0}}, {{3, 0b000},{3, 0b001},{2, 0b01},{1, 0b1},{0, 0b0},{0, 0b0}}, {{2, 0b00},{3, 0b010},{1, 0b1},{3, 0b011},{0, 0b0},{0, 0b0}}, {{2, 0b00},{2, 0b01},{2, 0b10},{2, 0b11},{0, 0b0},{0, 0b0}}, {{1, 0b0},{3, 0b100},{2, 0b11},{3, 0b101},{0, 0b0},{0, 0b0}}, {{1, 0b0},{2, 0b10},{3, 0b110},{3, 0b111},{0, 0b0},{0, 0b0}}, {{1, 0b0},{2, 0b10},{2, 0b11},{0, 0b0},{0, 0b0},{0, 0b0}}, {{1, 0b0},{1, 0b1},{0, 0b0},{0, 0b0},{0, 0b0},{0, 0b0}}, {{0, 0b0},{1, 0b1},{0, 0b0},{0, 0b0},{0, 0b0},{0, 0b0}} }; ForwardInputStream inputStream(_packedData,0,_packedData.size(),true); MSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; size_t rawSize=rawData.size(); ForwardOutputStream outputStream(rawData,0,rawSize); OptionalHuffmanDecoder decoder; { uint32_t treeIndex=readBits(5); for (uint32_t i=0;i<6;i++) { uint32_t length=treeDefinitions[treeIndex][i].length; uint32_t code=treeDefinitions[treeIndex][i].code; if (!length) { if (!i) decoder.setEmpty(0); break; } if (treeIndex==17 && i<2) decoder.insert(HuffmanCode{length,code,i+3}); else decoder.insert(HuffmanCode{length,code,i}); } } uint8_t dataMTFMap[256]; createMTFMap(dataMTFMap); auto processOutput=[&](uint8_t value)->uint8_t { updateMTF(value,dataMTFMap); return value; }; while (!outputStream.eof()) { bool doCopy=true; if (readBit()) { uint32_t count=readBits(2)+1; if (count==4) { count=readBits(3)+4; if (count==11) { count=readBits(4)+11; if (count==25) { count=readBits(6)+25; } else if (count==26) { count=readBits(7)+89; } } } count=std::min(count,uint32_t(rawSize-outputStream.getOffset())); for (uint32_t i=0;i=0x240?readBit():0) { code=4; if (offset<0x340) code=7; else if (offset<0x440) code=8; else if (offset<0x640) code=9; } else { code=offset>=0x40?readBit():0; count=2; } } else { if (offset>=0x40?!readBit():0) { code=3; if (offset<0x140) code=6; } else if (offset>=0xa40?readBit():1) { code=2; } else { code=5; if (offset<0xb40) code=10; else if (offset<0xc40) code=11; else if (offset<0xe40) code=12; else if (offset<0x1240) code=13; else if (offset<0x1a40) code=14; } } if (code>=2) { count=readBits(2)+3; if (count==6) { count=readBits(3)+6; if (count==11) { count=readBits(2)+11; } else if (count==12) { count=readBits(3)+15; } else if (count==13) { count=readBits(6)+23; if (count==85) { count=readBits(5)+85; } else if (count==86) count=readBits(7)+117; } } } static const uint32_t distanceAdditions[15]={ 1,0x41,1,0x41,0x241,0xa41,0x41,0x241,0x241,0x241,0xa41,0xa41,0xa41,0xa41,0xa41}; static const uint32_t distanceBits[15]={ 6,8,6,9,11,13,8,8,9,10,8,9,10,11,12}; uint32_t distance=readBits(distanceBits[code])+distanceAdditions[code]; count=std::min(count,uint32_t(rawSize-outputStream.getOffset())); outputStream.copy(distance,count,0x20); const uint8_t *block=outputStream.history(count); for (uint32_t i=0;i bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); OptionalHuffmanDecoder decoder; OptionalHuffmanDecoder distanceDecoder; auto createDecoder=[&]()->bool { decoder.reset(); // codes beyond 29 are going to fault anyway, but maybe they are not used? uint8_t symbols[31]; uint32_t numCodes=readBits(5); uint32_t minLength=readBits(3); bool ret=(numCodes>=10)&&(numCodes!=29||minLength); if (!minLength) { if (!numCodes) throw DecompressionError(); decoder.setEmpty(numCodes-1); } else { uint32_t codeLength=readBits(3); for (uint32_t i=0;iuint8_t { offset++; // we are interested offset after the update updateMTF(value,dataMTFMap); if (!(offset&0x3ffU)) { switch (offset>>10) { case 1: if (distanceTreeRequired) createDistanceDecoder(6); break; case 2: if (distanceTreeRequired) createDistanceDecoder(7); break; case 4: if (readBit()) distanceTreeRequired=createDecoder(); if (distanceTreeRequired) createDistanceDecoder(8); break; default: if (!(offset&0xfffU) && (offset>>12)>=2) { if (readBit()) { distanceTreeRequired=createDecoder(); if (distanceTreeRequired) createDistanceDecoder(8); } } break; } } return value; }; readBit(); // ignore first bit distanceTreeRequired=createDecoder(); if (distanceTreeRequired) createDistanceDecoder(5); while (!outputStream.eof()) { uint32_t code=decoder.decode(readBit); if (code<8) { static const uint32_t symbolAdditions[8]={0,8,16,32,64,96,128,192}; static const uint32_t symbolBits[8]={3,3,4,5,5,5,6,6}; outputStream.writeByte(processOutput(outputStream.getOffset(),decodeMTF(readBits(symbolBits[code])+symbolAdditions[code],dataMTFMap))); } else { code-=8; uint32_t count; if (code<15) { count=code+2; } else { if (code>=21) throw DecompressionError(); static const uint32_t countAdditions[6]={17,25,33,65,129,256}; static const uint32_t countBits[6]={3,3,5,6,7,0}; count=readBits(countBits[code-15])+countAdditions[code-15]; } uint32_t distance; if (!code) { distance=readBits(6)+1; } else if (code<20) { uint32_t tmp=distanceDecoder.decode(readBit); if (!tmp) distance=readBits(6)+1; else distance=readBits(tmp+5)+(1<<(tmp+5))+1; } else distance=1; outputStream.copy(distance,count,0x20); const uint8_t *block=outputStream.history(count); size_t offset=outputStream.getOffset()-count; for (uint32_t i=0;i #include #include "ImplodeDecompressor.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" #include "../HuffmanDecoder.hpp" namespace ancient::internal { ImplodeDecompressor::ImplodeDecompressor(const Buffer &packedData,uint32_t flags) : _packedData(packedData), _flags(flags) { } ImplodeDecompressor::~ImplodeDecompressor() { // nothing needed } size_t ImplodeDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t ImplodeDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &ImplodeDecompressor::getName() const noexcept { static std::string name="Zip: Implode"; return name; } void ImplodeDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); LSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); // Although Implode uses Fannon-Shano, Huffman decoder can be used typedef HuffmanDecoder ImplodeHuffmanDecoder; auto createDecoder=[&](ImplodeHuffmanDecoder &dec,uint32_t maxValue) { uint32_t length=readBits(8)+1; uint8_t bitLengths[257]; uint8_t counts[257]; uint8_t offsets[257]; uint8_t minDepth=32,maxDepth=0; for (uint32_t i=0;imaxDepth) maxDepth=bitLengths[i]; } if (static_cast(offsets[length-1]+counts[length-1])>maxValue) throw DecompressionError(); uint32_t code=0; for (uint32_t depth=maxDepth;depth>=minDepth;depth--) { for (int32_t i=length-1;i>=0;i--) { if (depth!=bitLengths[i]) continue; for (uint32_t j=counts[i];j>0;j--) { dec.insert(HuffmanCode{depth,code>>(maxDepth-depth),uint8_t(offsets[i]+j-1)}); code+=1<<(maxDepth-depth); } } } }; ImplodeHuffmanDecoder lengthDecoder,distanceDecoder,litDecoder; uint32_t extraDistanceBits=(_flags&2)?7:6; bool literalDecPresent=_flags&4; uint32_t lengthAddition=2; if (literalDecPresent) { createDecoder(litDecoder,256); lengthAddition++; } createDecoder(lengthDecoder,64); createDecoder(distanceDecoder,64); while (!outputStream.eof()) { if (readBit()) { if (literalDecPresent) outputStream.writeByte(litDecoder.decode(readBit)); else outputStream.writeByte(readBits(8)); } else { uint32_t distance=readBits(extraDistanceBits); distance|=uint32_t(distanceDecoder.decode(readBit))< #include #include "ReduceDecompressor.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" namespace ancient::internal { ReduceDecompressor::ReduceDecompressor(const Buffer &packedData,uint32_t mode) : _packedData(packedData), _mode(mode) { if (mode<2 || mode>5) throw InvalidFormatError(); } ReduceDecompressor::~ReduceDecompressor() { // nothing needed } size_t ReduceDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t ReduceDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &ReduceDecompressor::getName() const noexcept { static std::string name="Zip: Reduce"; return name; } void ReduceDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); LSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); uint8_t SLengths[256]; auto SBuf=std::make_unique(256*64); uint8_t (&S)[256][64]=*reinterpret_cast(SBuf.get()); static const uint8_t bitLengths[64]={ 8,1,1,2,2,3,3,3, 3,4,4,4,4,4,4,4, 4,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5, 5,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6}; for (int32_t i=255;i>=0;i--) { uint32_t length=SLengths[i]=readBits(6); for (uint32_t j=0;juint8_t { uint8_t ret; if (!SLengths[lastChar] || readBits(1)) { ret=readBits(8); } else { uint8_t tmp=readBits(bitLengths[SLengths[lastChar]]); if (tmp>=SLengths[lastChar]) throw DecompressionError(); ret=S[lastChar][tmp]; } lastChar=ret; return ret; }; uint8_t ch=readByte(); if (ch!=0x90U) { outputStream.writeByte(ch); } else { ch=readByte(); if (!ch) { outputStream.writeByte(0x90U); } else { uint8_t lengthMask=0xffU>>(8U-lengthBits); uint32_t count=ch&lengthMask; if (count==lengthMask) count+=uint32_t(readByte()); count+=3; uint32_t distance=(uint32_t(ch>>lengthBits)<<8)|uint32_t(readByte()); distance++; outputStream.copy(distance,count,0); } } } } } ancient-2.2.0/src/attic/Zip/ReduceDecompressor.hpp000066400000000000000000000012071463063262600221070ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef REDUCEDECOMPRESSOR_HPP #define REDUCEDECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class ReduceDecompressor : public Decompressor { public: ReduceDecompressor(const Buffer &packedData,uint32_t mode); virtual ~ReduceDecompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; uint32_t _mode; }; } #endif ancient-2.2.0/src/attic/Zip/ShrinkDecompressor.cpp000066400000000000000000000057451463063262600221440ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include "ShrinkDecompressor.hpp" #include "../InputStream.hpp" #include "../OutputStream.hpp" namespace ancient::internal { ShrinkDecompressor::ShrinkDecompressor(const Buffer &packedData) : _packedData(packedData) { } ShrinkDecompressor::~ShrinkDecompressor() { // nothing needed } size_t ShrinkDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t ShrinkDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &ShrinkDecompressor::getName() const noexcept { static std::string name="Zip: Shrink"; return name; } void ShrinkDecompressor::decompressImpl(Buffer &rawData,bool verify) { ForwardInputStream inputStream(_packedData,0,_packedData.size()); LSBBitReader bitReader(inputStream); auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; ForwardOutputStream outputStream(rawData,0,rawData.size()); const uint32_t maxCode=0x2000U; auto prefix=std::make_unique(maxCode-257); auto suffix=std::make_unique(maxCode-257); auto stack=std::make_unique(maxCode-257); uint32_t freeIndex,codeBits,prevCode,newCode; auto suffixLookup=[&](uint32_t code)->uint32_t { // main protection against negative values if (code>=maxCode) throw DecompressionError(); return (code<257)?code:suffix[code-257]; }; auto insert=[&](uint32_t code) { uint32_t stackPos=0; newCode=suffixLookup(code); while (code>=257) { if (stackPos+1>=maxCode-257) throw DecompressionError(); stack[stackPos++]=newCode; code=prefix[code-257]; newCode=suffixLookup(code); } stack[stackPos++]=newCode; while (stackPos) outputStream.writeByte(stack[--stackPos]); }; for (uint32_t i=0;i usageMap(freeIndex-257,false); for (uint32_t i=257;i=257) usageMap[tmp-257]=true; } uint32_t firstEmpty=freeIndex; for (uint32_t i=257;i=257 && (prefix[code-257]&0x8000'0000U)) { uint32_t tmp=newCode; insert(prevCode); outputStream.writeByte(tmp); } else insert(code); while (!(prefix[freeIndex-257]&0x8000'0000U) && freeIndex #include #include "ZIPDecompressor.hpp" #include "../BZIP2Decompressor.hpp" #include "../DEFLATEDecompressor.hpp" #include "ImplodeDecompressor.hpp" #include "ReduceDecompressor.hpp" #include "ShrinkDecompressor.hpp" namespace ancient::internal { ZIPDecompressor::ZIPDecompressor(const Buffer &packedData,uint32_t method,uint32_t flags) : _packedData(packedData), _method(method), _flags(flags) { } ZIPDecompressor::~ZIPDecompressor() { // nothing needed } size_t ZIPDecompressor::getRawSize() const noexcept { // N/A return 0; } size_t ZIPDecompressor::getPackedSize() const noexcept { // N/A return 0; } const std::string &ZIPDecompressor::getName() const noexcept { static std::string name="Zip"; return name; } void ZIPDecompressor::decompressImpl(Buffer &rawData,bool verify) { switch (_method) { case 0: if (rawData.size()!=_packedData.size()) throw DecompressionError(); std::memcpy(rawData.data(),_packedData.data(),rawData.size()); break; case 1: { ShrinkDecompressor dec(_packedData); dec.decompress(rawData,verify); } break; case 2: case 3: case 4: case 5: { ReduceDecompressor dec(_packedData,_method); dec.decompress(rawData,verify); } break; case 6: { ImplodeDecompressor dec(_packedData,_flags); dec.decompress(rawData,verify); } break; case 8: case 9: { DEFLATEDecompressor dec(_packedData,_packedData.size(),rawData.size(),false,verify,_method==9); dec.decompress(rawData,verify); } break; case 12: { BZIP2Decompressor dec(_packedData,true,verify); dec.decompress(rawData,verify); } break; case 14: // LZMA throw DecompressionError(); break; case 97: // WavPack throw DecompressionError(); break; case 98: // PPMd throw DecompressionError(); break; default: throw DecompressionError(); } } } ancient-2.2.0/src/attic/Zip/ZIPDecompressor.hpp000066400000000000000000000012351463063262600213430ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef ZIPDECOMPRESSOR_HPP #define ZIPDECOMPRESSOR_HPP #include "Decompressor.hpp" namespace ancient::internal { class ZIPDecompressor : public Decompressor { public: ZIPDecompressor(const Buffer &packedData,uint32_t method,uint32_t flags); virtual ~ZIPDecompressor(); virtual size_t getRawSize() const noexcept override final; virtual size_t getPackedSize() const noexcept override final; virtual const std::string &getName() const noexcept override final; virtual void decompressImpl(Buffer &rawData,bool verify) override final; private: const Buffer &_packedData; uint32_t _method; uint32_t _flags; }; } #endif ancient-2.2.0/src/common/000077500000000000000000000000001463063262600152235ustar00rootroot00000000000000ancient-2.2.0/src/common/Buffer.cpp000066400000000000000000000034671463063262600171520ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "Buffer.hpp" #include "OverflowCheck.hpp" namespace ancient::internal { void Buffer::resize(size_t newSize) { throw InvalidOperationError(); } uint8_t &Buffer::operator[](size_t i) { if (i>=size()) throw OutOfBoundsError(); return data()[i]; } const uint8_t &Buffer::operator[](size_t i) const { if (i>=size()) throw OutOfBoundsError(); return data()[i]; } uint32_t Buffer::readBE32(size_t offset) const { if (OverflowCheck::sum(offset,4U)>size()) throw OutOfBoundsError(); const uint8_t *ptr=data()+offset; return (uint32_t(ptr[0])<<24)|(uint32_t(ptr[1])<<16)|(uint32_t(ptr[2])<<8)|uint32_t(ptr[3]); } uint16_t Buffer::readBE16(size_t offset) const { if (OverflowCheck::sum(offset,2U)>size()) throw OutOfBoundsError(); const uint8_t *ptr=data()+offset; return (uint16_t(ptr[0])<<8)|uint16_t(ptr[1]); } uint64_t Buffer::readLE64(size_t offset) const { if (OverflowCheck::sum(offset,8U)>size()) throw OutOfBoundsError(); const uint8_t *ptr=data()+offset; return (uint64_t(ptr[7])<<56)|(uint64_t(ptr[6])<<48)|(uint64_t(ptr[5])<<40)|(uint64_t(ptr[4])<<32)| (uint64_t(ptr[3])<<24)|(uint64_t(ptr[2])<<16)|(uint64_t(ptr[1])<<8)|uint64_t(ptr[0]); } uint32_t Buffer::readLE32(size_t offset) const { if (OverflowCheck::sum(offset,4U)>size()) throw OutOfBoundsError(); const uint8_t *ptr=data()+offset; return (uint32_t(ptr[3])<<24)|(uint32_t(ptr[2])<<16)|(uint32_t(ptr[1])<<8)|uint32_t(ptr[0]); } uint16_t Buffer::readLE16(size_t offset) const { if (OverflowCheck::sum(offset,2U)>size()) throw OutOfBoundsError(); const uint8_t *ptr=data()+offset; return (uint16_t(ptr[1])<<8)|uint16_t(ptr[0]); } uint8_t Buffer::read8(size_t offset) const { if (offset>=size()) throw OutOfBoundsError(); const uint8_t *ptr=reinterpret_cast(data())+offset; return ptr[0]; } } ancient-2.2.0/src/common/Buffer.hpp000066400000000000000000000027151463063262600171520ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef BUFFER_HPP #define BUFFER_HPP #include #include #include namespace ancient::internal { class Buffer { protected: Buffer() noexcept=default; public: class Error : public std::exception { public: Error() noexcept=default; virtual ~Error() noexcept=default; }; class OutOfBoundsError : public Error { public: OutOfBoundsError() noexcept=default; virtual ~OutOfBoundsError() noexcept=default; }; class InvalidOperationError : public Error { public: InvalidOperationError() noexcept=default; virtual ~InvalidOperationError() noexcept=default; }; Buffer(const Buffer&)=delete; Buffer& operator=(const Buffer&)=delete; virtual ~Buffer() noexcept=default; virtual const uint8_t *data() const noexcept=0; virtual uint8_t *data()=0; virtual size_t size() const noexcept=0; template const T *cast() const noexcept { return reinterpret_cast(data()); } template T *cast() { return reinterpret_cast(data()); } virtual bool isResizable() const noexcept=0; virtual void resize(size_t newSize); uint8_t &operator[](size_t i); const uint8_t &operator[](size_t i) const; uint32_t readBE32(size_t offset) const; uint16_t readBE16(size_t offset) const; uint64_t readLE64(size_t offset) const; uint32_t readLE32(size_t offset) const; uint16_t readLE16(size_t offset) const; uint8_t read8(size_t offset) const; }; } #endif ancient-2.2.0/src/common/CRC16.cpp000066400000000000000000000046531463063262600165150ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "Buffer.hpp" #include "OverflowCheck.hpp" namespace ancient::internal { static const uint16_t CRC16Table[256]={ 0x0000,0xc0c1,0xc181,0x0140,0xc301,0x03c0,0x0280,0xc241,0xc601,0x06c0,0x0780,0xc741,0x0500,0xc5c1,0xc481,0x0440, 0xcc01,0x0cc0,0x0d80,0xcd41,0x0f00,0xcfc1,0xce81,0x0e40,0x0a00,0xcac1,0xcb81,0x0b40,0xc901,0x09c0,0x0880,0xc841, 0xd801,0x18c0,0x1980,0xd941,0x1b00,0xdbc1,0xda81,0x1a40,0x1e00,0xdec1,0xdf81,0x1f40,0xdd01,0x1dc0,0x1c80,0xdc41, 0x1400,0xd4c1,0xd581,0x1540,0xd701,0x17c0,0x1680,0xd641,0xd201,0x12c0,0x1380,0xd341,0x1100,0xd1c1,0xd081,0x1040, 0xf001,0x30c0,0x3180,0xf141,0x3300,0xf3c1,0xf281,0x3240,0x3600,0xf6c1,0xf781,0x3740,0xf501,0x35c0,0x3480,0xf441, 0x3c00,0xfcc1,0xfd81,0x3d40,0xff01,0x3fc0,0x3e80,0xfe41,0xfa01,0x3ac0,0x3b80,0xfb41,0x3900,0xf9c1,0xf881,0x3840, 0x2800,0xe8c1,0xe981,0x2940,0xeb01,0x2bc0,0x2a80,0xea41,0xee01,0x2ec0,0x2f80,0xef41,0x2d00,0xedc1,0xec81,0x2c40, 0xe401,0x24c0,0x2580,0xe541,0x2700,0xe7c1,0xe681,0x2640,0x2200,0xe2c1,0xe381,0x2340,0xe101,0x21c0,0x2080,0xe041, 0xa001,0x60c0,0x6180,0xa141,0x6300,0xa3c1,0xa281,0x6240,0x6600,0xa6c1,0xa781,0x6740,0xa501,0x65c0,0x6480,0xa441, 0x6c00,0xacc1,0xad81,0x6d40,0xaf01,0x6fc0,0x6e80,0xae41,0xaa01,0x6ac0,0x6b80,0xab41,0x6900,0xa9c1,0xa881,0x6840, 0x7800,0xb8c1,0xb981,0x7940,0xbb01,0x7bc0,0x7a80,0xba41,0xbe01,0x7ec0,0x7f80,0xbf41,0x7d00,0xbdc1,0xbc81,0x7c40, 0xb401,0x74c0,0x7580,0xb541,0x7700,0xb7c1,0xb681,0x7640,0x7200,0xb2c1,0xb381,0x7340,0xb101,0x71c0,0x7080,0xb041, 0x5000,0x90c1,0x9181,0x5140,0x9301,0x53c0,0x5280,0x9241,0x9601,0x56c0,0x5780,0x9741,0x5500,0x95c1,0x9481,0x5440, 0x9c01,0x5cc0,0x5d80,0x9d41,0x5f00,0x9fc1,0x9e81,0x5e40,0x5a00,0x9ac1,0x9b81,0x5b40,0x9901,0x59c0,0x5880,0x9841, 0x8801,0x48c0,0x4980,0x8941,0x4b00,0x8bc1,0x8a81,0x4a40,0x4e00,0x8ec1,0x8f81,0x4f40,0x8d01,0x4dc0,0x4c80,0x8c41, 0x4400,0x84c1,0x8581,0x4540,0x8701,0x47c0,0x4680,0x8641,0x8201,0x42c0,0x4380,0x8341,0x4100,0x81c1,0x8081,0x4040}; uint16_t CRC16(const Buffer &buffer,size_t offset,size_t len,uint16_t accumulator) { if (!len || OverflowCheck::sum(offset,len)>buffer.size()) throw Buffer::OutOfBoundsError(); const uint8_t *ptr=buffer.data()+offset; for (size_t i=0;i>8)^CRC16Table[(accumulator&0xff)^ptr[i]]; return accumulator; } uint16_t CRC16Byte(uint8_t ch,uint16_t accumulator) noexcept { return (accumulator>>8)^CRC16Table[(accumulator&0xff)^ch]; } } ancient-2.2.0/src/common/CRC16.hpp000066400000000000000000000005121463063262600165100ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef CRC16_HPP #define CRC16_HPP #include #include "Buffer.hpp" namespace ancient::internal { // The most common CRC16 uint16_t CRC16(const Buffer &buffer,size_t offset,size_t len,uint16_t accumulator); uint16_t CRC16Byte(uint8_t ch,uint16_t accumulator) noexcept; } #endif ancient-2.2.0/src/common/CRC32.cpp000066400000000000000000000167311463063262600165130ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include "Buffer.hpp" #include "OverflowCheck.hpp" namespace ancient::internal { static const uint32_t CRC32Table[256]={ 0x00000000U,0x77073096U,0xee0e612cU,0x990951baU,0x076dc419U,0x706af48fU,0xe963a535U,0x9e6495a3U, 0x0edb8832U,0x79dcb8a4U,0xe0d5e91eU,0x97d2d988U,0x09b64c2bU,0x7eb17cbdU,0xe7b82d07U,0x90bf1d91U, 0x1db71064U,0x6ab020f2U,0xf3b97148U,0x84be41deU,0x1adad47dU,0x6ddde4ebU,0xf4d4b551U,0x83d385c7U, 0x136c9856U,0x646ba8c0U,0xfd62f97aU,0x8a65c9ecU,0x14015c4fU,0x63066cd9U,0xfa0f3d63U,0x8d080df5U, 0x3b6e20c8U,0x4c69105eU,0xd56041e4U,0xa2677172U,0x3c03e4d1U,0x4b04d447U,0xd20d85fdU,0xa50ab56bU, 0x35b5a8faU,0x42b2986cU,0xdbbbc9d6U,0xacbcf940U,0x32d86ce3U,0x45df5c75U,0xdcd60dcfU,0xabd13d59U, 0x26d930acU,0x51de003aU,0xc8d75180U,0xbfd06116U,0x21b4f4b5U,0x56b3c423U,0xcfba9599U,0xb8bda50fU, 0x2802b89eU,0x5f058808U,0xc60cd9b2U,0xb10be924U,0x2f6f7c87U,0x58684c11U,0xc1611dabU,0xb6662d3dU, 0x76dc4190U,0x01db7106U,0x98d220bcU,0xefd5102aU,0x71b18589U,0x06b6b51fU,0x9fbfe4a5U,0xe8b8d433U, 0x7807c9a2U,0x0f00f934U,0x9609a88eU,0xe10e9818U,0x7f6a0dbbU,0x086d3d2dU,0x91646c97U,0xe6635c01U, 0x6b6b51f4U,0x1c6c6162U,0x856530d8U,0xf262004eU,0x6c0695edU,0x1b01a57bU,0x8208f4c1U,0xf50fc457U, 0x65b0d9c6U,0x12b7e950U,0x8bbeb8eaU,0xfcb9887cU,0x62dd1ddfU,0x15da2d49U,0x8cd37cf3U,0xfbd44c65U, 0x4db26158U,0x3ab551ceU,0xa3bc0074U,0xd4bb30e2U,0x4adfa541U,0x3dd895d7U,0xa4d1c46dU,0xd3d6f4fbU, 0x4369e96aU,0x346ed9fcU,0xad678846U,0xda60b8d0U,0x44042d73U,0x33031de5U,0xaa0a4c5fU,0xdd0d7cc9U, 0x5005713cU,0x270241aaU,0xbe0b1010U,0xc90c2086U,0x5768b525U,0x206f85b3U,0xb966d409U,0xce61e49fU, 0x5edef90eU,0x29d9c998U,0xb0d09822U,0xc7d7a8b4U,0x59b33d17U,0x2eb40d81U,0xb7bd5c3bU,0xc0ba6cadU, 0xedb88320U,0x9abfb3b6U,0x03b6e20cU,0x74b1d29aU,0xead54739U,0x9dd277afU,0x04db2615U,0x73dc1683U, 0xe3630b12U,0x94643b84U,0x0d6d6a3eU,0x7a6a5aa8U,0xe40ecf0bU,0x9309ff9dU,0x0a00ae27U,0x7d079eb1U, 0xf00f9344U,0x8708a3d2U,0x1e01f268U,0x6906c2feU,0xf762575dU,0x806567cbU,0x196c3671U,0x6e6b06e7U, 0xfed41b76U,0x89d32be0U,0x10da7a5aU,0x67dd4accU,0xf9b9df6fU,0x8ebeeff9U,0x17b7be43U,0x60b08ed5U, 0xd6d6a3e8U,0xa1d1937eU,0x38d8c2c4U,0x4fdff252U,0xd1bb67f1U,0xa6bc5767U,0x3fb506ddU,0x48b2364bU, 0xd80d2bdaU,0xaf0a1b4cU,0x36034af6U,0x41047a60U,0xdf60efc3U,0xa867df55U,0x316e8eefU,0x4669be79U, 0xcb61b38cU,0xbc66831aU,0x256fd2a0U,0x5268e236U,0xcc0c7795U,0xbb0b4703U,0x220216b9U,0x5505262fU, 0xc5ba3bbeU,0xb2bd0b28U,0x2bb45a92U,0x5cb36a04U,0xc2d7ffa7U,0xb5d0cf31U,0x2cd99e8bU,0x5bdeae1dU, 0x9b64c2b0U,0xec63f226U,0x756aa39cU,0x026d930aU,0x9c0906a9U,0xeb0e363fU,0x72076785U,0x05005713U, 0x95bf4a82U,0xe2b87a14U,0x7bb12baeU,0x0cb61b38U,0x92d28e9bU,0xe5d5be0dU,0x7cdcefb7U,0x0bdbdf21U, 0x86d3d2d4U,0xf1d4e242U,0x68ddb3f8U,0x1fda836eU,0x81be16cdU,0xf6b9265bU,0x6fb077e1U,0x18b74777U, 0x88085ae6U,0xff0f6a70U,0x66063bcaU,0x11010b5cU,0x8f659effU,0xf862ae69U,0x616bffd3U,0x166ccf45U, 0xa00ae278U,0xd70dd2eeU,0x4e048354U,0x3903b3c2U,0xa7672661U,0xd06016f7U,0x4969474dU,0x3e6e77dbU, 0xaed16a4aU,0xd9d65adcU,0x40df0b66U,0x37d83bf0U,0xa9bcae53U,0xdebb9ec5U,0x47b2cf7fU,0x30b5ffe9U, 0xbdbdf21cU,0xcabac28aU,0x53b39330U,0x24b4a3a6U,0xbad03605U,0xcdd70693U,0x54de5729U,0x23d967bfU, 0xb3667a2eU,0xc4614ab8U,0x5d681b02U,0x2a6f2b94U,0xb40bbe37U,0xc30c8ea1U,0x5a05df1bU,0x2d02ef8dU}; uint32_t CRC32(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator) { if (!len || OverflowCheck::sum(offset,len)>buffer.size()) throw Buffer::OutOfBoundsError(); const uint8_t *ptr=buffer.data()+offset; accumulator=~accumulator; for (size_t i=0;i>8)^CRC32Table[(accumulator&0xff)^ptr[i]]; return ~accumulator; } uint32_t CRC32Byte(uint8_t ch,uint32_t accumulator) noexcept { return ~((~accumulator>>8)^CRC32Table[(~accumulator&0xff)^ch]); } // instead of bit-twiddling lets have a separate implementation for reverse // same table as the previous one, but reflected static const uint32_t CRC32RevTable[256]={ 0x00000000U,0x04c11db7U,0x09823b6eU,0x0d4326d9U,0x130476dcU,0x17c56b6bU,0x1a864db2U,0x1e475005U, 0x2608edb8U,0x22c9f00fU,0x2f8ad6d6U,0x2b4bcb61U,0x350c9b64U,0x31cd86d3U,0x3c8ea00aU,0x384fbdbdU, 0x4c11db70U,0x48d0c6c7U,0x4593e01eU,0x4152fda9U,0x5f15adacU,0x5bd4b01bU,0x569796c2U,0x52568b75U, 0x6a1936c8U,0x6ed82b7fU,0x639b0da6U,0x675a1011U,0x791d4014U,0x7ddc5da3U,0x709f7b7aU,0x745e66cdU, 0x9823b6e0U,0x9ce2ab57U,0x91a18d8eU,0x95609039U,0x8b27c03cU,0x8fe6dd8bU,0x82a5fb52U,0x8664e6e5U, 0xbe2b5b58U,0xbaea46efU,0xb7a96036U,0xb3687d81U,0xad2f2d84U,0xa9ee3033U,0xa4ad16eaU,0xa06c0b5dU, 0xd4326d90U,0xd0f37027U,0xddb056feU,0xd9714b49U,0xc7361b4cU,0xc3f706fbU,0xceb42022U,0xca753d95U, 0xf23a8028U,0xf6fb9d9fU,0xfbb8bb46U,0xff79a6f1U,0xe13ef6f4U,0xe5ffeb43U,0xe8bccd9aU,0xec7dd02dU, 0x34867077U,0x30476dc0U,0x3d044b19U,0x39c556aeU,0x278206abU,0x23431b1cU,0x2e003dc5U,0x2ac12072U, 0x128e9dcfU,0x164f8078U,0x1b0ca6a1U,0x1fcdbb16U,0x018aeb13U,0x054bf6a4U,0x0808d07dU,0x0cc9cdcaU, 0x7897ab07U,0x7c56b6b0U,0x71159069U,0x75d48ddeU,0x6b93dddbU,0x6f52c06cU,0x6211e6b5U,0x66d0fb02U, 0x5e9f46bfU,0x5a5e5b08U,0x571d7dd1U,0x53dc6066U,0x4d9b3063U,0x495a2dd4U,0x44190b0dU,0x40d816baU, 0xaca5c697U,0xa864db20U,0xa527fdf9U,0xa1e6e04eU,0xbfa1b04bU,0xbb60adfcU,0xb6238b25U,0xb2e29692U, 0x8aad2b2fU,0x8e6c3698U,0x832f1041U,0x87ee0df6U,0x99a95df3U,0x9d684044U,0x902b669dU,0x94ea7b2aU, 0xe0b41de7U,0xe4750050U,0xe9362689U,0xedf73b3eU,0xf3b06b3bU,0xf771768cU,0xfa325055U,0xfef34de2U, 0xc6bcf05fU,0xc27dede8U,0xcf3ecb31U,0xcbffd686U,0xd5b88683U,0xd1799b34U,0xdc3abdedU,0xd8fba05aU, 0x690ce0eeU,0x6dcdfd59U,0x608edb80U,0x644fc637U,0x7a089632U,0x7ec98b85U,0x738aad5cU,0x774bb0ebU, 0x4f040d56U,0x4bc510e1U,0x46863638U,0x42472b8fU,0x5c007b8aU,0x58c1663dU,0x558240e4U,0x51435d53U, 0x251d3b9eU,0x21dc2629U,0x2c9f00f0U,0x285e1d47U,0x36194d42U,0x32d850f5U,0x3f9b762cU,0x3b5a6b9bU, 0x0315d626U,0x07d4cb91U,0x0a97ed48U,0x0e56f0ffU,0x1011a0faU,0x14d0bd4dU,0x19939b94U,0x1d528623U, 0xf12f560eU,0xf5ee4bb9U,0xf8ad6d60U,0xfc6c70d7U,0xe22b20d2U,0xe6ea3d65U,0xeba91bbcU,0xef68060bU, 0xd727bbb6U,0xd3e6a601U,0xdea580d8U,0xda649d6fU,0xc423cd6aU,0xc0e2d0ddU,0xcda1f604U,0xc960ebb3U, 0xbd3e8d7eU,0xb9ff90c9U,0xb4bcb610U,0xb07daba7U,0xae3afba2U,0xaafbe615U,0xa7b8c0ccU,0xa379dd7bU, 0x9b3660c6U,0x9ff77d71U,0x92b45ba8U,0x9675461fU,0x8832161aU,0x8cf30badU,0x81b02d74U,0x857130c3U, 0x5d8a9099U,0x594b8d2eU,0x5408abf7U,0x50c9b640U,0x4e8ee645U,0x4a4ffbf2U,0x470cdd2bU,0x43cdc09cU, 0x7b827d21U,0x7f436096U,0x7200464fU,0x76c15bf8U,0x68860bfdU,0x6c47164aU,0x61043093U,0x65c52d24U, 0x119b4be9U,0x155a565eU,0x18197087U,0x1cd86d30U,0x029f3d35U,0x065e2082U,0x0b1d065bU,0x0fdc1becU, 0x3793a651U,0x3352bbe6U,0x3e119d3fU,0x3ad08088U,0x2497d08dU,0x2056cd3aU,0x2d15ebe3U,0x29d4f654U, 0xc5a92679U,0xc1683bceU,0xcc2b1d17U,0xc8ea00a0U,0xd6ad50a5U,0xd26c4d12U,0xdf2f6bcbU,0xdbee767cU, 0xe3a1cbc1U,0xe760d676U,0xea23f0afU,0xeee2ed18U,0xf0a5bd1dU,0xf464a0aaU,0xf9278673U,0xfde69bc4U, 0x89b8fd09U,0x8d79e0beU,0x803ac667U,0x84fbdbd0U,0x9abc8bd5U,0x9e7d9662U,0x933eb0bbU,0x97ffad0cU, 0xafb010b1U,0xab710d06U,0xa6322bdfU,0xa2f33668U,0xbcb4666dU,0xb8757bdaU,0xb5365d03U,0xb1f740b4U}; uint32_t CRC32Rev(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator) { if (!len || OverflowCheck::sum(offset,len)>buffer.size()) throw Buffer::OutOfBoundsError(); const uint8_t *ptr=buffer.data()+offset; accumulator=~accumulator; for (size_t i=0;i>24)^ptr[i]]; return ~accumulator; } uint32_t CRC32RevByte(uint8_t ch,uint32_t &accumulator) noexcept { return ~((~accumulator<<8)^CRC32RevTable[(~accumulator>>24)^ch]); } } ancient-2.2.0/src/common/CRC32.hpp000066400000000000000000000010131463063262600165030ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef CRC32_HPP #define CRC32_HPP #include #include "Buffer.hpp" namespace ancient::internal { // The most common CRC32 uint32_t CRC32(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator); uint32_t CRC32Byte(uint8_t ch,uint32_t accumulator) noexcept; // Same polynomial, but in reverse... uint32_t CRC32Rev(const Buffer &buffer,size_t offset,size_t len,uint32_t accumulator); uint32_t CRC32RevByte(uint8_t ch,uint32_t accumulator) noexcept; } #endif ancient-2.2.0/src/common/Common.cpp000066400000000000000000000006441463063262600171630ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "Common.hpp" namespace ancient::internal { uint32_t rotateBits(uint32_t value,uint32_t count) noexcept { static const uint8_t rotateNibble[16]={ 0x0,0x8,0x4,0xc, 0x2,0xa,0x6,0xe, 0x1,0x9,0x5,0xd, 0x3,0xb,0x7,0xf }; uint32_t ret=0; for (uint32_t i=0;i>=4; } ret>>=(4-count)&3; return ret; } } ancient-2.2.0/src/common/Common.hpp000066400000000000000000000011661463063262600171700ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef COMMON_HPP #define COMMON_HPP #include #include #include namespace ancient::internal { constexpr uint16_t MultiChar2(const char (&cc)[3]) noexcept { return static_cast((static_cast(cc[0]) << 8) | static_cast(cc[1])); } constexpr uint32_t FourCC(const char (&cc)[5]) noexcept { return static_cast((static_cast(cc[0]) << 24) | (static_cast(cc[1]) << 16) | (static_cast(cc[2]) << 8) | static_cast(cc[3])); } uint32_t rotateBits(uint32_t value,uint32_t count) noexcept; } #endif ancient-2.2.0/src/common/MemoryBuffer.cpp000066400000000000000000000023011463063262600203250ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include #include #include #include #include "MemoryBuffer.hpp" #include "OverflowCheck.hpp" namespace ancient::internal { MemoryBuffer::MemoryBuffer(size_t size) : _data{reinterpret_cast(std::malloc(size))}, _size{size} { if (!_data) throw std::bad_alloc(); } MemoryBuffer::MemoryBuffer(const Buffer &src,size_t offset,size_t size) : MemoryBuffer{size} { if(OverflowCheck::sum(offset,size)>src.size()) throw InvalidOperationError(); std::memcpy(_data,src.data()+offset,size); } MemoryBuffer::~MemoryBuffer() noexcept { std::free(_data); } const uint8_t *MemoryBuffer::data() const noexcept { return _data; } uint8_t *MemoryBuffer::data() { return _data; } size_t MemoryBuffer::size() const noexcept { return _size; } bool MemoryBuffer::isResizable() const noexcept { return true; } void MemoryBuffer::resize(size_t newSize) { if (!newSize) { std::free(_data); _data=nullptr; _size=0; return; } uint8_t *newData=reinterpret_cast(std::realloc(_data,newSize)); if (!newData) { std::free(_data); _data=nullptr; _size=0; throw std::bad_alloc(); } _data=newData; _size=newSize; } } ancient-2.2.0/src/common/MemoryBuffer.hpp000066400000000000000000000010661463063262600203410ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef MEMORYBUFFER_HPP #define MEMORYBUFFER_HPP #include #include "Buffer.hpp" namespace ancient::internal { class MemoryBuffer : public Buffer { public: MemoryBuffer(size_t size); MemoryBuffer(const Buffer &src,size_t offset,size_t size); ~MemoryBuffer() noexcept; const uint8_t *data() const noexcept final; uint8_t *data() final; size_t size() const noexcept final; bool isResizable() const noexcept final; void resize(size_t newSize) final; private: uint8_t* _data; size_t _size; }; } #endif ancient-2.2.0/src/common/OverflowCheck.hpp000066400000000000000000000007661463063262600205060ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef OVERFLOW_CHECK_HPP #define OVERFLOW_CHECK_HPP #include "Buffer.hpp" namespace ancient::internal { class OverflowCheck { public: template static T sum(T a,U b) { // TODO: Add type traits to handle signed integers T ret=a+b; if (ret static T sum(T a,U b,Args... args) { return sum(sum(a,b),args...); } }; } #endif ancient-2.2.0/src/common/StaticBuffer.cpp000066400000000000000000000010171463063262600203070ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "StaticBuffer.hpp" namespace ancient::internal { ConstStaticBuffer::ConstStaticBuffer(const uint8_t *data,size_t length) noexcept : _data{data}, _length{length} { // nothing needed } const uint8_t *ConstStaticBuffer::data() const noexcept { return _data; } uint8_t *ConstStaticBuffer::data() { throw InvalidOperationError(); } size_t ConstStaticBuffer::size() const noexcept { return _length; } bool ConstStaticBuffer::isResizable() const noexcept { return false; } } ancient-2.2.0/src/common/StaticBuffer.hpp000066400000000000000000000022151463063262600203150ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef STATICBUFFER_HPP #define STATICBUFFER_HPP #include #include #include "Buffer.hpp" namespace ancient::internal { template class StaticBuffer : public Buffer { public: StaticBuffer(const StaticBuffer&)=delete; StaticBuffer& operator=(const StaticBuffer&)=delete; StaticBuffer() noexcept=default; ~StaticBuffer() noexcept=default; const uint8_t *data() const noexcept final { return _data; } uint8_t *data() final { return _data; } size_t size() const noexcept final { return N; } bool isResizable() const noexcept final { return false; } private: uint8_t _data[N]; }; class ConstStaticBuffer : public Buffer { public: ConstStaticBuffer(const ConstStaticBuffer&)=delete; ConstStaticBuffer& operator=(const ConstStaticBuffer&)=delete; ConstStaticBuffer(const uint8_t *data,size_t length) noexcept; ~ConstStaticBuffer() noexcept=default; const uint8_t *data() const noexcept final; uint8_t *data() final; size_t size() const noexcept final; bool isResizable() const noexcept final; private: const uint8_t *_data; size_t _length; }; } #endif ancient-2.2.0/src/common/SubBuffer.cpp000066400000000000000000000004301463063262600176070ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "SubBuffer.hpp" namespace ancient::internal { template <> uint8_t *GenericSubBuffer::data() { return _base.data()+_start; } template <> uint8_t *GenericSubBuffer::data() { throw InvalidOperationError(); } } ancient-2.2.0/src/common/SubBuffer.hpp000066400000000000000000000024321463063262600176200ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef SUBBUFFER_HPP #define SUBBUFFER_HPP #include #include #include "Buffer.hpp" #include "OverflowCheck.hpp" namespace ancient::internal { // helpers to splice Buffer template class GenericSubBuffer : public Buffer { public: GenericSubBuffer(const GenericSubBuffer&)=delete; GenericSubBuffer& operator=(const GenericSubBuffer&)=delete; GenericSubBuffer(T &base,size_t start,size_t length) : _base{base}, _start{start}, _length{length} { if (OverflowCheck::sum(start,length)>_base.size()) throw OutOfBoundsError(); } ~GenericSubBuffer() noexcept=default; const uint8_t *data() const noexcept final { return _base.data()+_start; } uint8_t *data() final; size_t size() const noexcept final { return _length; } bool isResizable() const noexcept final { return false; } // can only make the buffer smaller, can't run away from the current bounds void adjust(size_t start,size_t length) { if (start<_start || OverflowCheck::sum(start,length)>_start+_length) throw OutOfBoundsError(); _start=start; _length=length; } private: T &_base; size_t _start; size_t _length; }; typedef GenericSubBuffer SubBuffer; typedef GenericSubBuffer ConstSubBuffer; } #endif ancient-2.2.0/src/common/WrappedVectorBuffer.cpp000066400000000000000000000011551463063262600216500ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #include "WrappedVectorBuffer.hpp" namespace ancient::internal { WrappedVectorBuffer::WrappedVectorBuffer(std::vector &refdata) noexcept : _refdata{refdata} { // nothing needed } const uint8_t *WrappedVectorBuffer::data() const noexcept { return _refdata.data(); } uint8_t *WrappedVectorBuffer::data() { return _refdata.data(); } size_t WrappedVectorBuffer::size() const noexcept { return _refdata.size(); } bool WrappedVectorBuffer::isResizable() const noexcept { return true; } void WrappedVectorBuffer::resize(size_t newSize) { _refdata.resize(newSize); } } ancient-2.2.0/src/common/WrappedVectorBuffer.hpp000066400000000000000000000011441463063262600216530ustar00rootroot00000000000000/* Copyright (C) Teemu Suutari */ #ifndef WRAPPEDVECTORBUFFER_HPP #define WRAPPEDVECTORBUFFER_HPP #include #include #include #include "Buffer.hpp" namespace ancient::internal { class WrappedVectorBuffer : public Buffer { public: WrappedVectorBuffer(std::vector &refdata) noexcept; ~WrappedVectorBuffer() noexcept=default; const uint8_t *data() const noexcept final; uint8_t *data() final; size_t size() const noexcept final; bool isResizable() const noexcept final; void resize(size_t newSize) final; private: std::vector & _refdata; }; } #endif