tamp/Makefile at main · BrianPugh/tamp

780 lines (654 loc) · 34.3 KB
# Per-device on-device harness targets live next to their code; each fragment
# registers its help section in DEVICE_HELP_TARGETS. Shared data staging for
# these harnesses is under "Shared On-Device Data" below.
include $(wildcard devices/*/targets.mk)
# Default target - print help
.PHONY: help help-main help-extra
help: help-main $(DEVICE_HELP_TARGETS) help-extra
	@echo "Tamp - Low-memory compression library"
	@echo "Common targets:"
	@echo "  make test              Run Python and MicroPython tests"
	@echo "  make c-test            Run C unit tests"
	@echo "  make c-test-embedded   Run C unit tests with embedded find_best_match"
	@echo "  make clean             Clean all build artifacts"
	@echo "MicroPython native module:"
	@echo "  make mpy               Build native .mpy for RP2040 (requires MPY_DIR)"
	@echo "  make mpy ARCH=armv7m   Build for different architecture"
	@echo "On-device benchmarks (requires connected device):"
	@echo "  make on-device-compression-benchmark           Run Tamp compression benchmark (requires MPY_DIR)"
	@echo "  make on-device-decompression-benchmark         Run Tamp decompression benchmark (requires MPY_DIR)"
	@echo "  make on-device-deflate-compression-benchmark   Run MicroPython deflate compression benchmark"
	@echo "  make on-device-deflate-decompression-benchmark Run MicroPython deflate decompression benchmark"
help-extra:
	@echo "Fuzzing (requires LLVM clang; on macOS: brew install llvm):"
	@echo "  make fuzz-decompressor  Fuzz decompressor with random input"
	@echo "  make fuzz-round-trip    Fuzz compress->decompress round-trip"
	@echo "  make fuzz-clean         Clean fuzz artifacts and corpora"
	@echo "Other targets:"
	@echo "  make binary-size        Show binary sizes for README table"
	@echo "  make v1-compressed-datasets        Regenerate ground-truth v1 (--no-extended) .tamp binaries"
	@echo "  make extended-compressed-datasets  Regenerate ground-truth extended .tamp binaries"
	@echo "  make c-benchmark-stream Benchmark stream API with various temporary working buffer sizes"
	@echo "  make download-enwik8    Download enwik8 test dataset"
	@echo "  make download-sms       Download UCI SMS Spam Collection"
	@echo "  make download-tweets    Download Sentiment140 tweets (10K sample)"
	@echo "  make tamp-c-library     Build static C library"
	@echo "  make website-build      Build website for deployment"
.PHONY: clean test collect-data venv download
###########################
# MicroPython Native Module
###########################
# Build MicroPython native modules (.mpy files).
# Usage: make mpy ARCH=armv6m
#        (requires MPY_DIR environment variable pointing to MicroPython source)
# Supported architectures:
#     x86, x64, armv6m, armv7m, armv7emsp, armv7emdp, xtensa, xtensawin
#     TAMP_COMPRESSOR=0    Exclude compressor from build
#     TAMP_DECOMPRESSOR=0  Exclude decompressor from build
.PHONY: mpy
ARCH ?= armv6m
TAMP_COMPRESSOR ?= 1
TAMP_DECOMPRESSOR ?= 1
ifndef MPY_DIR
	$(error MPY_DIR must be set to MicroPython source directory)
# x86/x64 native modules can't be built on macOS (produces Mach-O, not ELF)
ifneq ($(filter $(ARCH),x86 x64),)
ifeq ($(shell uname),Darwin)
	$(error Cannot build $(ARCH) native modules on macOS - use Linux or specify an embedded ARCH (e.g., armv6m, armv7m))
	@$(MAKE) --no-print-directory _mpy-build \
		ARCH=$(ARCH) \
		TAMP_COMPRESSOR=$(TAMP_COMPRESSOR) \
		TAMP_DECOMPRESSOR=$(TAMP_DECOMPRESSOR)
# Internal: Only include dynruntime.mk when explicitly building MicroPython modules.
# This prevents accidental builds if MPY_DIR happens to be set in the environment.
ifeq ($(MAKECMDGOALS),_mpy-build)
ifdef MPY_DIR
# Override -Os with -O2 for better performance (last flag wins)
CFLAGS_EXTRA = -O2
CFLAGS += -Itamp/_c_src -DTAMP_COMPRESSOR=$(TAMP_COMPRESSOR) -DTAMP_DECOMPRESSOR=$(TAMP_DECOMPRESSOR) -DTAMP_STREAM=0 -DTAMP_USE_MEMSET=0
# Compiler-specific flags based on target architecture
ifeq ($(filter $(ARCH),x86 x64),)
# Cross-compiling for embedded (ARM, xtensa) - use GCC flags
CFLAGS += -fno-tree-loop-distribute-patterns
# Native x86/x64 - check if using clang (macOS)
ifneq ($(shell $(CC) --version 2>&1 | grep -q clang && echo clang),)
CFLAGS += -Wno-typedef-redefinition
CFLAGS += -fno-tree-loop-distribute-patterns
SRC = tamp/_c_src/tamp/common.c mpy_bindings/bindings.c mpy_bindings/bindings_common.py
ifeq ($(strip $(TAMP_COMPRESSOR)),1)
SRC += mpy_bindings/bindings_compressor.py tamp/_c_src/tamp/compressor.c
ifeq ($(strip $(TAMP_DECOMPRESSOR)),1)
SRC += mpy_bindings/bindings_decompressor.py tamp/_c_src/tamp/decompressor.c
MPY_CROSS_FLAGS = -s $(subst compressor,c,$(subst decompressor,d,$(subst bindings,m,$(notdir $<))))
include $(MPY_DIR)/py/dynruntime.mk
# _mpy-build delegates to 'all' which is defined by dynruntime.mk
.PHONY: _mpy-build
_mpy-build: all
endif  # MPY_DIR
endif  # MAKECMDGOALS == _mpy-build
################
# Common/Utility
################
.PHONY: venv clean-cython
	@. .venv/bin/activate
clean-cython:
	@rm -rf tamp/*.so
	@rm -rf tamp/_c_compressor.c
	@rm -rf tamp/_c_decompressor.c
	@rm -rf tamp/_c_common.c
# Only define 'clean' when not building MicroPython (dynruntime.mk has its own)
ifneq ($(MAKECMDGOALS),_mpy-build)
.PHONY: clean
clean: clean-cython clean-c-test fuzz-clean website-clean
	@rm -rf build
	@rm -rf dist
	@rm -f tamp.mpy
################
# Data Downloads
################
# Datasets are stored in datasets/ to persist across `make clean`
.PHONY: download-enwik8 download-silesia download-sms download-tweets download-micropython v1-compressed-datasets extended-compressed-datasets
datasets/enwik8.zip:
	@mkdir -p datasets
	curl -o datasets/enwik8.zip https://mattmahoney.net/dc/enwik8.zip
datasets/enwik8: | datasets/enwik8.zip
	cd datasets && unzip -q enwik8.zip
download-enwik8: datasets/enwik8
datasets/silesia:
	@mkdir -p datasets
	@if [ ! -d datasets/silesia ]; then \
		curl -o datasets/silesia.zip http://mattmahoney.net/dc/silesia.zip && \
		mkdir -p datasets/silesia && \
		unzip -q datasets/silesia.zip -d datasets/silesia && \
		rm datasets/silesia.zip; \
download-silesia: datasets/silesia
datasets/sms-spam-collection.zip:
	@mkdir -p datasets
	curl -sL -o datasets/sms-spam-collection.zip \
		https://archive.ics.uci.edu/static/public/228/sms+spam+collection.zip
datasets/sms.txt: | datasets/sms-spam-collection.zip
	@unzip -oq datasets/sms-spam-collection.zip SMSSpamCollection -d datasets
	@cut -f2- datasets/SMSSpamCollection > datasets/sms.txt
	@rm -f datasets/SMSSpamCollection datasets/readme
download-sms: datasets/sms.txt
datasets/sentiment140.zip:
	@mkdir -p datasets
	curl -sL -o datasets/sentiment140.zip \
		https://cs.stanford.edu/people/alecmgo/trainingandtestdata.zip
datasets/tweets.txt: | datasets/sentiment140.zip
	@unzip -oq datasets/sentiment140.zip training.1600000.processed.noemoticon.csv -d datasets
	@python3 -c "\
	import csv, random; random.seed(42); \
	tweets = [row[5] for row in csv.reader(open('datasets/training.1600000.processed.noemoticon.csv', encoding='latin-1')) if row[5]]; \
	open('datasets/tweets.txt', 'w').write('\n'.join(random.sample(tweets, 10000)))"
	@rm -f datasets/training.1600000.processed.noemoticon.csv
download-tweets: datasets/tweets.txt
.PHONY: download-tweets
# Regenerate the ground-truth compressed binaries. These are no longer committed
# (tests/test_dataset_regression.py stores only their SHA256s and re-derives them);
# use these targets when you actually need the binaries on disk. The recorded
# hashes identify the exact files these targets must produce.
v1-compressed-datasets: datasets/enwik8 datasets/silesia
	@mkdir -p datasets/v1-compressed
	@for f in datasets/*; do \
		[ -f "$$f" ] || continue; \
		base=$$(basename "$$f"); \
		echo "Compressing $$f -> datasets/v1-compressed/$$base.tamp"; \
		uv run tamp compress --no-extended "$$f" -o "datasets/v1-compressed/$$base.tamp"; \
	@mkdir -p datasets/v1-compressed/silesia
	@for f in datasets/silesia/*; do \
		[ -f "$$f" ] || continue; \
		base=$$(basename "$$f"); \
		echo "Compressing $$f -> datasets/v1-compressed/silesia/$$base.tamp"; \
		uv run tamp compress --no-extended "$$f" -o "datasets/v1-compressed/silesia/$$base.tamp"; \
extended-compressed-datasets: datasets/enwik8 datasets/silesia
	@mkdir -p datasets/extended-compressed
	@for f in datasets/* datasets/silesia/*; do \
		[ -f "$$f" ] || continue; \
		base=$$(basename "$$f"); \
		echo "Compressing $$f -> datasets/extended-compressed/$$base.tamp"; \
		uv run tamp compress --extended "$$f" -o "datasets/extended-compressed/$$base.tamp"; \
# Derived test files (OK to be in build/, quick to regenerate)
build/enwik8-100kb: download-enwik8
	@mkdir -p build
	@head -c 100000 datasets/enwik8 > build/enwik8-100kb
build/enwik8-100kb.tamp: build/enwik8-100kb
	@# Use Python implementation for extended format compression
	@uv run tamp compress --implementation=python build/enwik8-100kb -o build/enwik8-100kb.tamp
download-micropython:
	mkdir -p datasets
	cd datasets && curl -O https://micropython.org/resources/firmware/RPI_PICO-20250415-v1.25.0.uf2
download: download-enwik8 download-silesia download-sms download-tweets download-micropython
##################
# Python / Testing
##################
.PHONY: test collect-data
	@uv sync && uv run python -m pytest
	@uv run belay run micropython -m unittest tests/*.py
	@echo "All Tests Passed!"
collect-data: venv download-enwik8
	@python tools/collect-data.py 8
	@python tools/collect-data.py 9
	@python tools/collect-data.py 10
#######################
# MicroPython Utilities
#######################
.PHONY: on-device-compression-benchmark on-device-decompression-benchmark on-device-deflate-compression-benchmark on-device-deflate-decompression-benchmark mpy-viper-size mpy-native-size mpy-compression-benchmark
MPREMOTE := uv run mpremote
# Helper to sync a file only if hash differs: $(call mpremote-sync,local,remote)
define mpremote-sync
	@local_hash=$$(shasum -a 256 $(1) | cut -d' ' -f1); \
	remote_hash=$$($(MPREMOTE) sha256sum :$(2) 2>/dev/null | tail -1 || echo ""); \
	if [ "$$local_hash" != "$$remote_hash" ]; then \
		echo "Syncing $(1) -> :$(2)"; \
		$(MPREMOTE) cp $(1) :$(2); \
		echo "Skipping $(1) (unchanged)"; \
on-device-compression-benchmark: mpy build/enwik8-100kb
	$(MPREMOTE) rm :enwik8-100kb.tamp || true
	@# Remove any viper implementation that may exist from previous belay syncs
	$(MPREMOTE) rm :tamp/__init__.py :tamp/compressor_viper.py :tamp/decompressor_viper.py :tamp/compressor.py :tamp/decompressor.py :tamp/__main__.py :tamp/py.typed 2>/dev/null || true
	$(MPREMOTE) rmdir :tamp 2>/dev/null || true
	$(MPREMOTE) mkdir :lib || true
	$(call mpremote-sync,tamp.mpy,lib/tamp.mpy)
	$(call mpremote-sync,build/enwik8-100kb,enwik8-100kb)
	$(MPREMOTE) soft-reset
	$(MPREMOTE) run tools/on-device-compression-benchmark.py
	$(MPREMOTE) cp :enwik8-100kb.tamp build/on-device-enwik8-100kb.tamp
	uv run tamp decompress build/on-device-enwik8-100kb.tamp -o build/on-device-enwik8-100kb-decompressed
	cmp build/enwik8-100kb build/on-device-enwik8-100kb-decompressed
	@echo "Success!"
on-device-decompression-benchmark: mpy build/enwik8-100kb.tamp
	$(MPREMOTE) rm :enwik8-100kb-decompressed || true
	@# Remove any viper implementation that may exist from previous belay syncs
	$(MPREMOTE) rm :tamp/__init__.py :tamp/compressor_viper.py :tamp/decompressor_viper.py :tamp/compressor.py :tamp/decompressor.py :tamp/__main__.py :tamp/py.typed 2>/dev/null || true
	$(MPREMOTE) rmdir :tamp 2>/dev/null || true
	$(MPREMOTE) mkdir :lib || true
	$(call mpremote-sync,tamp.mpy,lib/tamp.mpy)
	$(call mpremote-sync,build/enwik8-100kb.tamp,enwik8-100kb.tamp)
	$(MPREMOTE) soft-reset
	$(MPREMOTE) run tools/on-device-decompression-benchmark.py
	$(MPREMOTE) cp :enwik8-100kb-decompressed build/on-device-enwik8-100kb-decompressed
	cmp build/enwik8-100kb build/on-device-enwik8-100kb-decompressed
	@echo "Success!"
on-device-deflate-compression-benchmark: build/enwik8-100kb
	$(MPREMOTE) rm :enwik8-100kb.deflate || true
	$(call mpremote-sync,build/enwik8-100kb,enwik8-100kb)
	$(MPREMOTE) soft-reset
	$(MPREMOTE) run tools/on-device-deflate-compression-benchmark.py
	$(MPREMOTE) cp :enwik8-100kb.deflate build/on-device-enwik8-100kb.deflate
	@echo "Success!"
on-device-deflate-decompression-benchmark: build/enwik8-100kb
	@# First ensure we have the deflate-compressed file on device
	@if ! $(MPREMOTE) sha256sum :enwik8-100kb.deflate >/dev/null 2>&1; then \
		echo "Error: enwik8-100kb.deflate not found on device. Run on-device-deflate-compression-benchmark first."; \
		exit 1; \
	$(MPREMOTE) rm :enwik8-100kb-decompressed || true
	$(MPREMOTE) soft-reset
	$(MPREMOTE) run tools/on-device-deflate-decompression-benchmark.py
	$(MPREMOTE) cp :enwik8-100kb-decompressed build/on-device-enwik8-100kb-decompressed
	cmp build/enwik8-100kb build/on-device-enwik8-100kb-decompressed
	@echo "Success!"
mpy-viper-size:
	@if [ -n "$(MPY_DIR)" ] && [ -x "$(MPY_DIR)/mpy-cross/build/mpy-cross" ]; then \
		MPY_CROSS="$(MPY_DIR)/mpy-cross/build/mpy-cross"; \
	elif command -v mpy-cross >/dev/null 2>&1; then \
		MPY_CROSS="mpy-cross"; \
		echo "Error: mpy-cross not found. Either set MPY_DIR or install mpy-cross."; \
		exit 1; \
	$$MPY_CROSS -O3 -march=armv6m -o /tmp/_tamp_init.mpy tamp/__init__.py; \
	$$MPY_CROSS -O3 -march=armv6m -o /tmp/_tamp_comp.mpy tamp/compressor_viper.py; \
	$$MPY_CROSS -O3 -march=armv6m -o /tmp/_tamp_decomp.mpy tamp/decompressor_viper.py; \
	size_init=$$(wc -c < /tmp/_tamp_init.mpy | tr -d ' '); \
	size_comp=$$(wc -c < /tmp/_tamp_comp.mpy | tr -d ' '); \
	size_decomp=$$(wc -c < /tmp/_tamp_decomp.mpy | tr -d ' '); \
	rm -f /tmp/_tamp_init.mpy /tmp/_tamp_comp.mpy /tmp/_tamp_decomp.mpy; \
	printf '%-34s %10d %12d %25d\n' "Tamp (MicroPython Viper)" \
		$$((size_init + size_comp)) $$((size_init + size_decomp)) $$((size_init + size_comp + size_decomp))
mpy-native-size:
ifndef MPY_DIR
	$(error MPY_DIR must be set for mpy-native-size)
	@rm -rf tamp.mpy build/tamp build/mpy_bindings build/tamp.native.mpy && \
		$(MAKE) -s _mpy-build MPY_DIR=$(MPY_DIR) ARCH=armv6m TAMP_COMPRESSOR=1 TAMP_DECOMPRESSOR=0 >/dev/null 2>&1 && \
		size_comp=$$(wc -c < tamp.mpy | tr -d ' ') && \
		rm -rf tamp.mpy build/tamp build/mpy_bindings build/tamp.native.mpy && \
		$(MAKE) -s _mpy-build MPY_DIR=$(MPY_DIR) ARCH=armv6m TAMP_COMPRESSOR=0 TAMP_DECOMPRESSOR=1 >/dev/null 2>&1 && \
		size_decomp=$$(wc -c < tamp.mpy | tr -d ' ') && \
		rm -rf tamp.mpy build/tamp build/mpy_bindings build/tamp.native.mpy && \
		$(MAKE) -s _mpy-build MPY_DIR=$(MPY_DIR) ARCH=armv6m TAMP_COMPRESSOR=1 TAMP_DECOMPRESSOR=1 >/dev/null 2>&1 && \
		size_both=$$(wc -c < tamp.mpy | tr -d ' ') && \
		printf '%-34s %10s %12s %25s\n' "Tamp (MicroPython Native)" $$size_comp $$size_decomp $$size_both
mpy-compression-benchmark:
	@time belay run micropython -X heapsize=300M tools/micropython-compression-benchmark.py
#######################
# Shared On-Device Data
#######################
# Data staging shared by the devices/ harnesses; the per-device targets that
# consume these live in devices/*/targets.mk (included at the top of this
# Makefile).
.PHONY: device-vectors
# Reference for the on-device byte-equality check. Must be v1 (non-extended)
# format because the harnesses compress with a default TampConf, and must use
# the python implementation because the device builds compile without
# TAMP_LAZY_MATCHING (the Cython build's lazy matching produces different bytes).
build/enwik8-100kb-v1.tamp: build/enwik8-100kb
	@uv run tamp compress --implementation=python --no-extended build/enwik8-100kb -o build/enwik8-100kb-v1.tamp
# Pack the shared regression vectors (devices/vectors/) for embedding.
device-vectors:
	@mkdir -p build
	uv run python tools/pack-device-vectors.py devices/vectors -o build/device-vectors.bin
# Unit tests using the Unity framework with AddressSanitizer enabled.
.PHONY: c-test clean-c-test
CTEST_CC = gcc
CTEST_SANITIZER_FLAGS = -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer -g -O0
CTEST_INCLUDES = -Ictests/Unity/src -Itamp/_c_src -Ictests -Ictests/littlefs -Ictests/fatfs/source
CTEST_DEFINES = -DTAMP_STREAM_STDIO=1 -DTAMP_STREAM_MEMORY=1 \
	-DTAMP_STREAM_LITTLEFS=1 -DTEST_LITTLEFS=1 \
	-DTAMP_STREAM_FATFS=1 -DTEST_FATFS=1 \
	-DTAMP_LAZY_MATCHING=1 \
	-DLFS_NO_DEBUG -DLFS_NO_WARN -DLFS_NO_ERROR
CTEST_CFLAGS = $(CTEST_INCLUDES) $(CTEST_SANITIZER_FLAGS) $(CTEST_DEFINES)
# Strict warnings applied only to first-party tamp sources, not third-party (Unity/LittleFS/FatFs)
CTEST_WARN_FLAGS = -Wall -Wextra -Wtype-limits -Werror
CTEST_LDFLAGS = $(CTEST_SANITIZER_FLAGS)
# Tamp library objects for testing
CTEST_TAMP_OBJS = \
	build/ctests/common.o \
	build/ctests/compressor.o \
	build/ctests/decompressor.o
# LittleFS objects
CTEST_LFS_OBJS = \
	build/ctests/lfs.o \
	build/ctests/lfs_util.o \
	build/ctests/lfs_rambd.o
# FatFs objects
CTEST_FATFS_OBJS = \
	build/ctests/ff.o \
	build/ctests/ffunicode.o \
	build/ctests/fatfs_ramdisk.o
# Test framework and test runner objects
CTEST_TEST_OBJS = \
	build/unity/unity.o \
	build/ctests/test_runner.o
# Build tamp source files for testing
build/ctests/%.o: tamp/_c_src/tamp/%.c
	@mkdir -p build/ctests
	$(CTEST_CC) $(CTEST_CFLAGS) $(CTEST_WARN_FLAGS) -c $< -o $@
# Build Unity framework
build/unity/unity.o: ctests/Unity/src/unity.c ctests/Unity/src/unity.h
	@mkdir -p build/unity
	$(CTEST_CC) $(CTEST_CFLAGS) -c $< -o $@
# Build LittleFS
build/ctests/lfs.o: ctests/littlefs/lfs.c
	@mkdir -p build/ctests
	$(CTEST_CC) $(CTEST_CFLAGS) -c $< -o $@
build/ctests/lfs_util.o: ctests/littlefs/lfs_util.c
	@mkdir -p build/ctests
	$(CTEST_CC) $(CTEST_CFLAGS) -c $< -o $@
build/ctests/lfs_rambd.o: ctests/lfs_rambd.c
	@mkdir -p build/ctests
	$(CTEST_CC) $(CTEST_CFLAGS) -c $< -o $@
# Build FatFs
build/ctests/ff.o: ctests/fatfs/source/ff.c
	@mkdir -p build/ctests
	$(CTEST_CC) $(CTEST_CFLAGS)  -c $< -o $@
build/ctests/ffunicode.o: ctests/fatfs/source/ffunicode.c
	@mkdir -p build/ctests
	$(CTEST_CC) $(CTEST_CFLAGS)  -c $< -o $@
build/ctests/fatfs_ramdisk.o: ctests/fatfs_ramdisk.c
	@mkdir -p build/ctests
	$(CTEST_CC) $(CTEST_CFLAGS)  -c $< -o $@
# Build test runner (includes test files via #include)
build/ctests/test_runner.o: ctests/test_runner.c ctests/test_compressor.c ctests/test_decompressor.c ctests/test_stream.c ctests/test_stream_filesystems.c
	@mkdir -p build/ctests
	$(CTEST_CC) $(CTEST_CFLAGS)  -c $< -o $@
# Link test executable
build/test_runner: $(CTEST_TAMP_OBJS) $(CTEST_LFS_OBJS) $(CTEST_FATFS_OBJS) $(CTEST_TEST_OBJS)
	$(CTEST_CC) $(CTEST_LDFLAGS) -o $@ $^
c-test: build/test_runner
	./build/test_runner
clean-c-test:
	@rm -f build/test_runner
	@rm -f build/test_runner_embedded
	@rm -f build/ctests/*.o
	@rm -f build/ctests-embedded/*.o
	@rm -f build/unity/*.o
# Embedded implementation tests (forces embedded find_best_match on desktop)
.PHONY: c-test-embedded
CTEST_EMBEDDED_TAMP_OBJS = \
	build/ctests-embedded/common.o \
	build/ctests-embedded/compressor.o \
	build/ctests-embedded/decompressor.o
CTEST_EMBEDDED_TEST_OBJS = \
	build/unity/unity.o \
	build/ctests-embedded/test_runner.o
build/ctests-embedded/%.o: tamp/_c_src/tamp/%.c
	@mkdir -p build/ctests-embedded
	$(CTEST_CC) $(CTEST_CFLAGS) $(CTEST_WARN_FLAGS) -DTAMP_USE_EMBEDDED_MATCH=1 -c $< -o $@
build/ctests-embedded/test_runner.o: ctests/test_runner.c ctests/test_compressor.c ctests/test_decompressor.c
	@mkdir -p build/ctests-embedded
	$(CTEST_CC) $(CTEST_CFLAGS) -DTAMP_USE_EMBEDDED_MATCH=1 -c $< -o $@
build/test_runner_embedded: $(CTEST_EMBEDDED_TAMP_OBJS) $(CTEST_EMBEDDED_TEST_OBJS) $(CTEST_LFS_OBJS) $(CTEST_FATFS_OBJS)
	$(CTEST_CC) $(CTEST_LDFLAGS) -o $@ $^
c-test-embedded: build/test_runner_embedded
	./build/test_runner_embedded
############
############
# Fuzz testing with libFuzzer (requires clang).
#   make fuzz-decompressor    # Fuzz the decompressor with random input
#   make fuzz-round-trip      # Fuzz compress->decompress round-trip
#   make fuzz-clean           # Remove fuzz build artifacts and corpora
# Stop with Ctrl-C. Crashes are saved to fuzz/corpus_*/crashes/.
# Reproduce a crash: ./build/fuzz_decompressor fuzz/corpus_decompressor/crashes/<file>
.PHONY: fuzz-decompressor fuzz-round-trip fuzz-clean
# Apple Clang does not ship the fuzzer runtime; use Homebrew LLVM if available.
# Override with: make fuzz-decompressor FUZZ_CC=/path/to/clang
# (if FUZZ_CC is a version-suffixed name like clang-18, also set FUZZ_CXX=clang++-18)
FUZZ_CC := $(shell \
	if [ -x /opt/homebrew/opt/llvm/bin/clang ]; then echo /opt/homebrew/opt/llvm/bin/clang; \
	elif [ -x /usr/local/opt/llvm/bin/clang ]; then echo /usr/local/opt/llvm/bin/clang; \
	else echo clang; fi)
FUZZ_CFLAGS = -g -O1 -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer \
	-Itamp/_c_src
FUZZ_LDFLAGS = -fsanitize=fuzzer,address,undefined
build/fuzz_decompressor: fuzz/fuzz_decompressor.c tamp/_c_src/tamp/decompressor.c tamp/_c_src/tamp/common.c
	@mkdir -p build
	$(FUZZ_CC) $(FUZZ_CFLAGS) -o $@ $^ $(FUZZ_LDFLAGS)
build/fuzz_round_trip: fuzz/fuzz_round_trip.c tamp/_c_src/tamp/compressor.c tamp/_c_src/tamp/decompressor.c tamp/_c_src/tamp/common.c
	@mkdir -p build
	$(FUZZ_CC) $(FUZZ_CFLAGS) -o $@ $^ $(FUZZ_LDFLAGS)
fuzz-decompressor: build/fuzz_decompressor
	@mkdir -p fuzz/corpus_decompressor
	./build/fuzz_decompressor fuzz/corpus_decompressor
fuzz-round-trip: build/fuzz_round_trip
	@mkdir -p fuzz/corpus_round_trip
	./build/fuzz_round_trip fuzz/corpus_round_trip
fuzz-clean:
	@rm -f build/fuzz_decompressor build/fuzz_round_trip
	@rm -rf fuzz/corpus_decompressor fuzz/corpus_round_trip
	@rm -rf build/esp32_host build/fuzz_round_trip_esp32 build/esp32_host_differential fuzz/corpus_round_trip_esp32
##################################
# ESP32-optimized compressor: host-side verification.
# The ESP32 build (TAMP_ESP32=1, espidf/tamp/compressor_esp32.cpp) compiles on
# desktop: all Xtensa asm/SIMD is behind `if constexpr` arch flags, so the
# scalar C++ search paths run natively and can be checked under ASan/UBSan
# against large inputs - something the device itself can't do.
#   make esp32-host-test        # differential checks + bounded round-trip fuzz
#   make fuzz-round-trip-esp32  # open-ended round-trip fuzzing (Ctrl-C to stop)
##################################
.PHONY: esp32-host-test fuzz-round-trip-esp32
FUZZ_CXX := $(FUZZ_CC)++
# -fno-sanitize=alignment: the ESP32 search code does unaligned word loads by
# design (its targets support them in hardware); don't drown out real findings.
ESP32_HOST_FLAGS = -DTAMP_ESP32=1 -DTAMP_LAZY_MATCHING=1 -fno-strict-aliasing -fno-sanitize=alignment \
	-Itamp/_c_src -Itamp/_c_src/tamp -Iespidf/tamp -Ifuzz/esp32_host
build/fuzz_round_trip_esp32: fuzz/fuzz_round_trip.c espidf/tamp/compressor_esp32.cpp \
		espidf/tamp/private/tamp_search.hpp espidf/tamp/private/copyutil.hpp \
		tamp/_c_src/tamp/compressor.c tamp/_c_src/tamp/decompressor.c tamp/_c_src/tamp/common.c
	@mkdir -p build/esp32_host
	$(FUZZ_CC) $(FUZZ_CFLAGS) $(ESP32_HOST_FLAGS) -c tamp/_c_src/tamp/compressor.c -o build/esp32_host/compressor.o
	$(FUZZ_CC) $(FUZZ_CFLAGS) $(ESP32_HOST_FLAGS) -c tamp/_c_src/tamp/decompressor.c -o build/esp32_host/decompressor.o
	$(FUZZ_CC) $(FUZZ_CFLAGS) $(ESP32_HOST_FLAGS) -c tamp/_c_src/tamp/common.c -o build/esp32_host/common.o
	$(FUZZ_CC) $(FUZZ_CFLAGS) $(ESP32_HOST_FLAGS) -c fuzz/fuzz_round_trip.c -o build/esp32_host/fuzz_round_trip.o
	$(FUZZ_CXX) -std=c++20 $(FUZZ_CFLAGS) $(ESP32_HOST_FLAGS) -c espidf/tamp/compressor_esp32.cpp -o build/esp32_host/compressor_esp32.o
	$(FUZZ_CXX) $(FUZZ_LDFLAGS) -o $@ build/esp32_host/compressor.o build/esp32_host/decompressor.o \
		build/esp32_host/common.o build/esp32_host/fuzz_round_trip.o build/esp32_host/compressor_esp32.o
build/esp32_host_differential: fuzz/esp32_host/differential.cpp espidf/tamp/compressor_esp32.cpp \
		espidf/tamp/private/tamp_search.hpp espidf/tamp/private/copyutil.hpp
	@mkdir -p build
	$(FUZZ_CXX) -std=c++20 -g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer $(ESP32_HOST_FLAGS) \
		-o $@ fuzz/esp32_host/differential.cpp espidf/tamp/compressor_esp32.cpp
# Seed the fuzz corpus with match-rich data; pure-random inputs rarely reach
# the RLE/extended-match paths.
fuzz/corpus_round_trip_esp32:
	@mkdir -p $@
	@head -c 4096 /dev/zero | tr '\0' 'a' > $@/seed_run
	@printf 'abcabcabcabc%.0s' $$(seq 1 300) > $@/seed_cycle
	@if [ -f build/enwik8-100kb ]; then head -c 16384 build/enwik8-100kb > $@/seed_text; fi
esp32-host-test: build/esp32_host_differential build/fuzz_round_trip_esp32 fuzz/corpus_round_trip_esp32
	./build/esp32_host_differential
	./build/fuzz_round_trip_esp32 -runs=100000 -max_len=8192 fuzz/corpus_round_trip_esp32
fuzz-round-trip-esp32: build/fuzz_round_trip_esp32 fuzz/corpus_round_trip_esp32
	./build/fuzz_round_trip_esp32 -max_len=16384 fuzz/corpus_round_trip_esp32
###############
# C Benchmarks
###############
# Benchmark the stream API with different temporary working buffer sizes.
# Uses the enwik8 dataset (100MB) for realistic performance measurements.
.PHONY: c-benchmark-stream
c-benchmark-stream: download-enwik8
	@echo "Stream API Benchmark (datasets/enwik8, 100MB)"
	@printf "%-20s %-15s %s\n" "Work Buffer Size" "Compress Time" "Decompress Time"
	@printf "%-20s %-15s %s\n" "----------------" "-------------" "---------------"
	@for size in 32 64 128 256 512 1024; do \
		gcc -O3 -DTAMP_STREAM_STDIO=1 -DTAMP_STREAM_WORK_BUFFER_SIZE=$$size \
			-Itamp/_c_src tools/benchmark_stream.c \
			tamp/_c_src/tamp/common.c \
			tamp/_c_src/tamp/compressor.c \
			tamp/_c_src/tamp/decompressor.c \
			-o build/benchmark_stream_$$size; \
		output=$$(./build/benchmark_stream_$$size datasets/enwik8 2>&1); \
		compress_time=$$(echo "$$output" | grep "Compression:" | sed 's/Compression: \([0-9.]*\)s.*/\1/'); \
		decompress_time=$$(echo "$$output" | grep "Decompression:" | sed 's/Decompression: \([0-9.]*\)s.*/\1/'); \
		printf "%-20s %-15s %s\n" "$$size bytes" "$${compress_time}s" "$${decompress_time}s"; \
	@rm -f build/benchmark_stream_*
#############
# C Library
#############
# Build a static library to check implementation size.
# Uses -Os for size optimization and -m32 for 32-bit target.
.PHONY: tamp-c-library
LIB_CC = $(CC)
LIB_CFLAGS = -Os -Wall -Itamp/_c_src -ffunction-sections -fdata-sections -m32
LIB_TAMP_OBJS = \
	build/lib/common.o \
	build/lib/compressor.o \
	build/lib/decompressor.o
build/lib/%.o: tamp/_c_src/tamp/%.c
	@mkdir -p build/lib
	$(LIB_CC) $(LIB_CFLAGS) -c $< -o $@
build/tamp.a: $(LIB_TAMP_OBJS)
	$(AR) rcs $@ $^
tamp-c-library: build/tamp.a
###############
# Binary Sizes
###############
# Generate binary size information for README table (armv6m with -O3).
.PHONY: binary-size c-size c-size-no-extended c-size-extended
ARM_CC := arm-none-eabi-gcc
ARM_AR := arm-none-eabi-ar
ARM_SIZE := arm-none-eabi-size
ARM_CFLAGS = -O3 -Wall -Itamp/_c_src -mcpu=cortex-m0plus -mthumb -ffunction-sections -fdata-sections
C_SRC_COMMON = tamp/_c_src/tamp/common.c
C_SRC_COMP = tamp/_c_src/tamp/compressor.c
C_SRC_DECOMP = tamp/_c_src/tamp/decompressor.c
# Flags to disable extended format support
NO_EXTENDED_FLAGS = -DTAMP_EXTENDED=0
c-size-no-extended:
	@rm -rf build/arm && mkdir -p build/arm
	@# No-extended without stream API
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=0 -DTAMP_STREAM=0 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=0 -DTAMP_STREAM=0 -c $(C_SRC_COMP) -o build/arm/compressor.o
	@$(ARM_AR) rcs build/arm/noext_comp.a build/arm/common.o build/arm/compressor.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=0 -DTAMP_DECOMPRESSOR=1 -DTAMP_STREAM=0 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=0 -DTAMP_DECOMPRESSOR=1 -DTAMP_STREAM=0 -c $(C_SRC_DECOMP) -o build/arm/decompressor.o
	@$(ARM_AR) rcs build/arm/noext_decomp.a build/arm/common.o build/arm/decompressor.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -DTAMP_STREAM=0 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -DTAMP_STREAM=0 -c $(C_SRC_COMP) -o build/arm/compressor.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -DTAMP_STREAM=0 -c $(C_SRC_DECOMP) -o build/arm/decompressor.o
	@$(ARM_AR) rcs build/arm/noext_full.a build/arm/common.o build/arm/compressor.o build/arm/decompressor.o
	@# No-extended with stream API
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=0 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=0 -c $(C_SRC_COMP) -o build/arm/compressor.o
	@$(ARM_AR) rcs build/arm/noext_comp_s.a build/arm/common.o build/arm/compressor.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=0 -DTAMP_DECOMPRESSOR=1 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=0 -DTAMP_DECOMPRESSOR=1 -c $(C_SRC_DECOMP) -o build/arm/decompressor.o
	@$(ARM_AR) rcs build/arm/noext_decomp_s.a build/arm/common.o build/arm/decompressor.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -c $(C_SRC_COMP) -o build/arm/compressor.o
	@$(ARM_CC) $(ARM_CFLAGS) $(NO_EXTENDED_FLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -c $(C_SRC_DECOMP) -o build/arm/decompressor.o
	@$(ARM_AR) rcs build/arm/noext_full_s.a build/arm/common.o build/arm/compressor.o build/arm/decompressor.o
	@size_comp=$$($(ARM_SIZE) -B --totals build/arm/noext_comp.a | grep TOTALS | awk '{print $$1+$$2}'); \
	size_decomp=$$($(ARM_SIZE) -B --totals build/arm/noext_decomp.a | grep TOTALS | awk '{print $$1+$$2}'); \
	size_full=$$($(ARM_SIZE) -B --totals build/arm/noext_full.a | grep TOTALS | awk '{print $$1+$$2}'); \
	printf '%-34s %10d %12d %25d\n' "Tamp (C, no extended, no stream)" $$size_comp $$size_decomp $$size_full
	@size_comp=$$($(ARM_SIZE) -B --totals build/arm/noext_comp_s.a | grep TOTALS | awk '{print $$1+$$2}'); \
	size_decomp=$$($(ARM_SIZE) -B --totals build/arm/noext_decomp_s.a | grep TOTALS | awk '{print $$1+$$2}'); \
	size_full=$$($(ARM_SIZE) -B --totals build/arm/noext_full_s.a | grep TOTALS | awk '{print $$1+$$2}'); \
	printf '%-34s %10d %12d %25d\n' "Tamp (C, no extended)" $$size_comp $$size_decomp $$size_full
c-size-extended:
	@rm -rf build/arm && mkdir -p build/arm
	@# Extended without stream API
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=0 -DTAMP_STREAM=0 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=0 -DTAMP_STREAM=0 -c $(C_SRC_COMP) -o build/arm/compressor.o
	@$(ARM_AR) rcs build/arm/ext_comp.a build/arm/common.o build/arm/compressor.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=0 -DTAMP_DECOMPRESSOR=1 -DTAMP_STREAM=0 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=0 -DTAMP_DECOMPRESSOR=1 -DTAMP_STREAM=0 -c $(C_SRC_DECOMP) -o build/arm/decompressor.o
	@$(ARM_AR) rcs build/arm/ext_decomp.a build/arm/common.o build/arm/decompressor.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -DTAMP_STREAM=0 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -DTAMP_STREAM=0 -c $(C_SRC_COMP) -o build/arm/compressor.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -DTAMP_STREAM=0 -c $(C_SRC_DECOMP) -o build/arm/decompressor.o
	@$(ARM_AR) rcs build/arm/ext_full.a build/arm/common.o build/arm/compressor.o build/arm/decompressor.o
	@# Extended with stream API
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=0 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=0 -c $(C_SRC_COMP) -o build/arm/compressor.o
	@$(ARM_AR) rcs build/arm/ext_comp_s.a build/arm/common.o build/arm/compressor.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=0 -DTAMP_DECOMPRESSOR=1 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=0 -DTAMP_DECOMPRESSOR=1 -c $(C_SRC_DECOMP) -o build/arm/decompressor.o
	@$(ARM_AR) rcs build/arm/ext_decomp_s.a build/arm/common.o build/arm/decompressor.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -c $(C_SRC_COMMON) -o build/arm/common.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -c $(C_SRC_COMP) -o build/arm/compressor.o
	@$(ARM_CC) $(ARM_CFLAGS) -DTAMP_COMPRESSOR=1 -DTAMP_DECOMPRESSOR=1 -c $(C_SRC_DECOMP) -o build/arm/decompressor.o
	@$(ARM_AR) rcs build/arm/ext_full_s.a build/arm/common.o build/arm/compressor.o build/arm/decompressor.o
	@size_comp=$$($(ARM_SIZE) -B --totals build/arm/ext_comp.a | grep TOTALS | awk '{print $$1+$$2}'); \
	size_decomp=$$($(ARM_SIZE) -B --totals build/arm/ext_decomp.a | grep TOTALS | awk '{print $$1+$$2}'); \
	size_full=$$($(ARM_SIZE) -B --totals build/arm/ext_full.a | grep TOTALS | awk '{print $$1+$$2}'); \
	printf '%-34s %10d %12d %25d\n' "Tamp (C, extended, no stream)" $$size_comp $$size_decomp $$size_full
	@size_comp=$$($(ARM_SIZE) -B --totals build/arm/ext_comp_s.a | grep TOTALS | awk '{print $$1+$$2}'); \
	size_decomp=$$($(ARM_SIZE) -B --totals build/arm/ext_decomp_s.a | grep TOTALS | awk '{print $$1+$$2}'); \
	size_full=$$($(ARM_SIZE) -B --totals build/arm/ext_full_s.a | grep TOTALS | awk '{print $$1+$$2}'); \
	printf '%-34s %10d %12d %25d\n' "Tamp (C, extended)" $$size_comp $$size_decomp $$size_full
c-size: c-size-no-extended c-size-extended
binary-size:
	@echo "Binary sizes for armv6m (bytes):"
	@printf '%-34s %10s %12s %25s\n' "" "Compressor" "Decompressor" "Compressor + Decompressor"
	@printf '%-34s %10s %12s %25s\n' "----------------------------------" "----------" "------------" "-------------------------"
	@output=$$($(MAKE) -s mpy-viper-size 2>&1) && echo "$$output" || echo "Tamp (MicroPython Viper)           (requires mpy-cross)"
	@output=$$($(MAKE) -s mpy-native-size 2>&1) && echo "$$output" || echo "Tamp (MicroPython Native)          (requires MPY_DIR)"
	@output=$$($(MAKE) -s c-size 2>&1) && echo "$$output" || echo "Tamp (C)                           (requires arm-none-eabi-gcc)"
.PHONY: website-build website-serve website-clean
website-build:
	cd website && npm install && npm run build
website-serve:
	cd website && npm install && npm run serve
website-clean:
	@rm -rf build/pages-deploy
	@rm -rf website/node_modules
Navigation Menu

Uh oh!

Uh oh!

FilesExpand file tree

Makefile

Latest commit

History

Makefile

File metadata and controls
