网传压缩算法对比表
算法 | 压缩率 | 压缩速度 | 解压速度 | 支持流式压缩 | 适用场景 |
---|---|---|---|---|---|
LZ4 | 低 | 极快 | 极快 | 是 | 实时数据压缩、日志压缩、内存缓存等 |
Zstandard | 高 | 快 | 快 | 是 | 文件压缩、网络传输、数据库备份等 |
Brotli | 很高 | 中等 | 快 | 是 | 静态资源压缩(HTML、CSS、JS)等 |
LZO | 低 | 极快 | 快 | 是 | 嵌入式系统、实时数据压缩 |
Snappy | 中等 | 快 | 快 | 否 | 日志压缩、内存缓存、网络传输 |
性能实测
./compress_test syslog.txt
LZ4 0 comp: 64.669 ms decom: 15.282 ms Size: 85.906 MB Ratio: 17.186 %
LZ4 1 comp: 65.662 ms decom: 15.046 ms Size: 85.906 MB Ratio: 17.186 %
LZO 0 comp: 65.834 ms decom: 51.929 ms Size: 85.906 MB Ratio: 17.936 %
LZO 1 comp: 61.834 ms decom: 50.462 ms Size: 85.906 MB Ratio: 17.936 %
Zstd 0 comp: 71.713 ms decom: 31.642 ms Size: 85.906 MB Ratio: 9.230 %
Zstd 1 comp: 93.255 ms decom: 47.457 ms Size: 85.906 MB Ratio: 9.230 %
Brotli 0 comp: 156.201 ms decom: 115.879 ms Size: 85.906 MB Ratio: 13.189 %
Brotli 1 comp: 127.775 ms decom: 104.258 ms Size: 85.906 MB Ratio: 13.189 %
Snappy 0 comp: 84.727 ms decom: 52.499 ms Size: 85.906 MB Ratio: 18.220 %
Snappy 1 comp: 82.503 ms decom: 52.898 ms Size: 85.906 MB Ratio: 18.220 %
对于一个系统日志文件,LZ4 LZO速度最快, 压缩率zstd遥遥领先,综合性价比 Zstd完胜。
安装与使用
LZ4
编译
git clone https://github.com/lz4/lz4.git
cd lz4
make
使用
ckun@DESKTOP-PODAH1A:~/ws/compress/test$ rm -f log.lz4
ckun@DESKTOP-PODAH1A:~/ws/compress/test$ time ./lz4 syslog -T1 log.lz4
Compressed 3290808339 bytes into 567183342 bytes ==> 17.24%
real 0m5.855s
user 0m4.636s
sys 0m1.910s
Zstd
git clone https://github.com/facebook/zstd.git
cd zstd
make
Brotli
git clone https://github.com/google/brotli.git
cd brotli
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make
LZO
wget http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz
tar -xvzf lzo-2.10.tar.gz
cd lzo-2.10
mkdir build
cd build
cmake -DENABLE_SHARED=ON ..
make
Snappy
git clone https://github.com/google/snappy.git
cd snappy
git submodule update --init
mkdir build
cd build
cmake ..
make
测试程序
#include <iostream>
#include <fstream>
#include <vector>
#include <chrono>
#include <cstring>
#include <lz4.h>
#include <lzo/lzo1x.h>
#include <zstd.h>
#include <brotli/encode.h>
#include <brotli/decode.h>
#include <snappy.h>
#include <iomanip>
// Helper function to calculate checksum (simple Adler-32 implementation)
uint32_t calculate_checksum(const std::vector<char>& data) {
uint32_t a = 1, b = 0;
for (char c : data) {
a = (a + static_cast<uint8_t>(c)) % 65521;
b = (b + a) % 65521;
}
return (b << 16) | a;
}
// Read file into memory
std::vector<char> read_file(const std::string& filename) {
std::ifstream file(filename, std::ios::binary | std::ios::ate);
if (!file) {
throw std::runtime_error("Failed to open file: " + filename);
}
std::streamsize size = file.tellg();
file.seekg(0, std::ios::beg);
std::vector<char> buffer(size);
if (!file.read(buffer.data(), size)) {
throw std::runtime_error("Failed to read file: " + filename);
}
return buffer;
}
// Test LZ4 compression and decompression
void test_lz4(const std::vector<char>& input, int i) {
//std::cout << "Testing LZ4..." << std::endl;
// Allocate buffers
int max_compressed_size = LZ4_compressBound(input.size());
std::vector<char> compressed(max_compressed_size);
std::vector<char> decompressed(input.size());
// Compress
auto start = std::chrono::high_resolution_clock::now();
int compressed_size = LZ4_compress_default(input.data(), compressed.data(), input.size(), max_compressed_size);
auto end = std::chrono::high_resolution_clock::now();
if (compressed_size <= 0) {
throw std::runtime_error("LZ4 compression failed");
}
auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// Decompress
start = std::chrono::high_resolution_clock::now();
int decompressed_size = LZ4_decompress_safe(compressed.data(), decompressed.data(), compressed_size, decompressed.size());
end = std::chrono::high_resolution_clock::now();
if (decompressed_size != input.size()) {
throw std::runtime_error("LZ4 decompression failed");
}
auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// Verify checksum
uint32_t original_checksum = calculate_checksum(input);
uint32_t decompressed_checksum = calculate_checksum(decompressed);
if (original_checksum != decompressed_checksum) {
throw std::runtime_error("LZ4 checksum mismatch");
}
// Output results
printf("LZ4 %d comp: %8.3lf ms decom: %8.3lf ms Size: %.3f MB Ratio: %.3f %%\n",
i, (compression_time / 1000.0), (decompression_time / 1000.0),
(input.size() / (1024.0 * 1024.0)),
(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}
// Test LZO compression and decompression
void test_lzo(const std::vector<char>& input, int i) {
//std::cout << "Testing LZO..." << std::endl;
// Initialize LZO
if (lzo_init() != LZO_E_OK) {
throw std::runtime_error("LZO initialization failed");
}
// Allocate buffers
std::vector<char> compressed(input.size() + input.size() / 16 + 64 + 3);
std::vector<char> decompressed(input.size());
std::vector<unsigned char> wrkmem(LZO1X_1_MEM_COMPRESS);
// Compress
auto start = std::chrono::high_resolution_clock::now();
lzo_uint compressed_size;
int r = lzo1x_1_compress(reinterpret_cast<const unsigned char*>(input.data()), input.size(),
reinterpret_cast<unsigned char*>(compressed.data()), &compressed_size, wrkmem.data());
auto end = std::chrono::high_resolution_clock::now();
if (r != LZO_E_OK) {
throw std::runtime_error("LZO compression failed");
}
auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// Decompress
start = std::chrono::high_resolution_clock::now();
lzo_uint decompressed_size;
r = lzo1x_decompress(reinterpret_cast<const unsigned char*>(compressed.data()), compressed_size,
reinterpret_cast<unsigned char*>(decompressed.data()), &decompressed_size, nullptr);
end = std::chrono::high_resolution_clock::now();
if (r != LZO_E_OK || decompressed_size != input.size()) {
throw std::runtime_error("LZO decompression failed");
}
auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// Verify checksum
uint32_t original_checksum = calculate_checksum(input);
uint32_t decompressed_checksum = calculate_checksum(decompressed);
if (original_checksum != decompressed_checksum) {
throw std::runtime_error("LZO checksum mismatch");
}
// Output results
// std::cout << "LZO " << i << " ";
// std::cout << "comp: " << (compression_time / 1000.0) << " ms "
// << "decom: " << (decompression_time / 1000.0) << " ms "
// << "Size: " << (input.size() / (1024.0 * 1024.0)) << " MB "
// << "Ratio: " << (static_cast<double>(compressed_size) / input.size()) * 100 << " %"
// << std::endl;
printf("LZO %d comp: %8.3lf ms decom: %8.3lf ms Size: %.3f MB Ratio: %.3f %%\n",
i, (compression_time / 1000.0), (decompression_time / 1000.0),
(input.size() / (1024.0 * 1024.0)),
(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}
void test_zstd(const std::vector<char>& input, int i) {
// Allocate buffers
size_t max_compressed_size = ZSTD_compressBound(input.size());
std::vector<char> compressed(max_compressed_size);
std::vector<char> decompressed(input.size());
// Compress
auto start = std::chrono::high_resolution_clock::now();
size_t compressed_size = ZSTD_compress(compressed.data(), max_compressed_size, input.data(), input.size(), 1);
auto end = std::chrono::high_resolution_clock::now();
if (ZSTD_isError(compressed_size)) {
throw std::runtime_error("Zstd compression failed: " + std::string(ZSTD_getErrorName(compressed_size)));
}
auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// Decompress
start = std::chrono::high_resolution_clock::now();
size_t decompressed_size = ZSTD_decompress(decompressed.data(), decompressed.size(), compressed.data(), compressed_size);
end = std::chrono::high_resolution_clock::now();
if (ZSTD_isError(decompressed_size) || decompressed_size != input.size()) {
throw std::runtime_error("Zstd decompression failed: " + std::string(ZSTD_getErrorName(decompressed_size)));
}
auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// Verify checksum
uint32_t original_checksum = calculate_checksum(input);
uint32_t decompressed_checksum = calculate_checksum(decompressed);
if (original_checksum != decompressed_checksum) {
throw std::runtime_error("Zstd checksum mismatch");
}
// Output results
printf("Zstd %d comp: %8.3lf ms decom: %8.3lf ms Size: %.3f MB Ratio: %.3f %%\n",
i, (compression_time / 1000.0), (decompression_time / 1000.0),
(input.size() / (1024.0 * 1024.0)),
(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}
void test_brotli(const std::vector<char>& input, int i) {
// Allocate buffers
size_t max_compressed_size = BrotliEncoderMaxCompressedSize(input.size());
std::vector<uint8_t> compressed(max_compressed_size);
std::vector<uint8_t> decompressed(input.size());
// Compress
auto start = std::chrono::high_resolution_clock::now();
size_t compressed_size = max_compressed_size;
if (!BrotliEncoderCompress(
0, BROTLI_DEFAULT_WINDOW, BROTLI_MODE_GENERIC,
input.size(), reinterpret_cast<const uint8_t*>(input.data()),
&compressed_size, compressed.data())) {
throw std::runtime_error("Brotli compression failed");
}
auto end = std::chrono::high_resolution_clock::now();
auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// Decompress
start = std::chrono::high_resolution_clock::now();
size_t decompressed_size = decompressed.size();
if (BrotliDecoderDecompress(
compressed_size, compressed.data(),
&decompressed_size, decompressed.data()) != BROTLI_DECODER_RESULT_SUCCESS) {
throw std::runtime_error("Brotli decompression failed");
}
end = std::chrono::high_resolution_clock::now();
auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// Verify checksum
uint32_t original_checksum = calculate_checksum(input);
uint32_t decompressed_checksum = calculate_checksum(std::vector<char>(decompressed.begin(), decompressed.end()));
if (original_checksum != decompressed_checksum) {
throw std::runtime_error("Brotli checksum mismatch");
}
// Output results
printf("Brotli %d comp: %8.3lf ms decom: %8.3lf ms Size: %.3f MB Ratio: %.3f %%\n",
i, (compression_time / 1000.0), (decompression_time / 1000.0),
(input.size() / (1024.0 * 1024.0)),
(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}
void test_snappy(const std::vector<char>& input, int i) {
// Allocate buffers
std::string compressed;
std::string decompressed;
// Compress
auto start = std::chrono::high_resolution_clock::now();
snappy::Compress(input.data(), input.size(), &compressed);
auto end = std::chrono::high_resolution_clock::now();
auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// Decompress
start = std::chrono::high_resolution_clock::now();
if (!snappy::Uncompress(compressed.data(), compressed.size(), &decompressed)) {
throw std::runtime_error("Snappy decompression failed");
}
end = std::chrono::high_resolution_clock::now();
auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// Verify checksum
uint32_t original_checksum = calculate_checksum(input);
uint32_t decompressed_checksum = calculate_checksum(std::vector<char>(decompressed.begin(), decompressed.end()));
if (original_checksum != decompressed_checksum) {
throw std::runtime_error("Snappy checksum mismatch");
}
// Output results
printf("Snappy %d comp: %8.3lf ms decom: %8.3lf ms Size: %.3f MB Ratio: %.3f %%\n",
i, (compression_time / 1000.0), (decompression_time / 1000.0),
(input.size() / (1024.0 * 1024.0)),
(double) (static_cast<double>(compressed.size()) / input.size()) * 100);
}
template <typename Func>
void run_test(Func func, const std::vector<char>& input, int times) {
for (int i = 0; i < times; ++i) {
func(input, i);
}
}
int main(int argc, char* argv[]) {
if (argc != 2) {
std::cerr << "Usage: " << argv[0] << " <file>" << std::endl;
return 1;
}
std::cout << std::fixed << std::setprecision(3);
try {
// Read file into memory
std::vector<char> input = read_file(argv[1]);
// Calculate checksum
uint32_t checksum = calculate_checksum(input);
std::cout << "Original Checksum: " << checksum << std::endl;
// Test LZ4
run_test(test_lz4, input, 2);
// Test LZO
run_test(test_lzo, input, 2);
run_test(test_zstd, input, 2);
// Test Brotli
run_test(test_brotli, input, 2);
// Test Snappy
run_test(test_snappy, input, 2);
} catch (const std::exception& e) {
std::cerr << "Error: " << e.what() << std::endl;
return 1;
}
return 0;
}
Makefile
TARGET = compress_test
SRCS = $(wildcard *.cpp)
CC = g++
CFLAGS += -g -O2 -I../lz4/lib -I../lzo-2.10/include -I../zstd/lib \
-I../brotli/c/include -I../snappy
LDFLAGS += -lyaml-cpp -L../lz4/lib -llz4 \
-L../lzo-2.10/build -llzo2 -lz \
-L../zstd/lib -lzstd \
-L../brotli/build -lbrotlicommon -lbrotlidec -lbrotlienc \
-L../snappy/build -lsnappy
all: $(TARGET)
$(TARGET): $(SRCS)
$(CC) $(CFLAGS) $(SRCS) -o $@ $(LDFLAGS)
.PHONY: clean
clean:
rm $(TARGET)
运行
export LD_LIBRARY_PATH=../lz4/lib:../lzo-2.10/build:../zstd/lib:../brotli/build/:../snappy/build
$ ldd compress_test
linux-vdso.so.1 (0x00007ffdb71cf000)
liblz4.so.1 => ../lz4/lib/liblz4.so.1 (0x00007fc26045a000)
liblzo2.so.2 => ../lzo-2.10/build/liblzo2.so.2 (0x00007fc260427000)
libzstd.so.1 => ../zstd/lib/libzstd.so.1 (0x00007fc260330000)
libbrotlidec.so.1 => ../brotli/build/libbrotlidec.so.1 (0x00007fc260321000)
libbrotlienc.so.1 => ../brotli/build/libbrotlienc.so.1 (0x00007fc260268000)
libsnappy.so.1 => ../snappy/build/libsnappy.so.1 (0x00007fc260259000)
libstdc++.so.6 => /lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007fc260026000)
libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007fc260006000)
libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007fc25fddd000)
libbrotlicommon.so.1 => ../brotli/build/libbrotlicommon.so.1 (0x00007fc25fdb7000)
libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007fc25fcd0000)
/lib64/ld-linux-x86-64.so.2 (0x00007fc260498000)
./compress_test file