Сжатие данных без потерь
作者
module CDeflateDemoTools
using Random, Statistics, DataFrames
include(joinpath(@__DIR__, "c_codec.jl"))
using .CDeflate
const MatrixDeflate=CDeflate
implementation_label()="C pure_deflate.c через Julia ccall; новые замеры текущего запуска"
matrix_bytes(a)=copy(reinterpret(UInt8,htol.(copy(reinterpret(UInt64,vec(Matrix(a)))))))
function pack_matrix(a; shuffle=true)
raw=matrix_bytes(a); data=deflate(raw;shuffle); use=length(data)<length(raw)
(; shape=size(a),raw_size=length(raw),codec=use ? (shuffle ? :shuffle_deflate : :deflate) : :raw,payload=use ? data : raw)
end
function unpack_matrix(p)
raw=p.codec==:raw ? p.payload : inflate(p.payload,p.raw_size;shuffle=p.codec==:shuffle_deflate)
reshape(copy(reinterpret(Float64,ltoh.(copy(reinterpret(UInt64,raw))))),p.shape)
end
export MatrixDeflate, deflate, inflate, matrix_bytes, byte_shuffle, byte_unshuffle,
pack_matrix, unpack_matrix, implementation_label, examples, measure_cases, mdtable,
size_table, link_table, block_table, special_values_test,
write_report
fmt(x) = string(round(x; digits=3))
function mdtable(caption, headers, rows)
return DataFrame([String(h) => [row[i] for row in rows]
for (i,h) in enumerate(headers)])
end
function examples(n=400; seed=42)
rng = MersenneTwister(seed)
sparse = [rand(rng) < .1 ? rand(rng,(-1.,1.,2.,10.)) : 0. for i in 1:n,j in 1:n]
smooth = [1000. + .5*(i+j-2)/max(n-1,1) for i in 1:n,j in 1:n]
random_near = 1000. .+ rand(rng,n,n)
return [("Нули и повторы",sparse), ("Плавная матрица",smooth),
("Случайные близкие значения",random_near)]
end
function measure_cases(cases; repeats=5)
rows = NamedTuple[]
for (name,a) in cases, shuffle in (false,true)
raw = matrix_bytes(a)
packed = deflate(raw; shuffle)
@assert inflate(packed,length(raw); shuffle) == raw
tc, td = Float64[], Float64[]
for _ in 1:repeats
GC.gc(false)
t = time_ns()
packed = deflate(raw; shuffle)
push!(tc,(time_ns()-t)/1e9)
t = time_ns()
restored = inflate(packed,length(raw); shuffle)
push!(td,(time_ns()-t)/1e9)
@assert restored == raw
end
push!(rows,(; name, method=shuffle ? "shuffle + DEFLATE" : "DEFLATE",
raw_bytes=length(raw), packed_bytes=length(packed), ratio=length(raw)/length(packed),
tc=median(tc), td=median(td), exact=true))
GC.gc(false)
end
return rows
end
size_table(rows) = mdtable("Таблица 1. Замеры на поде: матрицы 400 × 400, медиана трёх повторов.",
["Данные","Метод","До, КБ","После, КБ","Сжатие, раз","Упаковка, мс","Распаковка, мс","Биты совпали"],
[(r.name,r.method,fmt(r.raw_bytes/1000),fmt(r.packed_bytes/1000),fmt(r.ratio),
fmt(1000r.tc),fmt(1000r.td),r.exact) for r in rows])
function link_table(rows)
links = [("UART 115200, 8N1",115200/10), ("UART 1 Мбод, 8N1",1e6/10),
("SPI 8 Мбит/с, идеальный поток",8e6/8), ("Полезный поток 10 МБ/с",10e6)]
entries = [(r.name,r.method,label,fmt(r.raw_bytes/v),
fmt(r.tc+r.packed_bytes/v+r.td),
fmt((r.raw_bytes/v)/(r.tc+r.packed_bytes/v+r.td)))
for r in rows for (label,v) in links]
return mdtable("Таблица 2. Расчёт времени передачи с замеренными затратами пода.",
["Данные","Метод","Сценарий интерфейса","Без сжатия, с","Полный путь, с","Ускорение, раз"], entries)
end
function block_table(raw)
entries = []
for block_size in (8192,16384,65536,length(raw))
sent, blocks, raw_blocks = 0,0,0
for first in 1:block_size:length(raw)
last = min(first+block_size-1,length(raw))
block = raw[first:last]
compressed = deflate(block; shuffle=true)
@assert inflate(compressed,length(block); shuffle=true) == block
sent += min(length(block),length(compressed))
raw_blocks += length(compressed) >= length(block)
blocks += 1
end
push!(entries,(block_size,blocks,sent,fmt(length(raw)/sent),raw_blocks))
end
return mdtable("Таблица 3. Независимое сжатие блоков плавной матрицы, без заголовков протокола.",
["Исходный блок, байт","Блоков","Передаваемые данные, байт","Сжатие, раз","Блоков без сжатия"], entries)
end
function special_values_test()
words = UInt64[0,0x8000000000000000,0x7ff0000000000000,0xfff0000000000000,
0x7ff8000000000001,0x7ff8000000000042,1,0xffffffffffffffff]
a = reshape(copy(reinterpret(Float64,words)),2,4)
for shuffle in (false,true)
raw = matrix_bytes(a)
@assert inflate(deflate(raw;shuffle),length(raw);shuffle) == raw
@assert matrix_bytes(unpack_matrix(pack_matrix(a;shuffle))) == raw
end
return "Побитно проверены +0, −0, ±Inf, разные NaN и субнормальные числа."
end
function write_report(path, sections)
open(path,"w") do io
println(io,"# DEFLATE: новые замеры C-кодека через Julia\n")
println(io,"Julia $(VERSION); реализация $(implementation_label()). Передача по интерфейсу рассчитана, а не измерена.\n")
for (i,section) in enumerate(sections)
println(io,"Таблица $i\n")
println(io,"| ",join(names(section)," | ")," |")
println(io,"| ",join(fill("---",ncol(section))," | ")," |")
for row in eachrow(section)
println(io,"| ",join(string.(collect(row))," | ")," |")
end
println(io,"\n")
end
end
return path
end
end