feat: transparently append to compressed archives
When appending to a compressed archive (gzip, brotli, zstd), the tool now handles compression automatically. Since some compression formats don't support appending to compressed files in place, we write a new compressed file with all the data and atomically rename it to replace the original (assuming there is enough space on that filesystem). This means you can work with compressed archives the same way as uncompressed ones. Point the tool at your .json.gz file and append values. No manual decompression/recompression needed.
This commit is contained in:
parent
da0fed29de
commit
2ab1c31993
34 changed files with 4747 additions and 1099 deletions
55
tests/compression-integration/run_gzip_test.sh
Executable file
55
tests/compression-integration/run_gzip_test.sh
Executable file
|
|
@ -0,0 +1,55 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Test gzip compression workflow:
|
||||
# 1. Create archive from first state file
|
||||
# 2. Compress with gzip
|
||||
# 3. Append remaining state files to the compressed archive
|
||||
# 4. Decompress and show info
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")"
|
||||
BINARY="$PROJECT_DIR/target/debug/json-archive"
|
||||
DATA_DIR="$SCRIPT_DIR/data"
|
||||
OUT_DIR="$SCRIPT_DIR/out/gzip"
|
||||
|
||||
echo "=== Gzip Compression Test ==="
|
||||
|
||||
# Setup
|
||||
rm -rf "$OUT_DIR"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
# Create initial archive from first state file
|
||||
echo "Creating archive from state_1.json..."
|
||||
"$BINARY" "$DATA_DIR/state_1.json" -o "$OUT_DIR/test.json.archive"
|
||||
|
||||
# Compress with gzip
|
||||
echo "Compressing with gzip..."
|
||||
gzip "$OUT_DIR/test.json.archive"
|
||||
ls -la "$OUT_DIR/"
|
||||
|
||||
# Append remaining files to compressed archive
|
||||
for i in $(seq 2 9); do
|
||||
echo "Appending state_$i.json to compressed archive..."
|
||||
"$BINARY" "$OUT_DIR/test.json.archive.gz" "$DATA_DIR/state_$i.json"
|
||||
done
|
||||
|
||||
# Show info on the result
|
||||
echo ""
|
||||
echo "Final archive info:"
|
||||
"$BINARY" info "$OUT_DIR/test.json.archive.gz"
|
||||
|
||||
# Decompress for manual inspection
|
||||
echo ""
|
||||
echo "Decompressing for comparison..."
|
||||
gunzip -k "$OUT_DIR/test.json.archive.gz" 2>/dev/null || gunzip -c "$OUT_DIR/test.json.archive.gz" > "$OUT_DIR/test.json.archive"
|
||||
|
||||
echo ""
|
||||
echo "Decompressed archive info:"
|
||||
"$BINARY" info "$OUT_DIR/test.json.archive"
|
||||
|
||||
echo ""
|
||||
echo "Files in $OUT_DIR:"
|
||||
ls -la "$OUT_DIR/"
|
||||
Loading…
Add table
Add a link
Reference in a new issue