Commit ffcbc2d7 authored by Baptiste Daroussin's avatar Baptiste Daroussin
Browse files

Import Zstandard 1.2.0

Among changes: threading support is now default and supports freebsd
parent ab984b3e
# Object files
*.o
*.ko
*.dSYM
# Libraries
*.lib
*.a
# Shared objects (inc. Windows DLLs)
*.dll
*.so
*.so.*
*.dylib
# Executables
zstd
zstdmt
*.exe
*.out
*.app
# Test artefacts
tmp*
dictionary*
# Other files
.directory
_codelite/
_zstdbench/
.clang_complete
*.idea
*.swp
.DS_Store
googletest/
*.d
# Directories
bin/
.buckd/
buck-out/
# Long tests: run on commits to master branch/cron builds
language: c
sudo: required
dist: trusty
matrix:
include:
# Ubuntu 14.04
- env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-test'
- env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test32'
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test'
- env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
- env: Cmd='make valgrindinstall && make -C tests clean valgrindTest'
- env: Cmd='make arminstall && make armtest'
- env: Cmd='make arminstall && make aarch64test'
- env: Cmd='make ppcinstall && make ppctest'
- env: Cmd='make ppcinstall && make ppc64test'
- env: Cmd='make gpp6install valgrindinstall && make -C zlibWrapper test && make -C zlibWrapper valgrindTest'
- env: Cmd='make -C tests versionsTest'
- env: Cmd='make gpp6install && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan'
install:
- export CXX="g++-6" CC="gcc-6"
- env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zstd-nolegacy'
- env: Cmd='make gcc6install && CC=gcc-6 make uasan-test-zbuff'
# OS X Mavericks
- env: Cmd="make gnu90build && make clean && make test && make clean && make travis-install"
os: osx
git:
depth: 1
branches:
only:
- dev
- master
script:
- JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
- echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
- export FUZZERTEST=-T5mn;
export ZSTREAM_TESTTIME=-T5mn;
export DECODECORPUS_TESTTIME=-T1mn;
if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
git fetch origin dev;
git checkout -f FETCH_HEAD;
fi;
sh -c "$Cmd" || travis_terminate 1;
......@@ -90,6 +90,10 @@ examples:
manual:
$(MAKE) -C contrib/gen_html $@
.PHONY: cleanTabs
cleanTabs:
cd contrib; ./cleanTabs
.PHONY: clean
clean:
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
......@@ -105,9 +109,15 @@ clean:
# make install is validated only for Linux, OSX, Hurd and some BSD targets
#------------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
HOST_OS = POSIX
.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
.PHONY: list
list:
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
install:
@$(MAKE) -C $(ZSTDDIR) $@
@$(MAKE) -C $(PRGDIR) $@
......@@ -152,16 +162,16 @@ ppc64build: clean
CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch
armfuzz: clean
CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
aarch64fuzz: clean
CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
ppcfuzz: clean
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
ppc64fuzz: clean
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" $(MAKE) -C $(TESTDIR) fuzztest
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
gpptest: clean
CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
......@@ -180,19 +190,19 @@ clangtest: clean
armtest: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
aarch64test:
$(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
ppctest: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" FUZZER_FLAGS=--no-big-tests
ppc64test: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests
arm-ppc-compilation:
$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
......@@ -263,7 +273,7 @@ endif
ifneq (,$(filter MSYS%,$(shell uname)))
HOST_OS = MSYS
CMAKE_PARAMS = -G"MSYS Makefiles"
CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
endif
......@@ -275,7 +285,7 @@ cmakebuild:
cmake --version
$(RM) -r $(BUILDIR)/cmake/build
mkdir $(BUILDIR)/cmake/build
cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
c90build: clean
gcc -v
......
v1.2.0
cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
cli : new : zstdmt symlink hardwired to `zstd -T0`
cli : new : command --threads=# (#671)
cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
cli : fix : does not output compressed data on console
cli : fix : ignore symbolic links unless --force specified,
API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
API : improved: ZSTDMT_compressCCtx() reduced memory usage
API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
API : fix : src size stored in frame header is controlled at end of frame
API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
build: improved cmake script, by @Majlen
build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
new : contrib/linux-kernel version, by Nick Terrell
v1.1.4
cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
cli : new : advanced benchmark command --priority=rt
......@@ -5,13 +27,13 @@ cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
cli : fix : --rm remains silent when input is stdin
cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
arch : 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
build: new: meson build system in contrib/meson, by Dima Krasner
build: improved cmake script, by @Majlen
build: added -Wformat-security flag, as recommended by Padraig Brady
build : new: meson build system in contrib/meson, by Dima Krasner
build : improved cmake script, by @Majlen
build : added -Wformat-security flag, as recommended by Padraig Brady
doc : new : educational decoder, by Sean Purcell
v1.1.3
......
......@@ -12,13 +12,13 @@ you can consult a list of known ports on [Zstandard homepage](http://www.zstd.ne
|dev | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=dev)](https://travis-ci.org/facebook/zstd) |
As a reference, several fast compression algorithms were tested and compared
on a server running Linux Mint Debian Edition (`Linux version 4.8.0-1-amd64`),
on a server running Linux Debian (`Linux version 4.8.0-1-amd64`),
with a Core i7-6700K CPU @ 4.0GHz,
using [lzbench v1.6], an open-source in-memory benchmark by @inikep
using [lzbench], an open-source in-memory benchmark by @inikep
compiled with GCC 6.3.0,
on the [Silesia compression corpus].
[lzbench v1.6]: https://github.com/inikep/lzbench
[lzbench]: https://github.com/inikep/lzbench
[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
| Compressor name | Ratio | Compression| Decompress.|
......@@ -38,7 +38,12 @@ on the [Silesia compression corpus].
Zstd can also offer stronger compression ratios at the cost of compression speed.
Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.
The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.2.1, on the [Silesia compression corpus].
The following tests were run
on a server running Linux Debian (`Linux version 4.8.0-1-amd64`)
with a Core i7-6700K CPU @ 4.0GHz,
using [lzbench], an open-source in-memory benchmark by @inikep
compiled with GCC 6.3.0,
on the [Silesia compression corpus].
Compression Speed vs Ratio | Decompression Speed
---------------------------|--------------------
......
version: 1.0.{build}
environment:
matrix:
- COMPILER: "gcc"
PLATFORM: "mingw64"
MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"'
- COMPILER: "gcc"
PLATFORM: "mingw32"
MAKE_PARAMS: '"make -C tests test-zstd test-fullbench test-fuzzer test-invalidDictionaries"'
- COMPILER: "gcc"
PLATFORM: "clang"
MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion""'
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "x64"
- COMPILER: "visual"
CONFIGURATION: "Debug"
PLATFORM: "Win32"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "x64"
- COMPILER: "visual"
CONFIGURATION: "Release"
PLATFORM: "Win32"
-
version: 1.0.{build}
branches:
only:
- dev
- master
environment:
matrix:
- COMPILER: "gcc"
HOST: "mingw"
PLATFORM: "x64"
SCRIPT: "make allarch && make -C tests test-symbols fullbench-dll fullbench-lib"
ARTIFACT: "true"
BUILD: "true"
- COMPILER: "gcc"
HOST: "mingw"
PLATFORM: "x86"
SCRIPT: "make allarch"
ARTIFACT: "true"
BUILD: "true"
- COMPILER: "clang"
HOST: "mingw"
PLATFORM: "x64"
SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch"
BUILD: "true"
install:
- COMPILER: "gcc"
HOST: "mingw"
PLATFORM: "x64"
SCRIPT: ""
TEST: "cmake"
- COMPILER: "gcc"
HOST: "mingw"
PLATFORM: "x64"
SCRIPT: ""
TEST: "pzstd"
- COMPILER: "visual"
HOST: "visual"
PLATFORM: "x64"
CONFIGURATION: "Debug"
- COMPILER: "visual"
HOST: "visual"
PLATFORM: "Win32"
CONFIGURATION: "Debug"
- COMPILER: "visual"
HOST: "visual"
PLATFORM: "x64"
CONFIGURATION: "Release"
- COMPILER: "visual"
HOST: "visual"
PLATFORM: "Win32"
CONFIGURATION: "Release"
install:
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
- MKDIR bin
- if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
- if [%COMPILER%]==[gcc] (
SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
COPY C:\msys64\usr\bin\make.exe C:\MinGW\bin\make.exe &&
COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe
) else (
IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;")
- SET PATH_ORIGINAL=%PATH%
- if [%HOST%]==[mingw] (
SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
)
build_script:
- ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
- if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL%
- if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
- if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
- if [%COMPILER%]==[gcc] (
ECHO *** &&
ECHO *** Building %PLATFORM% &&
ECHO *** &&
make -v &&
cc -v &&
ECHO %MAKE_PARAMS% &&
sh -c %MAKE_PARAMS%
- IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
)
- if [%PLATFORM%]==[clang] COPY tests\fuzzer.exe tests\fuzzer_clang.exe
- if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] (
COPY programs\zstd.exe bin\zstd.exe &&
appveyor PushArtifact bin\zstd.exe
build_script:
- if [%HOST%]==[mingw] (
( if [%PLATFORM%]==[x64] (
SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
) else if [%PLATFORM%]==[x86] (
SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
) )
)
- if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw32] (
COPY programs\zstd.exe bin\zstd32.exe &&
appveyor PushArtifact bin\zstd32.exe
- if [%HOST%]==[mingw] if [%BUILD%]==[true] (
make -v &&
sh -c "%COMPILER% -v" &&
ECHO Building zlib to static link &&
SET "CC=%COMPILER%" &&
sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib" &&
sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
ECHO Building zstd &&
SET "CPPFLAGS=-I../../zlib" &&
SET "LDFLAGS=../../zlib/libz.a" &&
sh -c "%SCRIPT%" &&
( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true]
lib\dll\example\build_package.bat &&
make -C programs DEBUGFLAGS= clean zstd &&
cp programs\zstd.exe zstd_%PLATFORM%.exe &&
appveyor PushArtifact zstd_%PLATFORM%.exe &&
cp programs\zstd.exe bin\zstd.exe &&
make -C programs DEBUGFLAGS= clean zstdmt &&
cp programs\zstd.exe bin\zstdmt.exe &&
cd bin\ && 7z a -tzip zstd-win-release-%PLATFORM%.zip * &&
appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
)
)
- if [%COMPILER%]==[gcc] make clean
- if [%COMPILER%]==[visual] (
- if [%HOST%]==[visual] (
ECHO *** &&
ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% &&
ECHO *** &&
......@@ -111,29 +148,26 @@ build_script:
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
)
test_script:
test_script:
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
- SET FUZZERTEST=-T1mn
- if [%COMPILER%]==[gcc] if [%PLATFORM%]==[clang] (
tests\fuzzer_clang.exe %FUZZERTEST% &&
ECHO *** &&
ECHO *** Building cmake for %PLATFORM% &&
ECHO *** &&
- SET "CC=gcc"
- SET "CXX=g++"
- if [%TEST%]==[cmake] (
mkdir build\cmake\build &&
cd build\cmake\build &&
cmake -G "Visual Studio 14 2015 Win64" .. &&
cd ..\..\.. &&
make clean &&
ECHO *** &&
ECHO *** Building pzstd for %PLATFORM% &&
ECHO *** &&
make clean
)
- if [%TEST%]==[pzstd] (
make -C contrib\pzstd googletest-mingw64 &&
make -C contrib\pzstd pzstd.exe &&
make -C contrib\pzstd tests &&
make -C contrib\pzstd check &&
make -C contrib\pzstd clean
)
- if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
- SET "FUZZERTEST=-T30s"
- if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] (
CD tests &&
SET ZSTD=./zstd.exe &&
sh -e playTests.sh --test-large-data &&
......@@ -146,33 +180,76 @@ test_script:
fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
)
branches:
only:
- dev
- master
-
version: 1.0.{build}
environment:
matrix:
- COMPILER: "gcc"
HOST: "mingw"
PLATFORM: "x64"
SCRIPT: "make allarch"
- COMPILER: "gcc"
HOST: "mingw"
PLATFORM: "x86"
SCRIPT: "make allarch"
- COMPILER: "clang"
HOST: "mingw"
PLATFORM: "x64"
SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch"
artifacts:
- path: bin\zstd.exe
- path: bin\zstd32.exe
- COMPILER: "visual"
HOST: "visual"
PLATFORM: "x64"
CONFIGURATION: "Debug"
- COMPILER: "visual"
HOST: "visual"
PLATFORM: "Win32"
CONFIGURATION: "Debug"
- COMPILER: "visual"
HOST: "visual"
PLATFORM: "x64"
CONFIGURATION: "Release"
- COMPILER: "visual"
HOST: "visual"
PLATFORM: "Win32"
CONFIGURATION: "Release"
install:
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
- SET PATH_ORIGINAL=%PATH%
- if [%HOST%]==[mingw] (
SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
)
- IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
)
deploy:
- provider: GitHub
auth_token:
secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4
artifact: bin\zstd.exe
force_update: true
on:
branch: autobuild
COMPILER: gcc
PLATFORM: "mingw64"
appveyor_repo_tag: true
- provider: GitHub
auth_token:
secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4
artifact: bin\zstd32.exe
force_update: true
on:
branch: autobuild
COMPILER: gcc
PLATFORM: "mingw32"
appveyor_repo_tag: true
build_script:
- ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
- if [%HOST%]==[mingw] (
( if [%PLATFORM%]==[x64] (
SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
) else if [%PLATFORM%]==[x86] (
SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
) ) &&
make -v &&
sh -c "%COMPILER% -v" &&
set "CC=%COMPILER%" &&
sh -c "%SCRIPT%"
)
- if [%HOST%]==[visual] (
ECHO *** &&
ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
)
......@@ -3,7 +3,7 @@ dependencies:
- sudo dpkg --add-architecture i386
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
- sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
- sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6
- sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 zlib1g-dev liblzma-dev
- sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386
test:
......
#!/bin/sh
sed -i '' $'s/\t/ /g' ../lib/**/*.{h,c} ../programs/*.{h,c} ../tests/*.c ./**/*.{h,cpp} ../examples/*.c ../zlibWrapper/*.{h,c}
......@@ -91,7 +91,7 @@ void usage() {
std::fprintf(stderr, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel);
std::fprintf(stderr, " -d, --decompress : decompression\n");
std::fprintf(stderr, " -o file : result stored into `file` (only if 1 input file)\n");
std::fprintf(stderr, " -f, --force : overwrite output without prompting\n");
std::fprintf(stderr, " -f, --force : overwrite output without prompting, (de)compress links\n");
std::fprintf(stderr, " --rm : remove source file(s) after successful (de)compression\n");
std::fprintf(stderr, " -k, --keep : preserve source file(s) (default)\n");
std::fprintf(stderr, " -h, --help : display help and exit\n");
......@@ -121,6 +121,7 @@ Options::Status Options::parse(int argc, const char **argv) {
bool recursive = false;
bool ultra = false;
bool forceStdout = false;
bool followLinks = false;
// Local copy of input files, which are pointers into argv.
std::vector<const char *> localInputFiles;
for (int i = 1; i < argc; ++i) {
......@@ -255,6 +256,7 @@ Options::Status Options::parse(int argc, const char **argv) {
case 'f':
overwrite = true;
forceStdout = true;
followLinks = true;
break;
case 't':
test = true;
......@@ -328,13 +330,29 @@ Options::Status Options::parse(int argc, const char **argv) {
}
}
g_utilDisplayLevel = verbosity;
// Remove local input files that are symbolic links
if (!followLinks) {
std::remove_if(localInputFiles.begin(), localInputFiles.end(),
[&](const char *path) {
bool isLink = UTIL_isLink(path);
if (isLink && verbosity >= 2) {
std::fprintf(
stderr,
"Warning : %s is symbolic link, ignoring\n",
path);
}
return isLink;
});
}
// Translate input files/directories into files to (de)compress
if (recursive) {
char *scratchBuffer = nullptr;
unsigned numFiles = 0;
const char **files =
UTIL_createFileList(localInputFiles.data(), localInputFiles.size(),
&scratchBuffer, &numFiles);
&scratchBuffer, &numFiles, followLinks);
if (files == nullptr) {
std::fprintf(stderr, "Error traversing directories\n");
return Status::Failure;
......