From 7b6f849458e60fcca27f5fb91e30437d02bd2f1f Mon Sep 17 00:00:00 2001 From: Chris Ryder Date: Sat, 3 Mar 2018 09:33:44 +0000 Subject: [PATCH 1/4] Avoid double invoking ccache In Travis, when jobs have the 'cache: ccache' option set, Travis automatically adds the location of the ccache wrappers around g++/clang++ to the head of the PATH environment variable. This means that when you invoke 'g++' or 'clang++' what you are actually invoking is ccache. However, our jobs execute 'ccache g++', which means we are actually invoking ccache on itself. This was causing very poor ccache hit rates. --- .travis.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 48e1152ef9a..67e704a89cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,7 @@ jobs: - mkdir bin ; ln -s /usr/bin/gcc-5 bin/gcc # env: COMPILER=g++-5 SAN_FLAGS="-fsanitize=undefined -fno-sanitize-recover -fno-omit-frame-pointer" env: - - COMPILER="ccache g++-5" + - COMPILER="ccache /usr/bin/g++-5" - EXTRA_CXXFLAGS="-D_GLIBCXX_DEBUG" # OS X using g++ @@ -76,10 +76,8 @@ jobs: compiler: gcc cache: ccache before_install: - #we create symlink to non-ccache gcc, to be used in tests - - mkdir bin ; ln -s /usr/bin/gcc bin/gcc - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache - - export PATH=/usr/local/opt/ccache/libexec:$PATH + - export PATH=$PATH:/usr/local/opt/ccache/libexec env: COMPILER="ccache g++" # OS X using clang++ @@ -90,7 +88,7 @@ jobs: cache: ccache before_install: - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache - - export PATH=/usr/local/opt/ccache/libexec:$PATH + - export PATH=$PATH:/usr/local/opt/ccache/libexec env: - COMPILER="ccache clang++ -Qunused-arguments -fcolor-diagnostics" - CCACHE_CPP2=yes @@ -113,7 +111,7 @@ jobs: - mkdir bin ; ln -s /usr/bin/gcc-5 bin/gcc # env: COMPILER=g++-5 SAN_FLAGS="-fsanitize=undefined -fno-sanitize-recover -fno-omit-frame-pointer" env: - - COMPILER="ccache g++-5" + - COMPILER="ccache /usr/bin/g++-5" - EXTRA_CXXFLAGS="-DDEBUG" script: echo "Not running any tests for a debug build." @@ -138,7 +136,7 @@ jobs: - export CCACHE_CPP2=yes # env: COMPILER=clang++-3.7 SAN_FLAGS="-fsanitize=undefined -fno-sanitize-recover=undefined,integer -fno-omit-frame-pointer" env: - - COMPILER="ccache clang++-3.7 -Qunused-arguments -fcolor-diagnostics" + - COMPILER="ccache /usr/bin/clang++-3.7 -Qunused-arguments -fcolor-diagnostics" - CCACHE_CPP2=yes - EXTRA_CXXFLAGS="-DNDEBUG" @@ -163,7 +161,7 @@ jobs: - export CCACHE_CPP2=yes # env: COMPILER=clang++-3.7 SAN_FLAGS="-fsanitize=undefined -fno-sanitize-recover=undefined,integer -fno-omit-frame-pointer" env: - - COMPILER="ccache clang++-3.7 -Qunused-arguments -fcolor-diagnostics" + - COMPILER="ccache /usr/bin/clang++-3.7 -Qunused-arguments -fcolor-diagnostics" - CCACHE_CPP2=yes - EXTRA_CXXFLAGS="-DDEBUG -DUSE_STD_STRING" script: echo "Not running any tests for a debug build." @@ -180,10 +178,12 @@ jobs: - ubuntu-toolchain-r-test packages: - g++-5 + before_install: + - mkdir bin ; ln -s /usr/bin/gcc-5 bin/gcc install: - ccache -z - ccache --max-size=1G - - cmake -H. -Bbuild '-DCMAKE_BUILD_TYPE=Release' '-DCMAKE_CXX_COMPILER=g++-5' + - cmake -H. -Bbuild '-DCMAKE_BUILD_TYPE=Release' '-DCMAKE_CXX_COMPILER=/usr/bin/g++-5' - cmake --build build -- -j4 script: (cd build; ctest -V -L CORE -j2) @@ -192,7 +192,7 @@ jobs: cache: ccache before_install: - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache - - export PATH=/usr/local/opt/ccache/libexec:$PATH + - export PATH=$PATH:/usr/local/opt/ccache/libexec env: - BUILD_SYSTEM=cmake - CCACHE_CPP2=yes From 423cd4969d58fd7a6f0a1282bb991bdaf0b5e224 Mon Sep 17 00:00:00 2001 From: Chris Ryder Date: Sat, 3 Mar 2018 09:45:25 +0000 Subject: [PATCH 2/4] Ensure all compile jobs declare their compiler type Some of the travis compile jobs were not declaring the 'compiler' property. While this doesn't currently cause us any problems, we should be consistent between the jobs. It also slightly improves the chance of jobs for a new PR to pickup a useful seed cache, since Travis looks for caches in the following order: 1 - If the PR has previously been built, the cache from that build 2 - else, the cache for the target branch (e.g. develop) 3 - else, a cache for the compiler type (e.g. gcc) --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 67e704a89cb..1d522556cf7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -169,6 +169,7 @@ jobs: # cmake build using g++-5 - stage: Test different OS/CXX/Flags os: linux + compiler: gcc cache: ccache env: - BUILD_SYSTEM=cmake @@ -189,6 +190,7 @@ jobs: - stage: Test different OS/CXX/Flags os: osx + compiler: clang cache: ccache before_install: - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache From f0fc34592c3c0e594dd38fbbcdb1206145310e40 Mon Sep 17 00:00:00 2001 From: Chris Ryder Date: Sat, 3 Mar 2018 09:50:53 +0000 Subject: [PATCH 3/4] Increase ccache size for debug builds Currently we use a 1GB limit on the ccache size for all builds. However for the debug builds (those specifying '-g' to g++) object sizes are significantly larger due to the extra DWARF debug info. This is causing the debug builds to hit the ccache size limit. Local experiments suggest that with a 1GB limit, we can only get about 80% hit rate. Increasing the ccache size to 2GB increases the hit rate to 99%. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1d522556cf7..cb26baa9046 100644 --- a/.travis.yml +++ b/.travis.yml @@ -249,7 +249,7 @@ jobs: install: - ccache -z - - ccache --max-size=1G + - ccache --max-size=2G - make -C src minisat2-download - make -C src/ansi-c library_check - make -C src "CXX=${COMPILER}" "CXXFLAGS=-Wall -Werror -pedantic -O2 -g ${EXTRA_CXXFLAGS}" -j2 From 00ec070755756900a7a677205994d29278e6425e Mon Sep 17 00:00:00 2001 From: Chris Ryder Date: Sat, 3 Mar 2018 09:56:36 +0000 Subject: [PATCH 4/4] Slightly increase the number of parallel build jobs Travis hosts typically 2 cores available, slightly bump the number of parallel jobs to be (number of cores + 1) --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cb26baa9046..24840362fe3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -252,8 +252,8 @@ install: - ccache --max-size=2G - make -C src minisat2-download - make -C src/ansi-c library_check - - make -C src "CXX=${COMPILER}" "CXXFLAGS=-Wall -Werror -pedantic -O2 -g ${EXTRA_CXXFLAGS}" -j2 - - make -C src "CXX=${COMPILER}" "CXXFLAGS=-Wall -Werror -pedantic -O2 -g ${EXTRA_CXXFLAGS}" -j2 clobber.dir memory-models.dir + - make -C src "CXX=${COMPILER}" "CXXFLAGS=-Wall -Werror -pedantic -O2 -g ${EXTRA_CXXFLAGS}" -j3 + - make -C src "CXX=${COMPILER}" "CXXFLAGS=-Wall -Werror -pedantic -O2 -g ${EXTRA_CXXFLAGS}" -j3 clobber.dir memory-models.dir script: - if [ -e bin/gcc ] ; then export PATH=$PWD/bin:$PATH ; fi ;