squash merge pants3-in-ci

merge sha 2dfb39d and push to travis to ensure that PR works

Author: cosmicexplorer

.travis.yml

@@ -2,7 +2,7 @@
 
 # We use some subshell pipelines to collect target lists, make sure target collection failing
 # fails the build.
-set -o pipefail
+set -ox pipefail
 
 REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd "$(git rev-parse --show-toplevel)" && pwd)
 cd ${REPO_ROOT}
@@ -13,10 +13,9 @@ function usage() {
   cat <<EOF
 Runs commons tests for local or hosted CI.
 
-Usage: $0 (-h|-3fxbkmrjlpuneycitzs)
+Usage: $0 (-h|-2fxbkmrjlpuneycitzs)
  -h           print out this help message
- -3           After pants is bootstrapped, set --python-setup-interpreter-constraints such that any
-              python tests run with Python 3.
+ -2           Run using Python 2 (defaults to using Python 3).
  -f           run python code formatting checks
  -x           run bootstrap clean-all (assume bootstrapping from a
               fresh clone)
@@ -58,12 +57,12 @@ EOF
 python_unit_shard="0/1"
 python_contrib_shard="0/1"
 python_intg_shard="0/1"
-python_three="false"
+python_two="false"
 
-while getopts "h3fxbmrjlpeasu:ny:ci:tz" opt; do
+while getopts "h2fxbmrjlpeasu:ny:ci:tz" opt; do
   case ${opt} in
     h) usage ;;
-    3) python_three="true" ;;
+    2) python_two="true" ;;
     f) run_pre_commit_checks="true" ;;
     x) run_bootstrap_clean="true" ;;
     b) run_bootstrap="false" ;;
@@ -104,13 +103,26 @@ esac
 # We're running against a Pants clone.
 export PANTS_DEV=1
 
+# Determine interpreter for both under-the-hood and for subprocesses.
+# Order matters here. We must constrain subprocesses before running the bootstrap stage,
+# or we will encounter the _Py_Dealloc error when running `./pants.pex -V` with Python 3 under-the-hood.
+if [[ "${python_two:-false}" == "false" ]]; then
+  py_version_number="3"
+  pants_script="./pants3"
+  export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.6,<4"]'
+else
+  py_version_number="2"
+  pants_script="./pants"
+fi
+banner "Using Python ${py_version_number} to execute subprocesses."
+
 if [[ "${run_bootstrap:-true}" == "true" ]]; then
-  start_travis_section "Bootstrap" "Bootstrapping pants"
+  start_travis_section "Bootstrap" "Bootstrapping pants with Python ${py_version_number} under-the-hood."
   (
     if [[ "${run_bootstrap_clean:-false}" == "true" ]]; then
       ./build-support/python/clean.sh || die "Failed to clean before bootstrapping pants."
     fi
-    ./pants binary \
+    ${pants_script} binary \
       src/python/pants/bin:pants_local_binary && \
     mv dist/pants_local_binary.pex pants.pex && \
     ./pants.pex -V
@@ -124,21 +136,16 @@ fi
 # integration tests that shell out to `./pants`, so we set this env var for those cases.
 export RUN_PANTS_FROM_PEX=1
 
+# TODO: Clear interpreters, otherwise the above subprocess interpreter constraint does not end up 
+# applying due to a cache bug between the `./pants binary` and further runs.
+./pants.pex clean-all
+
 if [[ "${run_pre_commit_checks:-false}" == "true" ]]; then
   start_travis_section "PreCommit" "Running pre-commit checks"
   FULL_CHECK=1 ./build-support/bin/pre-commit.sh || exit 1
   end_travis_section
 fi
 
-# NB: Ordering matters here. We (currently) always bootstrap a Python 2 pex.
-if [[ "${python_three:-false}" == "true" ]]; then
-  banner "Setting interpreter constraints for 3!"
-  export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS='["CPython>=3.6,<4"]'
-  # TODO: Clear interpreters, otherwise this constraint does not end up applying due to a cache
-  # bug between the `./pants binary` and further runs.
-  ./pants.pex clean-all
-fi
-
 if [[ "${run_sanity_checks:-false}" == "true" ]]; then
   start_travis_section "SanityCheck" "Sanity checking bootstrapped pants and repo BUILD files"
   sanity_tests=(

build-support/bin/ci.sh

@@ -5,7 +5,7 @@
 # TravisCI-specific environment fixups can live in this script which forwards to the generic ci
 # script post fixups.
 
-set -euo pipefail
+set -euox pipefail
 
 cd "$(git rev-parse --show-toplevel)"
 

build-support/bin/travis-ci.sh

@@ -28,6 +28,24 @@ RUN mkdir -p /travis/home
 RUN echo >> /travis/home/.bashrc 'eval "$(pyenv init -)"'
 RUN echo >> /travis/home/.bashrc 'eval "$(pyenv virtualenv-init -)"'
 
+# -----------------------------------------------------------------------
+# This section belongs in centos6, but is here temporarily because
+# centos6's build must first be published to Docker.
+
+RUN mkdir -p /python-3-docker-build/python-3.6.8-{source,install}
+RUN curl -L -v https://www.python.org/ftp/python/3.6.8/Python-3.6.8.tgz \
+    | tar -zxvC /python-3-docker-build/python-3.6.8-source
+RUN ls -lAvF /python-3-docker-build/python-3.6.8-source
+RUN /usr/bin/scl enable devtoolset-7 -- \
+    bash -c 'pushd /python-3-docker-build/python-3.6.8-source/Python-3.6.8 \
+         && ./configure --prefix=/python-3-docker-build/python-3.6.8-install \
+                        --enable-optimizations \
+         && make install'
+ENV PATH "${PATH}:/python-3-docker-build/python-3.6.8-install/bin"
+RUN ls -lAvF /python-3-docker-build/python-3.6.8-install/bin
+
+# -----------------------------------------------------------------------
+
 # Setup mount points for the travis ci user & workdir.
 VOLUME /travis/home
 VOLUME /travis/workdir
@@ -42,9 +60,6 @@ RUN groupadd --gid ${TRAVIS_GID} ${TRAVIS_GROUP} || true
 RUN useradd -d /travis/home -g ${TRAVIS_GROUP} --uid ${TRAVIS_UID} ${TRAVIS_USER}
 USER ${TRAVIS_USER}:${TRAVIS_GROUP}
 
-# Ensure Pants runs under the 2.7.13 interpreter.
-ENV PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython==2.7.13']"
-
 # Our newly created user is unlikely to have a sane environment: set a locale at least.
 ENV LC_ALL="en_US.UTF-8"
 

build-support/docker/travis_ci/Dockerfile

@@ -15,6 +15,9 @@
 # GENERATED, DO NOT EDIT!
 # To change, edit build-support/travis/travis.yml.mustache and run
 # ./pants --quiet run build-support/travis:generate_travis_yml > .travis.yml
+#
+# Tip: Copy the generated `.travis.yml` into https://yamlvalidator.com to validate the YAML
+# and see how the entries resolve to normalized JSON (helpful to debug anchors).
 """