Sync ray_merge with master

.travis.yml

@@ -6,25 +6,25 @@ matrix:
   include:
     - os: linux
       dist: trusty
-      env: PYTHON=2.7
+      env: PYTHON=2.7 PYTHONWARNINGS=ignore
 
     - os: linux
       dist: trusty
-      env: PYTHON=3.5
+      env: PYTHON=3.5 PYTHONWARNINGS=ignore
 
     - os: osx
       osx_image: xcode7
-      env: PYTHON=2.7
+      env: PYTHON=2.7 PYTHONWARNINGS=ignore
 
     - os: osx
       osx_image: xcode7
-      env: PYTHON=3.5
+      env: PYTHON=3.5 PYTHONWARNINGS=ignore
 
     - os: linux
       dist: trusty
       env:
         - JDK='Oracle JDK 8'
-        - PYTHON=3.5
+        - PYTHON=3.5 PYTHONWARNINGS=ignore
       install:
         - ./.travis/install-dependencies.sh
         - export PATH="$HOME/miniconda/bin:$PATH"
@@ -33,7 +33,7 @@ matrix:
 
     - os: linux
       dist: trusty
-      env: LINT=1
+      env: LINT=1 PYTHONWARNINGS=ignore
       before_install:
         # In case we ever want to use a different version of clang-format:
         #- wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
@@ -55,10 +55,13 @@ matrix:
         # Run Python linting, ignore dict vs {} (C408), others are defaults
         - flake8 --exclude=python/ray/core/generated/,doc/source/conf.py,python/ray/cloudpickle/ --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
         - .travis/format.sh --all
+        # Make sure that the README is formatted properly.
+        - cd python
+        - python setup.py check --restructuredtext --strict --metadata
 
     - os: linux
       dist: trusty
-      env: VALGRIND=1 PYTHON=2.7
+      env: VALGRIND=1 PYTHON=2.7 PYTHONWARNINGS=ignore
       before_install:
         - sudo apt-get update -qq
         - sudo apt-get install -qq valgrind
@@ -75,7 +78,7 @@ matrix:
     # Build Linux wheels.
     - os: linux
       dist: trusty
-      env: LINUX_WHEELS=1
+      env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore
       install:
         - ./.travis/install-dependencies.sh
         # This command should be kept in sync with ray/python/README-building-wheels.md.
@@ -86,7 +89,7 @@ matrix:
     # Build MacOS wheels.
     - os: osx
       osx_image: xcode7
-      env: MAC_WHEELS=1
+      env: MAC_WHEELS=1 PYTHONWARNINGS=ignore
       install:
         - ./.travis/install-dependencies.sh
         # This command should be kept in sync with ray/python/README-building-wheels.md.
@@ -100,6 +103,7 @@ matrix:
       env:
         - PYTHON=3.5
         - RAY_USE_NEW_GCS=on
+        - PYTHONWARNINGS=ignore
 
 
 install:
@@ -131,45 +135,53 @@ script:
   # module is only found if the test directory is in the PYTHONPATH.
   - export PYTHONPATH="$PYTHONPATH:./test/"
 
-  - python -m pytest -v python/ray/test/test_global_state.py
-  - python -m pytest -v python/ray/test/test_queue.py
-  - python -m pytest -v python/ray/test/test_ray_init.py
-  - python -m pytest -v test/xray_test.py
-
-  - python -m pytest -v test/runtest.py
-  - python -m pytest -v test/array_test.py
-  - python -m pytest -v test/actor_test.py
-  - python -m pytest -v test/autoscaler_test.py
-  - python -m pytest -v test/tensorflow_test.py
-  - python -m pytest -v test/failure_test.py
-  - python -m pytest -v test/microbenchmarks.py
-  - python -m pytest -v test/stress_tests.py
-  - python -m pytest -v test/component_failures_test.py
-  - python -m pytest -v test/multi_node_test.py
-  - python -m pytest -v test/multi_node_test_2.py
-  - python -m pytest -v test/recursion_test.py
-  - python -m pytest -v test/monitor_test.py
-  - python -m pytest -v test/cython_test.py
-  - python -m pytest -v test/credis_test.py
-  - python -m pytest -v test/node_manager_test.py
-
   # ray tune tests
   - python python/ray/tune/test/dependency_test.py
-  - python -m pytest -v python/ray/tune/test/trial_runner_test.py
-  - python -m pytest -v python/ray/tune/test/trial_scheduler_test.py
-  - python -m pytest -v python/ray/tune/test/experiment_test.py
-  - python -m pytest -v python/ray/tune/test/tune_server_test.py
-  - python -m pytest -v python/ray/tune/test/ray_trial_executor_test.py
-  - python -m pytest -v python/ray/tune/test/automl_searcher_test.py
+  - python -m pytest -v --durations=10 python/ray/tune/test/trial_runner_test.py
+  - python -m pytest -v --durations=10 python/ray/tune/test/trial_scheduler_test.py
+  - python -m pytest -v --durations=10 python/ray/tune/test/experiment_test.py
+  - python -m pytest -v --durations=10 python/ray/tune/test/tune_server_test.py
+  - python -m pytest -v --durations=10 python/ray/tune/test/ray_trial_executor_test.py
+  - python -m pytest -v --durations=10 python/ray/tune/test/automl_searcher_test.py
 
   # ray rllib tests
-  - python -m pytest -v python/ray/rllib/test/test_catalog.py
-  - python -m pytest -v python/ray/rllib/test/test_filters.py
-  - python -m pytest -v python/ray/rllib/test/test_optimizers.py
-  - python -m pytest -v python/ray/rllib/test/test_evaluators.py
+  - python -m pytest -v --durations=10 python/ray/rllib/test/test_catalog.py
+  - python -m pytest -v --durations=10 python/ray/rllib/test/test_filters.py
+  - python -m pytest -v --durations=10 python/ray/rllib/test/test_optimizers.py
+  - python -m pytest -v --durations=10 python/ray/rllib/test/test_evaluators.py
+
+  # Python3.5+ only. Otherwise we will get `SyntaxError` regardless of how we set the tester.
+  - python -c 'import sys;exit(sys.version_info>=(3,5))' || python -m pytest -v --durations=10 python/ray/experimental/test/async_test.py
+
+  - python -m pytest -v --durations=10 python/ray/test/test_global_state.py
+  - python -m pytest -v --durations=10 python/ray/test/test_queue.py
+  - python -m pytest -v --durations=10 python/ray/test/test_ray_init.py
+  - python -m pytest -v --durations=10 test/xray_test.py
+
+  - python -m pytest -v --durations=10 test/runtest.py
+  - python -m pytest -v --durations=10 test/array_test.py
+  - python -m pytest -v --durations=10 test/actor_test.py
+  - python -m pytest -v --durations=10 test/autoscaler_test.py
+  - python -m pytest -v --durations=10 test/tensorflow_test.py
+  - python -m pytest -v --durations=10 test/failure_test.py
+  - python -m pytest -v --durations=10 test/microbenchmarks.py
+  - python -m pytest -v --durations=10 test/stress_tests.py
+  - python -m pytest -v --durations=10 test/component_failures_test.py
+  - python -m pytest -v --durations=10 test/multi_node_test.py
+  - python -m pytest -v --durations=10 test/multi_node_test_2.py
+  - python -m pytest -v --durations=10 test/recursion_test.py
+  - python -m pytest -v --durations=10 test/monitor_test.py
+  - python -m pytest -v --durations=10 test/cython_test.py
+  - python -m pytest -v --durations=10 test/credis_test.py
+  - python -m pytest -v --durations=10 test/node_manager_test.py
+  # TODO(yuhguo): object_manager_test.py requires a lot of CPU/memory, and
+  # better be put in Jenkins. However, it fails frequently in Jenkins, but
+  # works well in Travis. We should consider moving it back to Jenkins once
+  # we figure out the reason.
+  - python -m pytest -v --durations=10 test/object_manager_test.py
 
   # ray temp file tests
-  - python -m pytest -v test/tempfile_test.py
+  - python -m pytest -v --durations=10 test/tempfile_test.py
 
   # modin test files
   - python python/ray/test/test_modin.py

.travis/install-dependencies.sh

@@ -24,16 +24,16 @@ if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
-    feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout
+  pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+    feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
   sudo apt-get update
   sudo apt-get install -y cmake pkg-config python-dev python-numpy build-essential autoconf curl libtool unzip
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+  pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout
 elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
@@ -50,8 +50,8 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
-    feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout
+  pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+    feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
   which -s brew
@@ -67,7 +67,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+  pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout
 elif [[ "$LINT" == "1" ]]; then
   sudo apt-get update

CMakeLists.txt

@@ -101,7 +101,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   set(ray_file_list
         "src/ray/thirdparty/redis/src/redis-server"
         "src/ray/gcs/redis_module/libray_redis_module.so"
-        "src/ray/raylet/liblocal_scheduler_library_python.so"
+        "src/ray/raylet/libraylet_library_python.so"
         "src/ray/raylet/raylet_monitor"
         "src/ray/raylet/raylet")
 
@@ -128,8 +128,8 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   add_dependencies(copy_ray copy_ray_files)
 
   # Make sure that the Python extensions are built before copying the files.
-  get_local_scheduler_library("python" LOCAL_SCHEDULER_LIBRARY_PYTHON)
-  add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_PYTHON})
+  get_raylet_library("python" RAYLET_LIBRARY_PYTHON)
+  add_dependencies(copy_ray ${RAYLET_LIBRARY_PYTHON})
 
   foreach(file ${ray_file_list})
   add_custom_command(TARGET copy_ray POST_BUILD
@@ -146,8 +146,8 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
 endif()
 
 if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
-  get_local_scheduler_library("java" LOCAL_SCHEDULER_LIBRARY_JAVA)
-  add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_JAVA})
+  get_raylet_library("java" RAYLET_LIBRARY_JAVA)
+  add_dependencies(copy_ray ${RAYLET_LIBRARY_JAVA})
 
   # copy libplasma_java files
   add_custom_command(TARGET copy_ray POST_BUILD

README.rst

@@ -1,14 +1,12 @@
-.. raw:: html
-
-    <a href=http://ray.readthedocs.io/en/latest/index.html><img align="right" width="30%" src="https://github.com/devin-petersohn/ray/raw/docs/update_readme/doc/source/images/ray_logo.png"></a>
+.. image:: https://github.com/ray-project/ray/raw/master/doc/source/images/ray_header_logo.png
 
 .. image:: https://travis-ci.com/ray-project/ray.svg?branch=master
     :target: https://travis-ci.com/ray-project/ray
 
 .. image:: https://readthedocs.org/projects/ray/badge/?version=latest
     :target: http://ray.readthedocs.io/en/latest/?badge=latest
 
-.. image:: https://img.shields.io/badge/pypi-0.6.0-blue.svg
+.. image:: https://img.shields.io/badge/pypi-0.6.1-blue.svg
     :target: https://pypi.org/project/ray/
 
 |
@@ -41,12 +39,12 @@ Example Use
 
 Ray comes with libraries that accelerate deep learning and reinforcement learning development:
 
-- `Ray Tune`_: Hyperparameter Optimization Framework
-- `Ray RLlib`_: Scalable Reinforcement Learning
+- `Tune`_: Hyperparameter Optimization Framework
+- `RLlib`_: Scalable Reinforcement Learning
 - `Distributed Training <http://ray.readthedocs.io/en/latest/distributed_sgd.html>`__
 
-.. _`Ray Tune`: http://ray.readthedocs.io/en/latest/tune.html
-.. _`Ray RLlib`: http://ray.readthedocs.io/en/latest/rllib.html
+.. _`Tune`: http://ray.readthedocs.io/en/latest/tune.html
+.. _`RLlib`: http://ray.readthedocs.io/en/latest/rllib.html
 
 Installation
 ------------

cmake/Modules/ArrowExternalProject.cmake

@@ -14,11 +14,15 @@
 #  - PLASMA_STATIC_LIB
 #  - PLASMA_SHARED_LIB
 
-set(arrow_URL https://github.com/apache/arrow.git)
-# The PR for this commit is https://github.com/apache/arrow/pull/3061. We
+set(arrow_URL https://github.com/ray-project/arrow.git)
+# This commit is based on https://github.com/apache/arrow/pull/3197. We
 # include the link here to make it easier to find the right commit because
 # Arrow often rewrites git history and invalidates certain commits.
-set(arrow_TAG a667fca3b71772886bb2595986266d2039823dcc)
+# It has been patched to fix an upstream symbol clash with TensorFlow,
+# the patch is available at
+# https://github.com/ray-project/arrow/commit/c347cd571e51723fc8512922f1b3a8e45e45b169
+# See the discussion in https://github.com/apache/arrow/pull/3177
+set(arrow_TAG c347cd571e51723fc8512922f1b3a8e45e45b169)
 
 set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install)
 set(ARROW_HOME ${ARROW_INSTALL_PREFIX})

cmake/Modules/BoostExternalProject.cmake

@@ -27,14 +27,15 @@ else()
   set(BOOST_ROOT ${Boost_INSTALL_PREFIX})
   set(Boost_LIBRARY_DIR ${Boost_INSTALL_PREFIX}/lib)
   set(Boost_SYSTEM_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(Boost_THREAD_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX})
   set(Boost_FILESYSTEM_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX})
 
   #set(boost_URL https://github.com/boostorg/boost.git)
-  #set(boost_TAG boost-1.65.1)
+  #set(boost_TAG boost-1.68.0)
 
-  set(Boost_TAR_GZ_URL http://dl.bintray.com/boostorg/release/1.65.1/source/boost_1_65_1.tar.gz)
+  set(Boost_TAR_GZ_URL http://dl.bintray.com/boostorg/release/1.68.0/source/boost_1_68_0.tar.gz)
   set(Boost_BUILD_PRODUCTS ${Boost_SYSTEM_LIBRARY} ${Boost_FILESYSTEM_LIBRARY})
-  set(Boost_URL_MD5 "ee64fd29a3fe42232c6ac3c419e523cf")
+  set(Boost_URL_MD5 "5d8b4503582fffa9eefdb9045359c239")
 
   set(Boost_USE_STATIC_LIBS ON)
 
@@ -48,6 +49,6 @@ else()
     BUILD_IN_SOURCE 1
     BUILD_BYPRODUCTS ${Boost_BUILD_PRODUCTS}
     CONFIGURE_COMMAND ./bootstrap.sh
-    BUILD_COMMAND bash -c "./b2 cxxflags=-fPIC cflags=-fPIC variant=release link=static --with-filesystem --with-system --with-regex -j8 install --prefix=${Boost_INSTALL_PREFIX} > /dev/null"
+    BUILD_COMMAND bash -c "./b2 cxxflags=-fPIC cflags=-fPIC variant=release link=static --with-filesystem --with-system --with-thread --with-atomic --with-chrono --with-date_time --with-regex -j8 install --prefix=${Boost_INSTALL_PREFIX} > /dev/null"
     INSTALL_COMMAND "")
 endif ()

cmake/Modules/FlatBuffersExternalProject.cmake

@@ -18,9 +18,9 @@ if(DEFINED ENV{RAY_FLATBUFFERS_HOME} AND EXISTS $ENV{RAY_FLATBUFFERS_HOME})
 
   add_custom_target(flatbuffers_ep)
 else()
-  set(flatbuffers_VERSION "1.9.0")
+  set(flatbuffers_VERSION "1.10.0")
   set(flatbuffers_URL "https://github.com/google/flatbuffers/archive/v${flatbuffers_VERSION}.tar.gz")
-  set(flatbuffers_URL_MD5 "8be7513bf960034f6873326d09521a4b")
+  set(flatbuffers_URL_MD5 "f7d19a3f021d93422b0bc287d7148cd2")
 
   set(FLATBUFFERS_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/flatbuffers-install")
 

cmake/Modules/ThirdpartyToolchain.cmake

@@ -54,11 +54,14 @@ ADD_THIRDPARTY_LIB(boost_system
   STATIC_LIB ${Boost_SYSTEM_LIBRARY})
 ADD_THIRDPARTY_LIB(boost_filesystem
   STATIC_LIB ${Boost_FILESYSTEM_LIBRARY})
+ADD_THIRDPARTY_LIB(boost_thread
+  STATIC_LIB ${Boost_THREAD_LIBRARY})
 
 add_dependencies(boost_system boost_ep)
 add_dependencies(boost_filesystem boost_ep)
+add_dependencies(boost_thread boost_ep)
 
-add_custom_target(boost DEPENDS boost_system boost_filesystem)
+add_custom_target(boost DEPENDS boost_system boost_filesystem boost_thread)
 
 # flatbuffers
 include(FlatBuffersExternalProject)
@@ -120,6 +123,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
       "PYARROW_WITH_TENSORFLOW=1"
       "PYARROW_BUNDLE_ARROW_CPP=1"
       "PARQUET_HOME=${PARQUET_HOME}"
+      "BOOST_ROOT=${BOOST_ROOT}"
       "PYARROW_WITH_PARQUET=1"
       "PYARROW_PARALLEL=")