diff --git a/README.rst b/README.rst index 9eb76f3d03..bb8a011625 100644 --- a/README.rst +++ b/README.rst @@ -244,20 +244,9 @@ Starting a new Mars on Ray runtime locally via: .. code-block:: python - import ray - ray.init() import mars - mars.new_ray_session(worker_num=2) - import mars.tensor as mt - mt.random.RandomState(0).rand(1000_0000, 5).sum().execute() - -Or connecting to a Mars on Ray runtime which is already initialized. - -.. code-block:: python - - import mars - mars.new_ray_session('http://:') - # perform computation + mars.new_session(backend='ray') + # Perform compute Interact with Ray Dataset: diff --git a/docs/requirements-doc.txt b/docs/requirements-doc.txt index c4f0648a8b..f2bf58f323 100644 --- a/docs/requirements-doc.txt +++ b/docs/requirements-doc.txt @@ -10,7 +10,7 @@ pytest-cov>=2.5.0 pytest-timeout>=1.2.0 cloudpickle>=1.0.0 sqlalchemy>=1.2.0 -sphinx<6.0.0 +sphinx>=3.0.0 pydata-sphinx-theme>=0.3.0 sphinx-intl>=0.9.9 ipython>=4.0 diff --git a/docs/source/conf.py b/docs/source/conf.py index fa7b779e3a..41a531c7c4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -20,9 +20,9 @@ # -- Project information ----------------------------------------------------- from mars import __version__ -project = 'mars' -copyright = '1999-2020, The Alibaba Group Holding Ltd.' -author = 'jisheng, qinxing' +project = "mars" +copyright = "1999-2020, The Alibaba Group Holding Ltd." +author = "jisheng, qinxing" # The short X.Y version version = __version__ @@ -40,29 +40,29 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.intersphinx', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', - 'sphinx.ext.autosummary', - 'sphinx.ext.napoleon', - 'IPython.sphinxext.ipython_directive', - 'IPython.sphinxext.ipython_console_highlighting', - 'matplotlib.sphinxext.plot_directive', + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "IPython.sphinxext.ipython_directive", + "IPython.sphinxext.ipython_console_highlighting", + "matplotlib.sphinxext.plot_directive", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -77,7 +77,7 @@ exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # -- Options for HTML output ------------------------------------------------- @@ -85,8 +85,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'pydata_sphinx_theme' -html_logo = 'images/mars.svg' +html_theme = "pydata_sphinx_theme" +html_logo = "images/mars.svg" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -96,14 +96,21 @@ "github_url": "https://github.com/mars-project/mars", "twitter_url": "https://twitter.com/pymars_dev", "external_links": [ - {"name": "Release Notes", "url": "https://github.com/mars-project/mars/releases"}, - ] + { + "name": "Release Notes", + "url": "https://github.com/mars-project/mars/releases", + }, + ], + "logo": { + "image_light": "mars.svg", + "image_dark": "mars.svg", + }, } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['images'] +html_static_path = ["images"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -119,7 +126,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'marsdoc' +htmlhelp_basename = "marsdoc" # -- Options for LaTeX output ------------------------------------------------ @@ -128,15 +135,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -146,8 +150,7 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'mars.tex', 'mars Documentation', - 'jisheng,qinxing', 'manual'), + (master_doc, "mars.tex", "mars Documentation", "jisheng,qinxing", "manual"), ] @@ -155,10 +158,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'mars', 'mars Documentation', - [author], 1) -] +man_pages = [(master_doc, "mars", "mars Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -167,9 +167,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'mars', 'mars Documentation', - author, 'mars', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "mars", + "mars Documentation", + author, + "mars", + "One line description of project.", + "Miscellaneous", + ), ] autosummary_generate = True @@ -191,8 +197,8 @@ # -- Extension configuration ------------------------------------------------- -locale_dirs = ['locale/'] # path is example but recommended. -gettext_compact = False # optional. +locale_dirs = ["locale/"] # path is example but recommended. +gettext_compact = False # optional. ipython_warning_is_error = False diff --git a/docs/source/installation/ray.rst b/docs/source/installation/ray.rst index 7e117416d8..000e53b3fe 100644 --- a/docs/source/installation/ray.rst +++ b/docs/source/installation/ray.rst @@ -11,21 +11,21 @@ Install Ray locally: .. code-block:: bash - pip install ray>=1.8.0 + pip install ray -Start a Ray cluster: +(Optional) Start a Ray cluster or Mars starts a Ray cluster automatically: .. code-block:: python import ray ray.init() -Or connecting to a existing Ray cluster using `Ray client `_: +(Optional) Or connecting to a existing Ray cluster using `Ray client `_: .. code-block:: python import ray - ray.init(address="ray://:10001") + ray.init(address='ray://:10001') Creating a Mars on Ray runtime in the Ray cluster and do the computing: @@ -34,7 +34,8 @@ Creating a Mars on Ray runtime in the Ray cluster and do the computing: import mars import mars.tensor as mt import mars.dataframe as md - session = mars.new_ray_session(worker_num=2, worker_mem=2 * 1024 ** 3) + # This driver is the Mars supervisor. + session = mars.new_session(backend='ray') mt.random.RandomState(0).rand(1000_0000, 5).sum().execute() df = md.DataFrame( mt.random.rand(1000_0000, 4, chunk_size=500_0000), @@ -44,66 +45,82 @@ Creating a Mars on Ray runtime in the Ray cluster and do the computing: # Convert mars dataframe to ray dataset ds = md.to_ray_dataset(df) print(ds.schema(), ds.count()) - ds.filter(lambda row: row["a"] > 0.5).show(5) + ds.filter(lambda row: row['a'] > 0.5).show(5) # Convert ray dataset to mars dataframe df2 = md.read_ray_dataset(ds) print(df2.head(5).execute()) -Create a Mars on Ray runtime independently in the Ray cluster: +Stop the created Mars on Ray runtime: .. code-block:: python - import mars - import mars.tensor as mt - cluster = mars.new_cluster_in_ray(worker_num=2, worker_mem=2 * 1024 ** 3) + session.stop_server() + + +Customizing cluster +------------------- -Connect to the created Mars on Ray runtime and do the computing: +There are two ways to initialize a Mars on Ray session: + +- `mars.new_session(...) # Start Mars supervisor in current process.` + Recommend for most use cases. +- `mars.new_ray_session(...) # Start a Ray actor for Mars supervisor.` + Recommend for large scale compute or compute through Ray client. + + +Start a Ray actor for Mars supervisor: .. code-block:: python import mars - import mars.tensor as mt - session = mars.new_ray_session(address="http://ip:port", session_id="abcd", default=True) - session.execute(mt.random.RandomState(0).rand(100, 5).sum()) + # Start a Ray actor for Mars supervisor. + session = mars.new_ray_session(backend='ray') -Stop the created Mars on Ray runtime: +Connect to the created Mars on Ray runtime and do the computing, the supervisor virtual address is the name of Ray actor for Mars supervisor, +e.g. `ray://ray-cluster-1672904753/0/0`. .. code-block:: python - cluster.stop() - + import mars + import mars.tensor as mt + # Be aware that `mars.new_ray_session()` connects to an existing Mars + # cluster requires Ray runtime. + # e.g. Current process is a initialized Ray driver, client or worker. + session = mars.new_ray_session( + address='ray://', + session_id='abcd', + backend='ray', + default=True) + session.execute(mt.random.RandomState(0).rand(100, 5).sum()) -Customizing cluster -------------------- -``new_ray_session``/``new_cluster_in_ray`` function provides several keyword arguments for users to define +The ``new_ray_session`` function provides several keyword arguments for users to define the cluster. Arguments for supervisors: -+----------------------+-----------------------------------------------------------+ -| Argument | Description | -+======================+===========================================================+ -| supervisor_mem | Memory size for supervisor in the cluster, in bytes. | -+----------------------+-----------------------------------------------------------+ ++--------------------+-----------------------------------------------------------------+ +| Argument | Description | ++====================+=================================================================+ +| supervisor_cpu | Number of CPUs for supervisor, 1 by default. | ++--------------------+-----------------------------------------------------------------+ +| supervisor_mem | Memory size for supervisor in bytes, 1G by default. | ++--------------------+-----------------------------------------------------------------+ Arguments for workers: +--------------------+-----------------------------------------------------------------+ | Argument | Description | +====================+=================================================================+ -| worker_num | Number of workers in the cluster, 1 by default. | -+--------------------+-----------------------------------------------------------------+ | worker_cpu | Number of CPUs for every worker, 2 by default. | +--------------------+-----------------------------------------------------------------+ -| worker_mem | Memory size for workers in the cluster, in bytes, 2G by default.| +| worker_mem | Memory size for workers in bytes, 2G by default. | +--------------------+-----------------------------------------------------------------+ -For instance, if you want to create a Mars cluster with 100 workers, -each worker has 4 cores and 16GB memory, you can use the code below: +For instance, if you want to create a Mars cluster with a standalone supervisor, +you can use the code below (In this example, one Ray node has 16 CPUs in total): .. code-block:: python import mars - import mars.tensor as mt - cluster = mars.new_cluster_in_ray(worker_num=100, worker_cpu=4, worker_mem=16 * 1024 ** 3) + session = mars.new_ray_session(supervisor_cpu=16)