From 80d13d16fdeb7c591962192ee200bf2be30d9ecc Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Sat, 28 Jan 2023 18:34:44 +0100 Subject: [PATCH] [docs] simple web crawler example (#31900) --- doc/source/_static/css/custom.css | 5 + doc/source/_static/js/custom.js | 18 ++ doc/source/_toc.yml | 1 + doc/source/custom_directives.py | 4 +- doc/source/ray-air/user-guides.rst | 9 +- .../ray-core/examples/web-crawler.ipynb | 244 ++++++++++++++++++ doc/source/ray-overview/eco-gallery.yml | 10 +- doc/source/ray-overview/use-cases.rst | 122 ++++----- python/requirements_test.txt | 1 + 9 files changed, 322 insertions(+), 92 deletions(-) create mode 100644 doc/source/ray-core/examples/web-crawler.ipynb diff --git a/doc/source/_static/css/custom.css b/doc/source/_static/css/custom.css index 221eb33fa6a8..00c1677d1a83 100644 --- a/doc/source/_static/css/custom.css +++ b/doc/source/_static/css/custom.css @@ -316,6 +316,10 @@ img.horizontal-scroll { float: right; } +.card-body { + padding: 0.5rem !important; +} + /* Wrap code blocks instead of horizontal scrolling. */ pre { white-space: pre-wrap; @@ -325,6 +329,7 @@ pre { .cell .cell_output { max-height: 250px; overflow-y: auto; + font-weight: bold; } /* Yellow doesn't render well on light background */ diff --git a/doc/source/_static/js/custom.js b/doc/source/_static/js/custom.js index 7db383309c9b..13dcfc9c57d8 100644 --- a/doc/source/_static/js/custom.js +++ b/doc/source/_static/js/custom.js @@ -28,6 +28,24 @@ window.addEventListener("scroll", loadVisibleTermynals); createTermynals(); loadVisibleTermynals(); + +document.addEventListener("DOMContentLoaded", function() { + let images = document.getElementsByClassName("fixed-height-img"); + let maxHeight = 0; + + for (let i = 0; i < images.length; i++) { + if (images[i].height > maxHeight) { + maxHeight = images[i].height; + } + } + + for (let i = 0; i < images.length; i++) { + let margin = Math.floor((maxHeight - images[i].height) / 2); + images[i].style.cssText = "margin-top: " + margin + "px !important;" + + "margin-bottom: " + margin + "px !important;" + } +}); + // Remember the scroll position when the page is unloaded. window.onload = function() { let sidebar = document.querySelector("#bd-docs-nav"); diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index 5168a4923121..c21f9dc22177 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -32,6 +32,7 @@ parts: - file: ray-core/examples/batch_prediction - file: ray-core/examples/batch_training - file: ray-core/examples/automl_for_time_series + - file: ray-core/examples/web-crawler - file: ray-core/api - file: cluster/getting-started diff --git a/doc/source/custom_directives.py b/doc/source/custom_directives.py index d3f3bb6f547c..3e15ddbaa930 100644 --- a/doc/source/custom_directives.py +++ b/doc/source/custom_directives.py @@ -313,10 +313,10 @@ def build_gallery(app): --- :img-top: {item["image"]} - {item["description"]} - {gh_stars} + {item["description"]} + +++ .. link-button:: {item["website"]} {ref} diff --git a/doc/source/ray-air/user-guides.rst b/doc/source/ray-air/user-guides.rst index 4ab31a63f8bd..03d127f09630 100644 --- a/doc/source/ray-air/user-guides.rst +++ b/doc/source/ray-air/user-guides.rst @@ -12,14 +12,13 @@ AIR User Guides .. panels:: :container: text-center :column: col-md-4 px-2 py-2 - :img-top-cls: pt-5 w-75 d-block mx-auto + :img-top-cls: pt-5 w-75 d-block mx-auto fixed-height-img --- :img-top: /ray-air/images/preprocessors.svg .. https://docs.google.com/drawings/d/1ZIbsXv5vvwTVIEr2aooKxuYJ_VL7-8VMNlRinAiPaTI/edit - +++ .. link-button:: /ray-air/preprocessors :type: ref :text: Using Preprocessors @@ -30,7 +29,6 @@ AIR User Guides .. https://docs.google.com/drawings/d/15SXGHbKPWdrzx3aTAIFcO2uh_s6Q7jLU03UMuwKSzzM/edit - +++ .. link-button:: trainer :type: ref :text: Using Trainers @@ -41,7 +39,6 @@ AIR User Guides .. https://docs.google.com/drawings/d/10GZE_6s6ss8PSxLYyzcbj6yEalWO4N7MS7ao8KO7ne0/edit - +++ .. link-button:: air-ingest :type: ref :text: Configuring Training Datasets @@ -52,7 +49,6 @@ AIR User Guides .. https://docs.google.com/drawings/d/1yMd12iMkyo6DGrFoET1TIlKfFnXX9dfh2u3GSdTz6W4/edit - +++ .. link-button:: /ray-air/tuner :type: ref :text: Configuring Hyperparameter Tuning @@ -63,7 +59,6 @@ AIR User Guides .. https://docs.google.com/presentation/d/1jfkQk0tGqgkLgl10vp4-xjcbYG9EEtlZV_Vnve_NenQ/edit#slide=id.g131c21f5e88_0_549 - +++ .. link-button:: predictors :type: ref :text: Using Predictors for Inference @@ -74,7 +69,6 @@ AIR User Guides .. https://docs.google.com/drawings/d/1-rg77bV-vEMURXZw5_mIOUFM3FObIIYbFOiYzFJW_68/edit - +++ .. link-button:: /ray-air/examples/serving_guide :type: ref :text: Deploying Predictors with Serve @@ -85,7 +79,6 @@ AIR User Guides .. https://docs.google.com/drawings/d/1ja1RfNCEFn50B9FHWSemUzwhtPAmVyoak1JqEJUmxs4/edit - +++ .. link-button:: air-deployment :type: ref :text: How to Deploy AIR diff --git a/doc/source/ray-core/examples/web-crawler.ipynb b/doc/source/ray-core/examples/web-crawler.ipynb new file mode 100644 index 000000000000..920e2ed94ca9 --- /dev/null +++ b/doc/source/ray-core/examples/web-crawler.ipynb @@ -0,0 +1,244 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Speed up your web crawler by parallelizing it with Ray" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "In this example we'll quickly demonstrate how to build a simple web scraper in Python and\n", + "parallelize it with Ray Tasks with minimal code changes.\n", + "\n", + "To run this example locally on your machine, please first install `ray` and `beautifulsoup` with\n", + "\n", + "```\n", + "pip install \"beautifulsoup4==4.11.1\" \"ray>=2.2.0\"\n", + "```\n", + "\n", + "First, we'll define a function called `find_links` which takes a starting page (`start_url`) to crawl,\n", + "and we'll take the Ray documentation as example of such a starting point.\n", + "Our crawler simply extracts all available links from the starting URL that contain a given `base_url`\n", + "(e.g. in our example we only want to follow links on `http://docs.ray.io`, not any external links).\n", + "The `find_links` function is then called recursively with all the links we found this way, until a\n", + "certain depth is reached.\n", + "\n", + "To extract the links from HTML elements on a site, we define a little helper function called\n", + "`extract_links`, which takes care of handling relative URLs properly and sets a limit on the\n", + "number of links returned from a site (`max_results`) to control the runtime of the crawler more easily.\n", + "\n", + "Here's the full implementation:" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 154, + "outputs": [], + "source": [ + "import requests\n", + "from bs4 import BeautifulSoup\n", + "\n", + "def extract_links(elements, base_url, max_results=100):\n", + " links = []\n", + " for e in elements:\n", + " url = e[\"href\"]\n", + " if \"https://\" not in url:\n", + " url = base_url + url\n", + " if base_url in url:\n", + " links.append(url)\n", + " return set(links[:max_results])\n", + "\n", + "\n", + "def find_links(start_url, base_url, depth=2):\n", + " if depth == 0:\n", + " return set()\n", + "\n", + " page = requests.get(start_url)\n", + " soup = BeautifulSoup(page.content, \"html.parser\")\n", + " elements = soup.find_all(\"a\", href=True)\n", + " links = extract_links(elements, base_url)\n", + "\n", + " for url in links:\n", + " new_links = find_links(url, base_url, depth-1)\n", + " links = links.union(new_links)\n", + " return links" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "Let's define a starting and base URL and crawl the Ray docs to a `depth` of 2." + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 162, + "outputs": [], + "source": [ + "base = \"https://docs.ray.io/en/latest/\"\n", + "docs = base + \"index.html\"" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 163, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 19.3 s, sys: 340 ms, total: 19.7 s\n", + "Wall time: 25.8 s\n" + ] + }, + { + "data": { + "text/plain": "591" + }, + "execution_count": 163, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%time len(find_links(docs, base))" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "As you can see, crawling the documentation root recursively like this returns a\n", + "total of `591` pages and the wall time comes in at around 25 seconds.\n", + "\n", + "Crawling pages can be parallelized in many ways.\n", + "Probably the simplest way is to simple start with multiple starting URLs and call\n", + "`find_links` in parallel for each of them.\n", + "We can do this with [Ray Tasks](https://docs.ray.io/en/latest/ray-core/tasks.html) in a straightforward way.\n", + "We simply use the `ray.remote` decorator to wrap the `find_links` function in a task called `find_links_task` like this:" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 157, + "outputs": [], + "source": [ + "import ray\n", + "\n", + "@ray.remote\n", + "def find_links_task(start_url, base_url, depth=2):\n", + " return find_links(start_url, base_url, depth)" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "To use this task to kick off a parallel call, the only thing you have to do is use\n", + "`find_links_tasks.remote(...)` instead of calling the underlying Python function directly.\n", + "\n", + "Here's how you run six crawlers in parallel, the first three (redundantly) crawl\n", + "`docs.ray.io` again, the other three crawl the main entry points of the Ray RLlib,\n", + "Tune, and Serve libraries, respectively:" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 160, + "outputs": [], + "source": [ + "links = [find_links_task.remote(f\"{base}{lib}/index.html\", base)\n", + " for lib in [\"\", \"\", \"\", \"rllib\", \"tune\", \"serve\"]]" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 161, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "591\n", + "591\n", + "105\n", + "204\n", + "105\n", + "CPU times: user 65.5 ms, sys: 47.8 ms, total: 113 ms\n", + "Wall time: 27.2 s\n" + ] + } + ], + "source": [ + "%time for res in ray.get(links): print(len(res))" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "This parallel run crawls around four times the number of pages in roughly the same time as the initial, sequential run.\n", + "Note the use of `ray.get` in the timed run to retrieve the results from Ray (the `remote` call promise gets resolved with `get`).\n", + "\n", + "Of course, there are much smarter ways to create a crawler and efficiently parallelize it, and this example\n", + "gives you a starting point to work from." + ], + "metadata": { + "collapsed": false + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/doc/source/ray-overview/eco-gallery.yml b/doc/source/ray-overview/eco-gallery.yml index ff47b975a178..5876e8f372f9 100644 --- a/doc/source/ray-overview/eco-gallery.yml +++ b/doc/source/ray-overview/eco-gallery.yml @@ -1,8 +1,8 @@ meta: - section-titles: true - container: container pb-12 - column: col-md-12 px-2 py-2 - img-top-cls: pt-10 w-50 d-block mx-auto + section-titles: false + container: container pb-4 + column: col-md-4 px-1 py-1 + img-top-cls: p-2 w-75 d-block mx-auto fixed-height-img buttons: classes: btn-outline-info btn-block @@ -146,7 +146,7 @@ projects: random forests, gradient boosting, k-means and DBSCAN, and is designed to interoperate with the Python numerical and scientific libraries NumPy and SciPy. website: https://docs.ray.io/en/master/joblib.html - repo: https://docs.ray.io/en/master/joblib.html + repo: https://github.com/scikit-learn/scikit-learn image: ../images/scikit.png - name: Seldon Alibi Integration section_title: Seldon Alibi diff --git a/doc/source/ray-overview/use-cases.rst b/doc/source/ray-overview/use-cases.rst index acb326c335d3..751e0686b784 100644 --- a/doc/source/ray-overview/use-cases.rst +++ b/doc/source/ray-overview/use-cases.rst @@ -44,14 +44,13 @@ Batch Inference Batch inference refers to generating model predictions over a set of input observations. The model could be a regression model, neural network, or simply a Python function. Ray can scale batch inference from single GPU machines to large clusters. .. panels:: - :container: container pb-4 - :column: col-md-4 px-2 py-2 - :img-top-cls: pt-5 w-75 d-block mx-auto + :container: container pb-3 + :column: col-md-3 px-1 py-1 + :img-top-cls: p-2 w-75 d-block mx-auto fixed-height-img --- :img-top: /images/ray_logo.png - +++ .. link-button:: https://github.com/ray-project/ray-educational-materials/blob/main/Computer_vision_workloads/Semantic_segmentation/Scaling_batch_inference.ipynb :type: url :text: [Tutorial] Architectures for Scalable Batch Inference with Ray @@ -59,7 +58,6 @@ Batch inference refers to generating model predictions over a set of input obser --- :img-top: /images/ray_logo.png - +++ .. link-button:: https://www.anyscale.com/blog/model-batch-inference-in-ray-actors-actorpool-and-datasets :type: url :text: [Blog] Batch Inference in Ray: Actors, ActorPool, and Datasets @@ -67,7 +65,6 @@ Batch inference refers to generating model predictions over a set of input obser --- :img-top: /images/ray_logo.png - +++ .. link-button:: /ray-core/examples/batch_prediction :type: ref :text: [Example] Batch Prediction using Ray Core @@ -75,7 +72,6 @@ Batch inference refers to generating model predictions over a set of input obser --- :img-top: /images/ray_logo.png - +++ .. link-button:: /data/examples/nyc_taxi_basic_processing :type: ref :text: [Example] Batch Inference on NYC taxi data using Ray Data @@ -84,7 +80,6 @@ Batch inference refers to generating model predictions over a set of input obser --- :img-top: /images/ray_logo.png - +++ .. link-button:: /data/examples/ocr_example :type: ref :text: [Example] Batch OCR processing using Ray Data @@ -111,14 +106,13 @@ There are three ways of using Ray to express this workload. Add link to many model training blog. .. panels:: - :container: container pb-4 - :column: col-md-4 px-2 py-2 - :img-top-cls: pt-5 w-75 d-block mx-auto + :container: container pb-3 + :column: col-md-3 px-1 py-1 + :img-top-cls: p-2 w-75 d-block mx-auto fixed-height-img --- :img-top: /images/ray_logo.png - +++ .. link-button:: https://www.anyscale.com/blog/training-one-million-machine-learning-models-in-record-time-with-ray :type: url :text: [Blog] Training One Million ML Models in Record Time with Ray @@ -126,7 +120,6 @@ There are three ways of using Ray to express this workload. --- :img-top: /images/ray_logo.png - +++ .. link-button:: /ray-core/examples/batch_training :type: ref :text: [Example] Batch Training with Ray Core @@ -134,7 +127,6 @@ There are three ways of using Ray to express this workload. --- :img-top: /images/ray_logo.png - +++ .. link-button:: /data/examples/batch_training :type: ref :text: [Example] Batch Training with Ray Datasets @@ -142,7 +134,6 @@ There are three ways of using Ray to express this workload. --- :img-top: /images/tune.png - +++ .. link-button:: /tune/tutorials/tune-run :type: ref :text: [Guide] Tune Basic Parallel Experiments @@ -150,7 +141,6 @@ There are three ways of using Ray to express this workload. --- :img-top: /images/tune.png - +++ .. link-button:: /ray-air/examples/batch_tuning :type: ref :text: [Example] Batch Training and Tuning using Ray Tune @@ -158,7 +148,6 @@ There are three ways of using Ray to express this workload. --- :img-top: /images/carrot.png - +++ .. link-button:: https://www.youtube.com/watch?v=3t26ucTy0Rs :type: url :text: [Talk] Scaling Instacart fulfillment ML on Ray @@ -172,14 +161,13 @@ Ray Serve is particularly well suited for model composition, enabling you to bui .. panels:: - :container: container pb-4 - :column: col-md-4 px-2 py-2 - :img-top-cls: pt-5 w-75 d-block mx-auto + :container: container pb-3 + :column: col-md-3 px-1 py-1 + :img-top-cls: p-2 w-75 d-block mx-auto fixed-height-img --- :img-top: /images/serve.svg - +++ .. link-button:: https://www.youtube.com/watch?v=UtH-CMpmxvI :type: url :text: [Talk] Productionizing ML at Scale with Ray Serve @@ -187,7 +175,6 @@ Ray Serve is particularly well suited for model composition, enabling you to bui --- :img-top: /images/serve.svg - +++ .. link-button:: https://www.anyscale.com/blog/simplify-your-mlops-with-ray-and-ray-serve :type: url :text: [Blog] Simplify your MLOps with Ray & Ray Serve @@ -195,7 +182,6 @@ Ray Serve is particularly well suited for model composition, enabling you to bui --- :img-top: /images/serve.svg - +++ .. link-button:: /serve/getting_started :type: ref :text: [Guide] Getting Started with Ray Serve @@ -203,7 +189,6 @@ Ray Serve is particularly well suited for model composition, enabling you to bui --- :img-top: /images/serve.svg - +++ .. link-button:: /serve/model_composition :type: ref :text: [Guide] Model Composition in Serve @@ -211,7 +196,6 @@ Ray Serve is particularly well suited for model composition, enabling you to bui --- :img-top: /images/grid.png - +++ .. link-button:: /serve/tutorials/index :type: ref :text: [Gallery] Serve Examples Gallery @@ -219,7 +203,6 @@ Ray Serve is particularly well suited for model composition, enabling you to bui --- :img-top: /images/grid.png - +++ .. link-button:: https://www.anyscale.com/blog?tag=ray_serve :type: url :text: [Gallery] More Serve Use Cases on the Blog @@ -232,14 +215,13 @@ Ray's Tune library enables any parallel Ray workload to be run under a hyperpara Learn more about the Tune library with the following talks and user guides. .. panels:: - :container: container pb-4 - :column: col-md-4 px-2 py-2 - :img-top-cls: pt-5 w-75 d-block mx-auto + :container: container pb-3 + :column: col-md-3 px-1 py-1 + :img-top-cls: p-2 w-75 d-block mx-auto fixed-height-img --- :img-top: /images/tune.png - +++ .. link-button:: /tune/getting-started :type: ref :text: [Guide] Getting Started with Ray Tune @@ -247,7 +229,6 @@ Learn more about the Tune library with the following talks and user guides. --- :img-top: /images/tune.png - +++ .. link-button:: https://www.anyscale.com/blog/how-to-distribute-hyperparameter-tuning-using-ray-tune :type: url :text: [Blog] How to distribute hyperparameter tuning with Ray Tune @@ -255,7 +236,6 @@ Learn more about the Tune library with the following talks and user guides. --- :img-top: /images/tune.png - +++ .. link-button:: https://www.youtube.com/watch?v=KgYZtlbFYXE :type: url :text: [Talk] Simple Distributed Hyperparameter Optimization @@ -263,7 +243,6 @@ Learn more about the Tune library with the following talks and user guides. --- :img-top: /images/tune.png - +++ .. link-button:: https://www.anyscale.com/blog/hyperparameter-search-hugging-face-transformers-ray-tune :type: url :text: [Blog] Hyperparameter Search with đŸ¤— Transformers @@ -271,7 +250,6 @@ Learn more about the Tune library with the following talks and user guides. --- :img-top: /images/grid.png - +++ .. link-button:: /tune/examples/index :type: ref :text: [Gallery] Ray Tune Examples Gallery @@ -279,7 +257,6 @@ Learn more about the Tune library with the following talks and user guides. --- :img-top: /images/grid.png - +++ .. link-button:: https://www.anyscale.com/blog?tag=ray-tune :type: url :text: More Tune use cases on the Blog @@ -293,14 +270,13 @@ providing distributed orchestration and management capabilities out of the box. Learn more about the Train library with the following talks and user guides. .. panels:: - :container: container pb-4 - :column: col-md-4 px-2 py-2 - :img-top-cls: pt-5 w-75 d-block mx-auto + :container: container pb-3 + :column: col-md-3 px-1 py-1 + :img-top-cls: p-2 w-75 d-block mx-auto fixed-height-img --- :img-top: /images/ray_logo.png - +++ .. link-button:: https://www.youtube.com/watch?v=e-A93QftCfc :type: url :text: [Talk] Ray Train, PyTorch, TorchX, and distributed deep learning @@ -308,7 +284,6 @@ Learn more about the Train library with the following talks and user guides. --- :img-top: /images/uber.png - +++ .. link-button:: https://www.uber.com/blog/elastic-xgboost-ray/ :type: url :text: [Blog] Elastic Distributed Training with XGBoost on Ray @@ -316,7 +291,6 @@ Learn more about the Train library with the following talks and user guides. --- :img-top: /images/ray_logo.png - +++ .. link-button:: /train/train :type: ref :text: [Guide] Getting Started with Ray Train @@ -324,7 +298,6 @@ Learn more about the Train library with the following talks and user guides. --- :img-top: /images/ray_logo.png - +++ .. link-button:: /ray-air/examples/huggingface_text_classification :type: ref :text: [Example] Fine-tune a đŸ¤— Transformers model @@ -332,7 +305,6 @@ Learn more about the Train library with the following talks and user guides. --- :img-top: /images/grid.png - +++ .. link-button:: /train/examples :type: ref :text: [Gallery] Ray Train Examples Gallery @@ -340,7 +312,6 @@ Learn more about the Train library with the following talks and user guides. --- :img-top: /images/grid.png - +++ .. link-button:: https://www.anyscale.com/blog?tag=ray_train :type: url :text: [Gallery] More Train Use Cases on the Blog @@ -352,14 +323,13 @@ Reinforcement Learning RLlib is an open-source library for reinforcement learning (RL), offering support for production-level, highly distributed RL workloads while maintaining unified and simple APIs for a large variety of industry applications. RLlib is used by industry leaders in many different verticals, such as climate control, industrial control, manufacturing and logistics, finance, gaming, automobile, robotics, boat design, and many others. .. panels:: - :container: container pb-4 - :column: col-md-4 px-2 py-2 - :img-top-cls: pt-5 w-75 d-block mx-auto + :container: container pb-3 + :column: col-md-3 px-1 py-1 + :img-top-cls: p-2 w-75 d-block mx-auto fixed-height-img --- :img-top: /rllib/images/rllib-logo.png - +++ .. link-button:: https://applied-rl-course.netlify.app/ :type: url :text: [Course] Applied Reinforcement Learning with RLlib @@ -367,7 +337,6 @@ RLlib is an open-source library for reinforcement learning (RL), offering suppor --- :img-top: /rllib/images/rllib-logo.png - +++ .. link-button:: https://medium.com/distributed-computing-with-ray/intro-to-rllib-example-environments-3a113f532c70 :type: url :text: [Blog] Intro to RLlib: Example Environments @@ -375,7 +344,6 @@ RLlib is an open-source library for reinforcement learning (RL), offering suppor --- :img-top: /rllib/images/rllib-logo.png - +++ .. link-button:: /rllib/rllib-training :type: ref :text: [Guide] Getting Started with RLlib @@ -383,7 +351,6 @@ RLlib is an open-source library for reinforcement learning (RL), offering suppor --- :img-top: /images/riot.png - +++ .. link-button:: https://www.anyscale.com/events/2022/03/29/deep-reinforcement-learning-at-riot-games :type: url :text: [Talk] Deep reinforcement learning at Riot Games @@ -391,7 +358,6 @@ RLlib is an open-source library for reinforcement learning (RL), offering suppor --- :img-top: /images/grid.png - +++ .. link-button:: /rllib/rllib-examples :type: ref :text: [Gallery] RLlib Examples Gallery @@ -399,7 +365,6 @@ RLlib is an open-source library for reinforcement learning (RL), offering suppor --- :img-top: /images/grid.png - +++ .. link-button:: https://www.anyscale.com/blog?tag=rllib :type: url :text: [Gallery] More RL Use Cases on the Blog @@ -411,14 +376,13 @@ ML Platform The following highlights feature companies leveraging Ray's unified API to build simpler, more flexible ML platforms. .. panels:: - :container: container pb-4 - :column: col-md-4 px-2 py-2 - :img-top-cls: pt-5 w-75 d-block mx-auto + :container: container pb-3 + :column: col-md-3 px-1 py-1 + :img-top-cls: p-2 w-75 d-block mx-auto fixed-height-img --- :img-top: /images/shopify.png - +++ .. link-button:: https://shopify.engineering/merlin-shopify-machine-learning-platform :type: url :text: [Blog] The Magic of Merlin - Shopify's New ML Platform @@ -426,7 +390,6 @@ The following highlights feature companies leveraging Ray's unified API to build --- :img-top: /images/uber.png - +++ .. link-button:: https://drive.google.com/file/d/1BS5lfXfuG5bnI8UM6FdUrR7CiSuWqdLn/view :type: url :text: [Slides] Large Scale Deep Learning Training and Tuning with Ray @@ -434,7 +397,6 @@ The following highlights feature companies leveraging Ray's unified API to build --- :img-top: /images/carrot.png - +++ .. link-button:: https://www.instacart.com/company/how-its-made/griffin-how-instacarts-ml-platform-tripled-ml-applications-in-a-year/ :type: url :text: [Blog] Griffin: How Instacart’s ML Platform Tripled in a year @@ -442,7 +404,6 @@ The following highlights feature companies leveraging Ray's unified API to build --- :img-top: /images/predibase.png - +++ .. link-button:: https://www.youtube.com/watch?v=B5v9B5VSI7Q :type: url :text: [Talk] Predibase - A low-code deep learning platform built for scale @@ -450,7 +411,6 @@ The following highlights feature companies leveraging Ray's unified API to build --- :img-top: /images/gke.png - +++ .. link-button:: https://cloud.google.com/blog/products/ai-machine-learning/build-a-ml-platform-with-kubeflow-and-ray-on-gke :type: url :text: [Blog] Building a ML Platform with Kubeflow and Ray on GKE @@ -458,26 +418,25 @@ The following highlights feature companies leveraging Ray's unified API to build --- :img-top: /images/ray_logo.png - +++ .. link-button:: https://www.youtube.com/watch?v=_L0lsShbKaY :type: url :text: [Talk] Ray Summit Panel - ML Platform on Ray :classes: btn-link btn-block stretched-link summitMLPlatform + End-to-End ML Workflows ----------------------- The following are highlighted examples utilizing Ray AIR to implement end-to-end ML workflows. .. panels:: - :container: container pb-4 - :column: col-md-4 px-2 py-2 - :img-top-cls: pt-5 w-75 d-block mx-auto + :container: container pb-3 + :column: col-md-3 px-1 py-1 + :img-top-cls: p-2 w-75 d-block mx-auto fixed-height-img --- :img-top: /images/text-classification.png - +++ .. link-button:: /ray-air/examples/huggingface_text_classification :type: ref :text: [Example] Text classification with Ray @@ -485,7 +444,6 @@ The following are highlighted examples utilizing Ray AIR to implement end-to-end --- :img-top: /images/image-classification.webp - +++ .. link-button:: /ray-air/examples/torch_image_example :type: ref :text: [Example] Image classification with Ray @@ -493,7 +451,6 @@ The following are highlighted examples utilizing Ray AIR to implement end-to-end --- :img-top: /images/credit.png - +++ .. link-button:: /ray-air/examples/feast_example :type: ref :text: [Example] Credit scoring with Ray and Feast @@ -501,7 +458,6 @@ The following are highlighted examples utilizing Ray AIR to implement end-to-end --- :img-top: /images/tabular-data.png - +++ .. link-button:: /ray-air/examples/xgboost_example :type: ref :text: [Example] Machine learning on tabular data @@ -509,7 +465,6 @@ The following are highlighted examples utilizing Ray AIR to implement end-to-end --- :img-top: /images/timeseries.png - +++ .. link-button:: /ray-core/examples/automl_for_time_series :type: ref :text: [Example] AutoML for Time Series with Ray @@ -517,7 +472,6 @@ The following are highlighted examples utilizing Ray AIR to implement end-to-end --- :img-top: /images/grid.png - +++ .. link-button:: /ray-air/examples/index :type: ref :text: [Gallery] Full Ray AIR Examples Gallery @@ -529,14 +483,13 @@ Large Scale Workload Orchestration The following highlights feature companies leveraging Ray Core's distributed APIs to simplify the orchestration of large scale workloads. .. panels:: - :container: container pb-4 - :column: col-md-4 px-2 py-2 - :img-top-cls: pt-5 w-75 d-block mx-auto + :container: container pb-3 + :column: col-md-3 px-1 py-1 + :img-top-cls: p-2 w-75 d-block mx-auto fixed-height-img --- :img-top: /images/ray_logo.png - +++ .. link-button:: https://archive.ph/o/aNvFN/https://www.businessinsider.com/openai-chatgpt-trained-on-anyscale-ray-generative-lifelike-ai-models-2022-12 :type: url :text: [Blog] How OpenAI Uses Ray to Train Tools like ChatGPT @@ -544,7 +497,6 @@ The following highlights feature companies leveraging Ray Core's distributed API --- :img-top: /images/ray_logo.png - +++ .. link-button:: https://www.anyscale.com/blog/building-highly-available-and-scalable-online-applications-on-ray-at-ant :type: url :text: [Blog] Highly Available and Scalable Online Applications on Ray at Ant Group @@ -553,8 +505,24 @@ The following highlights feature companies leveraging Ray Core's distributed API --- :img-top: /images/ray_logo.png - +++ .. link-button:: https://www.anyscale.com/blog/ray-forward-2022 :type: url :text: [Blog] Ray Forward 2022 Conference: Hyper-scale Ray Application Use Cases :classes: btn-link btn-block stretched-link rayForward + + +Basic Examples +-------------- + +.. panels:: + :container: container pb-3 + :column: col-md-3 px-1 py-1 + :img-top-cls: p-2 w-75 d-block mx-auto fixed-height-img + + --- + :img-top: /images/ray_logo.png + + .. link-button:: /ray-core/examples/web-crawler + :type: ref + :text: Speed up your web crawler by parallelizing it with Ray + :classes: btn-link btn-block stretched-link webCrawler diff --git a/python/requirements_test.txt b/python/requirements_test.txt index 256c4d7130aa..8e994f34e43c 100644 --- a/python/requirements_test.txt +++ b/python/requirements_test.txt @@ -11,6 +11,7 @@ azure-mgmt-compute==23.1.0 azure-mgmt-network==19.0.0 azure-mgmt-resource==20.0.0 msrestazure==0.6.4 +beautifulsoup4==4.11.1 boto3==1.23.10 # Todo: investigate if we can get rid of this and exchange for ray.cloudpickle cloudpickle==2.2.0