From c5afcbcdf9deccbb7a21ddd82ae0fc305e79c008 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Tue, 17 Dec 2019 16:28:19 -0500 Subject: [PATCH] Linting and formatting fixes --- .python-version | 2 + Untitled.ipynb | 1289 ++++++++++++++++++++++++ dataproc/Untitled.ipynb | 671 ++++++++++++ dataproc/create_cluster.py | 5 +- dataproc/dataproc.iml | 9 + dataproc/quickstart/quickstart.py | 62 +- dataproc/quickstart/quickstart_test.py | 4 +- dataproc/run.py | 9 + dataproc/writeout.py | 4 + firestore/firestore.iml | 18 + vision/.python-version | 1 + 11 files changed, 2037 insertions(+), 37 deletions(-) create mode 100644 .python-version create mode 100644 Untitled.ipynb create mode 100644 dataproc/Untitled.ipynb create mode 100644 dataproc/dataproc.iml create mode 100644 dataproc/run.py create mode 100644 dataproc/writeout.py create mode 100644 firestore/firestore.iml create mode 100644 vision/.python-version diff --git a/.python-version b/.python-version new file mode 100644 index 000000000000..33d3ac7ac953 --- /dev/null +++ b/.python-version @@ -0,0 +1,2 @@ +python-docs-samples +3.6.8 diff --git a/Untitled.ipynb b/Untitled.ipynb new file mode 100644 index 000000000000..6525fd8b0170 --- /dev/null +++ b/Untitled.ipynb @@ -0,0 +1,1289 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from google.cloud import translate\n", + "\n", + "def sample_get_supported_languages_with_target(language_code, project_id):\n", + " \"\"\"\n", + " Listing supported languages with target language name\n", + "\n", + " Args:\n", + " language_code The language to use to return localized,\n", + " human readable names of supported languages.\n", + " \"\"\"\n", + "\n", + " client = translate.TranslationServiceClient()\n", + "\n", + " # TODO(developer): Uncomment and set the following variables\n", + " # language_code = 'en'\n", + " # project_id = '[Google Cloud Project ID]'\n", + " parent = client.location_path(project_id, \"global\")\n", + "\n", + " response = client.get_supported_languages(\n", + " display_language_code=language_code, parent=parent\n", + " )\n", + " return response\n", + " # List language codes of supported languages\n", + " for language in response.languages:\n", + " print(u\"Language Code: {}\".format(language.language_code))\n", + " print(u\"Display Name: {}\".format(language.display_name))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "x = sample_get_supported_languages_with_target(\"en\", \"bmiro-test\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "languages {\n", + " language_code: \"af\"\n", + " display_name: \"Afrikaans\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sq\"\n", + " display_name: \"Albanian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"am\"\n", + " display_name: \"Amharic\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ar\"\n", + " display_name: \"Arabic\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"hy\"\n", + " display_name: \"Armenian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"az\"\n", + " display_name: \"Azerbaijani\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"eu\"\n", + " display_name: \"Basque\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"be\"\n", + " display_name: \"Belarusian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"bn\"\n", + " display_name: \"Bengali\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"bs\"\n", + " display_name: \"Bosnian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"bg\"\n", + " display_name: \"Bulgarian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ca\"\n", + " display_name: \"Catalan\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ceb\"\n", + " display_name: \"Cebuano\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ny\"\n", + " display_name: \"Chichewa\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"zh-CN\"\n", + " display_name: \"Chinese (Simplified)\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"zh-TW\"\n", + " display_name: \"Chinese (Traditional)\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"co\"\n", + " display_name: \"Corsican\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"hr\"\n", + " display_name: \"Croatian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"cs\"\n", + " display_name: \"Czech\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"da\"\n", + " display_name: \"Danish\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"nl\"\n", + " display_name: \"Dutch\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"en\"\n", + " display_name: \"English\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"eo\"\n", + " display_name: \"Esperanto\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"et\"\n", + " display_name: \"Estonian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"tl\"\n", + " display_name: \"Filipino\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"fi\"\n", + " display_name: \"Finnish\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"fr\"\n", + " display_name: \"French\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"fy\"\n", + " display_name: \"Frisian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"gl\"\n", + " display_name: \"Galician\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ka\"\n", + " display_name: \"Georgian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"de\"\n", + " display_name: \"German\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"el\"\n", + " display_name: \"Greek\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"gu\"\n", + " display_name: \"Gujarati\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ht\"\n", + " display_name: \"Haitian Creole\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ha\"\n", + " display_name: \"Hausa\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"haw\"\n", + " display_name: \"Hawaiian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"iw\"\n", + " display_name: \"Hebrew\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"hi\"\n", + " display_name: \"Hindi\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"hmn\"\n", + " display_name: \"Hmong\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"hu\"\n", + " display_name: \"Hungarian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"is\"\n", + " display_name: \"Icelandic\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ig\"\n", + " display_name: \"Igbo\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"id\"\n", + " display_name: \"Indonesian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ga\"\n", + " display_name: \"Irish\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"it\"\n", + " display_name: \"Italian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ja\"\n", + " display_name: \"Japanese\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"jw\"\n", + " display_name: \"Javanese\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"kn\"\n", + " display_name: \"Kannada\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"kk\"\n", + " display_name: \"Kazakh\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"km\"\n", + " display_name: \"Khmer\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ko\"\n", + " display_name: \"Korean\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ku\"\n", + " display_name: \"Kurdish (Kurmanji)\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ky\"\n", + " display_name: \"Kyrgyz\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"lo\"\n", + " display_name: \"Lao\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"la\"\n", + " display_name: \"Latin\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"lv\"\n", + " display_name: \"Latvian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"lt\"\n", + " display_name: \"Lithuanian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"lb\"\n", + " display_name: \"Luxembourgish\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mk\"\n", + " display_name: \"Macedonian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mg\"\n", + " display_name: \"Malagasy\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ms\"\n", + " display_name: \"Malay\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ml\"\n", + " display_name: \"Malayalam\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mt\"\n", + " display_name: \"Maltese\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mi\"\n", + " display_name: \"Maori\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mr\"\n", + " display_name: \"Marathi\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mn\"\n", + " display_name: \"Mongolian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"my\"\n", + " display_name: \"Myanmar (Burmese)\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ne\"\n", + " display_name: \"Nepali\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"no\"\n", + " display_name: \"Norwegian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ps\"\n", + " display_name: \"Pashto\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"fa\"\n", + " display_name: \"Persian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"pl\"\n", + " display_name: \"Polish\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"pt\"\n", + " display_name: \"Portuguese\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"pa\"\n", + " display_name: \"Punjabi\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ro\"\n", + " display_name: \"Romanian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ru\"\n", + " display_name: \"Russian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sm\"\n", + " display_name: \"Samoan\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"gd\"\n", + " display_name: \"Scots Gaelic\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sr\"\n", + " display_name: \"Serbian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"st\"\n", + " display_name: \"Sesotho\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sn\"\n", + " display_name: \"Shona\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sd\"\n", + " display_name: \"Sindhi\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"si\"\n", + " display_name: \"Sinhala\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sk\"\n", + " display_name: \"Slovak\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sl\"\n", + " display_name: \"Slovenian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"so\"\n", + " display_name: \"Somali\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"es\"\n", + " display_name: \"Spanish\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"su\"\n", + " display_name: \"Sundanese\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sw\"\n", + " display_name: \"Swahili\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sv\"\n", + " display_name: \"Swedish\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"tg\"\n", + " display_name: \"Tajik\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ta\"\n", + " display_name: \"Tamil\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"te\"\n", + " display_name: \"Telugu\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"th\"\n", + " display_name: \"Thai\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"tr\"\n", + " display_name: \"Turkish\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"uk\"\n", + " display_name: \"Ukrainian\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ur\"\n", + " display_name: \"Urdu\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"uz\"\n", + " display_name: \"Uzbek\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"vi\"\n", + " display_name: \"Vietnamese\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"cy\"\n", + " display_name: \"Welsh\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"xh\"\n", + " display_name: \"Xhosa\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"yi\"\n", + " display_name: \"Yiddish\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"yo\"\n", + " display_name: \"Yoruba\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"zu\"\n", + " display_name: \"Zulu\"\n", + " support_source: true\n", + " support_target: true\n", + "}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def sample_get_supported_languages(project_id):\n", + " \"\"\"Getting a list of supported language codes\"\"\"\n", + "\n", + " client = translate.TranslationServiceClient()\n", + "\n", + " # TODO(developer): Uncomment and set the following variables\n", + " # project = '[Google Cloud Project ID]'\n", + " parent = client.location_path(project_id, \"global\")\n", + "\n", + " response = client.get_supported_languages(parent=parent)\n", + " return response\n", + " # List language codes of supported languages\n", + " print('Supported Languages:')\n", + " for language in response.languages:\n", + " print(u\"Language Code: {}\".format(language.language_code))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "x = sample_get_supported_languages(\"bmiro-test\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "languages {\n", + " language_code: \"af\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"am\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ar\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"az\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"be\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"bg\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"bn\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"bs\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ca\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ceb\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"co\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"cs\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"cy\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"da\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"de\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"el\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"en\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"eo\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"es\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"et\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"eu\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"fa\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"fi\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"fr\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"fy\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ga\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"gd\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"gl\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"gu\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ha\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"haw\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"hi\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"hmn\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"hr\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ht\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"hu\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"hy\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"id\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ig\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"is\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"it\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"iw\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ja\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"jw\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ka\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"kk\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"km\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"kn\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ko\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ku\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ky\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"la\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"lb\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"lo\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"lt\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"lv\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mg\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mi\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mk\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ml\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mn\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mr\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ms\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"mt\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"my\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ne\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"nl\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"no\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ny\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"pa\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"pl\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ps\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"pt\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ro\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ru\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sd\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"si\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sk\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sl\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sm\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sn\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"so\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sq\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sr\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"st\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"su\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sv\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"sw\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ta\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"te\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"tg\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"th\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"tl\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"tr\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"uk\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"ur\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"uz\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"vi\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"xh\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"yi\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"yo\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"zh-CN\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"zh-TW\"\n", + " support_source: true\n", + " support_target: true\n", + "}\n", + "languages {\n", + " language_code: \"zu\"\n", + " support_source: true\n", + " support_target: true\n", + "}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/dataproc/Untitled.ipynb b/dataproc/Untitled.ipynb new file mode 100644 index 000000000000..c3288712a755 --- /dev/null +++ b/dataproc/Untitled.ipynb @@ -0,0 +1,671 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from google.cloud import dataproc_v1 as dataproc" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "region = \"us-central1\"\n", + "cluster_name = \" test-cluster-70a1acce-87b6-42e4-a94d-b933f50711fc\"\n", + "project_id = \"bmiro-test\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "cluster_client = dataproc.ClusterControllerClient(client_options={\n", + " 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region)\n", + "})" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "cluster = {\n", + " 'project_id': project_id,\n", + " 'cluster_name': cluster_name,\n", + " 'config': {\n", + " 'master_config': {\n", + " 'num_instances': 1,\n", + " 'machine_type_uri': 'n1-standard-1'\n", + " },\n", + " 'worker_config': {\n", + " 'num_instances': 8,\n", + " 'machine_type_uri': 'n1-standard-1'\n", + " }\n", + " }\n", + "}\n", + "\n", + "update_mask = {\n", + " 'paths': {\n", + " 'config.worker_config.num_instances': 8\n", + " }\n", + "}\n", + "\n", + "response = cluster_client.update_cluster(project_id, region, cluster_name, cluster, update_mask)\n", + "result = response.result()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "8" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result.config.worker_config.num_instances" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cluster created successfully: test-regions-1\n" + ] + } + ], + "source": [ + "from google.cloud import dataproc_v1 as dataproc\n", + "\n", + "region = \"us-central1\"\n", + "cluster_name = \"test-regions-1\"\n", + "project_id = \"bmiro-test\"\n", + "\n", + "# Create a client with the endpoint set to the desired cluster region\n", + "cluster_client = dataproc.ClusterControllerClient(client_options={\n", + " 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region)\n", + "})\n", + "\n", + "# Create the cluster config\n", + "cluster = {\n", + " 'project_id': project_id,\n", + " 'cluster_name': cluster_name,\n", + " 'config': {\n", + " 'master_config': {\n", + " 'num_instances': 1,\n", + " 'machine_type_uri': 'n1-standard-1'\n", + " },\n", + " 'worker_config': {\n", + " 'num_instances': 2,\n", + " 'machine_type_uri': 'n1-standard-1'\n", + " }\n", + " }\n", + "}\n", + "\n", + "# Create the cluster\n", + "operation = cluster_client.create_cluster(project_id, region, cluster)\n", + "result = operation.result()\n", + "\n", + "# Output a success message\n", + "print('Cluster created successfully: {}'.format(result.cluster_name))\n", + "# [END dataproc_create_cluster]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [], + "source": [ + "with open('sort.py', 'w') as f:\n", + " f.write(sort_file)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from google.cloud import storage\n", + "storage_client = storage.Client()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "STAGING_BUCKET = \"bm-test-1234\"\n", + "bucket = storage_client.create_bucket(STAGING_BUCKET)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "EOL while scanning string literal (, line 5)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m5\u001b[0m\n\u001b[0;31m \"sum = rdd.reduce(lambda x, y: x + y)\\n\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m EOL while scanning string literal\n" + ] + } + ], + "source": [ + "sort_file = (\n", + "\"import pyspark\\n\"\n", + "\"sc = pyspark.SparkContext()\\n\"\n", + "\"rdd = sc.parallelize((1,2,3,4,5)\\n\"\n", + "\"sum = rdd.reduce(lambda x, y: x + y)\\n\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "blob = bucket.blob(\"sort.py\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "blob.upload_from_string(sort_file)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "blob.delete()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "ename": "NotFound", + "evalue": "404 Not found: Cluster projects/bmiro-test/regions/us-central1/clusters/test-23467", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31m_Rendezvous\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/grpc_helpers.py\u001b[0m in \u001b[0;36merror_remapped_callable\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 56\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mcallable_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 58\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mgrpc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRpcError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/grpc/_channel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, request, timeout, metadata, credentials, wait_for_ready, compression)\u001b[0m\n\u001b[1;32m 689\u001b[0m wait_for_ready, compression)\n\u001b[0;32m--> 690\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_end_unary_response_blocking\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 691\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/grpc/_channel.py\u001b[0m in \u001b[0;36m_end_unary_response_blocking\u001b[0;34m(state, call, with_call, deadline)\u001b[0m\n\u001b[1;32m 591\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 592\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0m_Rendezvous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeadline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 593\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31m_Rendezvous\u001b[0m: <_Rendezvous of RPC that terminated with:\n\tstatus = StatusCode.NOT_FOUND\n\tdetails = \"Not found: Cluster projects/bmiro-test/regions/us-central1/clusters/test-23467\"\n\tdebug_error_string = \"{\"created\":\"@1574793943.794004000\",\"description\":\"Error received from peer ipv6:[2607:f8b0:4006:805::200a]:443\",\"file\":\"src/core/lib/surface/call.cc\",\"file_line\":1055,\"grpc_message\":\"Not found: Cluster projects/bmiro-test/regions/us-central1/clusters/test-23467\",\"grpc_status\":5}\"\n>", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[0;31mNotFound\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcluster_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete_cluster\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproject_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcluster_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/cloud/dataproc_v1/gapic/cluster_controller_client.py\u001b[0m in \u001b[0;36mdelete_cluster\u001b[0;34m(self, project_id, region, cluster_name, cluster_uuid, request_id, retry, timeout, metadata)\u001b[0m\n\u001b[1;32m 583\u001b[0m )\n\u001b[1;32m 584\u001b[0m operation = self._inner_api_calls[\"delete_cluster\"](\n\u001b[0;32m--> 585\u001b[0;31m \u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretry\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mretry\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 586\u001b[0m )\n\u001b[1;32m 587\u001b[0m return google.api_core.operation.from_gapic(\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/gapic_v1/method.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"metadata\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 143\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 145\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/retry.py\u001b[0m in \u001b[0;36mretry_wrapped_func\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 275\u001b[0m \u001b[0msleep_generator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 276\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_deadline\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 277\u001b[0;31m \u001b[0mon_error\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon_error\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 278\u001b[0m )\n\u001b[1;32m 279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/retry.py\u001b[0m in \u001b[0;36mretry_target\u001b[0;34m(target, predicate, sleep_generator, deadline, on_error)\u001b[0m\n\u001b[1;32m 180\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msleep\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msleep_generator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 181\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 182\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 183\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 184\u001b[0m \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/timeout.py\u001b[0m in \u001b[0;36mfunc_with_timeout\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;34m\"\"\"Wrapped function that adds timeout.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 213\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"timeout\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 214\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 215\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 216\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc_with_timeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/grpc_helpers.py\u001b[0m in \u001b[0;36merror_remapped_callable\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mcallable_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mgrpc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRpcError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_grpc_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0merror_remapped_callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/six.py\u001b[0m in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n", + "\u001b[0;31mNotFound\u001b[0m: 404 Not found: Cluster projects/bmiro-test/regions/us-central1/clusters/test-23467" + ] + } + ], + "source": [ + "result = cluster_client.delete_cluster(project_id, region, cluster_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result.result()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'Operation' object has no attribute 'cluster_name'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcluster_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m: 'Operation' object has no attribute 'cluster_name'" + ] + } + ], + "source": [ + "result.result()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def write(f): \n", + " sort_file = (\n", + " \"import pyspark\\n\"\n", + " \"sc = pyspark.SparkContext()\\n\"\n", + " \"rdd = sc.parallelize((1,2,3,4,5)\\n\" \n", + " \"sum = rdd.reduce(lambda x, y: x + y)\\n\"\n", + " )\n", + " with open(\"writeout.py\", \"w\") as f:\n", + " f.write(sort_file)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "write(\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from google.cloud import dataproc_v1 as dataproc\n", + "region = \"us-central1\"\n", + "cluster_name = \"test-cluster-70a1acce-87b6-42e4-a94d-b933f50711fc\"\n", + "project_id = \"bmiro-test\"\n", + "job_file_path = STAGING_BUCKET + \"/\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "job_client = dataproc.JobControllerClient(client_options={\n", + " 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region)\n", + "})" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [], + "source": [ + "cluster_client = dataproc.ClusterControllerClient(client_options={\n", + " 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region)\n", + "})" + ] + }, + { + "cell_type": "code", + "execution_count": 118, + "metadata": {}, + "outputs": [ + { + "ename": "NotFound", + "evalue": "404 Not found: Cluster projects/bmiro-test/regions/us-central1/clusters/test", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31m_Rendezvous\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/grpc_helpers.py\u001b[0m in \u001b[0;36merror_remapped_callable\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 56\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mcallable_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 58\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mgrpc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRpcError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/grpc/_channel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, request, timeout, metadata, credentials, wait_for_ready, compression)\u001b[0m\n\u001b[1;32m 689\u001b[0m wait_for_ready, compression)\n\u001b[0;32m--> 690\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_end_unary_response_blocking\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 691\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/grpc/_channel.py\u001b[0m in \u001b[0;36m_end_unary_response_blocking\u001b[0;34m(state, call, with_call, deadline)\u001b[0m\n\u001b[1;32m 591\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 592\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0m_Rendezvous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeadline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 593\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31m_Rendezvous\u001b[0m: <_Rendezvous of RPC that terminated with:\n\tstatus = StatusCode.NOT_FOUND\n\tdetails = \"Not found: Cluster projects/bmiro-test/regions/us-central1/clusters/test\"\n\tdebug_error_string = \"{\"created\":\"@1575495942.170499000\",\"description\":\"Error received from peer ipv6:[2607:f8b0:4006:804::200a]:443\",\"file\":\"src/core/lib/surface/call.cc\",\"file_line\":1055,\"grpc_message\":\"Not found: Cluster projects/bmiro-test/regions/us-central1/clusters/test\",\"grpc_status\":5}\"\n>", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[0;31mNotFound\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcluster\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcluster_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_cluster\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproject_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"test\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/cloud/dataproc_v1/gapic/cluster_controller_client.py\u001b[0m in \u001b[0;36mget_cluster\u001b[0;34m(self, project_id, region, cluster_name, retry, timeout, metadata)\u001b[0m\n\u001b[1;32m 659\u001b[0m )\n\u001b[1;32m 660\u001b[0m return self._inner_api_calls[\"get_cluster\"](\n\u001b[0;32m--> 661\u001b[0;31m \u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretry\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mretry\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 662\u001b[0m )\n\u001b[1;32m 663\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/gapic_v1/method.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"metadata\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 143\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 145\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/retry.py\u001b[0m in \u001b[0;36mretry_wrapped_func\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 275\u001b[0m \u001b[0msleep_generator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 276\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_deadline\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 277\u001b[0;31m \u001b[0mon_error\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon_error\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 278\u001b[0m )\n\u001b[1;32m 279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/retry.py\u001b[0m in \u001b[0;36mretry_target\u001b[0;34m(target, predicate, sleep_generator, deadline, on_error)\u001b[0m\n\u001b[1;32m 180\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msleep\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msleep_generator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 181\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 182\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 183\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 184\u001b[0m \u001b[0;31m# pylint: disable=broad-except\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/timeout.py\u001b[0m in \u001b[0;36mfunc_with_timeout\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;34m\"\"\"Wrapped function that adds timeout.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 213\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"timeout\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 214\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 215\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 216\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc_with_timeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/google/api_core/grpc_helpers.py\u001b[0m in \u001b[0;36merror_remapped_callable\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mcallable_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mgrpc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRpcError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_grpc_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0merror_remapped_callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.pyenv/versions/3.6.7/envs/python-docs-samples/lib/python3.6/site-packages/six.py\u001b[0m in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n", + "\u001b[0;31mNotFound\u001b[0m: 404 Not found: Cluster projects/bmiro-test/regions/us-central1/clusters/test" + ] + } + ], + "source": [ + "cluster = cluster_client.get_cluster(project_id, region, \"test\")" + ] + }, + { + "cell_type": "code", + "execution_count": 129, + "metadata": {}, + "outputs": [], + "source": [ + "import uuid\n", + "\n", + "STAGING_BUCKET = 'test-bucket-{}'.format(str(uuid.uuid4()))\n", + "JOB_FILE_NAME = 'sum.py'\n", + "SORT_CODE = (\n", + " \"import pyspark\\n\"\n", + " \"import time\\n\"\n", + " \"sc = pyspark.SparkContext()\\n\"\n", + " \"rdd = sc.parallelize((1,2,3,4,5))\\n\"\n", + " \"sum = rdd.reduce(lambda x, y: x + y)\\n\"\n", + " \"print sum\\n\"\n", + ")\n", + "job_file_path = 'gs://{}/{}'.format(STAGING_BUCKET, JOB_FILE_NAME)\n", + "job = {\n", + " 'placement': {\n", + " 'cluster_name': cluster_name\n", + " },\n", + " 'pyspark_job': {\n", + " 'main_python_file_uri': job_file_path\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 130, + "metadata": {}, + "outputs": [], + "source": [ + "from google.cloud import storage\n", + "storage_client = storage.Client()\n", + "\n", + "bucket = storage_client.create_bucket(STAGING_BUCKET)\n", + "blob = bucket.blob(JOB_FILE_NAME)\n", + "blob.upload_from_string(SORT_CODE)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 131, + "metadata": {}, + "outputs": [], + "source": [ + "job_operation = job_client.submit_job(project_id, region, job)" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "''" + ] + }, + "execution_count": 88, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "job_operation.status.details" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "600.0" + ] + }, + "execution_count": 89, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bucket = client.get_bucket(output_bucket)\n", + "output_blob = (\n", + " ('google-cloud-dataproc-metainfo/{}/jobs/{}/driveroutput.000000000'.\n", + " format(cluster_id, job_id)))" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "5" + ] + }, + "execution_count": 104, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "job_id = job_operation.reference.job_id\n", + "job_info = job_client.get_job(project_id, region, job_id)\n", + "job_info.status.state" + ] + }, + { + "cell_type": "code", + "execution_count": 114, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "mappingproxy({'DESCRIPTOR': ,\n", + " '__module__': 'google.cloud.dataproc_v1.types',\n", + " '__doc__': \"The status of a cluster and its instances.\\n \\n \\n Attributes:\\n state:\\n Output only. The cluster's state.\\n detail:\\n Optional. Output only. Details of cluster's state.\\n state_start_time:\\n Output only. Time when this state was entered.\\n substate:\\n Output only. Additional state information that includes status\\n reported by the agent.\\n \",\n", + " '__slots__': (),\n", + " 'state': ,\n", + " 'detail': ,\n", + " 'state_start_time': ,\n", + " 'substate': ,\n", + " 'State': ,\n", + " 'UNKNOWN': 0,\n", + " 'CREATING': 1,\n", + " 'RUNNING': 2,\n", + " 'ERROR': 3,\n", + " 'DELETING': 4,\n", + " 'UPDATING': 5,\n", + " 'Substate': ,\n", + " 'UNSPECIFIED': 0,\n", + " 'UNHEALTHY': 1,\n", + " 'STALE_STATUS': 2})" + ] + }, + "execution_count": 114, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataproc.types.ClusterStatus.__dict__" + ] + }, + { + "cell_type": "code", + "execution_count": 117, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 117, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cluster.status.state" + ] + }, + { + "cell_type": "code", + "execution_count": 128, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'4fd83945-fdc0-4dd7-9abd-f8e547b97b2f'" + ] + }, + "execution_count": 128, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cluster.cluster_uuid" + ] + }, + { + "cell_type": "code", + "execution_count": 126, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "test-cluster-36ae17ed-9d55-4782-a8e1-472cf05a1425\n", + "test-cluster-cddb14dd-a168-4df3-b797-3b8c000e7f84\n", + "test-cluster-8f109c29-8099-4cc1-957c-8ad1569dc24f\n", + "test-cluster-70a1acce-87b6-42e4-a94d-b933f50711fc\n" + ] + } + ], + "source": [ + "for cluster in cluster_client.list_clusters(project_id, region):\n", + " print(cluster.cluster_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'9c99e969-bd5e-4705-83b9-950e101facc9'" + ] + }, + "execution_count": 57, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "job_operation.reference.job_id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/dataproc/create_cluster.py b/dataproc/create_cluster.py index 1280bed9126a..89752d538f52 100644 --- a/dataproc/create_cluster.py +++ b/dataproc/create_cluster.py @@ -14,15 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This sample walks a user through creating a Cloud Dataproc cluster using -# the Python client library. - # [START dataproc_create_cluster] from google.cloud import dataproc_v1 as dataproc def create_cluster(project_id, region, cluster_name): - """Creates a Cloud Dataproc cluster.""" + """Create a Cloud Dataproc cluster.""" # TODO(developer): Uncomment and set the following variables # project_id = 'YOUR_PROJECT_ID' # region = 'YOUR_CLUSTER_REGION' diff --git a/dataproc/dataproc.iml b/dataproc/dataproc.iml new file mode 100644 index 000000000000..ad3c0a365c8c --- /dev/null +++ b/dataproc/dataproc.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/dataproc/quickstart/quickstart.py b/dataproc/quickstart/quickstart.py index fbc062d5a5be..994f529b9c59 100644 --- a/dataproc/quickstart/quickstart.py +++ b/dataproc/quickstart/quickstart.py @@ -22,9 +22,9 @@ def quickstart(project_id, region, cluster_name, job_file_path): - """This quickstart sample walks a user through creating a Cloud Dataproc - cluster, submitting a PySpark job from Google Cloud Storage to the - cluster, reading the output of the job and deleting the cluster, all + """This quickstart sample walks a user through creating a Cloud Dataproc + cluster, submitting a PySpark job from Google Cloud Storage to the + cluster, reading the output of the job and deleting the cluster, all using the Python client library. """ # TODO(developer): Uncomment and set the following variables @@ -35,23 +35,23 @@ def quickstart(project_id, region, cluster_name, job_file_path): # Create the cluster client cluster_client = dataproc.ClusterControllerClient(client_options={ - 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region) + 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region) }) # Create the cluster config cluster = { - 'project_id': project_id, - 'cluster_name': cluster_name, - 'config': { - 'master_config': { - 'num_instances': 1, - 'machine_type_uri': 'n1-standard-1' - }, - 'worker_config': { - 'num_instances': 2, - 'machine_type_uri': 'n1-standard-1' + 'project_id': project_id, + 'cluster_name': cluster_name, + 'config': { + 'master_config': { + 'num_instances': 1, + 'machine_type_uri': 'n1-standard-1' + }, + 'worker_config': { + 'num_instances': 2, + 'machine_type_uri': 'n1-standard-1' + } } - } } # Create the cluster @@ -62,17 +62,17 @@ def quickstart(project_id, region, cluster_name, job_file_path): # Create the job client job_client = dataproc.JobControllerClient(client_options={ - 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region) + 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region) }) # Create the job config job = { - 'placement': { - 'cluster_name': cluster_name - }, - 'pyspark_job': { - 'main_python_file_uri': job_file_path - } + 'placement': { + 'cluster_name': cluster_name + }, + 'pyspark_job': { + 'main_python_file_uri': job_file_path + } } job_response = job_client.submit_job(project_id, region, job) @@ -82,14 +82,14 @@ def quickstart(project_id, region, cluster_name, job_file_path): # Termimal states for a job terminal_states = { - dataproc.types.JobStatus.ERROR, - dataproc.types.JobStatus.CANCELLED, - dataproc.types.JobStatus.DONE + dataproc.types.JobStatus.ERROR, + dataproc.types.JobStatus.CANCELLED, + dataproc.types.JobStatus.DONE } # Create a timeout such that the job gets cancelled if not in a # terminal state after a fixed period of time. - timeout = 600000 + timeout = 600000 time_start = time.time() # Wait for the job to complete @@ -102,8 +102,8 @@ def quickstart(project_id, region, cluster_name, job_file_path): # Poll for job termination once a second time.sleep(1) job_response = job_client.get_job(project_id, region, job_id) - - # Cloud Dataproc job output gets saved to a GCS bucket allocated to it. + + # Cloud Dataproc job output gets saved to a GCS bucket allocated to it. cluster_info = cluster_client.get_cluster(project_id, region, cluster_name) output_bucket = cluster_info.config.config_bucket cluster_uuid = cluster_info.cluster_uuid @@ -111,8 +111,8 @@ def quickstart(project_id, region, cluster_name, job_file_path): storage_client = storage.Client() bucket = storage_client.get_bucket(output_bucket) output_blob = ( - ('google-cloud-dataproc-metainfo/{}/jobs/{}/driveroutput.000000000'. - format(cluster_uuid, job_id))) + ('google-cloud-dataproc-metainfo/{}/jobs/{}/driveroutput.000000000'. + format(cluster_uuid, job_id))) output = bucket.blob(output_blob).download_as_string() print('Job {} finished with state {}:\n{}'.format( @@ -122,6 +122,6 @@ def quickstart(project_id, region, cluster_name, job_file_path): # Delete the cluster once the job has terminated cluster_client.delete_cluster(project_id, region, cluster_name).result() - + print('Cluster {} successfully deleted.'.format(cluster_name)) # [END dataproc_quickstart] diff --git a/dataproc/quickstart/quickstart_test.py b/dataproc/quickstart/quickstart_test.py index e9a32c38ab65..520322981479 100644 --- a/dataproc/quickstart/quickstart_test.py +++ b/dataproc/quickstart/quickstart_test.py @@ -49,8 +49,8 @@ def setup_teardown(): 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(REGION) }) - # The quickstart sample deletes the cluster, but in the event that the - # test fails before cluster deletion occurs, it can be manually deleted here. + # The quickstart sample deletes the cluster, but if the test fails + # before cluster deletion occurs, it can be manually deleted here. clusters = cluster_client.list_clusters(PROJECT_ID, REGION) for cluster in clusters: diff --git a/dataproc/run.py b/dataproc/run.py new file mode 100644 index 000000000000..dbc7f9a793c0 --- /dev/null +++ b/dataproc/run.py @@ -0,0 +1,9 @@ +import os +import create_cluster +import uuid + +project_id = os.environ['GOOGLE_CLOUD_PROJECT'] +region = 'us-central1' +cluster_name = 'test-cluster-{}'.format(str(uuid.uuid4())) + +create_cluster.create_cluster(project_id, region, cluster_name) diff --git a/dataproc/writeout.py b/dataproc/writeout.py new file mode 100644 index 000000000000..a96b5a564960 --- /dev/null +++ b/dataproc/writeout.py @@ -0,0 +1,4 @@ +import pyspark +sc = pyspark.SparkContext() +rdd = sc.parallelize((1,2,3,4,5) +sum = rdd.reduce(lambda x, y: x + y) diff --git a/firestore/firestore.iml b/firestore/firestore.iml new file mode 100644 index 000000000000..2851e5dfabfc --- /dev/null +++ b/firestore/firestore.iml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/vision/.python-version b/vision/.python-version new file mode 100644 index 000000000000..424e1794de6c --- /dev/null +++ b/vision/.python-version @@ -0,0 +1 @@ +3.6.8