From f227cf571d762f48250995bca0c1b4bfa364c688 Mon Sep 17 00:00:00 2001 From: ivanzvonkov Date: Mon, 3 Oct 2022 12:04:08 -0400 Subject: [PATCH 1/2] ee asset name duplication and gcloud project seting --- openmapflow/inference_utils.py | 2 +- openmapflow/notebooks/create_map.ipynb | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/openmapflow/inference_utils.py b/openmapflow/inference_utils.py index c73e00b6..e5c34833 100644 --- a/openmapflow/inference_utils.py +++ b/openmapflow/inference_utils.py @@ -44,7 +44,7 @@ def get_available_bboxes( """ if len(buckets_to_check) == 0: raise ValueError("No buckets to check") - client = storage.Client() + client = storage.Client(project=GCLOUD_PROJECT_ID) previous_matches = [] available_bboxes = [] bbox_regex = ( diff --git a/openmapflow/notebooks/create_map.ipynb b/openmapflow/notebooks/create_map.ipynb index 9ba276ef..aadd3fd3 100644 --- a/openmapflow/notebooks/create_map.ipynb +++ b/openmapflow/notebooks/create_map.ipynb @@ -21,7 +21,7 @@ "outputs": [], "source": [ "!pip install \"ipywidgets>=7,<8\" -q # https://github.com/googlecolab/colabtools/issues/3020\n", - "!pip install openmapflow[data]\n", + "!pip install openmapflow[data]==0.2.1rc1\n", "!pip install cmocean ipyleaflet==0.16.0 pyyaml==5.4.1 -q # Colab likes this version\n", "%env USE_AUTH_EPHEM=0" ] @@ -185,7 +185,12 @@ "def get_map_files(map_key):\n", " blobs = storage.Client().list_blobs(bucket_or_name=BucketNames.PREDS_MERGED, prefix=map_key)\n", " return [f\"gs://{BucketNames.PREDS_MERGED}/{b.name}\" for b in blobs]\n", - "existing_map_files = get_map_files(map_key)" + "\n", + "existing_map_files = get_map_files(map_key)\n", + "while len(existing_map_files) > 0:\n", + " print(f\"Map for {map_key} already exists: \\n{existing_map_files}\")\n", + " map_key += \"_\" + input(f\"Append to map key: {map_key}_\")\n", + " existing_map_files = get_map_files(map_key)" ] }, { @@ -384,7 +389,7 @@ "outputs": [], "source": [ "earthengine_user = input(\"Enter your earthengine username:\")\n", - "ee_safe_prefix = prefix.replace(\".\", \"-\").replace(\"=\", \"-\").replace(\"/\", \"-\")[:100]\n", + "ee_safe_prefix = input(\"EE asset name:\").replace(\".\", \"-\").replace(\"=\", \"-\").replace(\"/\", \"-\")[:100]\n", "request_id = ee.data.newTaskId()[0]\n", "params = {\n", " \"name\": f\"projects/earthengine-legacy/assets/users/{earthengine_user}/{ee_safe_prefix}\",\n", From e9ab490846820477725d01d9254d3c046dc98b2e Mon Sep 17 00:00:00 2001 From: ivanzvonkov Date: Wed, 5 Oct 2022 19:30:37 -0400 Subject: [PATCH 2/2] Addresses new ee credential issue --- openmapflow/constants.py | 2 +- openmapflow/ee_exporter.py | 12 ++++++++---- openmapflow/notebooks/create_map.ipynb | 7 ++++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/openmapflow/constants.py b/openmapflow/constants.py index 5e6f9321..6b371398 100644 --- a/openmapflow/constants.py +++ b/openmapflow/constants.py @@ -13,7 +13,7 @@ TEMPLATE_README = TEMPLATES_DIR / "README.md" TEMPLATE_DEPLOY_YML = TEMPLATES_DIR / "github-deploy.yaml" TEMPLATE_TEST_YML = TEMPLATES_DIR / "github-test.yaml" -VERSION = "0.2.1rc1" +VERSION = "0.2.1rc2" # -------------- Dataframe column names -------------------------------------- SOURCE = "source" diff --git a/openmapflow/ee_exporter.py b/openmapflow/ee_exporter.py index 26504ead..fde25eec 100644 --- a/openmapflow/ee_exporter.py +++ b/openmapflow/ee_exporter.py @@ -186,10 +186,14 @@ class EarthEngineExporter: """ def __init__( - self, dest_bucket: str, check_ee: bool = False, check_gcp: bool = False + self, + dest_bucket: str, + check_ee: bool = False, + check_gcp: bool = False, + credentials=None, ) -> None: self.dest_bucket = dest_bucket - ee.Initialize(get_ee_credentials()) + ee.Initialize(credentials=credentials if credentials else get_ee_credentials()) self.check_ee = check_ee self.ee_task_list = get_ee_task_list() if self.check_ee else [] self.check_gcp = check_gcp @@ -326,9 +330,9 @@ class EarthEngineAPI: the default credentials will be used """ - def __init__(self) -> None: + def __init__(self, credentials=None) -> None: ee.Initialize( - get_ee_credentials(), + ee.Initialize(credentials if credentials else get_ee_credentials()), opt_url="https://earthengine-highvolume.googleapis.com", ) diff --git a/openmapflow/notebooks/create_map.ipynb b/openmapflow/notebooks/create_map.ipynb index aadd3fd3..ae135385 100644 --- a/openmapflow/notebooks/create_map.ipynb +++ b/openmapflow/notebooks/create_map.ipynb @@ -21,7 +21,7 @@ "outputs": [], "source": [ "!pip install \"ipywidgets>=7,<8\" -q # https://github.com/googlecolab/colabtools/issues/3020\n", - "!pip install openmapflow[data]==0.2.1rc1\n", + "!pip install openmapflow[data]==0.2.1rc2\n", "!pip install cmocean ipyleaflet==0.16.0 pyyaml==5.4.1 -q # Colab likes this version\n", "%env USE_AUTH_EPHEM=0" ] @@ -61,6 +61,7 @@ "source": [ "import ee\n", "import google\n", + "import os\n", "import cmocean\n", "import rasterio as rio\n", "import matplotlib.pyplot as plt\n", @@ -110,7 +111,7 @@ " \"https://www.googleapis.com/auth/earthengine\",\n", "]\n", "CREDENTIALS, _ = google.auth.default(default_scopes=SCOPES)\n", - "ee.Initialize(CREDENTIALS, project=GCLOUD_PROJECT_ID)" + "os.environ[\"GOOGLE_CLOUD_PROJECT\"] = GCLOUD_PROJECT_ID" ] }, { @@ -239,7 +240,7 @@ "\n", " if not tifs_in_gcloud and ee_task_amount == 0:\n", " if confirmation(\"No existing data can be used, getting new data using EarthEngine\"):\n", - " EarthEngineExporter(check_ee=False, check_gcp=False, dest_bucket=BucketNames.INFERENCE_EO).export_for_bbox( \n", + " EarthEngineExporter(credentials=CREDENTIALS, check_ee=False, check_gcp=False, dest_bucket=BucketNames.INFERENCE_EO).export_for_bbox( \n", " bbox=bbox,\n", " bbox_name=map_key,\n", " start_date=start_date,\n",