Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cleaner create-map versioning + EE credential bug #111

Merged
merged 2 commits into from
Oct 6, 2022
Merged

Cleaner create-map versioning + EE credential bug #111

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f227cf5
ee asset name duplication and gcloud project seting
ivanzvonkov Oct 3, 2022
e9ab490
Addresses new ee credential issue
ivanzvonkov Oct 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion openmapflow/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
TEMPLATE_README = TEMPLATES_DIR / "README.md"
TEMPLATE_DEPLOY_YML = TEMPLATES_DIR / "github-deploy.yaml"
TEMPLATE_TEST_YML = TEMPLATES_DIR / "github-test.yaml"
VERSION = "0.2.1rc1"
VERSION = "0.2.1rc2"

# -------------- Dataframe column names --------------------------------------
SOURCE = "source"
Expand Down
12 changes: 8 additions & 4 deletions openmapflow/ee_exporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,10 +186,14 @@ class EarthEngineExporter:
"""

def __init__(
self, dest_bucket: str, check_ee: bool = False, check_gcp: bool = False
self,
dest_bucket: str,
check_ee: bool = False,
check_gcp: bool = False,
credentials=None,
) -> None:
self.dest_bucket = dest_bucket
ee.Initialize(get_ee_credentials())
ee.Initialize(credentials=credentials if credentials else get_ee_credentials())
self.check_ee = check_ee
self.ee_task_list = get_ee_task_list() if self.check_ee else []
self.check_gcp = check_gcp
Expand Down Expand Up @@ -326,9 +330,9 @@ class EarthEngineAPI:
the default credentials will be used
"""

def __init__(self) -> None:
def __init__(self, credentials=None) -> None:
ee.Initialize(
get_ee_credentials(),
ee.Initialize(credentials if credentials else get_ee_credentials()),
opt_url="https://earthengine-highvolume.googleapis.com",
)

Expand Down
2 changes: 1 addition & 1 deletion openmapflow/inference_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def get_available_bboxes(
"""
if len(buckets_to_check) == 0:
raise ValueError("No buckets to check")
client = storage.Client()
client = storage.Client(project=GCLOUD_PROJECT_ID)
previous_matches = []
available_bboxes = []
bbox_regex = (
Expand Down
16 changes: 11 additions & 5 deletions openmapflow/notebooks/create_map.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"outputs": [],
"source": [
"!pip install \"ipywidgets>=7,<8\" -q # https://github.com/googlecolab/colabtools/issues/3020\n",
"!pip install openmapflow[data]\n",
"!pip install openmapflow[data]==0.2.1rc2\n",
"!pip install cmocean ipyleaflet==0.16.0 pyyaml==5.4.1 -q # Colab likes this version\n",
"%env USE_AUTH_EPHEM=0"
]
Expand Down Expand Up @@ -61,6 +61,7 @@
"source": [
"import ee\n",
"import google\n",
"import os\n",
"import cmocean\n",
"import rasterio as rio\n",
"import matplotlib.pyplot as plt\n",
Expand Down Expand Up @@ -110,7 +111,7 @@
" \"https://www.googleapis.com/auth/earthengine\",\n",
"]\n",
"CREDENTIALS, _ = google.auth.default(default_scopes=SCOPES)\n",
"ee.Initialize(CREDENTIALS, project=GCLOUD_PROJECT_ID)"
"os.environ[\"GOOGLE_CLOUD_PROJECT\"] = GCLOUD_PROJECT_ID"
]
},
{
Expand Down Expand Up @@ -185,7 +186,12 @@
"def get_map_files(map_key):\n",
" blobs = storage.Client().list_blobs(bucket_or_name=BucketNames.PREDS_MERGED, prefix=map_key)\n",
" return [f\"gs://{BucketNames.PREDS_MERGED}/{b.name}\" for b in blobs]\n",
"existing_map_files = get_map_files(map_key)"
"\n",
"existing_map_files = get_map_files(map_key)\n",
"while len(existing_map_files) > 0:\n",
" print(f\"Map for {map_key} already exists: \\n{existing_map_files}\")\n",
" map_key += \"_\" + input(f\"Append to map key: {map_key}_\")\n",
" existing_map_files = get_map_files(map_key)"
]
},
{
Expand Down Expand Up @@ -234,7 +240,7 @@
"\n",
" if not tifs_in_gcloud and ee_task_amount == 0:\n",
" if confirmation(\"No existing data can be used, getting new data using EarthEngine\"):\n",
" EarthEngineExporter(check_ee=False, check_gcp=False, dest_bucket=BucketNames.INFERENCE_EO).export_for_bbox( \n",
" EarthEngineExporter(credentials=CREDENTIALS, check_ee=False, check_gcp=False, dest_bucket=BucketNames.INFERENCE_EO).export_for_bbox( \n",
" bbox=bbox,\n",
" bbox_name=map_key,\n",
" start_date=start_date,\n",
Expand Down Expand Up @@ -384,7 +390,7 @@
"outputs": [],
"source": [
"earthengine_user = input(\"Enter your earthengine username:\")\n",
"ee_safe_prefix = prefix.replace(\".\", \"-\").replace(\"=\", \"-\").replace(\"/\", \"-\")[:100]\n",
"ee_safe_prefix = input(\"EE asset name:\").replace(\".\", \"-\").replace(\"=\", \"-\").replace(\"/\", \"-\")[:100]\n",
"request_id = ee.data.newTaskId()[0]\n",
"params = {\n",
" \"name\": f\"projects/earthengine-legacy/assets/users/{earthengine_user}/{ee_safe_prefix}\",\n",
Expand Down