Skip to content

Commit

Permalink
updated v0.6.5
Browse files Browse the repository at this point in the history
- Fixed issue with iteritems for pandas >2.0.0
- Updated task running check and updated function
- Updated handling boolean for using last band as alpha mask
  • Loading branch information
samapriya committed Apr 15, 2023
1 parent 1fd3d96 commit 1f9e3c5
Show file tree
Hide file tree
Showing 6 changed files with 31 additions and 39 deletions.
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,11 @@ optional arguments:

# Changelog

### 0.6.5
- Fixed issue with iteritems for pandas >2.0.0
- Updated task running check and updated function
- Updated handling boolean for using last band as alpha mask

### 0.6.4
- Added masking option to use last band as mask

Expand Down
5 changes: 5 additions & 0 deletions docs/changelog.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# Changelog

### 0.6.5
- Fixed issue with iteritems for pandas >2.0.0
- Updated task running check and updated function
- Updated handling boolean for using last band as alpha mask

### 0.6.4
- Added masking option to use last band as mask

Expand Down
2 changes: 1 addition & 1 deletion geeup/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

__author__ = "Samapriya Roy"
__email__ = "samapriya.roy@gmail.com"
__version__ = "0.6.4"
__version__ = "0.6.5"
55 changes: 18 additions & 37 deletions geeup/batch_uploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

__copyright__ = """
Copyright 2016 Lukasz Tracewski
Copyright 2023 Samapriya Roy
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
Expand All @@ -19,32 +19,6 @@
"""
__license__ = "Apache 2.0"

__Modifications_copyright__ = """
Copyright 2022 Samapriya Roy
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
__license__ = "Apache 2.0"

"""
Modifications to file:
- Uses cookie based upload
- Removed multipart upload
- Uses polling
"""

import ast
import csv
import glob
Expand Down Expand Up @@ -84,6 +58,19 @@ def _format_message(self, field, error):
return "GEE file name & path cannot have spaces & can only have letters, numbers, hyphens and underscores"


def task_counter():
ee.Initialize()
status = ["RUNNING", "PENDING"]
task_count = len(
[
task
for task in ee.data.listOperations()
if task["metadata"]["state"] in status
]
)
return task_count


def upload(
user,
source_path,
Expand Down Expand Up @@ -133,19 +120,13 @@ def upload(
# logging.info(
# f"Processing image {current_image_no + 1} out of {no_images} : {image_path}"
# )
status = ["RUNNING", "PENDING"]
task_count = len(
[
task
for task in ee.data.listOperations()
if task["metadata"]["state"] in status
]
)
while task_count >= 2500:
task_count = task_counter()
while task_count >= 2800:
logging.info(
f"Total tasks running or submitted {task_count}: waiting for 5 minutes"
)
time.sleep(300)
task_count = task_counter()
filename = __get_filename_from_path(path=image_path)

destination_path = ee.data.getAsset(destination_path + "/")["name"]
Expand All @@ -165,7 +146,7 @@ def upload(

df = pd.read_csv(metadata_path)
dd = (df.applymap(type) == str).all(0)
for ind, val in dd.iteritems():
for ind, val in dd.items():
if val == True:
slist.append(ind)
intcol = list(df.select_dtypes(include=["int64"]).columns)
Expand Down
1 change: 1 addition & 0 deletions geeup/geeup.py
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,7 @@ def main(args=None):
optional_named.add_argument(
"--mask",
default=False,
choices=('True', 'False', 't', 'f'),
help="Binary to use last band for mask True or False",
)
optional_named.add_argument(
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def readme():

setuptools.setup(
name="geeup",
version="0.6.4",
version="0.6.5",
packages=find_packages(),
url="https://github.com/samapriya/geeup",
install_requires=[
Expand Down

0 comments on commit 1f9e3c5

Please sign in to comment.