Skip to content

Commit

Permalink
Merge pull request #275 from johntruckenbrodt/bugfix/osv_url
Browse files Browse the repository at this point in the history
change default S1 OSV download URL, add option to affected functions
  • Loading branch information
johntruckenbrodt authored Nov 21, 2023
2 parents 3c3cec1 + 85d2347 commit 4ec334e
Show file tree
Hide file tree
Showing 6 changed files with 38 additions and 32 deletions.
17 changes: 9 additions & 8 deletions pyroSAR/S1/auxil.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from spatialist.ancillary import finder

import logging

log = logging.getLogger(__name__)

try:
Expand Down Expand Up @@ -222,6 +223,9 @@ def __catch_step_auxdata(self, sensor, start, stop, osvtype='POE'):
url = 'https://step.esa.int/auxdata/orbits/Sentinel-1'
skeleton = '{url}/{osvtype}ORB/{sensor}/{year}/{month:02d}/'

if osvtype not in ['POE', 'RES']:
raise RuntimeError("osvtype must be either 'POE' or 'RES'")

if isinstance(sensor, str):
sensor = [sensor]

Expand Down Expand Up @@ -394,10 +398,9 @@ def catch(self, sensor, osvtype='POE', start=None, stop=None, url_option=1):
stop: str or None
the date to stop searching for files in format YYYYmmddTHHMMSS
url_option: int
the URL to query for OSV files
the OSV download URL option
- 1: https://scihub.copernicus.eu/gnss
- 2: https://step.esa.int/auxdata/orbits/Sentinel-1
- 1: https://step.esa.int/auxdata/orbits/Sentinel-1
Returns
-------
Expand All @@ -418,11 +421,9 @@ def catch(self, sensor, osvtype='POE', start=None, stop=None, url_option=1):
stop = datetime.now()

if url_option == 1:
items = self.__catch_gnss(sensor, start, stop, osvtype)
elif url_option == 2:
items = self.__catch_step_auxdata(sensor, start, stop, osvtype)
else:
raise ValueError("'url_option' must be either 1 or 2")
raise ValueError("unknown URL option")

if osvtype == 'RES' and self.maxdate('POE', 'stop') is not None:
items = [x for x in items
Expand Down Expand Up @@ -599,7 +600,7 @@ def retrieve(self, products, pbar=False):
response = requests.get(remote, auth=auth, timeout=self.timeout)
response.raise_for_status()
infile = response.content

# use a tempfile to allow atomic writes in the case of
# parallel executions dependent on the same orbit files
fd, tmp_path = tempfile.mkstemp(prefix=os.path.basename(local), dir=os.path.dirname(local))
Expand All @@ -623,7 +624,7 @@ def retrieve(self, products, pbar=False):
except Exception as e:
os.unlink(tmp_path)
raise

if pbar:
i += 1
progress.update(i)
Expand Down
16 changes: 4 additions & 12 deletions pyroSAR/drivers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1728,10 +1728,7 @@ def getOSV(self, osvdir=None, osvType='POE', returnMatch=False, useLocal=True, t
timeout: int or tuple or None
the timeout in seconds for downloading OSV files as provided to :func:`requests.get`
url_option: int
the URL to query for scenes
- 1: https://scihub.copernicus.eu/gnss
- 2: https://step.esa.int/auxdata/orbits/Sentinel-1
the OSV download URL option; see :meth:`pyroSAR.S1.OSV.catch` for options
Returns
-------
Expand All @@ -1742,11 +1739,6 @@ def getOSV(self, osvdir=None, osvType='POE', returnMatch=False, useLocal=True, t
--------
:class:`pyroSAR.S1.OSV`
"""
date = datetime.strptime(self.start, '%Y%m%dT%H%M%S')
# create a time span with one day before and one after the acquisition
before = (date - timedelta(days=1)).strftime('%Y%m%dT%H%M%S')
after = (date + timedelta(days=1)).strftime('%Y%m%dT%H%M%S')

with S1.OSV(osvdir, timeout=timeout) as osv:
if useLocal:
match = osv.match(sensor=self.sensor, timestamp=self.start,
Expand All @@ -1756,15 +1748,15 @@ def getOSV(self, osvdir=None, osvType='POE', returnMatch=False, useLocal=True, t

if osvType in ['POE', 'RES']:
files = osv.catch(sensor=self.sensor, osvtype=osvType,
start=before, stop=after,
start=self.start, stop=self.stop,
url_option=url_option)
elif sorted(osvType) == ['POE', 'RES']:
files = osv.catch(sensor=self.sensor, osvtype='POE',
start=before, stop=after,
start=self.start, stop=self.stop,
url_option=url_option)
if len(files) == 0:
files = osv.catch(sensor=self.sensor, osvtype='RES',
start=before, stop=after,
start=self.start, stop=self.stop,
url_option=url_option)
else:
msg = "osvType must either be 'POE', 'RES' or a list of both"
Expand Down
12 changes: 9 additions & 3 deletions pyroSAR/gamma/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ def convert2gamma(id, directory, S1_tnr=True, S1_bnr=True,


def correctOSV(id, directory, osvdir=None, osvType='POE', timeout=20,
logpath=None, outdir=None, shellscript=None):
logpath=None, outdir=None, shellscript=None, url_option=1):
"""
correct GAMMA parameter files with orbit state vector information from dedicated OSV files;
OSV files are downloaded automatically to either the defined `osvdir` or relative to the
Expand All @@ -463,6 +463,8 @@ def correctOSV(id, directory, osvdir=None, osvType='POE', timeout=20,
the directory to execute the command in
shellscript: str or None
a file to write the GAMMA commands to in shell format
url_option: int
the OSV download URL option; see :meth:`pyroSAR.S1.OSV.catch`
Returns
-------
Expand Down Expand Up @@ -506,7 +508,7 @@ def correctOSV(id, directory, osvdir=None, osvType='POE', timeout=20,
auxdatapath = os.path.join(os.path.expanduser('~'), '.snap', 'auxdata')
osvdir = os.path.join(auxdatapath, 'Orbits', 'Sentinel-1')
try:
id.getOSV(osvdir, osvType, timeout=timeout)
id.getOSV(osvdir, osvType, timeout=timeout, url_option=url_option)
except URLError:
log.warning('..no internet access')

Expand Down Expand Up @@ -543,7 +545,8 @@ def correctOSV(id, directory, osvdir=None, osvType='POE', timeout=20,
def geocode(scene, dem, tmpdir, outdir, spacing, scaling='linear', func_geoback=1,
nodata=(0, -99), update_osv=True, osvdir=None, allow_RES_OSV=False,
cleanup=True, export_extra=None, basename_extensions=None,
removeS1BorderNoiseMethod='gamma', refine_lut=False, rlks=None, azlks=None):
removeS1BorderNoiseMethod='gamma', refine_lut=False, rlks=None, azlks=None,
s1_osv_url_option=1):
"""
general function for radiometric terrain correction (RTC) and geocoding of SAR backscatter images with GAMMA.
Applies the RTC method by :cite:t:`Small2011` to retrieve gamma nought RTC backscatter.
Expand Down Expand Up @@ -619,6 +622,8 @@ def geocode(scene, dem, tmpdir, outdir, spacing, scaling='linear', func_geoback=
:func:`pyroSAR.ancillary.multilook_factors` based on the image pixel spacing and the target spacing.
azlks: int or None
the number of azimuth looks. Like `rlks`.
s1_osv_url_option: int
the OSV download URL option; see :meth:`pyroSAR.S1.OSV.catch`
Returns
-------
Expand Down Expand Up @@ -760,6 +765,7 @@ def geocode(scene, dem, tmpdir, outdir, spacing, scaling='linear', func_geoback=
osvtype = 'POE'
try:
correctOSV(id=scene, directory=tmpdir, osvdir=osvdir, osvType=osvtype,
url_option=s1_osv_url_option,
logpath=path_log, outdir=tmpdir, shellscript=shellscript)
except RuntimeError:
msg = 'orbit state vector correction failed for scene {}'
Expand Down
8 changes: 5 additions & 3 deletions pyroSAR/snap/auxil.py
Original file line number Diff line number Diff line change
Expand Up @@ -1619,7 +1619,7 @@ def mli_parametrize(scene, spacing=None, rlks=None, azlks=None, **kwargs):
return ml


def orb_parametrize(scene, formatName, allow_RES_OSV=True, **kwargs):
def orb_parametrize(scene, formatName, allow_RES_OSV=True, url_option=1, **kwargs):
"""
convenience function for parametrizing an `Apply-Orbit-File`.
Required Sentinel-1 orbit files are directly downloaded.
Expand All @@ -1636,6 +1636,8 @@ def orb_parametrize(scene, formatName, allow_RES_OSV=True, **kwargs):
the scene's data format
allow_RES_OSV: bool
(only applies to Sentinel-1) Also allow the less accurate RES orbit files to be used?
url_option: int
the OSV download URL option; see :meth:`pyroSAR.S1.OSV.catch`
kwargs
further keyword arguments for node parametrization. Known options:
Expand All @@ -1655,9 +1657,9 @@ def orb_parametrize(scene, formatName, allow_RES_OSV=True, **kwargs):

if formatName == 'SENTINEL-1':
osv_type = ['POE', 'RES'] if allow_RES_OSV else 'POE'
match = scene.getOSV(osvType=osv_type, returnMatch=True)
match = scene.getOSV(osvType=osv_type, returnMatch=True, url_option=url_option)
if match is None and allow_RES_OSV:
scene.getOSV(osvType='RES')
scene.getOSV(osvType='RES', url_option=url_option)
orbitType = 'Sentinel Restituted (Auto Download)'

orb = parse_node('Apply-Orbit-File')
Expand Down
14 changes: 10 additions & 4 deletions pyroSAR/snap/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def geocode(infile, outdir, t_srs=4326, spacing=20, polarizations='all', shapefi
demResamplingMethod='BILINEAR_INTERPOLATION', imgResamplingMethod='BILINEAR_INTERPOLATION',
alignToStandardGrid=False, standardGridOriginX=0, standardGridOriginY=0,
speckleFilter=False, refarea='gamma0', clean_edges=False, clean_edges_npixels=1,
rlks=None, azlks=None, dem_oversampling_multiple=2):
rlks=None, azlks=None, dem_oversampling_multiple=2, s1_osv_url_option=1):
"""
general function for geocoding of SAR backscatter images with SNAP.
Expand Down Expand Up @@ -203,6 +203,8 @@ def geocode(infile, outdir, t_srs=4326, spacing=20, polarizations='all', shapefi
Used only for terrain flattening.
The SNAP default of 1 has been found to be insufficient with stripe
artifacts remaining in the image.
s1_osv_url_option: int
the OSV download URL option; see :meth:`pyroSAR.S1.OSV.catch`
Returns
-------
Expand Down Expand Up @@ -372,7 +374,8 @@ def geocode(infile, outdir, t_srs=4326, spacing=20, polarizations='all', shapefi
last = deb
############################################
# Apply-Orbit-File node configuration
orb = orb_parametrize(scene=id, formatName=formatName, allow_RES_OSV=allow_RES_OSV)
orb = orb_parametrize(scene=id, formatName=formatName, allow_RES_OSV=allow_RES_OSV,
url_option=s1_osv_url_option)
workflow.insert_node(orb, before=last.id)
last = orb
############################################
Expand Down Expand Up @@ -654,7 +657,7 @@ def geocode(infile, outdir, t_srs=4326, spacing=20, polarizations='all', shapefi
def noise_power(infile, outdir, polarizations, spacing, t_srs, refarea='sigma0', tmpdir=None, test=False, cleanup=True,
demName='SRTM 1Sec HGT', externalDEMFile=None, externalDEMNoDataValue=None, externalDEMApplyEGM=True,
alignToStandardGrid=False, standardGridOriginX=0, standardGridOriginY=0, groupsize=1,
clean_edges=False, clean_edges_npixels=1, rlks=None, azlks=None):
clean_edges=False, clean_edges_npixels=1, rlks=None, azlks=None, osv_url_option=1):
"""
Generate Sentinel-1 noise power images for each polarization, calibrated to either beta, sigma or gamma nought.
The written GeoTIFF files will carry the suffix NEBZ, NESZ or NEGZ respectively.
Expand Down Expand Up @@ -720,6 +723,8 @@ def noise_power(infile, outdir, polarizations, spacing, t_srs, refarea='sigma0',
:func:`pyroSAR.ancillary.multilook_factors` based on the image pixel spacing and the target spacing.
azlks: int or None
the number of azimuth looks. Like `rlks`.
osv_url_option: int
the OSV download URL option; see :meth:`pyroSAR.S1.OSV.catch`
Returns
-------
Expand Down Expand Up @@ -750,7 +755,8 @@ def noise_power(infile, outdir, polarizations, spacing, t_srs, refarea='sigma0',
wf.insert_node(read)
############################################
orb = orb_parametrize(scene=id, workflow=wf, before=read.id,
formatName='SENTINEL-1', allow_RES_OSV=True)
formatName='SENTINEL-1', allow_RES_OSV=True,
url_option=osv_url_option)
############################################
cal = parse_node('Calibration')
wf.insert_node(cal, before=orb.id)
Expand Down
3 changes: 1 addition & 2 deletions tests/test_osv.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@ def test_scene_osv(tmpdir, testdata):
for item in osv.getLocals('POE')[1:3]:
os.remove(item)
assert len(osv.getLocals('POE')) == 1
# res = osv.catch(sensor='S1A', osvtype='RES', start='20180101T120000', stop='20180102T120000')
res = osv.catch(sensor='S1A', osvtype='RES', start='20210201T00000', stop='20210201T150000', url_option=2)
res = osv.catch(sensor='S1A', osvtype='RES', start='20210201T00000', stop='20210201T150000', url_option=1)
assert len(res) == 9
osv.retrieve(res[0:3])
assert len(osv.getLocals('RES')) == 3
Expand Down

0 comments on commit 4ec334e

Please sign in to comment.