Source code for

#!/usr/bin/env python

import argparse
import datetime
import ftplib
import math
import os
import subprocess
import tarfile

import pdb

This is a simple script to assist in the downloading of data from the ALOS mosaic product.

[docs]def generateURL(lat, lon, year, large_tile = False): """ Generates a URL to fetch ALOS mosaic data from JAXA FTP server. """ # Test that inputs are reasonable lats/lons if large_tile: assert lat % 5 == 0, "Latitudes must be a multiple of 5 degrees if downloading a 5x5 degree tile." assert lon % 5 == 0, "Longitudes must be a multiple of 5 degrees if downloading a 5x5 degree tile." # Test that years are reasonable assert (year >= 2007 and year <= 2010) or (year >= 2015 and year <=, "Years must be in the range 2007 - 2010 and 2015 - present. Your year was %s."%str(year) # Test that lat/lon is reasonable assert (lat >= -90 and lat <= 90) and (lon >= -180 and lon <= 180), "Latitudes must be between -90 and 90 degrees, and lontitudes between -180 and 180 degrees." # Get hemisphere hem_NS = 'S' if lat < 0 else 'N' hem_EW = 'W' if lon < 0 else 'E' tile_name = '%s%s%s%s'%(hem_NS, str(abs(lat)).zfill(2), hem_EW, str(abs(lon)).zfill(3)) # Get URL for a 5x5 tile if large_tile: # Filename patterns are different for ALOS-1/ALOS-2 if year <= 2010: url = '' else: url = '' url = url%(str(year), tile_name, str(year)[-2:]) # Get URL for a 1x1 tile else: # Filename patterns are different for ALOS-1/ALOS-2 if year <= 2010: url = '' else: url = '' # Special consideration for directory name of 1x1 tile lat_dir = int(5 * math.ceil(float(lat)/5)) lon_dir = int(5 * math.floor(float(lon)/5)) hem_NS_dir = 'S' if lat_dir < 0 else 'N' hem_EW_dir = 'W' if lon_dir < 0 else 'E' directory_name = '%s%s%s%s'%(hem_NS_dir, str(abs(lat_dir)).zfill(2), hem_EW_dir, str(abs(lon_dir)).zfill(3)) url = url%(str(year), directory_name, tile_name, str(year)[-2:]) return url
[docs]def download(url, output_dir = os.getcwd()): """ Download data from JAXA FTP server """ # Check that output directory exists assert os.path.isdir(output_dir), "The output directory (%s) does not exist. Create it, then try again."%output_dir # Check that output file doesn't already exist this_file = '%s/%s'%(output_dir,url.split('/')[-1]) if os.path.exists(this_file): print "WARNING: File %s already exists. Skipping."%this_file elif os.path.exists(this_file[:-7]): print "WARNING: File %s already exists. Skipping."%this_file[:-7] else: # Download exit_status =['wget', '-nc', url, '-P', output_dir]) if exit_status == 8: raise ValueError('Download failed: a tile for that location was not found on the FTP server.') if exit_status != 0: raise ValueError('Download failed with wget error code %s.'%str(exit_status)) # Determine absolute path of downloaded file filepath = '%s/%s'%(output_dir.rstrip('/'), url.split('/')[-1]) return filepath
[docs]def decompress(targz_file, remove = False): ''' Unzips .tar.gz ALOS mosaic files downloaded from JAXA, and removes original where requested. ''' assert targz_file.endswith("tar.gz"), "File name must end with .tar.gz to be decompressed." # Check that output file doesn't already exist if os.path.exists(targz_file.split('/')[-1].split('.')[0]): print 'WARNING: File %s already exists at output location. Not extracting.'%targz_file else: print 'Extracting %s'%targz_file tar =, "r:gz") tar.extractall(path = targz_file[:-7]) tar.close() if remove: removeTarGz(targz_file)
[docs]def removeTarGz(targz_file): """ Deletes ALOS-1/ALOS-2 .tar.gx files from disk. Input is a compress ALOS-1/ALOS-2 file from JAXA. """ assert targz_file.endswith('_MOS.tar.gz') or targz_file.endswith('_MOS_F02DAR.tar.gz'), "removeTarGz function should only be used to delete ALOS-1/ALOS2 .tar.gz files" os.remove(targz_file)
[docs]def checkYears(years): """ Reduces input years to those available for the ALOS mosaic (or may be available in future). """ years_cleansed = [] for y in years: # Remove years before ALOS-1 if y < 2007: raise ValueError("Can't download data for year %s; no data from ALOS are available before 2007."%str(y)) # Remove years between ALOS-1 and ALOS-2 elif y > 2010 and y < 2015: raise ValueError("Can't download data for year %s; no data from ALOS are available 2011 to 2014 (inclusive)."%str(y)) # Remove years from the future, which can't possibly exist yet. elif y > raise ValueError("Can't download data for %s; this year is in the future."%str(y))
[docs]def main(lat, lon, year, large_tile = False, output_dir = os.getcwd(), remove = False): ''' Run through data download and preparation chain ''' # Generate download URL url = generateURL(lat, lon, year, large_tile = large_tile) # Get file, sending it to output_dir filepath = download(url, output_dir = output_dir) # Decompress downloaded file, and delete original if remove == True decompress(filepath, remove = remove)
if __name__ == '__main__': # Set up command line parser parser = argparse.ArgumentParser(description = 'Download ALOS-1/2 data from JAXA, specifying a particular year and latitude/longitude.') # Required arguments parser.add_argument('-lat', '--latitude', type = int, help = "Latitude of tile upper-left corner.") parser.add_argument('-lon', '--longitude', type = int, help = "Longitude of tile upper-left corner.") # Optional arguments parser.add_argument('-l', '--large', action = 'store_true', default = False, help = "Download large tiles. ALOS mosaic tiles are available in 1x1 or 5x5 degree tiles. If downloading large volumes of data, it's usually better to use the latter. If this option is chosen, you must select a lat and lon that's a multiple of 5 degrees.") parser.add_argument('-y', '--years', type = int, nargs = '+', default = [2007, 2008, 2009, 2010, 2015, 2016], help = "Year of datat o download. Defaults to downloading all data.") parser.add_argument('-o', '--output_dir', type = str, default = os.getcwd(), help = "Optionally specify an output directory. Defaults to the present working directory.") parser.add_argument('-r', '--remove', action='store_true', default = False, help = "Optionally remove downloaded .tar.gz files after decompression.") # Get arguments from command line args = parser.parse_args() # Cleanse input years checkYears(args.years) # Run through entire processing sequence for year in args.years: main(args.latitude, args.longitude, year, large_tile = args.large, output_dir = args.output_dir, remove = args.remove)