diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6769e21 --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/README.md b/README.md index 02afa36..2807178 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ If the ephemeris release center is not provided, the CPF ephemeris files are dow ```python >>> from slrfield import cpf_download >>> cpf_dir_cddis,cpf_files_cddis = cpf_download() # From CDDIS by default; ->>> print(cpf_dir,cpf_files) +>>> print(cpf_dir_cddis,cpf_files_cddis) ``` A directory where CPF files are stored, such as *CPF/CDDIS/2020-10-02/*, will be automatically created. diff --git a/setup.py b/setup.py index 7e594a4..52cbccd 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup,find_packages +from setuptools import setup, find_packages setup( name='slrfield', @@ -10,8 +10,8 @@ license='MIT', long_description_content_type='text/markdown', long_description=open('README.md', 'rb').read().decode('utf-8'), - keywords = ['SLR','CPF'], - python_requires = '>=3.8', + keywords=['SLR', 'CPF'], + python_requires='>=3.8', classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Education', @@ -20,7 +20,7 @@ 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: MIT License', ], - packages = find_packages(), + packages=find_packages(), include_package_data=True, install_requires=[ 'scipy', @@ -28,6 +28,7 @@ 'astropy>=4.3.1', 'tqdm', 'colorama', - 'beautifulsoup4' + 'beautifulsoup4', + 'requests' ], ) diff --git a/slrfield/cpf/cpf_download.py b/slrfield/cpf/cpf_download.py index 778ac87..b708682 100644 --- a/slrfield/cpf/cpf_download.py +++ b/slrfield/cpf/cpf_download.py @@ -1,381 +1,526 @@ -from os import system,path,makedirs +from os import system, path, makedirs from pathlib import Path from ftplib import FTP import requests from bs4 import BeautifulSoup from astropy.time import Time +from astropy.units import day from warnings import warn -from ..utils.try_download import tqdm_ftp,tqdm_request_cpf +from ..utils.try_download import tqdm_ftp, tqdm_request_http -def download_bycurrent(source,satnames=None,keep=True): + +def download_bycurrent(source, satnames=None, keep=True): """ Download the latest CPF ephemeris files at the current moment. - Usage: + Usage: server,dir_cpf_from, dir_cpf_to,cpf_files = download_bycurrent('CDDIS') server,dir_cpf_from, dir_cpf_to,cpf_files = download_bycurrent('EDC') - server,dir_cpf_from, dir_cpf_to,cpf_files = download_bycurrent('CDDIS',lageos1') - server,dir_cpf_from, dir_cpf_to,cpf_files = download_bycurrent('EDC',['ajisai','lageos1','hy2a']) + + server,dir_cpf_from, dir_cpf_to,cpf_files + = download_bycurrent('CDDIS', lageos1') + + server,dir_cpf_from, dir_cpf_to,cpf_files + = download_bycurrent('EDC', ['ajisai','lageos1','hy2a']) Inputs: - source -> [str] source for CPF ephemeris files. Currently, only 'CDDIS' and 'EDC' are available. - + source -> [str] source for CPF ephemeris files. Currently, only 'CDDIS' + and 'EDC' are available. + Parameters: - satnames -> [str, list of str, default=None] target name or list of target names. If None, then all feasible targets at the current moment will be downloaded. - keep -> [Bool, default = True] If False, clear the data storage directory ahead of requesting CPF files. If True, then keep the data in storage directory. + satnames -> [str, list of str, default=None] target name or list of + target names. If None, then all feasible targets at the current + moment will be downloaded. + + keep -> [Bool, default = True] If False, clear the data storage + directory ahead of requesting CPF files. If True, then keep the + data in storage directory. Outputs: - server -> [str] server for downloading CPF ephemeris files. Currently, only 'cddis.nasa.gov' and 'edc.dgfi.tum.de' are available. - dir_cpf_from -> [str] directory for storing CPF ephemeris files in remote server. - dir_cpf_to -> [str] user's local directory for storing CPF ephemeris files + server -> [str] server for downloading CPF ephemeris files. Currently, + only 'cddis.nasa.gov' and 'edc.dgfi.tum.de' are available. + dir_cpf_from -> [str] directory for storing CPF ephemeris files in + remote server. + dir_cpf_to -> [str] user's local directory for storing CPF ephemeris + files cpf_files -> [str list] list of CPF ephemeris files - """ + """ cpf_files = [] date = Time.now().iso dir_cpf_to = 'CPF/'+source+'/'+date[:10] + '/' - - if not path.exists(dir_cpf_to): - makedirs(dir_cpf_to) + + if not path.exists(dir_cpf_to): + makedirs(dir_cpf_to) else: - if not keep: system('rm -rf %s/*' % dir_cpf_to) - + if not keep: + system('rm -rf %s/*' % dir_cpf_to) + if source == 'CDDIS': server = 'https://cddis.nasa.gov' - dir_cpf_from = '/archive/slr/cpf_predicts/current/' - cpf_files_dict,cpf_files_list = get_cpf_filelist(server,dir_cpf_from,'bycurrent') - + dir_cpf_from = '/archive/slr/cpf_predicts_v2/current/' + cpf_files_dict, cpf_files_list = get_cpf_filelist( + server, dir_cpf_from, 'bycurrent') + elif source == 'EDC': server = 'edc.dgfi.tum.de' - dir_cpf_from = '~/slr/cpf_predicts//current/' - ftp = FTP(server,timeout=200) + dir_cpf_from = '~/slr/cpf_predicts_v2/current/' + ftp = FTP(server, timeout=200) ftp.login() ftp.cwd(dir_cpf_from) - cpf_files_list = ftp.nlst('-t','*cpf*') # list files containing 'cpf' from newest to oldest - - ftp.quit() + # list files containing 'cpf' from newest to oldest + cpf_files_list = ftp.nlst('-t', '*cpf*') + + ftp.quit() ftp.close() - else: - raise Exception("Currently, for CPF prediction centers, only 'CDDIS' and 'EDC' are available.") + else: + raise Exception( + "Currently, for CPF prediction centers, " + + "only 'CDDIS' and 'EDC' are available.") if satnames is None: - reduplicates = set([cpf_file.split('_')[0] for cpf_file in cpf_files_list]) # remove duplicates + # Remove duplicates + reduplicates = set([cpf_file.split('_')[0] + for cpf_file in cpf_files_list]) elif type(satnames) is str: reduplicates = [satnames] elif type(satnames) is list: reduplicates = satnames else: - raise Exception('Type of satnames should be str or list.') - + raise Exception('Type of satnames should be str or list.') + for cpf_file in cpf_files_list: satname = cpf_file.split('_')[0] if satname in reduplicates: cpf_files.append(cpf_file) reduplicates.remove(satname) - if reduplicates == []: break - - return server,dir_cpf_from, dir_cpf_to,cpf_files + if reduplicates == []: + break + + return server, dir_cpf_from, dir_cpf_to, cpf_files + -def download_bydate(source,date,satnames,keep=True): +def download_bydate(source, date, satnames, keep=True): """ Download the latest CPF ephemeris files before a specific time. - Usage: - server,dirs_cpf_from, dir_cpf_to,cpf_files = download_bydate('CDDIS','2007-06-01 11:30:00',['ajisai','lageos1','hy2a']) - server,dirs_cpf_from, dir_cpf_to,cpf_files = download_bycurrent('EDC','2017-12-20 05:30:00','hy2a']) + Usage: + server,dirs_cpf_from, dir_cpf_to,cpf_files = download_bydate( + 'CDDIS', + '2007-06-01 11:30:00', + ['ajisai', + 'lageos1', + 'hy2a']) + server,dirs_cpf_from, dir_cpf_to,cpf_files = download_bycurrent( + 'EDC', + '2017-12-20 05:30:00', + 'hy2a']) Inputs: - source -> [str] source for CPF ephemeris files. Currently, only 'CDDIS' and 'EDC' are available. - date -> [str] 'iso-formatted' time, such as '2017-12-20 05:30:00'. It specifies a moment before which the latest CPF ephemeris files are downloaded. - satnames -> [str, str list] target name or list of target names. - + source -> [str] source for CPF ephemeris files. Currently, only 'CDDIS' + and 'EDC' are available. + + date -> [str] 'iso-formatted' time, such as '2017-12-20 05:30:00'. + It specifies a moment before which the latest CPF ephemeris files + are downloaded. + + satnames -> [str, str list] target name or list of target names. + Parameters: - keep -> [Bool, default = True] If False, clear the data storage directory ahead of requesting CPF files. If True, then keep the data storage directory. + keep -> [Bool, default = True] If False, clear the data storage + directory ahead of requesting CPF files. If True, then keep the + data storage directory. Outputs: - server -> [str] server for downloading CPF ephemeris files. Currently, only 'cddis.nasa.gov' and 'edc.dgfi.tum.de' are available. - dir_cpf_from -> [str] directory for storing CPF ephemeris files in remote server. - dir_cpf_to -> [str] user's local directory for storing CPF ephemeris files + server -> [str] server for downloading CPF ephemeris files. Currently, + only 'cddis.nasa.gov' and 'edc.dgfi.tum.de' are available. + + dir_cpf_from -> [str] directory for storing CPF ephemeris files in + remote server. + + dir_cpf_to -> [str] user's local directory for storing CPF ephemeris + files + cpf_files -> [str list] list of CPF ephemeris files - """ + """ if satnames is None: - raise Exception("satnames must be provided.") + raise Exception("satnames must be provided.") elif type(satnames) is str: reduplicates = [satnames] elif type(satnames) is list: reduplicates = satnames else: - raise Exception('Type of satname should be str or list.') - - dirs_cpf_from,cpf_files = [],[] + raise Exception('Type of satname should be str or list.') + + dirs_cpf_from, cpf_files = [], [] date_dir = date[:4] - date_str1 = Time(date).strftime('%y%m%d') - date_str2 = (Time(date)-7).strftime('%y%m%d') # ephemeris updates for some high-orbit satellites may take several days + date_str1 = Time(date).strftime('%y%m%d') + # ephemeris updates for some high-orbit satellites may take several days + date_str2 = (Time(date)-7*day).strftime('%y%m%d') date_str = Time(date).strftime('%Y%m%d%H%M%S') - dir_cpf_to = 'CPF/'+source+'/'+ date[:10] + '/' - - if not path.exists(dir_cpf_to): - makedirs(dir_cpf_to) + dir_cpf_to = 'CPF/'+source+'/' + date[:10] + '/' + + if not path.exists(dir_cpf_to): + makedirs(dir_cpf_to) else: - if not keep: system('rm -rf %s/*' % dir_cpf_to) - + if not keep: + system('rm -rf %s/*' % dir_cpf_to) + if source == 'CDDIS': - server = 'https://cddis.nasa.gov' + server = 'https://cddis.nasa.gov' + + if Time(date) < Time('2021-10-01'): + base_url = '/archive/slr/cpf_predicts/' + else: + base_url = '/archive/slr/cpf_predicts_v2/' for satname in reduplicates: cpf_files_list_reduced = [] find_flag = False - dir_cpf_from = '/archive/slr/cpf_predicts/' + date_dir + '/' + satname + '/' + dir_cpf_from = base_url + date_dir + '/' + satname + '/' dirs_cpf_from.append(dir_cpf_from) - cpf_files_dict,cpf_files_list = get_cpf_filelist(server,dir_cpf_from,'bydate') + cpf_files_dict, cpf_files_list = get_cpf_filelist( + server, + dir_cpf_from, + 'bydate') for cpf_file in cpf_files_list: cpf_file_date = cpf_file.split('_')[2] - if date_str2 <= cpf_file_date <= date_str1: cpf_files_list_reduced.append(cpf_file) - + if date_str2 <= cpf_file_date <= date_str1: + cpf_files_list_reduced.append(cpf_file) + if cpf_files_list_reduced: for cpf_file in cpf_files_list_reduced: # get the latest modification time for cpf files modified_time = cpf_files_dict[cpf_file] - # modify the time format from '2020:09:20 05:30:11' to '20200920053011' - for chara in [':',' ']: - modified_time = modified_time.replace(chara,'') + # modify the time format from '2020:09:20 05:30:11' + # to '20200920053011' + for chara in [':', ' ']: + modified_time = modified_time.replace(chara, '') - if modified_time < date_str: + if modified_time < date_str: find_flag = True - cpf_files.append(cpf_file) - break + cpf_files.append(cpf_file) + break - if not find_flag and date_str2[:2]>='05': + if not find_flag and date_str2[:2] >= '05': cpf_files_list_reduced = [] dirs_cpf_from.remove(dir_cpf_from) - dir_cpf_from = '/archive/slr/cpf_predicts/' + '20'+date_str2[:2] + '/' + satname + '/' + dir_cpf_from = '/archive/slr/cpf_predicts_v2/' + '20' + + date_str2[:2] + '/' + satname + '/' + dirs_cpf_from.append(dir_cpf_from) - cpf_files_dict,cpf_files_list = get_cpf_filelist(server,dir_cpf_from,'bydate') - + cpf_files_dict, cpf_files_list = get_cpf_filelist( + server, + dir_cpf_from, + 'bydate') + for cpf_file in cpf_files_list: cpf_file_date = cpf_file.split('_')[2] - if date_str2 <= cpf_file_date <= date_str1: cpf_files_list_reduced.append(cpf_file) - + if date_str2 <= cpf_file_date <= date_str1: + cpf_files_list_reduced.append(cpf_file) + for cpf_file in cpf_files_list_reduced: # get the latest modification time for cpf files modified_time = cpf_files_dict[cpf_file] - # modify the time format from '2020:09:20 05:30:11' to '20200920053011' - for chara in [':',' ']: - modified_time = modified_time.replace(chara,'') + # modify the time format from '2020:09:20 05:30:11' + # to '20200920053011' + for chara in [':', ' ']: + modified_time = modified_time.replace(chara, '') - if modified_time <= date_str: + if modified_time <= date_str: cpf_files.append(cpf_file) - break + break elif source == 'EDC': - server = 'edc.dgfi.tum.de' - ftp = FTP(server,timeout=200) - ftp.login() + server = 'edc.dgfi.tum.de' + + if Time(date) < Time('2021-10-01'): + base_url = '~/slr/cpf_predicts/' + else: + base_url = '~/slr/cpf_predicts_v2/' + + ftp = FTP(server, timeout=200) + ftp.login() for satname in reduplicates: cpf_files_list_reduced = [] find_flag = False - dir_cpf_from = '~/slr/cpf_predicts//' + date_dir + '/' + satname + '/' + dir_cpf_from = base_url + date_dir + '/' + satname + '/' dirs_cpf_from.append(dir_cpf_from) ftp.cwd(dir_cpf_from) - cpf_files_list = ftp.nlst('-t','*cpf*') # list files containing 'cpf' from newest to oldest - + # list files containing 'cpf' from newest to oldest + cpf_files_list = ftp.nlst('-t', '*cpf*') + for cpf_file in cpf_files_list: cpf_file_date = cpf_file.split('_')[2] - if date_str2 <= cpf_file_date <= date_str1: cpf_files_list_reduced.append(cpf_file) - + if date_str2 <= cpf_file_date <= date_str1: + cpf_files_list_reduced.append(cpf_file) + if cpf_files_list_reduced: for cpf_file in cpf_files_list_reduced: # get the latest modification time for cpf files modified_time = ftp.voidcmd('MDTM ' + cpf_file).split()[1] - if modified_time < date_str: + if modified_time < date_str: find_flag = True - cpf_files.append(cpf_file) - break - - if not find_flag and date_str2[:2]>='05': + cpf_files.append(cpf_file) + break + + if not find_flag and date_str2[:2] >= '05': cpf_files_list_reduced = [] dirs_cpf_from.remove(dir_cpf_from) - dir_cpf_from = '~/slr/cpf_predicts//' + '20'+date_str2[:2] + '/' + satname + '/' + dir_cpf_from = '~/slr/cpf_predicts_v2//' + '20'+date_str2[:2] + + '/' + satname + '/' dirs_cpf_from.append(dir_cpf_from) ftp.cwd(dir_cpf_from) - cpf_files_list = ftp.nlst('-t','*cpf*') # list files containing 'cpf' from newest to oldest - + # list files containing 'cpf' from newest to oldest + cpf_files_list = ftp.nlst('-t', '*cpf*') + for cpf_file in cpf_files_list: cpf_file_date = cpf_file.split('_')[2] - if date_str2 <= cpf_file_date <= date_str1: cpf_files_list_reduced.append(cpf_file) - + if date_str2 <= cpf_file_date <= date_str1: + cpf_files_list_reduced.append(cpf_file) + for cpf_file in cpf_files_list_reduced: # get the latest modification time for cpf files modified_time = ftp.voidcmd('MDTM ' + cpf_file).split()[1] - if modified_time <= date_str: + if modified_time <= date_str: cpf_files.append(cpf_file) - break - ftp.quit() - ftp.close() - - else: - raise Exception("Currently, CPF predictions only from 'CDDIS' and 'EDC' are available.") - - return server,dirs_cpf_from,dir_cpf_to,cpf_files - -def cpf_download_prior(satnames = None,date = None,source = 'CDDIS',keep=True): + break + ftp.quit() + ftp.close() + + else: + raise Exception( + "Currently, CPF predictions only from 'CDDIS'" + + " and 'EDC' are available.") + + return server, dirs_cpf_from, dir_cpf_to, cpf_files + + +def cpf_download_prior(satnames=None, date=None, source='CDDIS', keep=True): """ Download the latest CPF ephemeris files. - Usage: + Usage: dir_cpf_files = cpf_download() dir_cpf_files = cpf_download(source = 'EDC') dir_cpf_files = cpf_download('lageos1') dir_cpf_files = cpf_download('ajisai','2007-06-01 11:30:00') - dir_cpf_files = cpf_download(['ajisai','lageos1','hy2a'],'2007-06-01 11:30:00','EDC') + dir_cpf_files = cpf_download(['ajisai','lageos1','hy2a'], + '2007-06-01 11:30:00', + 'EDC') Parameters: - satnames -> [str, str list, default = None] target name or list of target names. If None, then all feasible targets at the current moment will be downloaded. In this case, 'date' must also be None. - date -> [str, default = None] 'iso-formatted' time, such as '2017-12-20 05:30:00'. It specifies a moment before which the latest CPF ephemeris files are downloaded. If None, then all feasible targets or targets in list at the current moment will be downloaded. - source -> [str, default = 'CDDIS'] source for CPF ephemeris files. Currently, only 'CDDIS' and 'EDC' are available. - keep -> [Bool, default = True] If False, clear the data storage directory ahead of requesting CPF files. If True, then keep the data in storage directory. - + satnames -> [str, str list, default = None] target name or list of + target names. If None, then all feasible targets at the current + moment will be downloaded. In this case, 'date' must also be None. + + date -> [str, default = None] 'iso-formatted' time, such as + '2017-12-20 05:30:00'. It specifies a moment before which the + latest CPF ephemeris files are downloaded. If None, then all + feasible targets or targets in list at the current moment will + be downloaded. + + source -> [str, default = 'CDDIS'] source for CPF ephemeris files. + Currently, only 'CDDIS' and 'EDC' are available. + + keep -> [Bool, default = True] If False, clear the data storage + directory ahead of requesting CPF files. If True, then keep the + data in storage directory. + Outputs: - dir_cpf_files -> [str list] list of paths for CPF ephemeris files in user's local directory - missing_cpf_files -> [str list or None] if not None, it lists files that are not responsed from the server + dir_cpf_files -> [str list] list of paths for CPF ephemeris files in + user's local directory + + missing_cpf_files -> [str list or None] if not None, it lists files + that are not responsed from the server Note: if 'date' is provided, then 'satnames' must be provided. """ - if source == 'CDDIS': # Need to create an Earthdata login account at https://urs.earthdata.nasa.gov/ + if source == 'CDDIS': + # Need to create an Earthdata login account at + # https://urs.earthdata.nasa.gov/ # Create a .netrc file in the home directory home = str(Path.home()) if not path.exists(home+'/.netrc'): - uid = input('Please input the Username for your EARTHDATA login account(which can be created at https://urs.earthdata.nasa.gov/): ') + uid = input( + 'Please input the Username for your EARTHDATA login account ' + + '(which can be created at https://urs.earthdata.nasa.gov/):') passwd = input('Please input the Password: ') - netrc_file = open(home+'/.netrc','w') - netrc_file.write('machine urs.earthdata.nasa.gov login '+uid+' password '+passwd) + netrc_file = open(home+'/.netrc', 'w') + netrc_file.write( + 'machine urs.earthdata.nasa.gov login ' + + uid + ' password ' + passwd) netrc_file.close() missing_cpf_files = [] - + if date is None: - server,dir_cpf_from, dir_cpf_to,cpf_files = download_bycurrent(source,satnames,keep) + server, dir_cpf_from, dir_cpf_to, cpf_files = download_bycurrent( + source, satnames, keep) if source == 'CDDIS': for cpf_file in cpf_files: url = server+dir_cpf_from+cpf_file desc = 'Downloading {:s}'.format(cpf_file) - missing_cpf_file = tqdm_request_cpf(url,dir_cpf_to,cpf_file,desc) - if missing_cpf_file is not None: missing_cpf_files.append(missing_cpf_file) - + missing_cpf_file = tqdm_request_http( + url, dir_cpf_to, cpf_file, desc) + if missing_cpf_file is not None: + missing_cpf_files.append(missing_cpf_file) + if source == 'EDC': - ftp = FTP(server,timeout=200) - ftp.login() - ftp.cwd(dir_cpf_from) + ftp = FTP(server, timeout=200) + ftp.login() + ftp.cwd(dir_cpf_from) for cpf_file in cpf_files: desc = 'Downloading {:s}'.format(cpf_file) - missing_cpf_file = tqdm_ftp(ftp,dir_cpf_to,cpf_file,desc) - if missing_cpf_file is not None: missing_cpf_files.append(missing_cpf_file) + missing_cpf_file = tqdm_ftp(ftp, dir_cpf_to, cpf_file, desc) + if missing_cpf_file is not None: + missing_cpf_files.append(missing_cpf_file) - ftp.quit() - ftp.close() - - else: - server,dirs_cpf_from, dir_cpf_to,cpf_files = download_bydate(source,date,satnames,keep) + ftp.quit() + ftp.close() + + else: + server, dirs_cpf_from, dir_cpf_to, cpf_files = download_bydate( + source, date, satnames, keep) if source == 'CDDIS': - for dir_cpf_from,cpf_file in zip(dirs_cpf_from,cpf_files): + for dir_cpf_from, cpf_file in zip(dirs_cpf_from, cpf_files): url = server+dir_cpf_from+cpf_file desc = 'Downloading {:s}'.format(cpf_file) - missing_cpf_file = tqdm_request_cpf(url,dir_cpf_to,cpf_file,desc) - if missing_cpf_file is not None: missing_cpf_files.append(missing_cpf_file) + missing_cpf_file = tqdm_request_http( + url, dir_cpf_to, cpf_file, desc) + if missing_cpf_file is not None: + missing_cpf_files.append(missing_cpf_file) - if source == 'EDC': - ftp = FTP(server,timeout=200) - ftp.login() + if source == 'EDC': + ftp = FTP(server, timeout=200) + ftp.login() - for dir_cpf_from,cpf_file in zip(dirs_cpf_from,cpf_files): + for dir_cpf_from, cpf_file in zip(dirs_cpf_from, cpf_files): ftp.cwd(dir_cpf_from) desc = 'Downloading {:s}'.format(cpf_file) - missing_cpf_file = tqdm_ftp(ftp,dir_cpf_to,cpf_file,desc) - if missing_cpf_file is not None: missing_cpf_files.append(missing_cpf_file) + missing_cpf_file = tqdm_ftp(ftp, dir_cpf_to, cpf_file, desc) + if missing_cpf_file is not None: + missing_cpf_files.append(missing_cpf_file) - ftp.quit() - ftp.close() + ftp.quit() + ftp.close() - return dir_cpf_to,cpf_files,missing_cpf_files - -def cpf_download(satnames = None,date = None,source = 'CDDIS',keep=True): + return dir_cpf_to, cpf_files, missing_cpf_files + + +def cpf_download(satnames=None, date=None, source='CDDIS', keep=True): """ Download the latest CPF ephemeris files. - Usage: + Usage: dir_cpf_files = cpf_download() dir_cpf_files = cpf_download(source = 'EDC') dir_cpf_files = cpf_download('lageos1') dir_cpf_files = cpf_download('ajisai','2007-06-01 11:30:00') - dir_cpf_files = cpf_download(['ajisai','lageos1','hy2a'],'2007-06-01 11:30:00','EDC') + dir_cpf_files = cpf_download(['ajisai','lageos1','hy2a'], + '2007-06-01 11:30:00','EDC') Parameters: - satnames -> [str, str list, default = None] target name or list of target names. If None, then all feasible targets at the current moment will be downloaded. In this case, 'date' must also be None. - date -> [str, default = None] 'iso-formatted' time, such as '2017-12-20 05:30:00'. It specifies a moment before which the latest CPF ephemeris files are downloaded. If None, then all feasible targets or targets in list at the current moment will be downloaded. - source -> [str, default = 'CDDIS'] source for CPF ephemeris files. Currently, only 'CDDIS' and 'EDC' are available. - keep -> [Bool, default = True] If False, clear the data storage directory ahead of requesting CPF files. If True, then keep the data in storage directory. - + satnames -> [str, str list, default = None] target name or list of + target names. If None, then all feasible targets at the current + moment will be downloaded. In this case, 'date' must also be None. + + date -> [str, default = None] 'iso-formatted' time, such as + '2017-12-20 05:30:00'. It specifies a moment before which the + latest CPF ephemeris files are downloaded. If None, then all + feasible targets or targets in list at the current moment will be + downloaded. + + source -> [str, default = 'CDDIS'] source for CPF ephemeris files. + Currently, only 'CDDIS' and 'EDC' are available. + keep -> [Bool, default = True] If False, clear the data storage + directory ahead of requesting CPF files. If True, then keep the + data in storage directory. + Outputs: dir_cpf_to -> [str] paths for storing CPF files cpf_files -> [list of str] list of CPF ephemeris files in CPF directory Note: if 'date' is provided, then 'satnames' must also be provided. - """ - dir_cpf_to, cpf_files, cpf_files_missed = cpf_download_prior(satnames,date,source,keep) - - if cpf_files_missed: warn('The following cpf files are faild to download:',cpf_files_missed) + """ + dir_cpf_to, cpf_files, cpf_files_missed = cpf_download_prior( + satnames, date, source, keep) - return dir_cpf_to, cpf_files + if cpf_files_missed: + warn( + 'The following cpf files are faild to download:', + cpf_files_missed) -def get_cpf_filelist(server,dir_cpf_from,mode): + return dir_cpf_to, cpf_files + + +def get_cpf_filelist(server, dir_cpf_from, mode): """ Generate CDDIS CPF files list sorted by date from the latest to the oldest. """ res = requests.get(server + dir_cpf_from) soup = BeautifulSoup(res.text, 'html.parser') + # Remove MD5SUMS and SHA512SUMS files manually + # Done this way as not every directory has the checksum files present + item_to_delete = soup.find(id="MD5SUMS") + if item_to_delete: + item_to_delete.parent.decompose() + item_to_delete = soup.find(id="SHA512SUMS") + if item_to_delete: + item_to_delete.parent.decompose() + # extract time infomation time_info = soup.find_all('span') - if mode == 'bycurrent': - time_list = [ele.get_text().split(' ')[0] for ele in time_info][2:] # Remove two extra items - elif mode == 'bydate': - time_list = [ele.get_text().split(' ')[0] for ele in time_info] + time_list = [ele.get_text().split(' ')[0] for ele in time_info] + n_time_list = len(time_list) # extract filename infomation filename_info = soup.find_all('a') - cpf_files_list_unsort = [ele.get_text() for ele in filename_info if '_cpf_' in ele.get_text()] + cpf_files_list_unsort = [ele.get_text() + for ele in filename_info if '_cpf_' + in ele.get_text()] n_cpf_files_list_unsort = len(cpf_files_list_unsort) if n_time_list != n_cpf_files_list_unsort: raise Exception('Timestamp and CPF files are not matched!') - cpf_files_dict = dict(zip(cpf_files_list_unsort,time_list)) - cpf_files_turple = sorted(cpf_files_dict.items(), key=lambda x: x[1],reverse=True) # Sort by release time + cpf_files_dict = dict(zip(cpf_files_list_unsort, time_list)) + cpf_files_turple = sorted(cpf_files_dict.items( + ), key=lambda x: x[1], reverse=True) # Sort by release time cpf_files_list = [ele[0] for ele in cpf_files_turple] res.close() - return cpf_files_dict, cpf_files_list + return cpf_files_dict, cpf_files_list -def get_cpf_satlist(source = 'CDDIS'): + +def get_cpf_satlist(source='CDDIS'): """ - Generate the CPF satellite list sorted by date from the latest to the oldest. + Generate the CPF satellite list sorted by date from the latest to + the oldest. """ - if source == 'CDDIS': # Need to create an Earthdata login account at https://urs.earthdata.nasa.gov/ + if source == 'CDDIS': + # Need to create an Earthdata login account + # at https://urs.earthdata.nasa.gov/ # Create a .netrc file in the home directory home = str(Path.home()) if not path.exists(home+'/.netrc'): - uid = input('Please input the Username for your EARTHDATA login account(which can be created at https://urs.earthdata.nasa.gov/): ') + uid = input( + 'Please input the Username for your EARTHDATA login account ' + + '(which can be created at https://urs.earthdata.nasa.gov/):') passwd = input('Please input the Password: ') - netrc_file = open(home+'/.netrc','w') - netrc_file.write('machine urs.earthdata.nasa.gov login '+uid+' password '+passwd) + netrc_file = open(home+'/.netrc', 'w') + netrc_file.write( + 'machine urs.earthdata.nasa.gov login ' + uid + ' password ' + + passwd) netrc_file.close() - - server,dir_cpf_from, dir_cpf_to,cpf_files = download_bycurrent(source) - cpf_satlist = [cpf_file.split('_cpf')[0] for cpf_file in cpf_files] - - return cpf_satlist \ No newline at end of file + + server, dir_cpf_from, dir_cpf_to, cpf_files = download_bycurrent(source) + cpf_satlist = [cpf_file.split('_cpf')[0] for cpf_file in cpf_files] + + return cpf_satlist diff --git a/slrfield/utils/data_download.py b/slrfield/utils/data_download.py index 32e405d..3e98a3b 100644 --- a/slrfield/utils/data_download.py +++ b/slrfield/utils/data_download.py @@ -1,48 +1,55 @@ -from datetime import datetime,timedelta -from os import path,makedirs,remove +from datetime import datetime, timedelta +from os import path, makedirs, remove from pathlib import Path -from ftplib import FTP -from .try_download import tqdm_ftp +from .try_download import tqdm_request_http -def download_eop(out_days=7,dir_to=None): + +def download_eop(out_days=7, dir_to=None): """ Download or update the Earth Orientation Parameters(EOP) file from IERS - Usage: + Usage: >>> eop_file = download_eop() - Parameters: + Parameters: out_days -> [int, default = 7] Update cycle of the EOP file dir_to -> [str, default = None] Directory for storing EOP file - - Outputs: + + Outputs: eop_file -> [str] Path of the EOP file """ if dir_to is None: home = str(Path.home()) dir_to = home + '/src/eop-data/' - - file = 'finals2000A.all' + + file = 'finals.all.iau2000.txt' dir_file = dir_to + file - server = 'ftp.iers.org' - src_dir = '~/products/eop/rapid/standard/' + url = 'https://datacenter.iers.org/data/latestVersion/finals.all.iau2000.txt' - if not path.exists(dir_to): makedirs(dir_to) - if not path.exists(dir_file): - ftp_access(server,src_dir) - desc = "Downloading the latest EOP '{:s}' from IERS".format(file) - tqdm_ftp(ftp,dir_to,file,desc) - else: + if not path.exists(dir_to): + makedirs(dir_to) + + if path.exists(dir_file): + # First check if the file needs updating modified_time = datetime.fromtimestamp(path.getmtime(dir_file)) if datetime.now() > modified_time + timedelta(days=out_days): remove(dir_file) - ftp_access(server,src_dir) desc = "Updating EOP '{:s}' from IERS".format(file) - tqdm_ftp(ftp,dir_to,file,desc) else: - print("EOP '{0:s}' in {1:s} is already the latest.".format(file,dir_to)) + print("EOP '{0:s}' in {1:s} is already the latest.".format( + file, dir_to)) + return dir_file + else: + desc = "Downloading the latest EOP '{:s}' from IERS".format(file) + + if tqdm_request_http(url, dir_to, file, desc) is not None: + print('SLRfield was unable to import Earth Orientation Parameters ' + + '(EOP) data. It is recommended to try again or to manually' + + 'download this data (finals.all.iau2000.txt) and place it ' + + f'in {dir_to}.') + raise ImportError('Unable to download EOP data') - return dir_file \ No newline at end of file + return dir_file diff --git a/slrfield/utils/data_prepare.py b/slrfield/utils/data_prepare.py index 1749236..fe243e8 100644 --- a/slrfield/utils/data_prepare.py +++ b/slrfield/utils/data_prepare.py @@ -1,11 +1,13 @@ from astropy.utils import iers as iers_astropy + from .data_download import download_eop -def time_load(): +def time_load(): # load the EOP file eop_file = download_eop() # for astropy - iers_a = iers_astropy.IERS_A.open(eop_file) - eop_table = iers_astropy.earth_orientation_table.set(iers_a) \ No newline at end of file + if eop_file is None: + iers_a = iers_astropy.IERS_A.open(eop_file) + iers_astropy.earth_orientation_table.set(iers_a) diff --git a/slrfield/utils/try_download.py b/slrfield/utils/try_download.py index 94f1d80..8a0f025 100644 --- a/slrfield/utils/try_download.py +++ b/slrfield/utils/try_download.py @@ -1,12 +1,14 @@ from os import remove -import requests from tqdm import tqdm from colorama import Fore from time import sleep +import requests +from ftplib import all_errors + -def tqdm_ftp(ftp,dir_to,file,desc): +def tqdm_ftp(ftp, dir_to, file, desc): """ - Try to download files from remote server by ftp with a colored progress bar. + Try to download files from remote server by ftp with a colored progress bar """ dir_file = dir_to + file total_size = int(ftp.size(file)) @@ -14,26 +16,30 @@ def tqdm_ftp(ftp,dir_to,file,desc): for idownload in range(5): try: - local_file = open(dir_file, 'wb') + local_file = open(dir_file, 'wb') pos = local_file.tell() - pbar = tqdm(desc = desc,total=total_size,unit='B',unit_scale=True,position=0,initial=pos,bar_format=bar_format) + pbar = tqdm(desc=desc, total=total_size, unit='B', unit_scale=True, + position=0, initial=pos, bar_format=bar_format) + def progressbar(chunk): local_file.write(chunk) pbar.update(len(chunk)) - res = ftp.retrbinary('RETR ' + file, progressbar) # RETR is an FTP command - break - except: + # RETR is an FTP command + ftp.retrbinary('RETR ' + file, progressbar) + break + except all_errors: sleep(2) if idownload == 4: remove(dir_file) print('No response, skip {:s}'.format(file)) finally: pbar.close() - local_file.close() + local_file.close() + -def tqdm_request_cpf(url,dir_cpf_to,cpf_file,desc): +def tqdm_request_http(url, dir_cpf_to, cpf_file, desc): """ - Try to download cpf files from url with a colored progress bar. + Try to download CPF or OEP files from url with a colored progress bar. """ block_size = 1024*10 missing_flag = False @@ -42,25 +48,31 @@ def tqdm_request_cpf(url,dir_cpf_to,cpf_file,desc): try: local_file = open(dir_cpf_to + cpf_file, 'wb') pos = local_file.tell() - res = requests.get(url,stream=True,timeout=200,headers={'Accept-Encoding': None,'Range': f'bytes={pos}-'}) + res = requests.get(url, stream=True, timeout=200, headers={ + 'Accept-Encoding': None, + 'Range': f'bytes={pos}-'}) + res.raise_for_status() total_size = int(res.headers.get('Content-Length')) - pbar = tqdm(desc = desc,total=total_size,unit='B',unit_scale=True,bar_format = bar_format,position=0,initial=pos) + pbar = tqdm(desc=desc, total=total_size, unit='B', unit_scale=True, + bar_format=bar_format, position=0, initial=pos) for chunk in res.iter_content(block_size): - local_file.write(chunk) - pbar.update(len(chunk)) - pbar.close() - res.close() + local_file.write(chunk) + pbar.update(len(chunk)) + pbar.close() + res.close() break - except: + except (requests.HTTPError, + requests.ConnectionError, + requests.Timeout) as e: sleep(3) if idownload == 4: remove(dir_cpf_to + cpf_file) - print('No response, skip {:s}'.format(cpf_file)) + print(f'No response {str(e)}, skip {cpf_file}') missing_flag = True - finally: - local_file.close() - if missing_flag: - return cpf_file - else: - return None - + finally: + local_file.close() + + if missing_flag: + return cpf_file + else: + return None