[109] | 1 | #!/usr/bin/env python |
---|
| 2 | # -*- coding: iso-8859-1 -*- |
---|
| 3 | |
---|
| 4 | """ |
---|
| 5 | DESCRIPTION |
---|
| 6 | =========== |
---|
| 7 | |
---|
| 8 | Toolbox to acces to ECMWF server |
---|
| 9 | |
---|
| 10 | SEE ALSO |
---|
| 11 | ======== |
---|
| 12 | |
---|
| 13 | called by get_erai.py (experimental) |
---|
| 14 | |
---|
| 15 | EVOLUTIONS |
---|
| 16 | ========== |
---|
| 17 | |
---|
| 18 | $Id$ |
---|
| 19 | |
---|
| 20 | $URL$ |
---|
| 21 | |
---|
| 22 | - fplod 20111130T153147Z cratos (Linux) |
---|
| 23 | |
---|
| 24 | * got from http://data-portal.ecmwf.int/data/d/token/interim_full_daily/ |
---|
| 25 | * add docstring |
---|
| 26 | |
---|
| 27 | """ |
---|
| 28 | import urllib |
---|
| 29 | import urllib2 |
---|
| 30 | import time |
---|
| 31 | import datetime |
---|
| 32 | |
---|
| 33 | class ECMWFDataServer: |
---|
| 34 | def __init__(self,portal,token,email): |
---|
| 35 | self.version = '0.3' |
---|
| 36 | self.portal = portal |
---|
| 37 | self.token = token |
---|
| 38 | self.email = email |
---|
| 39 | |
---|
| 40 | def _call(self,action,args): |
---|
| 41 | |
---|
| 42 | params = {'_token' : self.token, |
---|
| 43 | '_email' : self.email, |
---|
| 44 | '_action' : action, |
---|
| 45 | '_version' : self.version} |
---|
| 46 | params.update(args) |
---|
| 47 | |
---|
| 48 | data = urllib.urlencode(params) |
---|
| 49 | req = urllib2.Request(self.portal, data) |
---|
| 50 | response = urllib2.urlopen(req) |
---|
| 51 | |
---|
| 52 | json = response.read(); |
---|
| 53 | |
---|
| 54 | undef = None; |
---|
| 55 | json = eval(json) |
---|
| 56 | |
---|
| 57 | if json != None: |
---|
| 58 | if 'error' in json: |
---|
| 59 | raise RuntimeError(json['error']) |
---|
| 60 | if 'message' in json: |
---|
| 61 | self.put(json['message']) |
---|
| 62 | |
---|
| 63 | return json |
---|
| 64 | |
---|
| 65 | def put(self,*args): |
---|
| 66 | print datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), |
---|
| 67 | for a in args: |
---|
| 68 | print a, |
---|
| 69 | print |
---|
| 70 | |
---|
| 71 | |
---|
| 72 | def retrieve(self,args): |
---|
| 73 | self.put("ECMWF data server batch tool version",self.version); |
---|
| 74 | user = self._call("user_info",{}); |
---|
| 75 | self.put("Welcome to",user['name'], "from" , user['organisation']); |
---|
| 76 | |
---|
| 77 | r = self._call('retrieve',args) |
---|
| 78 | rid = r['request'] |
---|
| 79 | |
---|
| 80 | last = '' |
---|
| 81 | sleep = 0 |
---|
| 82 | while r['status'] != 'complete' and r['status'] != 'aborted': |
---|
| 83 | text = r['status'] + '.' |
---|
| 84 | if 'info' in r and r['info'] != None: |
---|
| 85 | text = text + ' ' + r['info'] |
---|
| 86 | |
---|
| 87 | if text != last: |
---|
| 88 | self.put("Request",text) |
---|
| 89 | last = text |
---|
| 90 | |
---|
| 91 | time.sleep(sleep) |
---|
| 92 | r = self._call('status',{'request':rid}) |
---|
| 93 | if sleep < 60: |
---|
| 94 | sleep = sleep + 1 |
---|
| 95 | |
---|
| 96 | if r['status'] != last: |
---|
| 97 | self.put("Request",r['status']) |
---|
| 98 | |
---|
| 99 | if 'reason' in r: |
---|
| 100 | for m in r['reason']: |
---|
| 101 | self.put(m) |
---|
| 102 | |
---|
| 103 | if 'result' in r: |
---|
| 104 | size = long(r['size']) |
---|
| 105 | self.put("Downloading",self._bytename(size)) |
---|
| 106 | done = self._transfer(r['result'],args['target']) |
---|
| 107 | self.put("Done") |
---|
| 108 | if done != size: |
---|
| 109 | raise RuntimeError("Size mismatch: " + str(done) + " and " + str(size)) |
---|
| 110 | |
---|
| 111 | self._call('delete',{'request':rid}) |
---|
| 112 | |
---|
| 113 | if r['status'] == 'aborted': |
---|
| 114 | raise RuntimeError("Request aborted") |
---|
| 115 | |
---|
| 116 | def _transfer(self,url,path): |
---|
| 117 | result = urllib.urlretrieve(url,path) |
---|
| 118 | return long(result[1]['content-length']) |
---|
| 119 | |
---|
| 120 | def _bytename(self,size): |
---|
| 121 | next = {'':'K','K':'M','M':'G','G':'T','T':'P'} |
---|
| 122 | l = '' |
---|
| 123 | size = size*1.0 |
---|
| 124 | while 1024 < size: |
---|
| 125 | l = next[l] |
---|
| 126 | size = size / 1024 |
---|
| 127 | return "%g %sbyte%s" % (size,l,'s') |
---|
| 128 | |
---|