source: XIOS/dev/branch_openmp/extern/src_netcdf4/ochttp.c @ 1501

Last change on this file since 1501 was 409, checked in by ymipsl, 11 years ago

Add improved nectdf internal library src

YM

  • Property svn:eol-style set to native
File size: 8.3 KB
Line 
1/* Copyright 2009, UCAR/Unidata and OPeNDAP, Inc.
2 See the COPYRIGHT file for more information. */
3
4#include "config.h"
5#include <sys/stat.h>
6#ifdef HAVE_UNISTD_H
7#include <unistd.h>
8#endif
9#include <fcntl.h>
10#include "ocinternal.h"
11#include "ocdebug.h"
12#include "ochttp.h"
13#include "ocrc.h"
14
15static size_t WriteFileCallback(void*, size_t, size_t, void*);
16static size_t WriteMemoryCallback(void*, size_t, size_t, void*);
17
18struct Fetchdata {
19        FILE* stream;
20        size_t size;
21};
22
23long
24ocfetchhttpcode(CURL* curl)
25{
26    long httpcode;
27    CURLcode cstat = CURLE_OK;
28    /* Extract the http code */
29    cstat = curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE,&httpcode);
30    if(cstat != CURLE_OK) httpcode = 0;
31    return httpcode;
32}
33
34int
35ocfetchurl_file(CURL* curl, const char* url, FILE* stream,
36                unsigned long* sizep, long* filetime)
37{
38        int stat = OC_NOERR;
39        CURLcode cstat = CURLE_OK;
40        struct Fetchdata fetchdata;
41
42        /* Set the URL */
43        cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)url);
44        if (cstat != CURLE_OK)
45                goto fail;
46
47        /* send all data to this function  */
48        cstat = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteFileCallback);
49        if (cstat != CURLE_OK)
50                goto fail;
51
52        /* we pass our file to the callback function */
53        cstat = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&fetchdata);
54        if (cstat != CURLE_OK)
55                goto fail;
56
57        /* One last thing; always try to get the last modified time */
58        cstat = curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1);
59        if (cstat != CURLE_OK)
60                goto fail;
61
62        fetchdata.stream = stream;
63        fetchdata.size = 0;
64        cstat = curl_easy_perform(curl);
65        if (cstat != CURLE_OK)
66            goto fail;
67
68        if (stat == OC_NOERR) {
69            /* return the file size*/
70#ifdef OCDEBUG
71            oc_log(LOGNOTE,"filesize: %lu bytes",fetchdata.size);
72#endif
73            if (sizep != NULL)
74                *sizep = fetchdata.size;
75            /* Get the last modified time */
76            if(filetime != NULL)
77                cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime);
78            if(cstat != CURLE_OK) goto fail;
79        }
80        return OCTHROW(stat);
81
82fail:
83        oc_log(LOGERR, "curl error: %s", curl_easy_strerror(cstat));
84        return OCTHROW(OC_ECURL);
85}
86
87int
88ocfetchurl(CURL* curl, const char* url, OCbytes* buf, long* filetime)
89{
90        int stat = OC_NOERR;
91        CURLcode cstat = CURLE_OK;
92        size_t len;
93
94        /* Set the URL */
95        cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)url);
96        if (cstat != CURLE_OK)
97                goto fail;
98
99        /* send all data to this function  */
100        cstat = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
101        if (cstat != CURLE_OK)
102                goto fail;
103
104        /* we pass our file to the callback function */
105        cstat = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)buf);
106        if (cstat != CURLE_OK)
107                goto fail;
108
109        /* One last thing; always try to get the last modified time */
110        cstat = curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1);
111
112        cstat = curl_easy_perform(curl);
113        if(cstat == CURLE_PARTIAL_FILE) {
114            /* Log it but otherwise ignore */
115            oc_log(LOGWARN, "curl error: %s; ignored",
116                   curl_easy_strerror(cstat));
117            cstat = CURLE_OK;
118        }
119        if(cstat != CURLE_OK) goto fail;
120
121        /* Get the last modified time */
122        if(filetime != NULL)
123            cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime);
124        if(cstat != CURLE_OK) goto fail;
125
126        /* Null terminate the buffer*/
127        len = ocbyteslength(buf);
128        ocbytesappend(buf, '\0');
129        ocbytessetlength(buf, len); /* dont count null in buffer size*/
130#ifdef OCDEBUG
131        oc_log(LOGNOTE,"buffersize: %lu bytes",(unsigned long)ocbyteslength(buf));
132#endif
133
134        return OCTHROW(stat);
135
136fail:
137        oc_log(LOGERR, "curl error: %s", curl_easy_strerror(cstat));
138        return OCTHROW(OC_ECURL);
139}
140
141static size_t
142WriteFileCallback(void* ptr, size_t size, size_t nmemb, void* data)
143{
144        size_t realsize = size * nmemb;
145        size_t count;
146        struct Fetchdata* fetchdata;
147        fetchdata = (struct Fetchdata*) data;
148        if(realsize == 0)
149            oc_log(LOGWARN,"WriteFileCallback: zero sized chunk");
150        count = fwrite(ptr, size, nmemb, fetchdata->stream);
151        if (count > 0) {
152                fetchdata->size += (count * size);
153        } else {
154            oc_log(LOGWARN,"WriteFileCallback: zero sized write");
155        }
156#ifdef OCPROGRESS
157        oc_log(LOGNOTE,"callback: %lu bytes",(unsigned long)realsize);
158#endif
159        return count;
160}
161
162static size_t
163WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data)
164{
165        size_t realsize = size * nmemb;
166        OCbytes* buf = (OCbytes*) data;
167        if(realsize == 0)
168            oc_log(LOGWARN,"WriteMemoryCallback: zero sized chunk");
169        /* Optimize for reading potentially large dods datasets */
170        if(!ocbytesavail(buf,realsize)) {
171            /* double the size of the packet */
172            ocbytessetalloc(buf,2*ocbytesalloc(buf));
173        }
174        ocbytesappendn(buf, ptr, realsize);
175#ifdef OCPROGRESS
176        oc_log(LOGNOTE,"callback: %lu bytes",(unsigned long)realsize);
177#endif
178        return realsize;
179}
180
181#if 0
182static void
183assembleurl(DAPURL* durl, OCbytes* buf, int what)
184{
185        encodeurltext(durl->url,buf);
186        if(what & WITHPROJ) {
187                ocbytescat(buf,"?");
188                encodeurltext(durl->projection,buf);
189        }
190        if(what & WITHSEL) encodeurltext(durl->selection,buf);
191
192}
193
194static char mustencode="";
195static char hexchars[16] = {
196        '0', '1', '2', '3',
197        '4', '5', '6', '7',
198        '8', '9', 'a', 'b',
199        'c', 'd', 'e', 'f',
200};
201
202static void
203encodeurltext(char* text, OCbytes* buf)
204{
205        /* Encode the URL to handle illegal characters */
206        len = strlen(url);
207        encoded = ocmalloc(len*4+1); /* should never be larger than this*/
208        if(encoded==NULL) return;
209        p = url; q = encoded;
210        while((c=*p++)) {
211                if(strchr(mustencode,c) != NULL) {
212                        char tmp[8];
213                        int hex1, hex2;
214                        hex1 = (c & 0x0F);
215                        hex2 = (c & 0xF0) >> 4;
216                        tmp[0] = '0'; tmp[1] = 'x';
217                        tmp[2] = hexchars[hex2]; tmp[3] = hexchars[hex1];
218                        tmp[4] = '\0';
219                        ocbytescat(buf,tmp);
220                } else *q++ = (char)c;
221        }
222
223}
224
225#endif
226
227int
228occurlopen(CURL** curlp)
229{
230        int stat = OC_NOERR;
231        CURLcode cstat = CURLE_OK;
232        CURL* curl;
233        /* initialize curl*/
234        curl = curl_easy_init();
235        if (curl == NULL)
236                stat = OC_ECURL;
237        else {
238                cstat = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
239                if (cstat != CURLE_OK)
240                        stat = OC_ECURL;
241                /* some servers don't like requests that are made without a user-agent */
242                cstat = curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0");
243                if (cstat != CURLE_OK)
244                        stat = OC_ECURL;
245        }
246        if (curlp)
247                *curlp = curl;
248        return OCTHROW(stat);
249}
250
251void
252occurlclose(CURL* curl)
253{
254        if (curl != NULL)
255                curl_easy_cleanup(curl);
256}
257
258int
259ocfetchlastmodified(CURL* curl, char* url, long* filetime)
260{
261    int stat = OC_NOERR;
262    CURLcode cstat = CURLE_OK;
263
264    /* Set the URL */
265    cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)url);
266    if (cstat != CURLE_OK)
267        goto fail;
268
269    /* Ask for head */
270    cstat = curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30); /* 30sec timeout*/
271    cstat = curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 2);
272    cstat = curl_easy_setopt(curl, CURLOPT_HEADER, 1);
273    cstat = curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
274    cstat = curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
275    cstat = curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1);
276
277    cstat = curl_easy_perform(curl);
278    if(cstat != CURLE_OK) goto fail;
279    if(filetime != NULL)
280        cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime);
281    if(cstat != CURLE_OK) goto fail;
282
283    return OCTHROW(stat);
284
285fail:
286    oc_log(LOGERR, "curl error: %s", curl_easy_strerror(cstat));
287    return OCTHROW(OC_ECURL);
288}
289
290int
291ocping(const char* url)
292{
293    int stat = OC_NOERR;
294    CURLcode cstat = CURLE_OK;
295    CURL* curl = NULL;
296    OCbytes* buf = NULL;
297
298    /* Create a CURL instance */
299    stat = occurlopen(&curl);
300    if(stat != OC_NOERR) return stat;   
301
302    /* use a very short timeout: 10 seconds */
303    cstat = curl_easy_setopt(curl, CURLOPT_TIMEOUT, (long)10);
304    if (cstat != CURLE_OK)
305        goto done;
306
307    /* fail on HTTP 400 code errors */
308    cstat = curl_easy_setopt(curl, CURLOPT_FAILONERROR, (long)1);
309    if (cstat != CURLE_OK)
310        goto done;
311
312    /* Try to get the file */
313    buf = ocbytesnew();
314    stat = ocfetchurl(curl,url,buf,NULL);
315    if(stat == OC_NOERR) {
316        /* Don't trust curl to return an error when request gets 404 */
317        long http_code = 0;
318        cstat = curl_easy_getinfo(curl,CURLINFO_RESPONSE_CODE, &http_code);
319        if (cstat != CURLE_OK)
320            goto done;
321        if(http_code >= 400) {
322            cstat = CURLE_HTTP_RETURNED_ERROR;
323            goto done;
324        }
325    } else
326        goto done;
327
328done:
329    ocbytesfree(buf);
330    occurlclose(curl);
331    if(cstat != CURLE_OK) {
332        oc_log(LOGERR, "curl error: %s", curl_easy_strerror(cstat));
333        stat = OC_EDAPSVC;
334    }
335    return OCTHROW(stat);
336}
337
338
Note: See TracBrowser for help on using the repository browser.