Skip to content

Commit 589bfcc

Browse files
bachuebachue
committed
now upload APIs could switch upHosts
1 parent 2b6d3ed commit 589bfcc

15 files changed

+465
-222
lines changed

‎qiniu/base.c

+12
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,18 @@ void Qiniu_FreeV2(void **addr)
3131
*addr = NULL;
3232
}
3333
}
34+
void Qiniu_Multi_Free(int n, ...)
35+
{
36+
void *p = NULL;
37+
va_list v1;
38+
va_start(v1, n);
39+
for (int i = 0; i < n; i++)
40+
{
41+
p = va_arg(v1, void *);
42+
free(p);
43+
}
44+
va_end(v1);
45+
}
3446

3547
/*============================================================================*/
3648
/* type Qiniu_Count */

‎qiniu/base.h

+1
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ typedef unsigned long long Qiniu_Uint64;
110110

111111
QINIU_DLLAPI extern void Qiniu_Free(void *addr);
112112
QINIU_DLLAPI extern void Qiniu_FreeV2(void **addr);
113+
QINIU_DLLAPI extern void Qiniu_Multi_Free(int n, ...);
113114
/*============================================================================*/
114115
/* type Qiniu_Count */
115116

‎qiniu/code.c

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#include <curl/curl.h>
2+
#include "private/code.h"
3+
4+
Qiniu_Retry_Decision _Qiniu_Should_Retry(int code)
5+
{
6+
if (code / 100 == 4 && code != 406 && code != 429)
7+
{
8+
return QINIU_DONT_RETRY;
9+
}
10+
switch (code)
11+
{
12+
case CURLE_COULDNT_RESOLVE_HOST:
13+
case CURLE_COULDNT_CONNECT:
14+
case CURLE_WEIRD_SERVER_REPLY:
15+
case CURLE_PARTIAL_FILE:
16+
case CURLE_UPLOAD_FAILED:
17+
case CURLE_OPERATION_TIMEDOUT:
18+
case CURLE_SSL_CONNECT_ERROR:
19+
case CURLE_SEND_ERROR:
20+
case CURLE_RECV_ERROR:
21+
case CURLE_SSL_CERTPROBLEM:
22+
case CURLE_SSL_CIPHER:
23+
case CURLE_PEER_FAILED_VERIFICATION:
24+
case CURLE_HTTP2_STREAM:
25+
case CURLE_HTTP3:
26+
case CURLE_QUIC_CONNECT_ERROR:
27+
case 406:
28+
case 429:
29+
case 500:
30+
case 502:
31+
case 503:
32+
case 504:
33+
case 509:
34+
case 571:
35+
case 573:
36+
case 599:
37+
return QINIU_TRY_NEXT_DOMAIN;
38+
39+
default:
40+
return QINIU_DONT_RETRY;
41+
}
42+
}

‎qiniu/http.c

+7-1
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,7 @@ void Qiniu_Client_InitEx(Qiniu_Client *self, Qiniu_Auth auth, size_t bufSize)
348348
Qiniu_Zero_Ptr(self);
349349
self->curl = curl_easy_init();
350350
self->auth = auth;
351+
self->retriesMax = SIZE_MAX;
351352

352353
Qiniu_Buffer_Init(&self->b, bufSize);
353354
Qiniu_Buffer_Init(&self->respHeader, bufSize);
@@ -396,6 +397,11 @@ void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long
396397
self->lowSpeedTime = lowSpeedTime;
397398
} // Qiniu_Client_SetLowSpeedLimit
398399

400+
void Qiniu_Client_SetMaximumRetries(Qiniu_Client *self, size_t retriesMax)
401+
{
402+
self->retriesMax = retriesMax;
403+
} // Qiniu_Client_SetMaximumRetries
404+
399405
void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs)
400406
{
401407
self->timeoutMs = timeoutMs;
@@ -475,7 +481,7 @@ static Qiniu_Error Qiniu_Client_callWithBody(
475481
Qiniu_Error err;
476482
const char *ctxType;
477483
char ctxLength[64], userAgent[64];
478-
Qiniu_Header *headers = NULL;
484+
Qiniu_Header *headers;
479485
CURL *curl = (CURL *)self->curl;
480486
err = Qiniu_Client_config(self);
481487
if (err.code != Qiniu_OK.code)

‎qiniu/http.h

+4
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,17 @@ extern "C"
139139

140140
// Millisecond timeout for the connection phase.
141141
long connectTimeoutMs;
142+
143+
// Max retries count.
144+
size_t retriesMax;
142145
};
143146
typedef struct _Qiniu_Client Qiniu_Client;
144147

145148
QINIU_DLLAPI extern void Qiniu_Client_InitEx(Qiniu_Client *self, Qiniu_Auth auth, size_t bufSize);
146149
QINIU_DLLAPI extern void Qiniu_Client_Cleanup(Qiniu_Client *self);
147150
QINIU_DLLAPI extern void Qiniu_Client_BindNic(Qiniu_Client *self, const char *nic);
148151
QINIU_DLLAPI extern void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long lowSpeedTime);
152+
QINIU_DLLAPI extern void Qiniu_Client_SetMaximumRetries(Qiniu_Client *self, size_t retries);
149153
QINIU_DLLAPI extern void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs);
150154
QINIU_DLLAPI extern void Qiniu_Client_SetConnectTimeout(Qiniu_Client *self, long connectTimeoutMs);
151155
QINIU_DLLAPI extern void Qiniu_Client_EnableAutoQuery(Qiniu_Client *self, Qiniu_Bool useHttps);

‎qiniu/io.c

+87-52
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "reader.h"
1212
#include "recorder_utils.h"
1313
#include "private/region.h"
14+
#include "private/code.h"
1415
#include <curl/curl.h>
1516

1617
/*============================================================================*/
@@ -58,20 +59,27 @@ CURL *Qiniu_Client_reset(Qiniu_Client *self);
5859

5960
Qiniu_Error Qiniu_callex(CURL *curl, Qiniu_Buffer *resp, Qiniu_Json **ret, Qiniu_Bool simpleError, Qiniu_Buffer *resph);
6061

61-
static Qiniu_Error Get_Qiniu_UpHost(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char **upHost)
62+
static Qiniu_Error Get_Qiniu_UpHosts(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char *const **upHosts, size_t *upHostsCount)
6263
{
6364
if (extra && extra->ipCount != 0)
6465
{
6566
Qiniu_Count oldIndex = Qiniu_Count_Inc(&extra->ipIndex);
66-
*upHost = extra->upIps[abs(oldIndex % extra->ipCount)];
67+
*upHosts = &extra->upIps[labs(oldIndex % extra->ipCount)];
68+
*upHostsCount = 1;
6769
}
6870
else if (extra && extra->upHost != NULL)
6971
{
70-
*upHost = extra->upHost;
72+
*upHosts = &extra->upHost;
73+
*upHostsCount = 1;
74+
}
75+
else if (extra && extra->upHosts != NULL && extra->upHostsCount != 0)
76+
{
77+
*upHosts = extra->upHosts;
78+
*upHostsCount = extra->upHostsCount;
7179
}
7280
else
7381
{
74-
Qiniu_Error err = _Qiniu_Region_Get_Up_Host(client, accessKey, bucketName, upHost);
82+
Qiniu_Error err = _Qiniu_Region_Get_Up_Hosts(client, accessKey, bucketName, upHosts, upHostsCount);
7583
if (err.code != Qiniu_OK.code)
7684
{
7785
return err;
@@ -150,48 +158,61 @@ static Qiniu_Error Qiniu_Io_call(
150158
{
151159
int retCode = 0;
152160
Qiniu_Error err;
153-
struct curl_slist *headers = NULL;
154-
const char *upHost = NULL;
161+
struct curl_slist *headers;
162+
const char *const *upHosts;
163+
const char *defaultUpHosts[] = {QINIU_UP_HOST};
164+
size_t upHostsCount;
155165

156-
err = Qiniu_Client_config(self);
166+
headers = curl_slist_append(NULL, "Expect:");
167+
err = Get_Qiniu_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount);
157168
if (err.code != Qiniu_OK.code)
158169
{
159170
return err;
160171
}
161-
err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost);
162-
if (err.code != Qiniu_OK.code)
172+
if (upHostsCount == 0)
163173
{
164-
return err;
174+
upHosts = defaultUpHosts;
175+
upHostsCount = 1;
165176
}
166177

167-
headers = curl_slist_append(NULL, "Expect:");
168-
169-
CURL *curl = Qiniu_Client_reset(self);
170-
curl_easy_setopt(curl, CURLOPT_URL, upHost);
171-
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
172-
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
173-
174-
//// For aborting uploading file.
175-
if (extra->upAbortCallback)
178+
for (size_t retries = 0; retries < upHostsCount && retries <= self->retriesMax; retries++)
176179
{
177-
curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback);
178-
} // if
180+
CURL *curl = Qiniu_Client_reset(self);
181+
err = Qiniu_Client_config(self);
182+
if (err.code != Qiniu_OK.code)
183+
{
184+
return err;
185+
}
186+
curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries]);
187+
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
188+
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
179189

180-
err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
181-
if (err.code == Qiniu_OK.code && ret != NULL)
182-
{
183-
if (extra->callbackRetParser != NULL)
190+
//// For aborting uploading file.
191+
if (extra->upAbortCallback)
184192
{
185-
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
193+
curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback);
194+
} // if
195+
196+
err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
197+
if (err.code == Qiniu_OK.code)
198+
{
199+
if (extra->callbackRetParser != NULL)
200+
{
201+
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
202+
}
203+
else if (ret != NULL)
204+
{
205+
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
206+
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
207+
ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL);
208+
}
209+
break;
186210
}
187-
else
211+
else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
188212
{
189-
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
190-
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
191-
ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL);
213+
break;
192214
}
193215
}
194-
195216
curl_formfree(formpost);
196217
curl_slist_free_all(headers);
197218
return err;
@@ -327,42 +348,56 @@ static Qiniu_Error Qiniu_Io_call_with_callback(
327348
{
328349
int retCode = 0;
329350
Qiniu_Error err;
330-
struct curl_slist *headers = NULL;
331-
const char *upHost;
351+
struct curl_slist *headers;
352+
const char *const *upHosts;
353+
const char *defaultUpHosts[] = {QINIU_UP_HOST};
354+
size_t upHostsCount;
332355

333-
CURL *curl = Qiniu_Client_reset(self);
334-
err = Qiniu_Client_config(self);
356+
headers = curl_slist_append(NULL, "Expect:");
357+
err = Get_Qiniu_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount);
335358
if (err.code != Qiniu_OK.code)
336359
{
337360
return err;
338361
}
339-
err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost);
340-
if (err.code != Qiniu_OK.code)
362+
if (upHostsCount == 0)
341363
{
342-
return err;
364+
upHosts = defaultUpHosts;
365+
upHostsCount = 1;
343366
}
344367

345-
headers = curl_slist_append(NULL, "Expect:");
368+
for (size_t retries = 0; retries < upHostsCount && retries <= self->retriesMax; retries++)
369+
{
370+
CURL *curl = Qiniu_Client_reset(self);
371+
err = Qiniu_Client_config(self);
372+
if (err.code != Qiniu_OK.code)
373+
{
374+
return err;
375+
}
346376

347-
curl_easy_setopt(curl, CURLOPT_URL, upHost);
348-
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
349-
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
350-
curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr);
377+
curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries]);
378+
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
379+
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
380+
curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr);
351381

352-
err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
353-
if (err.code == Qiniu_OK.code && ret != NULL)
354-
{
355-
if (extra->callbackRetParser != NULL)
382+
err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
383+
if (err.code == Qiniu_OK.code)
356384
{
357-
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
385+
if (extra->callbackRetParser != NULL)
386+
{
387+
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
388+
}
389+
else if (ret != NULL)
390+
{
391+
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
392+
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
393+
}
394+
break;
358395
}
359-
else
396+
else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
360397
{
361-
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
362-
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
398+
break;
363399
}
364400
}
365-
366401
curl_formfree(formpost);
367402
curl_slist_free_all(headers);
368403
return err;

‎qiniu/io.h

+6-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
Name : io.h
44
Author : Qiniu.com
55
Copyright : 2012(c) Shanghai Qiniu Information Technologies Co., Ltd.
6-
Description :
6+
Description :
77
============================================================================
88
*/
99

@@ -51,9 +51,14 @@ extern "C"
5151
Qiniu_Rd_FnAbort upAbortCallback;
5252

5353
const char *upHost;
54+
5455
const char **upIps;
5556
Qiniu_Count ipCount;
5657
Qiniu_Count ipIndex;
58+
59+
// Specify multiple upHosts
60+
const char *const *upHosts;
61+
size_t upHostsCount;
5762
} Qiniu_Io_PutExtra;
5863

5964
/*============================================================================*/

0 commit comments

Comments
 (0)