LibCurl实现多线程下载
libcurl版本: curl-7.61.1
特性:
- 多线程下载,平均速度比单线程快很多
- 失败多次重试,能适应网络较差环境
- 支持http/https
// CurlDownload.cpp
#include "stdafx.h"
#include "CurlDownload.h"
CRITICAL_SECTION CurlDownload::m_csFileLock = { 0 };
CurlDownload::CurlDownload(void)
{
m_sharedns_handle = NULL;
Init();
}
CurlDownload::~CurlDownload(void)
{
}
CurlDownload* CurlDownload::GetInstense()
{
static CurlDownload* pCurlDownld = NULL;
if (pCurlDownld == NULL)
{
pCurlDownld = new CurlDownload();
}
return pCurlDownld;
}
BOOL CurlDownload::Init()
{
CURLcode code = curl_global_init(CURL_GLOBAL_ALL);
if (m_sharedns_handle == NULL)
{
m_sharedns_handle = curl_share_init();
curl_share_setopt(m_sharedns_handle, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
}
return TRUE;
}
CURL* CurlDownload::CreateShareCurl()
{
CURL *curl = curl_easy_init();
if (curl == NULL)
{
return NULL;
}
curl_easy_setopt(curl, CURLOPT_SHARE, m_sharedns_handle);
curl_easy_setopt(curl, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 5);
return curl;
}
CURL* CurlDownload::CreateNodeCurl(PCHAR szUrl, DownNode* pDownNode)
{
CURL* curl = CreateShareCurl();
if (curl == NULL)
{
return NULL;
}
CURLcode code;
code = curl_easy_setopt(curl, CURLOPT_URL, szUrl);
if (code != CURLE_OK)
{
goto END;
}
CheckSSL(curl, string(szUrl));
code = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &CurlDownload::WriteData);
if (code != CURLE_OK)
{
goto END;
}
code = curl_easy_setopt(curl, CURLOPT_WRITEDATA, pDownNode);
if (code != CURLE_OK)
{
goto END;
}
code = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
if (code != CURLE_OK)
{
goto END;
}
code = curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
if (code != CURLE_OK)
{
goto END;
}
// 速度过慢,终止连接
code = curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1L);
if (code != CURLE_OK)
{
goto END;
}
code = curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 30L);
if (code != CURLE_OK)
{
goto END;
}
//限制连接超时
code = curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, MM_TIMEOUT);
if (code != CURLE_OK)
{
goto END;
}
return curl;
END:
if (curl != NULL)
{
curl_easy_cleanup(curl);
}
return NULL;
}
BOOL CurlDownload::CheckSSL(CURL* pCurl, const std::string& strUrl)
{
if (pCurl == NULL || strUrl.empty())
{
return FALSE;
}
string strHead = strUrl.substr(0, 5);
transform(strHead.begin(), strHead.end(), strHead.begin(), tolower);
if (strHead != "https")
{
return FALSE;
}
curl_easy_setopt(pCurl, CURLOPT_USE_SSL, CURLUSESSL_TRY);
curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYPEER, 0);
curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYHOST, 1);
return TRUE;
}
CURL_RET CurlDownload::GetFileLength(const string& strFileUrl, string& strRedirectUrl, INT64* pnDownloadFileLenth)
{
CURL_RET ret = CURL_RET_FAILD;
if (pnDownloadFileLenth == NULL)
{
return CURL_RET_FAILD;
}
CURL *curl = CreateShareCurl();
if (curl == NULL)
{
return CURL_RET_FAILD;
}
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET");
curl_easy_setopt(curl, CURLOPT_URL, strFileUrl.c_str());
CheckSSL(curl, strFileUrl);
curl_easy_setopt(curl, CURLOPT_HEADER, 1);
curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, MM_TIMEOUT);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
LONG lHttpCode = 0;
DOUBLE dbDownloadFileLenth = -1;
CURLcode code = curl_easy_perform(curl);
if (code == CURLE_OK)
{
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &lHttpCode);
//遇到重定向,获取重定向后的url
if (lHttpCode == 301 || lHttpCode == 302)
{
char *redirecturl = NULL;
curl_easy_getinfo(curl, CURLINFO_REDIRECT_URL, &redirecturl);
if (redirecturl != NULL)
{
strRedirectUrl = redirecturl;
ret = CURL_RET_REDIRECT;
}
else
{
ret = CURL_RET_NEED_RETRY;
}
}
else if (lHttpCode >= 200 && lHttpCode < 300)
{
curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dbDownloadFileLenth);
ret = CURL_RET_SUCCESS;
}
//资源不存在,直接返回
else if (lHttpCode == 404)
{
ret = CURL_RET_FAILD;
}
}
else
{
ret = CURL_RET_NEED_RETRY;
}
if (ret == CURL_RET_SUCCESS)
{
INT64 nDownloadFileLenth = (INT64)dbDownloadFileLenth;
if (nDownloadFileLenth == -1 || nDownloadFileLenth == 0)
{
ret = CURL_RET_NEED_RETRY;
}
else
{
*pnDownloadFileLenth = nDownloadFileLenth;
}
}
curl_easy_cleanup(curl);
return ret;
}
CURL_RET CurlDownload::TryGetFileLength(const string& strFileUrl, string& strRedirectUrl, INT64* pnDownloadFileLenth)
{
CURL_RET ret = CURL_RET_FAILD;
strRedirectUrl.clear();
string strFileUrlTemp = strFileUrl;
string strRedirectUrlTemp;
for (UINT i = 0; i < MAX_RETRY_TIMES; i++)
{
ret = GetFileLength(strFileUrlTemp, strRedirectUrlTemp, pnDownloadFileLenth);
if (ret == CURL_RET_SUCCESS ||
ret == CURL_RET_FAILD)
{
break;
}
else if (ret == CURL_RET_REDIRECT)
{
strFileUrlTemp = strRedirectUrlTemp;
}
if (ret == CURL_RET_NEED_RETRY &&
i != MAX_RETRY_TIMES - 1)
{
float fDelaySec = pow(2.0, (INT)i);
Sleep(fDelaySec * 1000);
}
}
if (ret == CURL_RET_SUCCESS)
{
//被重定向了
if (strFileUrl != strFileUrlTemp)
{
strRedirectUrl = strFileUrlTemp;
}
}
return ret;
}
size_t CurlDownload::HeaderInfo(char *ptr, size_t size, size_t nmemb, void *userdata)
{
std::string* pHead = (std::string*)userdata;
if (pHead != NULL)
{
pHead->append(std::string((char*)ptr, nmemb));
}
return size * nmemb;
}
CURL_RET CurlDownload::CheckIsSupportRange(const string& strUrl, BOOL *pbSupportMultiDown)
{
if (pbSupportMultiDown == NULL)
{
return CURL_RET_FAILD;
}
CURL* curl = CreateShareCurl();
if (curl == NULL)
{
return CURL_RET_FAILD;
}
curl_easy_setopt(curl, CURLOPT_URL, strUrl.c_str());
CheckSSL(curl, strUrl);
curl_easy_setopt(curl, CURLOPT_HEADER, 1);
curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
string strHeader;
curl_easy_setopt(curl, CURLOPT_HEADERDATA, &strHeader);
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &CurlDownload::HeaderInfo);
curl_easy_setopt(curl, CURLOPT_RANGE, "0-");
curl_easy_setopt(curl, CURLOPT_TIMEOUT, MM_TIMEOUT);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
CURLcode code = curl_easy_perform(curl);
curl_easy_cleanup(curl);
if (code == CURLE_OK)
{
if (strHeader.find("Content-Range: bytes") != std::string::npos)
{
*pbSupportMultiDown = TRUE;
}
else
{
pbSupportMultiDown = FALSE;
}
return CURL_RET_SUCCESS;
}
return CURL_RET_NEED_RETRY;
}
CURL_RET CurlDownload::TryCheckIsSupportRange(const string& strUrl, BOOL *pbSupportMultiDown)
{
CURL_RET ret = CURL_RET_FAILD;
for (UINT i = 0; i < MAX_RETRY_TIMES; i++)
{
ret = CheckIsSupportRange(strUrl, pbSupportMultiDown);
if (ret == CURL_RET_SUCCESS ||
ret == CURL_RET_FAILD)
{
break;
}
if (ret == CURL_RET_NEED_RETRY &&
i != MAX_RETRY_TIMES - 1)
{
float fDelaySec = pow(2.0, (INT)i);
Sleep(fDelaySec * 1000);
}
}
return ret;
}
size_t CurlDownload::WriteData(char *ptr, size_t size, size_t nmemb, void *userdata)
{
DownNode *pNode = (DownNode *)userdata;
if (pNode == NULL)
{
return 0;
}
//已经接收完成
if (pNode->nRecvSize >= pNode->nBlockSize)
{
return size * nmemb;
}
size_t uWriteLen = 0;
EnterCriticalSection(&m_csFileLock);
if (_fseeki64(pNode->pFile, pNode->nStartPos + pNode->nRecvSize, SEEK_SET) != 0)
{
goto END;
}
INT64 nUnRecvSize = pNode->nBlockSize - pNode->nRecvSize;
size_t uToWriteCount = 0;
if (nUnRecvSize > size * nmemb)
{
uToWriteCount = nmemb;
}
else
{
uToWriteCount = nUnRecvSize / size;
}
uWriteLen = fwrite (ptr, size, uToWriteCount, pNode->pFile);
pNode->nRecvSize += uWriteLen * size;
END:
LeaveCriticalSection(&m_csFileLock);
return size * nmemb;
}
DWORD WINAPI CurlDownload::DownWorkThread(PVOID pParam)
{
DownNode* pNode = (DownNode*)pParam;
if (pNode == NULL || pNode->curl == NULL)
{
return FALSE;
}
CURLcode code;
BOOL bRet = FALSE;
BOOL bNeedRetry = FALSE;
CURL* curl = NULL;
DWORD dwRetryTimes = 0;
INT64 nStartPos = 0;
INT64 nUnRecvSize = 0;
INT64 nRangeSize = 0;
CHAR szRange[32] = { 0 };
while(pNode->nRecvSize < pNode->nBlockSize)
{
if (dwRetryTimes > MAX_RETRY_TIMES)
{
goto END;
}
else if (bNeedRetry)
{
float fDelaySec = pow(2.0, (INT)dwRetryTimes);
Sleep(fDelaySec * 1000);
dwRetryTimes++;
bNeedRetry = FALSE;
}
if (pNode->bSupportMultiDown)
{
nStartPos = pNode->nStartPos + pNode->nRecvSize;
nUnRecvSize = pNode->nBlockSize - pNode->nRecvSize;
nRangeSize = pNode->nBlockSize - pNode->nRecvSize;
if (nRangeSize > MMBLOCK_RANGE_SIZE)
{
nRangeSize = MMBLOCK_RANGE_SIZE;
}
StringCchPrintfA(szRange, sizeof(szRange), "%lld-%lld", nStartPos, nStartPos + nRangeSize);
code = curl_easy_setopt(pNode->curl, CURLOPT_RANGE, szRange);
}
else
{
//单线程总是从0开始下载
pNode->nRecvSize = 0;
code = CURLE_OK;
}
if (code != CURLE_OK)
{
bNeedRetry = TRUE;
continue;
}
code = curl_easy_perform(pNode->curl);
if (code != CURLE_OK)
{
if (pNode->nRecvSize > nRangeSize)
{
pNode->nRecvSize -= nRangeSize;
}
bNeedRetry = TRUE;
continue;
}
}
END:
if (pNode->curl != NULL)
{
curl_easy_cleanup(pNode->curl);
}
if (pNode != NULL)
{
free(pNode);
}
return bRet;
}
BOOL CurlDownload::DownLoad(const std::string &strUrl, const std::string& strDownloadPath)
{
CURL_RET ret = CURL_RET_FAILD;
InitializeCriticalSection(&m_csFileLock);
string strRedirectUrl;
INT64 nDownloadFileLenth = 0;
ret = TryGetFileLength(strUrl, strRedirectUrl, &nDownloadFileLenth);
if (ret != CURL_RET_SUCCESS)
{
return FALSE;
}
DeleteFileA(strDownloadPath.c_str());
FILE *pFile = fopen (strDownloadPath.c_str (), "wb");
if (pFile == NULL)
{
return FALSE;
}
string strUrlFinal;
if (strRedirectUrl.empty())
{
strUrlFinal = strUrl;
}
else
{
strUrlFinal = strRedirectUrl;
}
//确定下载线程个数
DWORD dwThreadCount = (DWORD)ceil(1.0 * nDownloadFileLenth / (MMBLOCK_SIZE));
if (dwThreadCount > MAX_DOWN_THREAD_COUNT)
{
dwThreadCount = MAX_DOWN_THREAD_COUNT;
}
BOOL bSupportMultiDown = FALSE;
if (dwThreadCount > 1)
{
TryCheckIsSupportRange(strUrlFinal, &bSupportMultiDown);
if (!bSupportMultiDown)
{
dwThreadCount = 1;
}
}
DWORD dwRet = WAIT_FAILED;
//文件分块大小
INT64 nBlockSize = nDownloadFileLenth / dwThreadCount;
HANDLE hDownloadThread[MAX_DOWN_THREAD_COUNT] = { 0 };
for (UINT i = 0; i < dwThreadCount; i++)
{
DownNode *pNode = new DownNode();
if (pNode == NULL)
{
return FALSE;
}
memset(pNode, 0, sizeof(DownNode));
pNode->nStartPos = i * nBlockSize;
if (i != dwThreadCount - 1)
{
pNode->nBlockSize = nBlockSize;
}
else
{
pNode->nBlockSize = nDownloadFileLenth - i * nBlockSize;
}
pNode->curl = CreateNodeCurl((PCHAR)strUrlFinal.c_str(), pNode);
if (pNode->curl == NULL)
{
goto END;
}
pNode->pFile = pFile;
pNode->bSupportMultiDown = bSupportMultiDown;
pNode->dwThreadIndex = i;
hDownloadThread[i] = CreateThread(NULL, 0, DownWorkThread, pNode, 0, NULL);
if (hDownloadThread[i] == NULL)
{
goto END;
}
}
dwRet = WaitForMultipleObjects(dwThreadCount, hDownloadThread, TRUE, 120 * 60 * 1000);
END:
BOOL bDownLoadRet = FALSE;
if (pFile != NULL)
{
_fseeki64(pFile, 0, SEEK_END);
INT64 nFileLenth = _ftelli64(pFile);
if (nFileLenth == nDownloadFileLenth)
{
bDownLoadRet = TRUE;
}
fclose(pFile);
}
return bDownLoadRet ;
}
BOOL CurlDownload::DownLoad(const std::wstring &strUrl, const std::wstring&strDownloadPath)
{
string strUrlA(strUrl.length(), ' ');
string strDownloadPathA(strDownloadPath.length(), ' ');
copy(strUrl.begin(), strUrl.end(), strUrlA.begin());
copy(strDownloadPath.begin(), strDownloadPath.end(), strDownloadPathA.begin());
return DownLoad(strUrlA, strDownloadPathA);
}
头文件:
// CurlDownload.h
#pragma once
#include "include/curl/curl.h"
//最大线程数
#define MAX_DOWN_THREAD_COUNT 10
//超时时间
#define MM_TIMEOUT 60L
//失败重试次数
#define MAX_RETRY_TIMES 8
//每个线程负责传输的数据块的大小:10M
#define MMBLOCK_SIZE 1024 * 1024 * 10
//线程内部下载每次尝试下载的块大小:2M
#define MMBLOCK_RANGE_SIZE 1024 * 1024 * 2
enum CURL_RET
{
CURL_RET_SUCCESS = 0,
CURL_RET_FAILD,
CURL_RET_NEED_RETRY,
CURL_RET_REDIRECT,
};
struct DownNode
{
CURL *curl;
FILE *pFile;
DWORD dwThreadIndex;
BOOL bSupportMultiDown;
INT64 nStartPos;
INT64 nBlockSize;
INT64 nRecvSize;
};
class CurlDownload
{
public:
CurlDownload(void);
~CurlDownload(void);
static CurlDownload* GetInstense();
BOOL DownLoad(const std::string &strUrl, const std::string&strDownloadPath);
BOOL DownLoad(const std::wstring &strUrl, const std::wstring&strDownloadPath);
private:
BOOL Init();
CURL* CreateShareCurl();
CURL* CreateNodeCurl(PCHAR szUrl, DownNode* pNode);
static DWORD WINAPI CurlDownload::DownWorkThread(PVOID pParam);
BOOL CheckSSL(CURL* pCurl, const std::string& strUrl);
CURL_RET GetFileLength(const string& strFileUrl, string& strRedirectUrl, INT64* pnDownloadFileLenth);
CURL_RET TryGetFileLength(const string& strFileUrl, string& strRedirectUrl, INT64* pnDownloadFileLenth);
static size_t WriteData(char *ptr, size_t size, size_t nmemb, void *userdata);
static size_t HeaderInfo(char *ptr, size_t size, size_t nmemb, void *userdata);
CURL_RET CheckIsSupportRange(const string& strUrl, BOOL *pbSupportMultiDown);
CURL_RET TryCheckIsSupportRange(const string& strUrl, BOOL *pbSupportMultiDown);
CURLSH* m_sharedns_handle;
static CRITICAL_SECTION m_csFileLock;
};
使用方法:
CurlDownload* pDown = CurlDownload::GetInstense();
if (pDown != NULL)
{
pDown->DownLoad("https://xxx.com/1.exe", "c:\\2.exe");
}
发表评论
要发表评论,您必须先登录。