LibCurl实现多线程下载

  • 内容
  • 评论
  • 相关

libcurl版本: curl-7.61.1

特性:

  • 多线程下载，平均速度比单线程快很多
  • 失败多次重试，能适应网络较差环境
  • 支持http/https
// CurlDownload.cpp

#include "stdafx.h"
#include "CurlDownload.h"

CRITICAL_SECTION CurlDownload::m_csFileLock = { 0 };

CurlDownload::CurlDownload(void)
{
    m_sharedns_handle = NULL;
    Init();
}

CurlDownload::~CurlDownload(void)
{

}

CurlDownload* CurlDownload::GetInstense()
{
    static CurlDownload* pCurlDownld = NULL;
    if (pCurlDownld == NULL)
    {
        pCurlDownld = new CurlDownload();
    }
    return pCurlDownld;
}

BOOL CurlDownload::Init()
{
    CURLcode code = curl_global_init(CURL_GLOBAL_ALL);
    if (m_sharedns_handle == NULL)  
    {  
        m_sharedns_handle = curl_share_init();  
        curl_share_setopt(m_sharedns_handle, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);  
    }
    return TRUE;
}

CURL* CurlDownload::CreateShareCurl()
{  
    CURL *curl = curl_easy_init();
    if (curl == NULL)
    {
        return NULL;
    }

    curl_easy_setopt(curl, CURLOPT_SHARE, m_sharedns_handle); 
    curl_easy_setopt(curl, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 5);
    return curl;
}

CURL* CurlDownload::CreateNodeCurl(PCHAR szUrl, DownNode* pDownNode)
{
    CURL* curl = CreateShareCurl();
    if (curl == NULL)
    {
        return NULL;
    }

    CURLcode code;

    code = curl_easy_setopt(curl, CURLOPT_URL, szUrl);
    if (code != CURLE_OK)
    {
        goto END;
    }

    CheckSSL(curl, string(szUrl));

    code = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &CurlDownload::WriteData);  
    if (code != CURLE_OK) 
    {
        goto END;
    }

    code = curl_easy_setopt(curl, CURLOPT_WRITEDATA, pDownNode); 
    if (code != CURLE_OK) 
    {
        goto END;
    }

    code = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); 
    if (code != CURLE_OK) 
    {
        goto END;
    }
    code = curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);  
    if (code != CURLE_OK) 
    {
        goto END;
    }

    // 速度过慢，终止连接
    code = curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1L);  
    if (code != CURLE_OK) 
    {
        goto END;
    }
    code = curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 30L); 
    if (code != CURLE_OK) 
    {
        goto END;
    }

    //限制连接超时
    code = curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, MM_TIMEOUT);
    if (code != CURLE_OK) 
    {
        goto END;
    }

    return curl;

END:
    if (curl != NULL)
    {
        curl_easy_cleanup(curl);
    }
    return NULL;
}

BOOL CurlDownload::CheckSSL(CURL* pCurl, const std::string& strUrl)
{
    if (pCurl == NULL || strUrl.empty())
    {
        return FALSE;
    }

    string strHead = strUrl.substr(0, 5);
    transform(strHead.begin(), strHead.end(), strHead.begin(), tolower);
    if (strHead != "https")
    {
        return FALSE;
    }

    curl_easy_setopt(pCurl, CURLOPT_USE_SSL, CURLUSESSL_TRY);
    curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYPEER, 0);
    curl_easy_setopt(pCurl, CURLOPT_SSL_VERIFYHOST, 1);
    return TRUE;
}

CURL_RET CurlDownload::GetFileLength(const string& strFileUrl, string& strRedirectUrl, INT64* pnDownloadFileLenth)
{
    CURL_RET ret = CURL_RET_FAILD;

    if (pnDownloadFileLenth == NULL)
    {
        return CURL_RET_FAILD;
    }

    CURL *curl = CreateShareCurl(); 
    if (curl == NULL)
    {
        return CURL_RET_FAILD;
    }

    curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET");       
    curl_easy_setopt(curl, CURLOPT_URL, strFileUrl.c_str());
    CheckSSL(curl, strFileUrl);
    curl_easy_setopt(curl, CURLOPT_HEADER, 1); 
    curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
    curl_easy_setopt(curl, CURLOPT_TIMEOUT, MM_TIMEOUT);
    curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);

    LONG lHttpCode = 0;
    DOUBLE dbDownloadFileLenth  = -1;

    CURLcode code = curl_easy_perform(curl);
    if (code == CURLE_OK)  
    {
        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &lHttpCode);

        //遇到重定向，获取重定向后的url
        if (lHttpCode == 301 || lHttpCode == 302)
        {
            char *redirecturl = NULL;
            curl_easy_getinfo(curl, CURLINFO_REDIRECT_URL, &redirecturl);

            if (redirecturl != NULL)
            {
                strRedirectUrl = redirecturl;
                ret = CURL_RET_REDIRECT;
            }
            else
            {
                ret = CURL_RET_NEED_RETRY;
            }
        }
        else if (lHttpCode >= 200 && lHttpCode < 300)
        {
            curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dbDownloadFileLenth); 
            ret = CURL_RET_SUCCESS;
        }
        //资源不存在，直接返回
        else if (lHttpCode == 404)
        {
            ret = CURL_RET_FAILD;
        }
    }
    else
    {
        ret = CURL_RET_NEED_RETRY;
    }

    if (ret == CURL_RET_SUCCESS)
    {
        INT64 nDownloadFileLenth = (INT64)dbDownloadFileLenth;   
        if (nDownloadFileLenth == -1 || nDownloadFileLenth == 0)
        {
            ret = CURL_RET_NEED_RETRY;
        }
        else
        {
            *pnDownloadFileLenth = nDownloadFileLenth;
        }
    }

    curl_easy_cleanup(curl);
    return ret;
}

CURL_RET CurlDownload::TryGetFileLength(const string& strFileUrl, string& strRedirectUrl, INT64* pnDownloadFileLenth)
{
    CURL_RET ret = CURL_RET_FAILD;

    strRedirectUrl.clear();
    string strFileUrlTemp = strFileUrl;
    string strRedirectUrlTemp;

    for (UINT i = 0; i < MAX_RETRY_TIMES; i++)
    {
        ret = GetFileLength(strFileUrlTemp, strRedirectUrlTemp, pnDownloadFileLenth);
        if (ret == CURL_RET_SUCCESS ||
            ret == CURL_RET_FAILD)
        {
            break;
        }
        else if (ret == CURL_RET_REDIRECT)
        {
            strFileUrlTemp = strRedirectUrlTemp;
        }

        if (ret == CURL_RET_NEED_RETRY &&
            i != MAX_RETRY_TIMES - 1)
        {
            float fDelaySec = pow(2.0, (INT)i);
            Sleep(fDelaySec * 1000);
        }
    }

    if (ret == CURL_RET_SUCCESS)
    {
        //被重定向了
        if (strFileUrl != strFileUrlTemp)
        {
            strRedirectUrl = strFileUrlTemp;
        }
    }
    return ret;
}

size_t CurlDownload::HeaderInfo(char *ptr, size_t size, size_t nmemb, void *userdata)
{
    std::string* pHead = (std::string*)userdata;
    if (pHead != NULL)
    {
        pHead->append(std::string((char*)ptr, nmemb));
    }
    return size * nmemb;
}

CURL_RET CurlDownload::CheckIsSupportRange(const string& strUrl, BOOL *pbSupportMultiDown)
{
    if (pbSupportMultiDown == NULL)
    {
        return CURL_RET_FAILD;
    }

    CURL* curl = CreateShareCurl();
    if (curl == NULL)
    {
        return CURL_RET_FAILD;
    }

    curl_easy_setopt(curl, CURLOPT_URL, strUrl.c_str());  
    CheckSSL(curl, strUrl);
    curl_easy_setopt(curl, CURLOPT_HEADER, 1); 
    curl_easy_setopt(curl, CURLOPT_NOBODY, 1);

    string strHeader;
    curl_easy_setopt(curl, CURLOPT_HEADERDATA, &strHeader);
    curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &CurlDownload::HeaderInfo);
    curl_easy_setopt(curl, CURLOPT_RANGE, "0-");
    curl_easy_setopt(curl, CURLOPT_TIMEOUT, MM_TIMEOUT);
    curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
    CURLcode code = curl_easy_perform(curl);
    curl_easy_cleanup(curl);

    if (code == CURLE_OK)
    {
        if (strHeader.find("Content-Range: bytes") != std::string::npos)
        {
            *pbSupportMultiDown = TRUE;
        }
        else
        {
            pbSupportMultiDown = FALSE;
        }
        return CURL_RET_SUCCESS;
    }

    return CURL_RET_NEED_RETRY;
}


CURL_RET CurlDownload::TryCheckIsSupportRange(const string& strUrl, BOOL *pbSupportMultiDown)
{
    CURL_RET ret = CURL_RET_FAILD;

    for (UINT i = 0; i < MAX_RETRY_TIMES; i++)
    {
        ret = CheckIsSupportRange(strUrl, pbSupportMultiDown);
        if (ret == CURL_RET_SUCCESS ||
            ret == CURL_RET_FAILD)
        {
            break;
        }

        if (ret == CURL_RET_NEED_RETRY &&
            i != MAX_RETRY_TIMES - 1)
        {
            float fDelaySec = pow(2.0, (INT)i);
            Sleep(fDelaySec * 1000);
        }
    }
    return ret;
}

size_t CurlDownload::WriteData(char *ptr, size_t size, size_t nmemb, void *userdata)
{
    DownNode *pNode = (DownNode *)userdata;
    if (pNode == NULL)
    {
        return 0;
    }

    //已经接收完成
    if (pNode->nRecvSize >= pNode->nBlockSize)
    {
        return size * nmemb;
    }
    size_t uWriteLen = 0;

    EnterCriticalSection(&m_csFileLock);

    if (_fseeki64(pNode->pFile, pNode->nStartPos + pNode->nRecvSize, SEEK_SET) != 0)
    {
        goto END;
    }

    INT64 nUnRecvSize = pNode->nBlockSize - pNode->nRecvSize;

    size_t uToWriteCount = 0;
    if (nUnRecvSize > size * nmemb)
    {
        uToWriteCount = nmemb;
    }
    else
    {
        uToWriteCount = nUnRecvSize / size;
    }
    uWriteLen = fwrite (ptr, size, uToWriteCount, pNode->pFile);
    pNode->nRecvSize += uWriteLen * size;

END:
    LeaveCriticalSection(&m_csFileLock);
    return size * nmemb; 
}


DWORD WINAPI CurlDownload::DownWorkThread(PVOID pParam)
{
    DownNode* pNode = (DownNode*)pParam;
    if (pNode == NULL || pNode->curl == NULL)
    {
        return FALSE;
    }

    CURLcode code;
    BOOL bRet           = FALSE;
    BOOL bNeedRetry     = FALSE;
    CURL* curl          = NULL;
    DWORD dwRetryTimes  = 0;
    INT64 nStartPos     = 0;
    INT64 nUnRecvSize   = 0;
    INT64 nRangeSize    = 0;
    CHAR szRange[32]    = { 0 };

    while(pNode->nRecvSize < pNode->nBlockSize)
    {
        if (dwRetryTimes > MAX_RETRY_TIMES)
        {
            goto END;
        }
        else if (bNeedRetry)
        {
            float fDelaySec = pow(2.0, (INT)dwRetryTimes);
            Sleep(fDelaySec * 1000);

            dwRetryTimes++;
            bNeedRetry = FALSE;
        }

        if (pNode->bSupportMultiDown)
        {
            nStartPos = pNode->nStartPos + pNode->nRecvSize;
            nUnRecvSize = pNode->nBlockSize - pNode->nRecvSize;

            nRangeSize = pNode->nBlockSize - pNode->nRecvSize;
            if (nRangeSize > MMBLOCK_RANGE_SIZE)
            {
                nRangeSize = MMBLOCK_RANGE_SIZE;
            }

            StringCchPrintfA(szRange, sizeof(szRange), "%lld-%lld", nStartPos, nStartPos + nRangeSize);
            code = curl_easy_setopt(pNode->curl, CURLOPT_RANGE, szRange);
        }
        else
        {
            //单线程总是从0开始下载
            pNode->nRecvSize = 0;
            code = CURLE_OK;
        } 
        if (code != CURLE_OK)
        {
            bNeedRetry = TRUE;
            continue;
        }

        code = curl_easy_perform(pNode->curl);
        if (code != CURLE_OK)
        {  
            if (pNode->nRecvSize > nRangeSize)
            {
                pNode->nRecvSize -= nRangeSize;
            }
            bNeedRetry = TRUE;
            continue;
        }
    }

END:

    if (pNode->curl != NULL)
    {
        curl_easy_cleanup(pNode->curl);
    }
    if (pNode != NULL)
    {
        free(pNode);
    }
    return bRet;
}

BOOL CurlDownload::DownLoad(const std::string &strUrl, const std::string& strDownloadPath)
{
    CURL_RET ret = CURL_RET_FAILD;
    InitializeCriticalSection(&m_csFileLock);

    string strRedirectUrl;
    INT64 nDownloadFileLenth = 0;

    ret = TryGetFileLength(strUrl, strRedirectUrl, &nDownloadFileLenth);
    if (ret != CURL_RET_SUCCESS)
    {
        return FALSE;
    }
    DeleteFileA(strDownloadPath.c_str());

    FILE *pFile = fopen (strDownloadPath.c_str (), "wb");
    if (pFile == NULL)
    {
        return FALSE;
    }

    string strUrlFinal;
    if (strRedirectUrl.empty())
    {
        strUrlFinal = strUrl;
    }
    else
    {
        strUrlFinal = strRedirectUrl;
    }

    //确定下载线程个数
    DWORD dwThreadCount = (DWORD)ceil(1.0 * nDownloadFileLenth / (MMBLOCK_SIZE));
    if (dwThreadCount > MAX_DOWN_THREAD_COUNT)
    {
        dwThreadCount = MAX_DOWN_THREAD_COUNT;
    }

    BOOL bSupportMultiDown = FALSE;
    if (dwThreadCount > 1)
    {
        TryCheckIsSupportRange(strUrlFinal, &bSupportMultiDown);
        if (!bSupportMultiDown)
        {
            dwThreadCount = 1;
        }
    }

    DWORD dwRet = WAIT_FAILED;

    //文件分块大小
    INT64 nBlockSize = nDownloadFileLenth / dwThreadCount;
    HANDLE hDownloadThread[MAX_DOWN_THREAD_COUNT] = { 0 };

    for (UINT i = 0; i < dwThreadCount; i++)
    {
        DownNode *pNode = new DownNode();
        if (pNode == NULL)
        {
            return FALSE;
        }
        memset(pNode, 0, sizeof(DownNode));

        pNode->nStartPos = i * nBlockSize;
        if (i != dwThreadCount - 1)
        {
            pNode->nBlockSize = nBlockSize;
        }
        else
        {
            pNode->nBlockSize = nDownloadFileLenth - i * nBlockSize;
        }

        pNode->curl = CreateNodeCurl((PCHAR)strUrlFinal.c_str(), pNode);
        if (pNode->curl == NULL)
        {
            goto END;
        }

        pNode->pFile                = pFile;
        pNode->bSupportMultiDown    = bSupportMultiDown;
        pNode->dwThreadIndex        = i;

        hDownloadThread[i] = CreateThread(NULL, 0, DownWorkThread, pNode, 0, NULL); 
        if (hDownloadThread[i] == NULL)
        {
            goto END;
        }
    }

    dwRet = WaitForMultipleObjects(dwThreadCount, hDownloadThread, TRUE, 120 * 60 * 1000);
END:

    BOOL bDownLoadRet = FALSE;

    if (pFile != NULL)
    {
        _fseeki64(pFile, 0, SEEK_END);
        INT64 nFileLenth = _ftelli64(pFile);
        if (nFileLenth == nDownloadFileLenth)
        {
            bDownLoadRet = TRUE;
        }
        fclose(pFile);
    }

    return bDownLoadRet ;
}

BOOL CurlDownload::DownLoad(const std::wstring &strUrl, const std::wstring&strDownloadPath)
{
    string strUrlA(strUrl.length(), ' ');
    string strDownloadPathA(strDownloadPath.length(), ' ');

    copy(strUrl.begin(), strUrl.end(), strUrlA.begin());
    copy(strDownloadPath.begin(), strDownloadPath.end(), strDownloadPathA.begin());
    return DownLoad(strUrlA, strDownloadPathA);
}

头文件:

// CurlDownload.h

#pragma once
#include "include/curl/curl.h"

//最大线程数
#define MAX_DOWN_THREAD_COUNT   10
//超时时间
#define MM_TIMEOUT              60L
//失败重试次数
#define MAX_RETRY_TIMES         8
//每个线程负责传输的数据块的大小:10M
#define MMBLOCK_SIZE            1024 * 1024 * 10
//线程内部下载每次尝试下载的块大小:2M
#define MMBLOCK_RANGE_SIZE      1024 * 1024 * 2

enum CURL_RET
{
    CURL_RET_SUCCESS = 0,
    CURL_RET_FAILD,
    CURL_RET_NEED_RETRY,
    CURL_RET_REDIRECT,
};

struct DownNode
{
    CURL            *curl;
    FILE            *pFile;

    DWORD           dwThreadIndex;
    BOOL            bSupportMultiDown;

    INT64           nStartPos;
    INT64           nBlockSize;
    INT64           nRecvSize;
};

class CurlDownload
{
public:
    CurlDownload(void);
    ~CurlDownload(void);
    static CurlDownload* GetInstense();

    BOOL DownLoad(const std::string &strUrl, const std::string&strDownloadPath);
    BOOL DownLoad(const std::wstring &strUrl, const std::wstring&strDownloadPath);

private:
    BOOL Init();
    CURL* CreateShareCurl();
    CURL* CreateNodeCurl(PCHAR szUrl, DownNode* pNode);
    static DWORD WINAPI CurlDownload::DownWorkThread(PVOID pParam);
    BOOL CheckSSL(CURL* pCurl, const std::string& strUrl);
    CURL_RET GetFileLength(const string& strFileUrl, string& strRedirectUrl, INT64* pnDownloadFileLenth);
    CURL_RET TryGetFileLength(const string& strFileUrl, string& strRedirectUrl, INT64* pnDownloadFileLenth);

    static size_t WriteData(char *ptr, size_t size, size_t nmemb, void *userdata);
    static size_t HeaderInfo(char *ptr, size_t size, size_t nmemb, void *userdata);    
    CURL_RET CheckIsSupportRange(const string& strUrl, BOOL *pbSupportMultiDown);
    CURL_RET TryCheckIsSupportRange(const string& strUrl, BOOL *pbSupportMultiDown);

    CURLSH*                     m_sharedns_handle;
    static CRITICAL_SECTION     m_csFileLock;
};

使用方法：

CurlDownload* pDown = CurlDownload::GetInstense();
if (pDown != NULL)
{
    pDown->DownLoad("https://xxx.com/1.exe", "c:\\2.exe");
}