I'm working on a pretty big project that also involves making some get requests, pulling data from websites etc. For that this library: [CHttpClient] is used.
Sometimes when web requests are made like shown below, the program just crashes without any verbal error message or anything like that. When I looked at the previously used strings in memory in ollydbg, there were some blocks that had tons of '??????' in them. I suspect that somewhere there is an overflow occuring, but I cannot for the life of it figure out where. Also setting a try / catch around the code segment in question does not help. It would be great if you could point me into a direction, as the last resort I see would be to switch up the library.
This right below is the "top level" call that's being made.
if (!HttpUtils::makeGetRequest(L"https://calendar.yahoo.com", outCookies, response))
return false;
(outCookies is a NameValueMap (defined below) with the users cookies, response is a std::wstring in which the response is stored)
typedef std::map<std::wstring, std::wstring> NameValueMap;
I have a HttpUtils.cpp class that handles the HTTP requests. Source code for that is as follows (the commented out MessageBoxes are for debugging purposes)
The makeRequest method is the one in question
#include "HttpUtils.h"
#include <memory>
using namespace andrivet::ADVobfuscator;
bool Utilities::HttpUtils::makePostRequest(const wchar_t* url, NameValueMap& postData, NameValueMap& cookies, std::wstring& outResponse)
{
return makeRequest(url, REQUEST_TYPE::POST, postData, cookies, NameValueMap(), outResponse);
}
bool Utilities::HttpUtils::makePostRequest(const wchar_t* url, NameValueMap& postData, std::wstring& outResponse)
{
return makeRequest(url, REQUEST_TYPE::POST, postData, NameValueMap(), NameValueMap(), outResponse);
}
bool Utilities::HttpUtils::makeGetRequest(const wchar_t* url, std::wstring& outResponse)
{
return makeRequest(url, REQUEST_TYPE::GET, NameValueMap(), NameValueMap(), NameValueMap(), outResponse);
}
bool Utilities::HttpUtils::makeGetRequest(const wchar_t* url, NameValueMap& cookies, std::wstring& outResponse)
{
return makeRequest(url, REQUEST_TYPE::GET, NameValueMap(), cookies, NameValueMap(), outResponse);
}
bool Utilities::HttpUtils::makeRequest(const wchar_t* url, REQUEST_TYPE type, NameValueMap& postData, NameValueMap& cookies, NameValueMap& headers, std::wstring& outResponse)
{
CHttpClientW httpClient;
std::unique_ptr<CHttpResponseW> response;
std::wstringstream responseStream;
bool redirect_flag = true;
std::wstring cur_url = url;
try
{
//Add default user agent
httpClient.SetInternet(L"Mozilla/5.0 (compatible; Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1");
//Add post data if needed
for (auto postEntry : postData)
httpClient.AddParam(postEntry.first.c_str(), postEntry.second.c_str());
//Add header if needed
for (auto header : headers)
httpClient.AddHeader(header.first.c_str(), header.second.c_str());
while (redirect_flag)
{
//Add cookies if needed
for (int i = 0; i < httpClient.GetHeaderCount(L"Cookie"); i++)
httpClient.RemoveHeader(L"Cookie"), i;
if (cookies.size())
{
std::wstring cookieString = getCookieString(cookies);
httpClient.AddHeader(L"Cookie", cookieString.c_str());
}
//Check if cur_url is good
if (cur_url.size() > 4 && cur_url.substr(0, 4) != L"http")
{
std::wstring baseUrl;
if (cur_url[0] == L'/')
{
//Need to construct URL adding baseURL
if (!getBaseUrl(url, baseUrl))
return false;
cur_url = baseUrl + cur_url;
}
else
{
// TODO: What shall I do ?!
return false;
}
}
//Perform request
if (type == GET)
{
//MessageBox(0, cur_url.c_str(), L"CUR_URL REQ", 0);
try
{
response.reset(httpClient.RequestGet(cur_url.c_str()));
}
catch (std::overflow_error &err)
{
MessageBoxA(0, "ERROR", "Debug", 0);
}
}
else
response.reset(httpClient.RequestPost(cur_url.c_str()));
if (response->GetStatus() == 302)
{
MessageBoxA(0, "Redirect", "Debug", 0);
redirect_flag = true;
if (response->GetHeaderCount(L"Location") == 0)
return false;
//Update next url
cur_url = response->GetHeader(L"Location", 0);
//Update cookies if needed
handleSetCookie(response.get(), cookies);
}
else
{
redirect_flag = false;
//Get response text
if (Ryeol::Utils::getResponseContent(response.get(), responseStream, 8192))
{
outResponse = responseStream.str();
//MessageBox(0, outResponse.c_str(), L"Printing outResponse", 0);
responseStream.clear();
return true;
}
}
}
}
catch (httpclientexception& e) {
return false;
}
return false;
}
void Utilities::HttpUtils::handleSetCookie(CHttpResponseW* response, NameValueMap& inOutCookies)
{
int count = response->GetHeaderCount(L"Set-Cookie");
for (int i = 0; i < count; i++)
{
std::wstring curSetCookie = response->GetHeader(L"Set-Cookie", i);
std::wstring name, value;
if (getCookieFromSetCookie(curSetCookie, name, value))
inOutCookies[name] = value;
}
}
bool Utilities::HttpUtils::getBaseUrl(const std::wstring& url, std::wstring& outBase)
{
const std::wstring protocol_end = L"://";
const int offs1 = url.find(protocol_end);
if (offs1)
{
//Get next '/'
const int offs2 = url.find(L'/', offs1 + protocol_end.size());
outBase = url.substr(0, offs2);
return true;
}
return false;
}
std::wstring Utilities::HttpUtils::getCookieString(NameValueMap& cookies)
{
CHttpEncoderW encoder;
std::wstring cookieString;
for (auto cookie : cookies)
{
size_t encodedSize = encoder.UrlEncodeLenW(cookie.second.c_str());
//PWSTR encodedBuf = new WCHAR[encodedSize];
//Add name as it is
cookieString += cookie.first;
cookieString += L"=";
//URL encode cookie value
//encoder.UrlEncodeW(encodedBuf, cookie.second.c_str());
//Add encoded value to cookieString
cookieString += cookie.second;
cookieString += L"; ";
//Release encodedBuf
//delete[] encodedBuf;
}
return cookieString;
}
bool Utilities::HttpUtils::getCookieFromSetCookie(std::wstring& setCookie_str, std::wstring& outName, std::wstring& outValue)
{
int offset1 = setCookie_str.find(L"=");
int offset2 = setCookie_str.find(L";", offset1);
if (offset1 != std::wstring::npos && offset2 != std::wstring::npos)
{
outName = setCookie_str.substr(0, offset1);
outValue = setCookie_str.substr(offset1 + 1, offset2 - offset1 - 1);
return true;
}
return false;
}
Thanks in advance for your help!
