http: Support redirects for load URL too.

Might as well, especially to keep old recent entries working.
This commit is contained in:
Unknown W. Brackets 2019-06-23 13:12:13 -07:00
parent c5e2c0e6cd
commit 94e6950d48
4 changed files with 110 additions and 49 deletions

View File

@ -27,43 +27,53 @@ HTTPFileLoader::HTTPFileLoader(const std::string &filename)
void HTTPFileLoader::Prepare() {
std::call_once(preparedFlag_, [this](){
if (!url_.Valid()) {
ERROR_LOG(LOADER, "HTTP request failed, invalid URL");
latestError_ = "Invalid URL";
return;
}
if (!client_.Resolve(url_.Host().c_str(), url_.Port())) {
ERROR_LOG(LOADER, "HTTP request failed, unable to resolve: %s port %d", url_.Host().c_str(), url_.Port());
latestError_ = "Could not connect (name not resolved)";
return;
}
client_.SetDataTimeout(20.0);
Connect();
if (!connected_) {
ERROR_LOG(LOADER, "HTTP request failed, failed to connect: %s port %d", url_.Host().c_str(), url_.Port());
latestError_ = "Could not connect (refused to connect)";
return;
}
int err = client_.SendRequest("HEAD", url_.Resource().c_str());
if (err < 0) {
ERROR_LOG(LOADER, "HTTP request failed, failed to send request: %s port %d", url_.Host().c_str(), url_.Port());
latestError_ = "Could not connect (could not request data)";
Disconnect();
return;
}
Buffer readbuf;
std::vector<std::string> responseHeaders;
int code = client_.ReadResponseHeaders(&readbuf, responseHeaders);
if (code != 200) {
// Leave size at 0, invalid.
ERROR_LOG(LOADER, "HTTP request failed, got %03d for %s", code, filename_.c_str());
latestError_ = "Could not connect (invalid response)";
Disconnect();
return;
Url resourceURL = url_;
int redirectsLeft = 20;
while (redirectsLeft > 0) {
responseHeaders.clear();
int code = SendHEAD(resourceURL, responseHeaders);
if (code == -400) {
// Already reported the error.
return;
}
if (code == 301 || code == 302 || code == 303 || code == 307 || code == 308) {
Disconnect();
std::string redirectURL;
if (http::GetHeaderValue(responseHeaders, "Location", &redirectURL)) {
Url url(resourceURL);
url = url.Relative(redirectURL);
if (url.ToString() == url_.ToString() || url.ToString() == resourceURL.ToString()) {
ERROR_LOG(LOADER, "HTTP request failed, hit a redirect loop");
latestError_ = "Could not connect (redirect loop)";
return;
}
resourceURL = url;
redirectsLeft--;
continue;
}
// No Location header?
ERROR_LOG(LOADER, "HTTP request failed, invalid redirect");
latestError_ = "Could not connect (invalid response)";
return;
}
if (code != 200) {
// Leave size at 0, invalid.
ERROR_LOG(LOADER, "HTTP request failed, got %03d for %s", code, filename_.c_str());
latestError_ = "Could not connect (invalid response)";
Disconnect();
return;
}
// We got a good, non-redirect response.
redirectsLeft = 0;
url_ = resourceURL;
}
// TODO: Expire cache via ETag, etc.
@ -102,6 +112,39 @@ void HTTPFileLoader::Prepare() {
});
}
int HTTPFileLoader::SendHEAD(const Url &url, std::vector<std::string> &responseHeaders) {
if (!url.Valid()) {
ERROR_LOG(LOADER, "HTTP request failed, invalid URL");
latestError_ = "Invalid URL";
return -400;
}
if (!client_.Resolve(url.Host().c_str(), url.Port())) {
ERROR_LOG(LOADER, "HTTP request failed, unable to resolve: |%s| port %d", url.Host().c_str(), url.Port());
latestError_ = "Could not connect (name not resolved)";
return -400;
}
client_.SetDataTimeout(20.0);
Connect();
if (!connected_) {
ERROR_LOG(LOADER, "HTTP request failed, failed to connect: %s port %d", url.Host().c_str(), url.Port());
latestError_ = "Could not connect (refused to connect)";
return -400;
}
int err = client_.SendRequest("HEAD", url.Resource().c_str());
if (err < 0) {
ERROR_LOG(LOADER, "HTTP request failed, failed to send request: %s port %d", url.Host().c_str(), url.Port());
latestError_ = "Could not connect (could not request data)";
Disconnect();
return -400;
}
Buffer readbuf;
return client_.ReadResponseHeaders(&readbuf, responseHeaders);
}
HTTPFileLoader::~HTTPFileLoader() {
Disconnect();
}

View File

@ -18,6 +18,7 @@
#pragma once
#include <mutex>
#include <vector>
#include "net/http_client.h"
#include "net/resolve.h"
@ -54,6 +55,7 @@ public:
private:
void Prepare();
int SendHEAD(const Url &url, std::vector<std::string> &responseHeaders);
void Connect();

View File

@ -174,6 +174,31 @@ Client::~Client() {
Disconnect();
}
// Ignores line folding (deprecated), but respects field combining.
// Don't use for Set-Cookie, which is a special header per RFC 7230.
bool GetHeaderValue(const std::vector<std::string> &responseHeaders, const std::string &header, std::string *value) {
std::string search = header + ":";
bool found = false;
value->clear();
for (const std::string &line : responseHeaders) {
auto stripped = StripSpaces(line);
if (startsWithNoCase(stripped, search)) {
size_t value_pos = search.length();
size_t after_white = stripped.find_first_not_of(" \t", value_pos);
if (after_white != stripped.npos)
value_pos = after_white;
if (!found)
*value = stripped.substr(value_pos);
else
*value += "," + stripped.substr(value_pos);
found = true;
}
}
return found;
}
void DeChunk(Buffer *inbuffer, Buffer *outbuffer, int contentLength, float *progress) {
int dechunkedBytes = 0;
@ -450,19 +475,8 @@ int Download::PerformGET(const std::string &url) {
}
std::string Download::RedirectLocation(const std::string &baseUrl) {
std::string redirectUrl = "";
for (const std::string &line : responseHeaders_) {
if (startsWithNoCase(line, "Location:")) {
size_t url_pos = strlen("Location:");
size_t after_white = line.find_first_not_of(" \t", url_pos);
if (after_white != line.npos)
url_pos = after_white;
redirectUrl = line.substr(url_pos);
}
}
if (!redirectUrl.empty()) {
std::string redirectUrl;
if (GetHeaderValue(responseHeaders_, "Location", &redirectUrl)) {
Url url(baseUrl);
url = url.Relative(redirectUrl);
redirectUrl = url.ToString();

View File

@ -54,6 +54,8 @@ private:
namespace http {
bool GetHeaderValue(const std::vector<std::string> &responseHeaders, const std::string &header, std::string *value);
class Client : public net::Connection {
public:
Client();