From 94e6950d486b8341f8e88165b6f42c3b880d7bb1 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Jun 2019 13:12:13 -0700 Subject: [PATCH] http: Support redirects for load URL too. Might as well, especially to keep old recent entries working. --- Core/FileLoaders/HTTPFileLoader.cpp | 115 +++++++++++++++++++--------- Core/FileLoaders/HTTPFileLoader.h | 2 + ext/native/net/http_client.cpp | 40 ++++++---- ext/native/net/http_client.h | 2 + 4 files changed, 110 insertions(+), 49 deletions(-) diff --git a/Core/FileLoaders/HTTPFileLoader.cpp b/Core/FileLoaders/HTTPFileLoader.cpp index 0731c617fb..7d710dbe52 100644 --- a/Core/FileLoaders/HTTPFileLoader.cpp +++ b/Core/FileLoaders/HTTPFileLoader.cpp @@ -27,43 +27,53 @@ HTTPFileLoader::HTTPFileLoader(const std::string &filename) void HTTPFileLoader::Prepare() { std::call_once(preparedFlag_, [this](){ - if (!url_.Valid()) { - ERROR_LOG(LOADER, "HTTP request failed, invalid URL"); - latestError_ = "Invalid URL"; - return; - } - - if (!client_.Resolve(url_.Host().c_str(), url_.Port())) { - ERROR_LOG(LOADER, "HTTP request failed, unable to resolve: %s port %d", url_.Host().c_str(), url_.Port()); - latestError_ = "Could not connect (name not resolved)"; - return; - } - - client_.SetDataTimeout(20.0); - Connect(); - if (!connected_) { - ERROR_LOG(LOADER, "HTTP request failed, failed to connect: %s port %d", url_.Host().c_str(), url_.Port()); - latestError_ = "Could not connect (refused to connect)"; - return; - } - - int err = client_.SendRequest("HEAD", url_.Resource().c_str()); - if (err < 0) { - ERROR_LOG(LOADER, "HTTP request failed, failed to send request: %s port %d", url_.Host().c_str(), url_.Port()); - latestError_ = "Could not connect (could not request data)"; - Disconnect(); - return; - } - - Buffer readbuf; std::vector responseHeaders; - int code = client_.ReadResponseHeaders(&readbuf, responseHeaders); - if (code != 200) { - // Leave size at 0, invalid. - ERROR_LOG(LOADER, "HTTP request failed, got %03d for %s", code, filename_.c_str()); - latestError_ = "Could not connect (invalid response)"; - Disconnect(); - return; + Url resourceURL = url_; + int redirectsLeft = 20; + while (redirectsLeft > 0) { + responseHeaders.clear(); + int code = SendHEAD(resourceURL, responseHeaders); + if (code == -400) { + // Already reported the error. + return; + } + + if (code == 301 || code == 302 || code == 303 || code == 307 || code == 308) { + Disconnect(); + + std::string redirectURL; + if (http::GetHeaderValue(responseHeaders, "Location", &redirectURL)) { + Url url(resourceURL); + url = url.Relative(redirectURL); + + if (url.ToString() == url_.ToString() || url.ToString() == resourceURL.ToString()) { + ERROR_LOG(LOADER, "HTTP request failed, hit a redirect loop"); + latestError_ = "Could not connect (redirect loop)"; + return; + } + + resourceURL = url; + redirectsLeft--; + continue; + } + + // No Location header? + ERROR_LOG(LOADER, "HTTP request failed, invalid redirect"); + latestError_ = "Could not connect (invalid response)"; + return; + } + + if (code != 200) { + // Leave size at 0, invalid. + ERROR_LOG(LOADER, "HTTP request failed, got %03d for %s", code, filename_.c_str()); + latestError_ = "Could not connect (invalid response)"; + Disconnect(); + return; + } + + // We got a good, non-redirect response. + redirectsLeft = 0; + url_ = resourceURL; } // TODO: Expire cache via ETag, etc. @@ -102,6 +112,39 @@ void HTTPFileLoader::Prepare() { }); } +int HTTPFileLoader::SendHEAD(const Url &url, std::vector &responseHeaders) { + if (!url.Valid()) { + ERROR_LOG(LOADER, "HTTP request failed, invalid URL"); + latestError_ = "Invalid URL"; + return -400; + } + + if (!client_.Resolve(url.Host().c_str(), url.Port())) { + ERROR_LOG(LOADER, "HTTP request failed, unable to resolve: |%s| port %d", url.Host().c_str(), url.Port()); + latestError_ = "Could not connect (name not resolved)"; + return -400; + } + + client_.SetDataTimeout(20.0); + Connect(); + if (!connected_) { + ERROR_LOG(LOADER, "HTTP request failed, failed to connect: %s port %d", url.Host().c_str(), url.Port()); + latestError_ = "Could not connect (refused to connect)"; + return -400; + } + + int err = client_.SendRequest("HEAD", url.Resource().c_str()); + if (err < 0) { + ERROR_LOG(LOADER, "HTTP request failed, failed to send request: %s port %d", url.Host().c_str(), url.Port()); + latestError_ = "Could not connect (could not request data)"; + Disconnect(); + return -400; + } + + Buffer readbuf; + return client_.ReadResponseHeaders(&readbuf, responseHeaders); +} + HTTPFileLoader::~HTTPFileLoader() { Disconnect(); } diff --git a/Core/FileLoaders/HTTPFileLoader.h b/Core/FileLoaders/HTTPFileLoader.h index d853153177..cf3441df9c 100644 --- a/Core/FileLoaders/HTTPFileLoader.h +++ b/Core/FileLoaders/HTTPFileLoader.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include "net/http_client.h" #include "net/resolve.h" @@ -54,6 +55,7 @@ public: private: void Prepare(); + int SendHEAD(const Url &url, std::vector &responseHeaders); void Connect(); diff --git a/ext/native/net/http_client.cpp b/ext/native/net/http_client.cpp index 1e2d7c90b7..19bff66f48 100644 --- a/ext/native/net/http_client.cpp +++ b/ext/native/net/http_client.cpp @@ -174,6 +174,31 @@ Client::~Client() { Disconnect(); } +// Ignores line folding (deprecated), but respects field combining. +// Don't use for Set-Cookie, which is a special header per RFC 7230. +bool GetHeaderValue(const std::vector &responseHeaders, const std::string &header, std::string *value) { + std::string search = header + ":"; + bool found = false; + + value->clear(); + for (const std::string &line : responseHeaders) { + auto stripped = StripSpaces(line); + if (startsWithNoCase(stripped, search)) { + size_t value_pos = search.length(); + size_t after_white = stripped.find_first_not_of(" \t", value_pos); + if (after_white != stripped.npos) + value_pos = after_white; + + if (!found) + *value = stripped.substr(value_pos); + else + *value += "," + stripped.substr(value_pos); + found = true; + } + } + + return found; +} void DeChunk(Buffer *inbuffer, Buffer *outbuffer, int contentLength, float *progress) { int dechunkedBytes = 0; @@ -450,19 +475,8 @@ int Download::PerformGET(const std::string &url) { } std::string Download::RedirectLocation(const std::string &baseUrl) { - std::string redirectUrl = ""; - for (const std::string &line : responseHeaders_) { - if (startsWithNoCase(line, "Location:")) { - size_t url_pos = strlen("Location:"); - size_t after_white = line.find_first_not_of(" \t", url_pos); - if (after_white != line.npos) - url_pos = after_white; - - redirectUrl = line.substr(url_pos); - } - } - - if (!redirectUrl.empty()) { + std::string redirectUrl; + if (GetHeaderValue(responseHeaders_, "Location", &redirectUrl)) { Url url(baseUrl); url = url.Relative(redirectUrl); redirectUrl = url.ToString(); diff --git a/ext/native/net/http_client.h b/ext/native/net/http_client.h index 1cf3a2e0ed..3d2d97eb20 100644 --- a/ext/native/net/http_client.h +++ b/ext/native/net/http_client.h @@ -54,6 +54,8 @@ private: namespace http { +bool GetHeaderValue(const std::vector &responseHeaders, const std::string &header, std::string *value); + class Client : public net::Connection { public: Client();