Break out the http::RequestManager from HTTPClient.h

This commit is contained in:
Henrik Rydgård committed 2025-09-04 17:12:18 +02:00
1 parent 37a32e2c99
commit e65c5bfdb6
19 files changed
+266 -223

No files matched your search

+2
View File
@@ -453,6 +453,7 @@
<ClInclude Include="Math\Statistics.h" />
<ClInclude Include="Net\Connection.h" />
<ClInclude Include="Net\HTTPNaettRequest.h" />
<ClInclude Include="Net\HTTPRequestManager.h" />
<ClInclude Include="Net\NetBuffer.h" />
<ClInclude Include="Net\HTTPClient.h" />
<ClInclude Include="Net\HTTPHeaders.h" />
@@ -909,6 +910,7 @@
<ClCompile Include="Math\Statistics.cpp" />
<ClCompile Include="Net\Connection.cpp" />
<ClCompile Include="Net\HTTPNaettRequest.cpp" />
<ClCompile Include="Net\HTTPRequestManager.cpp" />
<ClCompile Include="Net\NetBuffer.cpp" />
<ClCompile Include="Net\HTTPClient.cpp" />
<ClCompile Include="Net\HTTPHeaders.cpp" />
+6
View File
@@ -692,6 +692,9 @@
<ClInclude Include="Net\Connection.h">
<Filter>Net</Filter>
</ClInclude>
<ClInclude Include="Net\HTTPRequestManager.h">
<Filter>Net</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="ABI.cpp" />
@@ -1296,6 +1299,9 @@
<ClCompile Include="Net\Connection.cpp">
<Filter>Net</Filter>
</ClCompile>
<ClCompile Include="Net\HTTPRequestManager.cpp">
<Filter>Net</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<Filter Include="Crypto">
-16
View File
@@ -92,20 +92,4 @@ private:
net::ResolveFunc customResolve_;
};
// Fake request for cache hits.
// The download manager uses this when caching was requested, and a new-enough file was present in the cache directory.
// This is simply a finished request, that can still be queried like a normal one so users don't know it came from the cache.
class CachedRequest : public Request {
public:
CachedRequest(RequestMethod method, std::string_view url, std::string_view name, bool *cancelled, RequestFlags flags, std::string_view responseData)
: Request(method, url, name, cancelled, flags)
{
buffer_.Append(responseData);
}
void Start() override {}
void Join() override {}
bool Done() override { return true; }
bool Failed() const override { return false; }
};
} // namespace http
-151
View File
@@ -36,155 +36,4 @@ Request::Request(RequestMethod method, std::string_view url, std::string_view na
};
}
static bool IsHttpsUrl(std::string_view url) {
return startsWith(url, "https:");
}
Path UrlToCachePath(const Path &cacheDir, std::string_view url) {
std::string fn = "DLCACHE_";
for (auto c : url) {
if (isalnum(c) || c == '.' || c == '-' || c == '_') {
fn.push_back(tolower(c));
} else {
fn.push_back('_');
}
}
return cacheDir / fn;
}
Path RequestManager::UrlToCachePath(const std::string_view url) {
if (cacheDir_.empty()) {
return Path();
}
return http::UrlToCachePath(cacheDir_, url);
}
static std::shared_ptr<Request> CreateRequest(RequestMethod method, std::string_view url, std::string_view postdata, std::string_view postMime, const Path &outfile, RequestFlags flags, net::ResolveFunc customResolve, std::string_view name) {
if (IsHttpsUrl(url) && System_GetPropertyBool(SYSPROP_SUPPORTS_HTTPS)) {
#ifndef HTTPS_NOT_AVAILABLE
return std::make_shared<HTTPSRequest>(method, url, postdata, postMime, outfile, flags, name);
#else
return std::shared_ptr<Request>();
#endif
} else {
return std::make_shared<HTTPRequest>(method, url, postdata, postMime, outfile, flags, customResolve, name);
}
}
std::shared_ptr<Request> RequestManager::StartDownload(std::string_view url, const Path &outfile, RequestFlags flags, const char *acceptMime) {
const bool enableCache = !cacheDir_.empty() && (flags & RequestFlags::Cached24H);
// Come up with a cache file path.
Path cacheFile = UrlToCachePath(url);
if (enableCache) {
_dbg_assert_(outfile.empty()); // It's automatically replaced below
// TODO: This should be done on the thread, maybe. But let's keep it simple for now.
time_t cacheFileTime;
if (File::GetModifTimeT(cacheFile, &cacheFileTime)) {
time_t now = (time_t)time_now_unix_utc();
if (cacheFileTime > now - 24 * 60 * 60) {
// The file is new enough. Let's construct a fake, already finished download so we don't need
// to modify the calling code.
std::string contents;
if (File::ReadBinaryFileToString(cacheFile, &contents)) {
INFO_LOG(Log::sceNet, "Returning cached file for %.*s: %s", (int)url.size(), url.data(), cacheFile.c_str());
// All is well, but we've indented a bit much here.
std::shared_ptr<Request> dl(new CachedRequest(RequestMethod::GET, url, "infra-dns.json", nullptr, flags, contents));
newDownloads_.push_back(dl);
return dl;
} else {
INFO_LOG(Log::sceNet, "Failed reading from cache, proceeding with request");
}
} else {
INFO_LOG(Log::sceNet, "Cached file too old, proceeding with request");
}
} else {
INFO_LOG(Log::sceNet, "Failed to check time modified. Proceeding with request.");
}
}
std::shared_ptr<Request> dl = CreateRequest(RequestMethod::GET, url, "", "", outfile, flags, nullptr, "");
// OK, didn't get it from cache, so let's continue with the download, putting it in the cache.
if (enableCache) {
dl->OverrideOutFile(cacheFile);
dl->AddFlag(RequestFlags::KeepInMemory);
}
if (!userAgent_.empty()) {
dl->SetUserAgent(userAgent_);
}
if (acceptMime) {
dl->SetAccept(acceptMime);
}
newDownloads_.push_back(dl);
dl->Start();
return dl;
}
std::shared_ptr<Request> RequestManager::StartDownloadWithCallback(
std::string_view url,
const Path &outfile,
RequestFlags flags,
std::function<void(Request &)> callback,
std::string_view name,
const char *acceptMime) {
std::shared_ptr<Request> dl = CreateRequest(RequestMethod::GET, url, "", "", outfile, flags, nullptr, name);
if (!userAgent_.empty())
dl->SetUserAgent(userAgent_);
if (acceptMime)
dl->SetAccept(acceptMime);
dl->SetCallback(callback);
newDownloads_.push_back(dl);
dl->Start();
return dl;
}
std::shared_ptr<Request> RequestManager::AsyncPostWithCallback(
std::string_view url,
std::string_view postData,
std::string_view postMime,
RequestFlags flags,
std::function<void(Request &)> callback,
std::string_view name) {
std::shared_ptr<Request> dl = CreateRequest(RequestMethod::POST, url, postData, postMime, Path(), flags, nullptr, name);
if (!userAgent_.empty())
dl->SetUserAgent(userAgent_);
dl->SetCallback(callback);
newDownloads_.push_back(dl);
dl->Start();
return dl;
}
void RequestManager::Update() {
for (auto &iter : newDownloads_) {
downloads_.push_back(iter);
}
newDownloads_.clear();
restart:
for (size_t i = 0; i < downloads_.size(); i++) {
auto dl = downloads_[i];
if (dl->Done()) {
dl->RunCallback();
dl->Join();
downloads_.erase(downloads_.begin() + i);
goto restart;
}
}
}
void RequestManager::CancelAll() {
for (size_t i = 0; i < downloads_.size(); i++) {
downloads_[i]->Cancel();
}
for (size_t i = 0; i < downloads_.size(); i++) {
downloads_[i]->Join();
}
downloads_.clear();
}
} // namespace
-51
View File
@@ -96,57 +96,6 @@ private:
std::function<void(Request &)> callback_;
};
using std::shared_ptr;
class RequestManager {
public:
~RequestManager() {
CancelAll();
}
// NOTE: This is the only version that supports the cache flag (for now).
std::shared_ptr<Request> StartDownload(std::string_view url, const Path &outfile, RequestFlags flags, const char *acceptMime = nullptr);
std::shared_ptr<Request> StartDownloadWithCallback(
std::string_view url,
const Path &outfile,
RequestFlags flags,
std::function<void(Request &)> callback,
std::string_view name = "",
const char *acceptMime = nullptr);
std::shared_ptr<Request> AsyncPostWithCallback(
std::string_view url,
std::string_view postData,
std::string_view postMime, // Use postMime = "application/x-www-form-urlencoded" for standard form-style posts, such as used by retroachievements. For encoding form data manually we have MultipartFormDataEncoder.
RequestFlags flags,
std::function<void(Request &)> callback,
std::string_view name = "");
// Drops finished downloads from the list.
void Update();
void CancelAll();
void SetUserAgent(std::string_view userAgent) {
userAgent_ = userAgent;
}
void SetCacheDir(const Path &path) {
cacheDir_ = path;
}
Path UrlToCachePath(const std::string_view url);
private:
std::vector<std::shared_ptr<Request>> downloads_;
// These get copied to downloads_ in Update(). It's so that callbacks can add new downloads
// while running.
std::vector<std::shared_ptr<Request>> newDownloads_;
std::string userAgent_;
Path cacheDir_;
};
inline const char *RequestMethodToString(RequestMethod method) {
switch (method) {
case RequestMethod::GET: return "GET";
+164
View File
@@ -0,0 +1,164 @@
#include "Common/StringUtils.h"
#include "Common/System/System.h"
#include "Common/System/Request.h"
#include "Common/Net/HTTPClient.h"
#include "Common/Net/HTTPRequest.h"
#include "Common/Net/HTTPRequestManager.h"
#include "Common/Net/HTTPNaettRequest.h"
#include "Common/File/FileUtil.h"
#include "Common/TimeUtil.h"
namespace http {
static bool IsHttpsUrl(std::string_view url) {
return startsWith(url, "https:");
}
Path UrlToCachePath(const Path &cacheDir, std::string_view url) {
std::string fn = "DLCACHE_";
for (auto c : url) {
if (isalnum(c) || c == '.' || c == '-' || c == '_') {
fn.push_back(tolower(c));
} else {
fn.push_back('_');
}
}
return cacheDir / fn;
}
Path RequestManager::UrlToCachePath(const std::string_view url) {
if (cacheDir_.empty()) {
return Path();
}
return http::UrlToCachePath(cacheDir_, url);
}
static std::shared_ptr<Request> CreateRequest(RequestMethod method, std::string_view url, std::string_view postdata, std::string_view postMime, const Path &outfile, RequestFlags flags, net::ResolveFunc customResolve, std::string_view name) {
if (IsHttpsUrl(url) && System_GetPropertyBool(SYSPROP_SUPPORTS_HTTPS)) {
#ifndef HTTPS_NOT_AVAILABLE
return std::make_shared<HTTPSRequest>(method, url, postdata, postMime, outfile, flags, name);
#else
return std::shared_ptr<Request>();
#endif
} else {
return std::make_shared<HTTPRequest>(method, url, postdata, postMime, outfile, flags, customResolve, name);
}
}
std::shared_ptr<Request> RequestManager::StartDownload(std::string_view url, const Path &outfile, RequestFlags flags, const char *acceptMime) {
const bool enableCache = !cacheDir_.empty() && (flags & RequestFlags::Cached24H);
// Come up with a cache file path.
Path cacheFile = UrlToCachePath(url);
if (enableCache) {
_dbg_assert_(outfile.empty()); // It's automatically replaced below
// TODO: This should be done on the thread, maybe. But let's keep it simple for now.
time_t cacheFileTime;
if (File::GetModifTimeT(cacheFile, &cacheFileTime)) {
time_t now = (time_t)time_now_unix_utc();
if (cacheFileTime > now - 24 * 60 * 60) {
// The file is new enough. Let's construct a fake, already finished download so we don't need
// to modify the calling code.
std::string contents;
if (File::ReadBinaryFileToString(cacheFile, &contents)) {
INFO_LOG(Log::sceNet, "Returning cached file for %.*s: %s", (int)url.size(), url.data(), cacheFile.c_str());
// All is well, but we've indented a bit much here.
std::shared_ptr<Request> dl(new CachedRequest(RequestMethod::GET, url, "infra-dns.json", nullptr, flags, contents));
newDownloads_.push_back(dl);
return dl;
} else {
INFO_LOG(Log::sceNet, "Failed reading from cache, proceeding with request");
}
} else {
INFO_LOG(Log::sceNet, "Cached file too old, proceeding with request");
}
} else {
INFO_LOG(Log::sceNet, "Failed to check time modified. Proceeding with request.");
}
}
std::shared_ptr<Request> dl = CreateRequest(RequestMethod::GET, url, "", "", outfile, flags, nullptr, "");
// OK, didn't get it from cache, so let's continue with the download, putting it in the cache.
if (enableCache) {
dl->OverrideOutFile(cacheFile);
dl->AddFlag(RequestFlags::KeepInMemory);
}
if (!userAgent_.empty()) {
dl->SetUserAgent(userAgent_);
}
if (acceptMime) {
dl->SetAccept(acceptMime);
}
newDownloads_.push_back(dl);
dl->Start();
return dl;
}
std::shared_ptr<Request> RequestManager::StartDownloadWithCallback(
std::string_view url,
const Path &outfile,
RequestFlags flags,
std::function<void(Request &)> callback,
std::string_view name,
const char *acceptMime) {
std::shared_ptr<Request> dl = CreateRequest(RequestMethod::GET, url, "", "", outfile, flags, nullptr, name);
if (!userAgent_.empty())
dl->SetUserAgent(userAgent_);
if (acceptMime)
dl->SetAccept(acceptMime);
dl->SetCallback(callback);
newDownloads_.push_back(dl);
dl->Start();
return dl;
}
std::shared_ptr<Request> RequestManager::AsyncPostWithCallback(
std::string_view url,
std::string_view postData,
std::string_view postMime,
RequestFlags flags,
std::function<void(Request &)> callback,
std::string_view name) {
std::shared_ptr<Request> dl = CreateRequest(RequestMethod::POST, url, postData, postMime, Path(), flags, nullptr, name);
if (!userAgent_.empty())
dl->SetUserAgent(userAgent_);
dl->SetCallback(callback);
newDownloads_.push_back(dl);
dl->Start();
return dl;
}
void RequestManager::Update() {
for (auto &iter : newDownloads_) {
downloads_.push_back(iter);
}
newDownloads_.clear();
restart:
for (size_t i = 0; i < downloads_.size(); i++) {
auto dl = downloads_[i];
if (dl->Done()) {
dl->RunCallback();
dl->Join();
downloads_.erase(downloads_.begin() + i);
goto restart;
}
}
}
void RequestManager::CancelAll() {
for (size_t i = 0; i < downloads_.size(); i++) {
downloads_[i]->Cancel();
}
for (size_t i = 0; i < downloads_.size(); i++) {
downloads_[i]->Join();
}
downloads_.clear();
}
} // namespace http
+75
View File
@@ -0,0 +1,75 @@
#pragma once
#include <memory>
#include <string_view>
#include "Common/Net/HTTPRequest.h"
#include "Common/Net/NetBuffer.h"
namespace http {
class RequestManager {
public:
~RequestManager() {
CancelAll();
}
// NOTE: This is the only version that supports the cache flag (for now).
std::shared_ptr<Request> StartDownload(std::string_view url, const Path &outfile, RequestFlags flags, const char *acceptMime = nullptr);
std::shared_ptr<Request> StartDownloadWithCallback(
std::string_view url,
const Path &outfile,
RequestFlags flags,
std::function<void(Request &)> callback,
std::string_view name = "",
const char *acceptMime = nullptr);
std::shared_ptr<Request> AsyncPostWithCallback(
std::string_view url,
std::string_view postData,
std::string_view postMime, // Use postMime = "application/x-www-form-urlencoded" for standard form-style posts, such as used by retroachievements. For encoding form data manually we have MultipartFormDataEncoder.
RequestFlags flags,
std::function<void(Request &)> callback,
std::string_view name = "");
// Drops finished downloads from the list.
void Update();
void CancelAll();
void SetUserAgent(std::string_view userAgent) {
userAgent_ = userAgent;
}
void SetCacheDir(const Path &path) {
cacheDir_ = path;
}
Path UrlToCachePath(const std::string_view url);
private:
std::vector<std::shared_ptr<Request>> downloads_;
// These get copied to downloads_ in Update(). It's so that callbacks can add new downloads
// while running.
std::vector<std::shared_ptr<Request>> newDownloads_;
std::string userAgent_;
Path cacheDir_;
};
// Fake request for cache hits.
// The download manager uses this when caching was requested, and a new-enough file was present in the cache directory.
// This is simply a finished request, that can still be queried like a normal one so users don't know it came from the cache.
class CachedRequest : public Request {
public:
CachedRequest(RequestMethod method, std::string_view url, std::string_view name, bool *cancelled, RequestFlags flags, std::string_view responseData)
: Request(method, url, name, cancelled, flags)
{
buffer_.Append(responseData);
}
void Start() override {}
void Join() override {}
bool Done() override { return true; }
bool Failed() const override { return false; }
};
} // namespace http
+1 -1
View File
@@ -41,11 +41,11 @@
#include "Common/File/VFS/VFS.h"
#include "Common/Log/LogManager.h"
#include "Common/Math/CrossSIMD.h"
#include "Common/Net/HTTPRequestManager.h"
#include "Common/OSVersion.h"
#include "Common/System/Display.h"
#include "Common/System/System.h"
#include "Common/StringUtils.h"
#include "Common/Thread/ThreadUtil.h"
#include "Common/GPU/Vulkan/VulkanLoader.h"
#include "Common/VR/PPSSPPVR.h"
#include "Common/System/OSD.h"
+1
View File
@@ -25,6 +25,7 @@
#include "Common/Data/Text/I18n.h"
#include "Common/File/VFS/VFS.h"
#include "Common/File/FileUtil.h"
#include "Common/Net/HTTPRequestManager.h"
#include "Common/System/OSD.h"
#include "Common/Serialize/Serializer.h"
#include "Common/Serialize/SerializeFuncs.h"
+1
View File
@@ -40,6 +40,7 @@ extern "C" {
#include "Common/System/OSD.h"
#include "Common/Data/Text/I18n.h"
#include "Common/Net/HTTPClient.h"
#include "Common/Net/HTTPRequestManager.h"
#include "Common/Net/Resolve.h"
#include "Common/Net/URL.h"
#include "Common/Thread/ThreadUtil.h"
+1
View File
@@ -31,6 +31,7 @@
#include "Common/Log.h"
#include "Common/File/Path.h"
#include "Common/Net/HTTPRequest.h"
#include "Common/Net/HTTPRequestManager.h"
#include "Common/System/OSD.h"
#include "Common/System/System.h"
#include "Common/System/NativeApp.h"
+1
View File
@@ -38,6 +38,7 @@
#include "Common/Log.h"
#include "Common/System/OSD.h"
#include "Common/File/FileUtil.h"
#include "Common/Net/HTTPRequestManager.h"
#include "Common/StringUtils.h"
#include "Common/Thread/ThreadUtil.h"
#include "Core/Config.h"
+1 -1
View File
@@ -39,7 +39,7 @@
#include "Common/Data/Text/I18n.h"
#include "Common/Data/Text/Parsers.h"
#include "Common/Data/Encoding/Utf8.h"
#include "Common/Net/HTTPClient.h"
#include "Common/Net/HTTPRequestManager.h"
#include "Common/UI/Context.h"
#include "Common/UI/View.h"
#include "Common/UI/ViewGroup.h"
+2 -2
View File
@@ -45,7 +45,7 @@
#include "Windows/CaptureDevice.h"
#endif
#include "Common/Net/HTTPClient.h"
#include "Common/Net/HTTPRequestManager.h"
#include "Common/Net/Resolve.h"
#include "Common/Net/URL.h"
#include "Common/Render/TextureAtlas.h"
@@ -77,7 +77,6 @@
#include "Common/File/VFS/DirectoryReader.h"
#include "Common/CPUDetect.h"
#include "Common/File/FileUtil.h"
#include "Common/TimeUtil.h"
#include "Common/StringUtils.h"
#include "Common/Log/LogManager.h"
#include "Common/MemArena.h"
@@ -87,6 +86,7 @@
#include "Common/VR/PPSSPPVR.h"
#include "Common/Thread/ThreadManager.h"
#include "Common/Audio/AudioBackend.h"
#include "Common/TimeUtil.h"
#include "Core/ControlMapper.h"
#include "Core/Config.h"
+1 -1
View File
@@ -28,7 +28,7 @@
#include "Common/Data/Format/JSONReader.h"
#include "Common/StringUtils.h"
#include "Common/Render/ManagedTexture.h"
#include "Common/Net/NetBuffer.h"
#include "Common/Net/HTTPRequestManager.h"
#include "Core/Config.h"
#include "Core/System.h"
#include "Core/Util/GameManager.h"
+2
View File
@@ -102,6 +102,7 @@
<ClInclude Include="..\..\Common\Net\Connection.h" />
<ClInclude Include="..\..\Common\Net\HTTPNaettRequest.h" />
<ClInclude Include="..\..\Common\Net\HTTPRequest.h" />
<ClInclude Include="..\..\Common\Net\HTTPRequestManager.h" />
<ClInclude Include="..\..\Common\Net\NetBuffer.h" />
<ClInclude Include="..\..\Common\Data\Collections\ConstMap.h" />
<ClInclude Include="..\..\Common\Data\Collections\FixedSizeQueue.h" />
@@ -289,6 +290,7 @@
<ClCompile Include="..\..\Common\Net\Connection.cpp" />
<ClCompile Include="..\..\Common\Net\HTTPNaettRequest.cpp" />
<ClCompile Include="..\..\Common\Net\HTTPRequest.cpp" />
<ClCompile Include="..\..\Common\Net\HTTPRequestManager.cpp" />
<ClCompile Include="..\..\Common\Net\NetBuffer.cpp" />
<ClCompile Include="..\..\Common\Data\Color\RGBAUtil.cpp" />
<ClCompile Include="..\..\Common\Data\Convert\SmallDataConvert.cpp" />
+6
View File
@@ -523,6 +523,9 @@
<ClCompile Include="..\..\Common\Net\Connection.cpp">
<Filter>Net</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\Net\HTTPRequestManager.cpp">
<Filter>Net</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="targetver.h" />
@@ -1005,6 +1008,9 @@
<ClInclude Include="..\..\Common\Net\Connection.h">
<Filter>Net</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Net\HTTPRequestManager.h">
<Filter>Net</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="..\..\Common\Math\fast\fast_matrix_neon.S">
+1
View File
@@ -340,6 +340,7 @@ EXEC_AND_LIB_FILES := \
$(SRC)/Common/Net/HTTPClient.cpp \
$(SRC)/Common/Net/HTTPHeaders.cpp \
$(SRC)/Common/Net/HTTPRequest.cpp \
$(SRC)/Common/Net/HTTPRequestManager.cpp \
$(SRC)/Common/Net/HTTPNaettRequest.cpp \
$(SRC)/Common/Net/HTTPServer.cpp \
$(SRC)/Common/Net/NetBuffer.cpp \
+1
View File
@@ -447,6 +447,7 @@ SOURCES_CXX += \
$(COMMONDIR)/Net/HTTPServer.cpp \
$(COMMONDIR)/Net/HTTPRequest.cpp \
$(COMMONDIR)/Net/HTTPNaettRequest.cpp \
$(COMMONDIR)/Net/HTTPRequestManager.cpp \
$(COMMONDIR)/Net/NetBuffer.cpp \
$(COMMONDIR)/Net/Resolve.cpp \
$(COMMONDIR)/Net/Sinks.cpp \