diff --git a/CMakeLists.txt b/CMakeLists.txt index 73eed44006..99b390d14b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -575,6 +575,8 @@ add_library(Common STATIC Common/Net/HTTPHeaders.h Common/Net/HTTPServer.cpp Common/Net/HTTPServer.h + Common/Net/NetBuffer.cpp + Common/Net/NetBuffer.h Common/Net/Resolve.cpp Common/Net/Resolve.h Common/Net/Sinks.cpp diff --git a/Common/Buffer.cpp b/Common/Buffer.cpp index 68c7e7682e..fd25b0ad97 100644 --- a/Common/Buffer.cpp +++ b/Common/Buffer.cpp @@ -1,24 +1,7 @@ +#include +#include +#include -#include -#include -#include - -#ifdef _WIN32 -#include -#undef min -#undef max -#else -#include -#include -#endif - -#ifndef MSG_NOSIGNAL -// Default value to 0x00 (do nothing) in systems where it's not supported. -#define MSG_NOSIGNAL 0x00 -#endif - -#include "Common/File/FileDescriptor.h" -#include "Common/TimeUtil.h" #include "Common/Buffer.h" #include "Common/Log.h" @@ -36,14 +19,14 @@ char *Buffer::Append(size_t length) { } void Buffer::Append(const std::string &str) { - char *ptr = Append(str.size()); - memcpy(ptr, str.data(), str.size()); + char *ptr = Append(str.size()); + memcpy(ptr, str.data(), str.size()); } void Buffer::Append(const char *str) { - size_t len = strlen(str); - char *dest = Append(len); - memcpy(dest, str, len); + size_t len = strlen(str); + char *dest = Append(len); + memcpy(dest, str, len); } void Buffer::Append(const Buffer &other) { @@ -55,10 +38,10 @@ void Buffer::Append(const Buffer &other) { } void Buffer::AppendValue(int value) { - char buf[16]; - // This is slow. - sprintf(buf, "%i", value); - Append(buf); + char buf[16]; + // This is slow. + sprintf(buf, "%i", value); + Append(buf); } void Buffer::Take(size_t length, std::string *dest) { @@ -78,14 +61,14 @@ void Buffer::Take(size_t length, char *dest) { } int Buffer::TakeLineCRLF(std::string *dest) { - int after_next_line = OffsetToAfterNextCRLF(); - if (after_next_line < 0) - return after_next_line; - else { - Take(after_next_line - 2, dest); - Skip(2); // Skip the CRLF - return after_next_line - 2; - } + int after_next_line = OffsetToAfterNextCRLF(); + if (after_next_line < 0) { + return after_next_line; + } else { + Take(after_next_line - 2, dest); + Skip(2); // Skip the CRLF + return after_next_line - 2; + } } void Buffer::Skip(size_t length) { @@ -97,48 +80,39 @@ void Buffer::Skip(size_t length) { } int Buffer::SkipLineCRLF() { - int after_next_line = OffsetToAfterNextCRLF(); - if (after_next_line < 0) - return after_next_line; - else { - Skip(after_next_line); - return after_next_line - 2; - } + int after_next_line = OffsetToAfterNextCRLF(); + if (after_next_line < 0) { + return after_next_line; + } else { + Skip(after_next_line); + return after_next_line - 2; + } } int Buffer::OffsetToAfterNextCRLF() { - for (int i = 0; i < (int)data_.size() - 1; i++) { - if (data_[i] == '\r' && data_[i + 1] == '\n') { - return i + 2; - } - } - return -1; + for (int i = 0; i < (int)data_.size() - 1; i++) { + if (data_[i] == '\r' && data_[i + 1] == '\n') { + return i + 2; + } + } + return -1; } void Buffer::Printf(const char *fmt, ...) { - char buffer[2048]; - va_list vl; - va_start(vl, fmt); - size_t retval = vsnprintf(buffer, sizeof(buffer), fmt, vl); - if ((int)retval >= (int)sizeof(buffer)) { - // Output was truncated. TODO: Do something. - ERROR_LOG(IO, "Buffer::Printf truncated output"); - } - if (retval < 0) { - ERROR_LOG(IO, "Buffer::Printf failed"); - } - va_end(vl); - char *ptr = Append(retval); - memcpy(ptr, buffer, retval); -} - -bool Buffer::Flush(int fd) { - // Look into using send() directly. - bool success = data_.size() == fd_util::WriteLine(fd, &data_[0], data_.size()); - if (success) { - data_.resize(0); - } - return success; + char buffer[2048]; + va_list vl; + va_start(vl, fmt); + size_t retval = vsnprintf(buffer, sizeof(buffer), fmt, vl); + if ((int)retval >= (int)sizeof(buffer)) { + // Output was truncated. TODO: Do something. + ERROR_LOG(IO, "Buffer::Printf truncated output"); + } + if (retval < 0) { + ERROR_LOG(IO, "Buffer::Printf failed"); + } + va_end(vl); + char *ptr = Append(retval); + memcpy(ptr, buffer, retval); } bool Buffer::FlushToFile(const char *filename) { @@ -152,116 +126,6 @@ bool Buffer::FlushToFile(const char *filename) { return true; } -bool Buffer::FlushSocket(uintptr_t sock, double timeout, bool *cancelled) { - static constexpr float CANCEL_INTERVAL = 0.25f; - for (size_t pos = 0, end = data_.size(); pos < end; ) { - bool ready = false; - double leftTimeout = timeout; - while (!ready && (leftTimeout >= 0 || cancelled)) { - if (cancelled && *cancelled) - return false; - ready = fd_util::WaitUntilReady(sock, CANCEL_INTERVAL, true); - if (!ready && leftTimeout >= 0.0) { - leftTimeout -= CANCEL_INTERVAL; - if (leftTimeout < 0) { - ERROR_LOG(IO, "FlushSocket timed out"); - return false; - } - } - } - int sent = send(sock, &data_[pos], (int)(end - pos), MSG_NOSIGNAL); - if (sent < 0) { - ERROR_LOG(IO, "FlushSocket failed"); - return false; - } - pos += sent; - - // Buffer full, don't spin. - if (sent == 0 && timeout < 0.0) { - sleep_ms(1); - } - } - data_.resize(0); - return true; -} - -bool Buffer::ReadAll(int fd, int hintSize) { - std::vector buf; - if (hintSize >= 65536 * 16) { - buf.resize(65536); - } else if (hintSize >= 1024 * 16) { - buf.resize(hintSize / 16); - } else { - buf.resize(4096); - } - - while (true) { - int retval = recv(fd, &buf[0], (int)buf.size(), MSG_NOSIGNAL); - if (retval == 0) { - break; - } else if (retval < 0) { - ERROR_LOG(IO, "Error reading from buffer: %i", retval); - return false; - } - char *p = Append((size_t)retval); - memcpy(p, &buf[0], retval); - } - return true; -} - -bool Buffer::ReadAllWithProgress(int fd, int knownSize, float *progress, bool *cancelled) { - static constexpr float CANCEL_INTERVAL = 0.25f; - std::vector buf; - if (knownSize >= 65536 * 16) { - buf.resize(65536); - } else if (knownSize >= 1024 * 16) { - buf.resize(knownSize / 16); - } else { - buf.resize(1024); - } - - int total = 0; - while (true) { - bool ready = false; - while (!ready && cancelled) { - if (*cancelled) - return false; - ready = fd_util::WaitUntilReady(fd, CANCEL_INTERVAL, false); - } - int retval = recv(fd, &buf[0], (int)buf.size(), MSG_NOSIGNAL); - if (retval == 0) { - return true; - } else if (retval < 0) { - ERROR_LOG(IO, "Error reading from buffer: %i", retval); - return false; - } - char *p = Append((size_t)retval); - memcpy(p, &buf[0], retval); - total += retval; - if (progress) - *progress = (float)total / (float)knownSize; - } - return true; -} - -int Buffer::Read(int fd, size_t sz) { - char buf[1024]; - int retval; - size_t received = 0; - while ((retval = recv(fd, buf, (int)std::min(sz, sizeof(buf)), MSG_NOSIGNAL)) > 0) { - if (retval < 0) { - return retval; - } - char *p = Append((size_t)retval); - memcpy(p, buf, retval); - sz -= retval; - received += retval; - if (sz == 0) - return 0; - } - return (int)received; -} - void Buffer::PeekAll(std::string *dest) { dest->resize(data_.size()); memcpy(&(*dest)[0], &data_[0], data_.size()); diff --git a/Common/Buffer.h b/Common/Buffer.h index ef93a2945c..7a271bf8ba 100644 --- a/Common/Buffer.h +++ b/Common/Buffer.h @@ -59,25 +59,17 @@ public: // Writes the entire buffer to the file descriptor. Also resets the // size to zero. On failure, data remains in buffer and nothing is // written. - bool Flush(int fd); bool FlushToFile(const char *filename); - bool FlushSocket(uintptr_t sock, double timeout = -1.0, bool *cancelled = nullptr); // Windows portability - - bool ReadAll(int fd, int hintSize = 0); - bool ReadAllWithProgress(int fd, int knownSize, float *progress, bool *cancelled); - - // < 0: error - // >= 0: number of bytes read - int Read(int fd, size_t sz); // Utilities. Try to avoid checking for size. size_t size() const { return data_.size(); } bool empty() const { return size() == 0; } void clear() { data_.resize(0); } -private: +protected: // TODO: Find a better internal representation, like a cord. std::vector data_; +private: DISALLOW_COPY_AND_ASSIGN(Buffer); }; diff --git a/Common/Common.vcxproj b/Common/Common.vcxproj index ed732b280a..384ac4b8b6 100644 --- a/Common/Common.vcxproj +++ b/Common/Common.vcxproj @@ -452,6 +452,7 @@ + @@ -775,6 +776,7 @@ + diff --git a/Common/Common.vcxproj.filters b/Common/Common.vcxproj.filters index 94649bd4b1..f341effcae 100644 --- a/Common/Common.vcxproj.filters +++ b/Common/Common.vcxproj.filters @@ -68,7 +68,6 @@ - ext\libpng17 @@ -388,6 +387,10 @@ Data\Convert + + + Net + @@ -434,7 +437,6 @@ - ext\libpng17 @@ -748,6 +750,10 @@ Data\Convert + + + Net + diff --git a/Common/Net/HTTPClient.cpp b/Common/Net/HTTPClient.cpp index d4f4eb8df7..fb86f96d6f 100644 --- a/Common/Net/HTTPClient.cpp +++ b/Common/Net/HTTPClient.cpp @@ -31,7 +31,7 @@ #include "Common/File/FileDescriptor.h" #include "Common/Thread/ThreadUtil.h" #include "Common/Data/Encoding/Compression.h" -#include "Common/Buffer.h" +#include "Common/Net/NetBuffer.h" #include "Common/Log.h" namespace net { @@ -253,7 +253,7 @@ int Client::GET(const char *resource, Buffer *output, std::vector & return err; } - Buffer readbuf; + net::Buffer readbuf; int code = ReadResponseHeaders(&readbuf, responseHeaders, progress, cancelled); if (code < 0) { return code; @@ -284,7 +284,7 @@ int Client::POST(const char *resource, const std::string &data, const std::strin return err; } - Buffer readbuf; + net::Buffer readbuf; std::vector responseHeaders; int code = ReadResponseHeaders(&readbuf, responseHeaders, progress); if (code < 0) { @@ -311,7 +311,7 @@ int Client::SendRequestWithData(const char *method, const char *resource, const *progress = 0.01f; } - Buffer buffer; + net::Buffer buffer; const char *tpl = "%s %s HTTP/%s\r\n" "Host: %s\r\n" @@ -333,7 +333,7 @@ int Client::SendRequestWithData(const char *method, const char *resource, const return 0; } -int Client::ReadResponseHeaders(Buffer *readbuf, std::vector &responseHeaders, float *progress, bool *cancelled) { +int Client::ReadResponseHeaders(net::Buffer *readbuf, std::vector &responseHeaders, float *progress, bool *cancelled) { // Snarf all the data we can into RAM. A little unsafe but hey. static constexpr float CANCEL_INTERVAL = 0.25f; bool ready = false; @@ -389,7 +389,7 @@ int Client::ReadResponseHeaders(Buffer *readbuf, std::vector &respo return code; } -int Client::ReadResponseEntity(Buffer *readbuf, const std::vector &responseHeaders, Buffer *output, float *progress, bool *cancelled) { +int Client::ReadResponseEntity(net::Buffer *readbuf, const std::vector &responseHeaders, Buffer *output, float *progress, bool *cancelled) { bool gzip = false; bool chunked = false; int contentLength = 0; diff --git a/Common/Net/HTTPClient.h b/Common/Net/HTTPClient.h index 9281863f1b..342f8f5604 100644 --- a/Common/Net/HTTPClient.h +++ b/Common/Net/HTTPClient.h @@ -5,10 +5,9 @@ #include #include +#include "Common/Net/NetBuffer.h" #include "Common/Net/Resolve.h" -#include "Common/Buffer.h" - namespace net { class Connection { @@ -61,9 +60,9 @@ public: int SendRequest(const char *method, const char *resource, const char *otherHeaders = nullptr, float *progress = nullptr, bool *cancelled = nullptr); int SendRequestWithData(const char *method, const char *resource, const std::string &data, const char *otherHeaders = nullptr, float *progress = nullptr, bool *cancelled = nullptr); - int ReadResponseHeaders(Buffer *readbuf, std::vector &responseHeaders, float *progress = nullptr, bool *cancelled = nullptr); + int ReadResponseHeaders(net::Buffer *readbuf, std::vector &responseHeaders, float *progress = nullptr, bool *cancelled = nullptr); // If your response contains a response, you must read it. - int ReadResponseEntity(Buffer *readbuf, const std::vector &responseHeaders, Buffer *output, float *progress = nullptr, bool *cancelled = nullptr); + int ReadResponseEntity(net::Buffer *readbuf, const std::vector &responseHeaders, Buffer *output, float *progress = nullptr, bool *cancelled = nullptr); void SetDataTimeout(double t) { dataTimeout_ = t; diff --git a/Common/Net/HTTPHeaders.h b/Common/Net/HTTPHeaders.h index dec2501d69..5900b01874 100644 --- a/Common/Net/HTTPHeaders.h +++ b/Common/Net/HTTPHeaders.h @@ -3,7 +3,7 @@ #include #include -#include "Common/Buffer.h" +#include "Common/Net/NetBuffer.h" namespace net { class InputSink; diff --git a/Common/Net/HTTPServer.cpp b/Common/Net/HTTPServer.cpp index 4a4227d423..882d49a965 100644 --- a/Common/Net/HTTPServer.cpp +++ b/Common/Net/HTTPServer.cpp @@ -35,10 +35,9 @@ #include #include "Common/Net/HTTPServer.h" +#include "Common/Net/NetBuffer.h" #include "Common/Net/Sinks.h" #include "Common/File/FileDescriptor.h" - -#include "Common/Buffer.h" #include "Common/Log.h" void NewThreadExecutor::Run(std::function &&func) { diff --git a/Common/Net/NetBuffer.cpp b/Common/Net/NetBuffer.cpp new file mode 100644 index 0000000000..08059e76c8 --- /dev/null +++ b/Common/Net/NetBuffer.cpp @@ -0,0 +1,134 @@ +#ifdef _WIN32 +#include +#undef min +#undef max +#else +#include +#include +#endif +#include +#include + +#ifndef MSG_NOSIGNAL +// Default value to 0x00 (do nothing) in systems where it's not supported. +#define MSG_NOSIGNAL 0x00 +#endif + +#include "Common/File/FileDescriptor.h" +#include "Common/Log.h" +#include "Common/Net/NetBuffer.h" +#include "Common/TimeUtil.h" + +namespace net { + +bool Buffer::FlushSocket(uintptr_t sock, double timeout, bool *cancelled) { + static constexpr float CANCEL_INTERVAL = 0.25f; + for (size_t pos = 0, end = data_.size(); pos < end; ) { + bool ready = false; + double leftTimeout = timeout; + while (!ready && (leftTimeout >= 0 || cancelled)) { + if (cancelled && *cancelled) + return false; + ready = fd_util::WaitUntilReady(sock, CANCEL_INTERVAL, true); + if (!ready && leftTimeout >= 0.0) { + leftTimeout -= CANCEL_INTERVAL; + if (leftTimeout < 0) { + ERROR_LOG(IO, "FlushSocket timed out"); + return false; + } + } + } + int sent = send(sock, &data_[pos], (int)(end - pos), MSG_NOSIGNAL); + if (sent < 0) { + ERROR_LOG(IO, "FlushSocket failed"); + return false; + } + pos += sent; + + // Buffer full, don't spin. + if (sent == 0 && timeout < 0.0) { + sleep_ms(1); + } + } + data_.resize(0); + return true; +} + +bool Buffer::ReadAll(int fd, int hintSize) { + std::vector buf; + if (hintSize >= 65536 * 16) { + buf.resize(65536); + } else if (hintSize >= 1024 * 16) { + buf.resize(hintSize / 16); + } else { + buf.resize(4096); + } + + while (true) { + int retval = recv(fd, &buf[0], (int)buf.size(), MSG_NOSIGNAL); + if (retval == 0) { + break; + } else if (retval < 0) { + ERROR_LOG(IO, "Error reading from buffer: %i", retval); + return false; + } + char *p = Append((size_t)retval); + memcpy(p, &buf[0], retval); + } + return true; +} + +bool Buffer::ReadAllWithProgress(int fd, int knownSize, float *progress, bool *cancelled) { + static constexpr float CANCEL_INTERVAL = 0.25f; + std::vector buf; + if (knownSize >= 65536 * 16) { + buf.resize(65536); + } else if (knownSize >= 1024 * 16) { + buf.resize(knownSize / 16); + } else { + buf.resize(1024); + } + + int total = 0; + while (true) { + bool ready = false; + while (!ready && cancelled) { + if (*cancelled) + return false; + ready = fd_util::WaitUntilReady(fd, CANCEL_INTERVAL, false); + } + int retval = recv(fd, &buf[0], (int)buf.size(), MSG_NOSIGNAL); + if (retval == 0) { + return true; + } else if (retval < 0) { + ERROR_LOG(IO, "Error reading from buffer: %i", retval); + return false; + } + char *p = Append((size_t)retval); + memcpy(p, &buf[0], retval); + total += retval; + if (progress) + *progress = (float)total / (float)knownSize; + } + return true; +} + +int Buffer::Read(int fd, size_t sz) { + char buf[1024]; + int retval; + size_t received = 0; + while ((retval = recv(fd, buf, (int)std::min(sz, sizeof(buf)), MSG_NOSIGNAL)) > 0) { + if (retval < 0) { + return retval; + } + char *p = Append((size_t)retval); + memcpy(p, buf, retval); + sz -= retval; + received += retval; + if (sz == 0) + return 0; + } + return (int)received; +} + +} diff --git a/Common/Net/NetBuffer.h b/Common/Net/NetBuffer.h new file mode 100644 index 0000000000..3575dceddd --- /dev/null +++ b/Common/Net/NetBuffer.h @@ -0,0 +1,19 @@ +#pragma once + +#include "Common/Buffer.h" + +namespace net { + +class Buffer : public ::Buffer { +public: + bool FlushSocket(uintptr_t sock, double timeout = -1.0, bool *cancelled = nullptr); + + bool ReadAll(int fd, int hintSize = 0); + bool ReadAllWithProgress(int fd, int knownSize, float *progress, bool *cancelled); + + // < 0: error + // >= 0: number of bytes read + int Read(int fd, size_t sz); +}; + +} diff --git a/Core/FileLoaders/HTTPFileLoader.cpp b/Core/FileLoaders/HTTPFileLoader.cpp index 5aa692b0d6..0535ee024a 100644 --- a/Core/FileLoaders/HTTPFileLoader.cpp +++ b/Core/FileLoaders/HTTPFileLoader.cpp @@ -145,7 +145,7 @@ int HTTPFileLoader::SendHEAD(const Url &url, std::vector &responseH return -400; } - Buffer readbuf; + net::Buffer readbuf; return client_.ReadResponseHeaders(&readbuf, responseHeaders); } @@ -203,7 +203,7 @@ size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data, Flags f return 0; } - Buffer readbuf; + net::Buffer readbuf; std::vector responseHeaders; int code = client_.ReadResponseHeaders(&readbuf, responseHeaders); if (code != 206) { @@ -235,7 +235,7 @@ size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data, Flags f } // TODO: Would be nice to read directly. - Buffer output; + net::Buffer output; int res = client_.ReadResponseEntity(&readbuf, responseHeaders, &output); if (res != 0) { ERROR_LOG(LOADER, "Unable to read HTTP response entity: %d", res); diff --git a/UWP/CommonUWP/CommonUWP.vcxproj b/UWP/CommonUWP/CommonUWP.vcxproj index 1e9d6615d8..4f5deb5d70 100644 --- a/UWP/CommonUWP/CommonUWP.vcxproj +++ b/UWP/CommonUWP/CommonUWP.vcxproj @@ -386,6 +386,7 @@ + @@ -513,6 +514,7 @@ + diff --git a/UWP/CommonUWP/CommonUWP.vcxproj.filters b/UWP/CommonUWP/CommonUWP.vcxproj.filters index 4bf921ae9f..851b917b4c 100644 --- a/UWP/CommonUWP/CommonUWP.vcxproj.filters +++ b/UWP/CommonUWP/CommonUWP.vcxproj.filters @@ -124,6 +124,7 @@ Crypto + ext\libpng17 @@ -415,6 +416,7 @@ Crypto + diff --git a/android/jni/Android.mk b/android/jni/Android.mk index 82c9624c6a..a9e52cc2a9 100644 --- a/android/jni/Android.mk +++ b/android/jni/Android.mk @@ -274,6 +274,7 @@ EXEC_AND_LIB_FILES := \ $(SRC)/Common/Net/HTTPClient.cpp \ $(SRC)/Common/Net/HTTPHeaders.cpp \ $(SRC)/Common/Net/HTTPServer.cpp \ + $(SRC)/Common/Net/NetBuffer.cpp \ $(SRC)/Common/Net/Resolve.cpp \ $(SRC)/Common/Net/Sinks.cpp \ $(SRC)/Common/Net/URL.cpp \ diff --git a/libretro/Makefile.common b/libretro/Makefile.common index 6ed5e666ee..3bba3a5ab0 100644 --- a/libretro/Makefile.common +++ b/libretro/Makefile.common @@ -197,6 +197,7 @@ SOURCES_CXX += \ $(COMMONDIR)/Net/HTTPClient.cpp \ $(COMMONDIR)/Net/HTTPHeaders.cpp \ $(COMMONDIR)/Net/HTTPServer.cpp \ + $(COMMONDIR)/Net/NetBuffer.cpp \ $(COMMONDIR)/Net/Resolve.cpp \ $(COMMONDIR)/Net/Sinks.cpp \ $(COMMONDIR)/Net/URL.cpp \