Skip to content

Commit 4a1c14f

Browse files
poor-circlepoor-circle
authored
[breakchange][feat] async_upload support offset (#621)
1 parent 03a5cf8 commit 4a1c14f

File tree

2 files changed

+111
-19
lines changed

2 files changed

+111
-19
lines changed

‎include/cinatra/coro_http_client.hpp‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
164164
: coro_http_client(executor->get_asio_executor()) {}
165165

166166
bool init_config(const config &conf) {
167+
config_ = conf;
167168
if (conf.conn_timeout_duration.has_value()) {
168169
set_conn_timeout(*conf.conn_timeout_duration);
169170
}
@@ -207,6 +208,8 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
207208

208209
coro_io::ExecutorWrapper<> &get_executor() { return executor_wrapper_; }
209210

211+
const config &get_config() { return config_; }
212+
210213
#ifdef CINATRA_ENABLE_SSL
211214
bool init_ssl(int verify_mode, const std::string &base_path,
212215
const std::string &cert_file, const std::string &sni_hostname) {
@@ -875,14 +878,16 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
875878
}
876879

877880
async_simple::coro::Lazy<void> send_file_no_chunked_with_copy(
878-
std::string_view source, std::error_code &ec, std::size_t length) {
881+
std::string_view source, std::error_code &ec, std::size_t length,
882+
std::size_t offset) {
879883
if (length <= 0) {
880884
co_return;
881885
}
882886
std::string file_data;
883887
detail::resize(file_data, std::min(max_single_part_size_, length));
884888
coro_io::coro_file file{};
885889
file.open(source, std::ios::in);
890+
file.seek(offset, std::ios::cur);
886891
if (!file.is_open()) {
887892
ec = std::make_error_code(std::errc::bad_file_descriptor);
888893
co_return;
@@ -920,15 +925,15 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
920925
};
921926
async_simple::coro::Lazy<void> send_file_without_copy(
922927
const std::filesystem::path &source, std::error_code &ec,
923-
std::size_t length) {
928+
std::size_t length, std::size_t offset) {
924929
fd_guard guard(source.c_str());
925930
if (guard.fd < 0) [[unlikely]] {
926931
ec = std::make_error_code(std::errc::bad_file_descriptor);
927932
co_return;
928933
}
929934
std::size_t actual_len = 0;
930-
std::tie(ec, actual_len) =
931-
co_await coro_io::async_sendfile(socket_->impl_, guard.fd, 0, length);
935+
std::tie(ec, actual_len) = co_await coro_io::async_sendfile(
936+
socket_->impl_, guard.fd, offset, length);
932937
if (ec) [[unlikely]] {
933938
co_return;
934939
}
@@ -1006,7 +1011,8 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
10061011
template <typename S, typename Source>
10071012
async_simple::coro::Lazy<resp_data> async_upload(
10081013
S uri, http_method method, Source source /* file */,
1009-
int64_t content_length = -1,
1014+
uint64_t offset = 0 /*file offset*/,
1015+
int64_t content_length = -1 /*upload size*/,
10101016
req_content_type content_type = req_content_type::text,
10111017
std::unordered_map<std::string, std::string> headers = {}) {
10121018
std::error_code ec{};
@@ -1050,6 +1056,12 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
10501056
co_return resp_data{std::make_error_code(std::errc::invalid_argument),
10511057
404};
10521058
}
1059+
content_length -= offset;
1060+
if (content_length < 0) {
1061+
CINATRA_LOG_ERROR << "the offset is larger than the end of file";
1062+
co_return resp_data{std::make_error_code(std::errc::invalid_argument),
1063+
404};
1064+
}
10531065
}
10541066

10551067
assert(content_length >= 0);
@@ -1088,6 +1100,7 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
10881100
}
10891101

10901102
if constexpr (is_stream_file) {
1103+
source->seekg(offset, std::ios::cur);
10911104
std::string file_data;
10921105
detail::resize(file_data, std::min<std::size_t>(max_single_part_size_,
10931106
content_length));
@@ -1116,15 +1129,17 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
11161129
if (!has_init_ssl_) {
11171130
#endif
11181131
co_await send_file_without_copy(std::filesystem::path{source}, ec,
1119-
content_length);
1132+
content_length, offset);
11201133
#ifdef CINATRA_ENABLE_SSL
11211134
}
11221135
else {
1123-
co_await send_file_no_chunked_with_copy(source, ec, content_length);
1136+
co_await send_file_no_chunked_with_copy(source, ec, content_length,
1137+
offset);
11241138
}
11251139
#endif
11261140
#else
1127-
co_await send_file_no_chunked_with_copy(source, ec, content_length);
1141+
co_await send_file_no_chunked_with_copy(source, ec, content_length,
1142+
offset);
11281143
#endif
11291144
}
11301145
else {
@@ -2432,6 +2447,7 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
24322447
std::string resp_chunk_str_;
24332448
std::span<char> out_buf_;
24342449
bool should_reset_ = false;
2450+
config config_;
24352451

24362452
#ifdef CINATRA_ENABLE_GZIP
24372453
bool enable_ws_deflate_ = false;

‎tests/test_cinatra.cpp‎

Lines changed: 87 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,10 @@ bool create_file(std::string_view filename, size_t file_size = 1024) {
271271
return false;
272272
}
273273

274-
std::string str(file_size, 'A');
274+
std::string str;
275+
for (int i = 0; i < file_size; ++i) {
276+
str.push_back(rand() % 26 + 'A');
277+
}
275278
out.write(str.data(), str.size());
276279
return true;
277280
}
@@ -1146,40 +1149,45 @@ TEST_CASE("test coro_http_client multipart upload") {
11461149

11471150
TEST_CASE("test coro_http_client upload") {
11481151
auto test_upload_by_file_path = [](std::string filename,
1152+
std::size_t offset = 0,
11491153
std::size_t r_size = SIZE_MAX,
11501154
bool should_failed = false) {
11511155
coro_http_client client{};
11521156
client.add_header("filename", filename);
1157+
client.add_header("offset", std::to_string(offset));
11531158
if (r_size != SIZE_MAX)
11541159
client.add_header("filesize", std::to_string(r_size));
11551160
std::string uri = "http://127.0.0.1:8090/upload";
11561161
cinatra::resp_data result;
11571162
if (r_size != SIZE_MAX) {
1158-
auto lazy = client.async_upload(uri, http_method::PUT, filename, r_size);
1163+
auto lazy =
1164+
client.async_upload(uri, http_method::PUT, filename, offset, r_size);
11591165
result = async_simple::coro::syncAwait(lazy);
11601166
}
11611167
else {
1162-
auto lazy = client.async_upload(uri, http_method::PUT, filename);
1168+
auto lazy = client.async_upload(uri, http_method::PUT, filename, offset);
11631169
result = async_simple::coro::syncAwait(lazy);
11641170
}
11651171
CHECK(((result.status == 200) ^ should_failed));
11661172
};
1167-
auto test_upload_by_stream = [](std::string filename,
1173+
auto test_upload_by_stream = [](std::string filename, std::size_t offset = 0,
11681174
std::size_t r_size = SIZE_MAX,
11691175
bool should_failed = false) {
11701176
coro_http_client client{};
11711177
client.add_header("filename", filename);
1178+
client.add_header("offset", std::to_string(offset));
11721179
if (r_size != SIZE_MAX)
11731180
client.add_header("filesize", std::to_string(r_size));
11741181
std::string uri = "http://127.0.0.1:8090/upload";
11751182
std::ifstream ifs(filename, std::ios::binary);
11761183
cinatra::resp_data result;
11771184
if (r_size != SIZE_MAX) {
1178-
auto lazy = client.async_upload(uri, http_method::PUT, filename, r_size);
1185+
auto lazy =
1186+
client.async_upload(uri, http_method::PUT, filename, offset, r_size);
11791187
result = async_simple::coro::syncAwait(lazy);
11801188
}
11811189
else {
1182-
auto lazy = client.async_upload(uri, http_method::PUT, filename);
1190+
auto lazy = client.async_upload(uri, http_method::PUT, filename, offset);
11831191
result = async_simple::coro::syncAwait(lazy);
11841192
}
11851193
CHECK(((result.status == 200) ^ should_failed));
@@ -1189,6 +1197,7 @@ TEST_CASE("test coro_http_client upload") {
11891197
bool should_failed = false) {
11901198
coro_http_client client{};
11911199
client.add_header("filename", filename);
1200+
client.add_header("offset", "0");
11921201
if (r_size != SIZE_MAX)
11931202
client.add_header("filesize", std::to_string(r_size));
11941203
std::string uri = "http://127.0.0.1:8090/upload";
@@ -1210,7 +1219,7 @@ TEST_CASE("test coro_http_client upload") {
12101219
}
12111220
else {
12121221
auto lazy =
1213-
client.async_upload(uri, http_method::PUT, async_read, r_size);
1222+
client.async_upload(uri, http_method::PUT, async_read, 0, r_size);
12141223
result = async_simple::coro::syncAwait(lazy);
12151224
CHECK(((result.status == 200) ^ should_failed));
12161225
}
@@ -1231,15 +1240,31 @@ TEST_CASE("test coro_http_client upload") {
12311240
file.write(req.get_body().data(), req.get_body().size());
12321241
file.flush();
12331242
file.close();
1243+
1244+
size_t offset = 0;
1245+
std::string offset_s = std::string{req.get_header_value("offset")};
1246+
if (!offset_s.empty()) {
1247+
offset = stoull(offset_s);
1248+
}
1249+
12341250
std::string filesize = std::string{req.get_header_value("filesize")};
1251+
12351252
if (!filesize.empty()) {
12361253
sz = stoull(filesize);
12371254
}
12381255
else {
12391256
sz = std::filesystem::file_size(oldpath);
1257+
sz -= offset;
12401258
}
1259+
12411260
CHECK(!filename.empty());
12421261
CHECK(sz == std::filesystem::file_size(newpath));
1262+
std::ifstream ifs(oldpath);
1263+
ifs.seekg(offset, std::ios::cur);
1264+
std::string str;
1265+
str.resize(sz);
1266+
ifs.read(str.data(), sz);
1267+
CHECK(str == req.get_body());
12431268
resp.set_status_and_content(status_type::ok, std::string(filename));
12441269
co_return;
12451270
});
@@ -1274,8 +1299,8 @@ TEST_CASE("test coro_http_client upload") {
12741299
}
12751300
bool r = create_file(filename, size);
12761301
CHECK(r);
1277-
test_upload_by_file_path(filename, r_size);
1278-
test_upload_by_stream(filename, r_size);
1302+
test_upload_by_file_path(filename, 0, r_size);
1303+
test_upload_by_stream(filename, 0, r_size);
12791304
test_upload_by_coro(filename, r_size);
12801305
}
12811306
}
@@ -1292,11 +1317,62 @@ TEST_CASE("test coro_http_client upload") {
12921317
}
12931318
bool r = create_file(filename, size);
12941319
CHECK(r);
1295-
test_upload_by_file_path(filename, r_size, true);
1296-
test_upload_by_stream(filename, r_size, true);
1320+
test_upload_by_file_path(filename, 0, r_size, true);
1321+
test_upload_by_stream(filename, 0, r_size, true);
12971322
test_upload_by_coro(filename, r_size, true);
12981323
}
12991324
}
1325+
// upload with offset
1326+
{
1327+
auto sizes = {std::pair{1024 * 1024, 1'000'000},
1328+
std::pair{2'000'000, 1'999'999}, std::pair{200, 1},
1329+
std::pair{100, 0}, std::pair{0, 0}};
1330+
for (auto [size, offset] : sizes) {
1331+
std::error_code ec{};
1332+
fs::remove(filename, ec);
1333+
if (ec) {
1334+
std::cout << ec << "\n";
1335+
}
1336+
bool r = create_file(filename, size);
1337+
CHECK(r);
1338+
test_upload_by_file_path(filename, offset);
1339+
test_upload_by_stream(filename, offset);
1340+
}
1341+
}
1342+
// upload with size & offset
1343+
{
1344+
auto sizes = {std::tuple{1024 * 1024, 500'000, 500'000},
1345+
std::tuple{2'000'000, 1'999'999, 1}, std::tuple{200, 1, 199},
1346+
std::tuple{100, 100, 0}};
1347+
for (auto [size, offset, r_size] : sizes) {
1348+
std::error_code ec{};
1349+
fs::remove(filename, ec);
1350+
if (ec) {
1351+
std::cout << ec << "\n";
1352+
}
1353+
bool r = create_file(filename, size);
1354+
CHECK(r);
1355+
test_upload_by_file_path(filename, offset, r_size);
1356+
test_upload_by_stream(filename, offset, r_size);
1357+
}
1358+
}
1359+
// upload with too large size & offset
1360+
{
1361+
auto sizes = {std::tuple{1024 * 1024, 1'000'000, 50'000},
1362+
std::tuple{2'000'000, 1'999'999, 2}, std::tuple{200, 1, 200},
1363+
std::tuple{100, 100, 1}};
1364+
for (auto [size, offset, r_size] : sizes) {
1365+
std::error_code ec{};
1366+
fs::remove(filename, ec);
1367+
if (ec) {
1368+
std::cout << ec << "\n";
1369+
}
1370+
bool r = create_file(filename, size);
1371+
CHECK(r);
1372+
test_upload_by_file_path(filename, offset, r_size, true);
1373+
test_upload_by_stream(filename, offset, r_size, true);
1374+
}
1375+
}
13001376
}
13011377

13021378
TEST_CASE("test coro_http_client chunked upload and download") {

0 commit comments

Comments
 (0)