Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add configuration options to HTTP server #9765

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+387 −26
Open

Add configuration options to HTTP server #9765

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion contrib/epee/include/net/abstract_tcp_server2.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
#define MONERO_DEFAULT_LOG_CATEGORY "net"

#define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000
#define ABSTRACT_SERVER_SEND_QUE_MAX_BYTES_DEFAULT 100 * 1024 * 1024

namespace epee
{
Expand Down Expand Up @@ -170,6 +171,7 @@ namespace net_utils
} read;
struct {
std::deque<epee::byte_slice> queue;
std::size_t total_bytes;
bool wait_consume;
} write;
};
Expand Down Expand Up @@ -268,11 +270,17 @@ namespace net_utils
struct shared_state : connection_basic_shared_state, t_protocol_handler::config_type
{
shared_state()
: connection_basic_shared_state(), t_protocol_handler::config_type(), pfilter(nullptr), plimit(nullptr), stop_signal_sent(false)
: connection_basic_shared_state(),
t_protocol_handler::config_type(),
pfilter(nullptr),
plimit(nullptr),
response_soft_limit(ABSTRACT_SERVER_SEND_QUE_MAX_BYTES_DEFAULT),
stop_signal_sent(false)
{}

i_connection_filter* pfilter;
i_connection_limit* plimit;
std::size_t response_soft_limit;
bool stop_signal_sent;
};

Expand Down Expand Up @@ -380,6 +388,7 @@ namespace net_utils

void set_connection_filter(i_connection_filter* pfilter);
void set_connection_limit(i_connection_limit* plimit);
void set_response_soft_limit(std::size_t limit);

void set_default_remote(epee::net_utils::network_address remote)
{
Expand Down
57 changes: 48 additions & 9 deletions contrib/epee/include/net/abstract_tcp_server2.inl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,10 +497,12 @@ namespace net_utils
if (m_state.socket.cancel_write) {
m_state.socket.cancel_write = false;
m_state.data.write.queue.clear();
m_state.data.write.total_bytes = 0;
state_status_check();
}
else if (ec.value()) {
m_state.data.write.queue.clear();
m_state.data.write.total_bytes = 0;
interrupt();
}
else {
Expand All @@ -525,8 +527,11 @@ namespace net_utils

start_timer(get_default_timeout(), true);
}
assert(bytes_transferred == m_state.data.write.queue.back().size());
const std::size_t byte_count = m_state.data.write.queue.back().size();
assert(bytes_transferred == byte_count);
m_state.data.write.queue.pop_back();
m_state.data.write.total_bytes -=
std::min(m_state.data.write.total_bytes, byte_count);
m_state.condition.notify_all();
start_write();
}
Expand Down Expand Up @@ -670,8 +675,9 @@ namespace net_utils
return;
if (m_state.timers.throttle.out.wait_expire)
return;
if (m_state.socket.wait_write)
return;
// \NOTE See on_terminating() comments
//if (m_state.socket.wait_write)
// return;
if (m_state.socket.wait_shutdown)
return;
if (m_state.protocol.wait_init)
Expand Down Expand Up @@ -729,8 +735,13 @@ namespace net_utils
return;
if (m_state.timers.throttle.out.wait_expire)
return;
if (m_state.socket.wait_write)
return;
// Writes cannot be canceled due to `async_write` being a "composed"
// handler. ASIO has new cancellation routines, not available in 1.66, to
// handle this situation. The problem is that if cancel is called after an
// intermediate handler is queued, the op will not check the cancel flag in
// our code, and will instead queue up another write.
//if (m_state.socket.wait_write)
// return;
if (m_state.socket.wait_shutdown)
return;
if (m_state.protocol.wait_init)
Expand All @@ -757,6 +768,8 @@ namespace net_utils
std::lock_guard<std::mutex> guard(m_state.lock);
if (m_state.status != status_t::RUNNING || m_state.socket.wait_handshake)
return false;
if (std::numeric_limits<std::size_t>::max() - m_state.data.write.total_bytes < message.size())
return false;

// Wait for the write queue to fall below the max. If it doesn't after a
// randomized delay, drop the connection.
Expand All @@ -774,7 +787,14 @@ namespace net_utils
std::uniform_int_distribution<>(5000, 6000)(rng)
);
};
if (m_state.data.write.queue.size() <= ABSTRACT_SERVER_SEND_QUE_MAX_COUNT)

// The bytes check intentionally does not include incoming message size.
// This allows for a soft overflow; a single http response will never fail
// this check, but multiple responses could. Clients can avoid this case
// by reading the entire response before making another request. P2P
// should never hit the MAX_BYTES check (when using default values).
if (m_state.data.write.queue.size() <= ABSTRACT_SERVER_SEND_QUE_MAX_COUNT &&
m_state.data.write.total_bytes <= static_cast<shared_state&>(connection_basic::get_state()).response_soft_limit)
return true;
m_state.data.write.wait_consume = true;
bool success = m_state.condition.wait_for(
Expand All @@ -783,14 +803,23 @@ namespace net_utils
[this]{
return (
m_state.status != status_t::RUNNING ||
m_state.data.write.queue.size() <=
ABSTRACT_SERVER_SEND_QUE_MAX_COUNT
(
m_state.data.write.queue.size() <=
ABSTRACT_SERVER_SEND_QUE_MAX_COUNT &&
m_state.data.write.total_bytes <=
static_cast<shared_state&>(connection_basic::get_state()).response_soft_limit
)
);
}
);
m_state.data.write.wait_consume = false;
if (!success) {
terminate();
// synchronize with intermediate writes on `m_strand`
auto self = connection<T>::shared_from_this();
boost::asio::post(m_strand, [this, self] {
std::lock_guard<std::mutex> guard(m_state.lock);
terminate();
});
return false;
}
else
Expand All @@ -816,7 +845,9 @@ namespace net_utils
) {
if (!wait_consume())
return false;
const std::size_t byte_count = message.size();
m_state.data.write.queue.emplace_front(std::move(message));
m_state.data.write.total_bytes += byte_count;
start_write();
}
else {
Expand All @@ -826,6 +857,7 @@ namespace net_utils
m_state.data.write.queue.emplace_front(
message.take_slice(CHUNK_SIZE)
);
m_state.data.write.total_bytes += m_state.data.write.queue.front().size();
start_write();
}
}
Expand Down Expand Up @@ -1369,6 +1401,13 @@ namespace net_utils
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
void boosted_tcp_server<t_protocol_handler>::set_response_soft_limit(const std::size_t limit)
{
assert(m_state != nullptr); // always set in constructor
m_state->response_soft_limit = limit;
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
bool boosted_tcp_server<t_protocol_handler>::run_server(size_t threads_count, bool wait, const boost::thread::attributes& attrs)
{
TRY_ENTRY();
Expand Down
18 changes: 9 additions & 9 deletions contrib/epee/include/net/http_protocol_handler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

#include <boost/optional/optional.hpp>
#include <string>
#include <unordered_map>
#include "net_utils_base.h"
#include "http_auth.h"
#include "http_base.h"
Expand All @@ -54,8 +55,13 @@ namespace net_utils
{
std::string m_folder;
std::vector<std::string> m_access_control_origins;
std::unordered_map<std::string, std::size_t> m_connections;
boost::optional<login> m_user;
size_t m_max_content_length{std::numeric_limits<size_t>::max()};
std::size_t m_connection_count{0};
std::size_t m_max_public_ip_connections{3};
std::size_t m_max_private_ip_connections{25};
std::size_t m_max_connections{100};
critical_section m_lock;
};

Expand All @@ -70,7 +76,7 @@ namespace net_utils
typedef http_server_config config_type;

simple_http_connection_handler(i_service_endpoint* psnd_hndlr, config_type& config, t_connection_context& conn_context);
virtual ~simple_http_connection_handler(){}
virtual ~simple_http_connection_handler();

bool release_protocol()
{
Expand All @@ -86,10 +92,7 @@ namespace net_utils
{
return true;
}
bool after_init_connection()
{
return true;
}
bool after_init_connection();
virtual bool handle_recv(const void* ptr, size_t cb);
virtual bool handle_request(const http::http_request_info& query_info, http_response_info& response);

Expand Down Expand Up @@ -146,6 +149,7 @@ namespace net_utils
protected:
i_service_endpoint* m_psnd_hndlr;
t_connection_context& m_conn_context;
bool m_initialized;
};

template<class t_connection_context>
Expand Down Expand Up @@ -212,10 +216,6 @@ namespace net_utils
}
void handle_qued_callback()
{}
bool after_init_connection()
{
return true;
}

private:
//simple_http_connection_handler::config_type m_stub_config;
Expand Down
37 changes: 36 additions & 1 deletion contrib/epee/include/net/http_protocol_handler.inl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,11 +208,46 @@ namespace net_utils
m_newlines(0),
m_bytes_read(0),
m_psnd_hndlr(psnd_hndlr),
m_conn_context(conn_context)
m_conn_context(conn_context),
m_initialized(false)
{

}
//--------------------------------------------------------------------------------------------
template<class t_connection_context>
simple_http_connection_handler<t_connection_context>::~simple_http_connection_handler()
{
try
{
if (m_initialized)
{
CRITICAL_REGION_LOCAL(m_config.m_lock);
if (m_config.m_connection_count)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor style issue (not material for approval): If GitHub browser view is correct, the indent here appears to differ from elsewhere.

--m_config.m_connection_count;
auto elem = m_config.m_connections.find(m_conn_context.m_remote_address.host_str());
if (elem != m_config.m_connections.end())
{
if (elem->second == 1 || elem->second == 0)
m_config.m_connections.erase(elem);
else
--(elem->second);
}
}
}
catch (...)
{}
}
//--------------------------------------------------------------------------------------------
template<class t_connection_context>
bool simple_http_connection_handler<t_connection_context>::after_init_connection()
{
CRITICAL_REGION_LOCAL(m_config.m_lock);
++m_config.m_connections[m_conn_context.m_remote_address.host_str()];
++m_config.m_connection_count;
m_initialized = true;
return true;
}
//--------------------------------------------------------------------------------------------
template<class t_connection_context>
bool simple_http_connection_handler<t_connection_context>::set_ready_state()
{
Expand Down
34 changes: 32 additions & 2 deletions contrib/epee/include/net/http_server_impl_base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ namespace epee
{

template<class t_child_class, class t_connection_context = epee::net_utils::connection_context_base>
class http_server_impl_base: public net_utils::http::i_http_server_handler<t_connection_context>
class http_server_impl_base: public net_utils::http::i_http_server_handler<t_connection_context>,
net_utils::i_connection_limit
{

public:
Expand All @@ -60,7 +61,11 @@ namespace epee
const std::string& bind_ipv6_address = "::", bool use_ipv6 = false, bool require_ipv4 = true,
std::vector<std::string> access_control_origins = std::vector<std::string>(),
boost::optional<net_utils::http::login> user = boost::none,
net_utils::ssl_options_t ssl_options = net_utils::ssl_support_t::e_ssl_support_autodetect)
net_utils::ssl_options_t ssl_options = net_utils::ssl_support_t::e_ssl_support_autodetect,
const std::size_t max_public_ip_connections = 3,
const std::size_t max_private_ip_connections = 25,
const std::size_t max_connections = 100,
const std::size_t response_soft_limit = 25 * 1024 * 1024)
{

//set self as callback handler
Expand All @@ -75,6 +80,11 @@ namespace epee
m_net_server.get_config_object().m_access_control_origins = std::move(access_control_origins);

m_net_server.get_config_object().m_user = std::move(user);
m_net_server.get_config_object().m_max_public_ip_connections = max_public_ip_connections;
m_net_server.get_config_object().m_max_private_ip_connections = max_private_ip_connections;
m_net_server.get_config_object().m_max_connections = max_connections;
m_net_server.set_response_soft_limit(response_soft_limit);
m_net_server.set_connection_limit(this);

MGINFO("Binding on " << bind_ip << " (IPv4):" << bind_port);
if (use_ipv6)
Expand Down Expand Up @@ -131,6 +141,26 @@ namespace epee
}

protected:

virtual bool is_host_limit(const net_utils::network_address& na) override final
{
auto& config = m_net_server.get_config_object();
CRITICAL_REGION_LOCAL(config.m_lock);
if (config.m_max_connections <= config.m_connection_count)
return true;

const bool is_private = na.is_loopback() || na.is_local();
const auto elem = config.m_connections.find(na.host_str());
if (elem != config.m_connections.end())
{
if (is_private)
return config.m_max_private_ip_connections <= elem->second;
else
return config.m_max_public_ip_connections <= elem->second;
}
return false;
}

net_utils::boosted_tcp_server<net_utils::http::http_custom_handler<t_connection_context> > m_net_server;
};
}
Loading
Loading