client.cc

Go to the documentation of this file.

// Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC")
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include <config.h>

#include <asiolink/asio_wrapper.h>
#include <asiolink/interval_timer.h>
#include <asiolink/tcp_socket.h>
#include <http/client.h>
#include <http/http_log.h>
#include <http/http_messages.h>
#include <http/response_json.h>
#include <http/response_parser.h>
#include <boost/bind.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/weak_ptr.hpp>
#include <array>
#include <map>
#include <queue>

#include <iostream>

using namespace isc;
using namespace isc::asiolink;
using namespace http;

namespace {

constexpr size_t MAX_LOGGED_MESSAGE_SIZE = 1024;

typedef boost::function<void(boost::system::error_code ec, size_t length)>
SocketCallbackFunction;

class SocketCallback {
public:

    SocketCallback(SocketCallbackFunction socket_callback)
        : callback_(socket_callback) {
    }

    void operator()(boost::system::error_code ec, size_t length = 0) {
        if (ec.value() == boost::asio::error::operation_aborted) {
            return;
        }
        callback_(ec, length);
    }

private:

    SocketCallbackFunction callback_;

};

class ConnectionPool;

typedef boost::shared_ptr<ConnectionPool> ConnectionPoolPtr;

class Connection : public boost::enable_shared_from_this<Connection> {
public:

    explicit Connection(IOService& io_service, const ConnectionPoolPtr& conn_pool,
                        const Url& url);

    ~Connection();

    void doTransaction(const HttpRequestPtr& request, const HttpResponsePtr& response,
                       const long request_timeout, const HttpClient::RequestHandler& callback,
                       const HttpClient::ConnectHandler& connect_callback);

    void close();

    bool isTransactionOngoing() const;

private:

    void resetState();

    void terminate(const boost::system::error_code& ec,
                   const std::string& parsing_error = "");

    void scheduleTimer(const long request_timeout);

    void doSend();

    void doReceive();

    void connectCallback(HttpClient::ConnectHandler connect_callback,
                         const boost::system::error_code& ec);

    void sendCallback(const boost::system::error_code& ec, size_t length);

    void receiveCallback(const boost::system::error_code& ec, size_t length);

    void timerCallback();

    boost::weak_ptr<ConnectionPool> conn_pool_;

    Url url_;

    TCPSocket<SocketCallback> socket_;

    IntervalTimer timer_;

    HttpRequestPtr current_request_;

    HttpResponsePtr current_response_;

    HttpResponseParserPtr parser_;

    HttpClient::RequestHandler current_callback_;

    std::string buf_;

    std::array<char, 32768> input_buf_;
};

typedef boost::shared_ptr<Connection> ConnectionPtr;

class ConnectionPool : public boost::enable_shared_from_this<ConnectionPool> {
public:

    explicit ConnectionPool(IOService& io_service)
        : io_service_(io_service), conns_(), queue_() {
    }

    ~ConnectionPool() {
        closeAll();
    }

    bool getNextRequest(const Url& url,
                        HttpRequestPtr& request,
                        HttpResponsePtr& response,
                        long& request_timeout,
                        HttpClient::RequestHandler& callback,
                        HttpClient::ConnectHandler& connect_callback) {
        // Check if there is a queue for this URL. If there is no queue, there
        // is no request queued either.
        auto it = queue_.find(url);
        if (it != queue_.end()) {
            // If the queue is non empty, we take the oldest request.
            if (!it->second.empty()) {
                RequestDescriptor desc = it->second.front();
                it->second.pop();
                request = desc.request_;
                response = desc.response_;
                request_timeout = desc.request_timeout_,
                callback = desc.callback_;
                connect_callback = desc.connect_callback_;
                return (true);
            }
        }

        return (false);
    }

    void queueRequest(const Url& url,
                      const HttpRequestPtr& request,
                      const HttpResponsePtr& response,
                      const long request_timeout,
                      const HttpClient::RequestHandler& request_callback,
                      const HttpClient::ConnectHandler& connect_callback) {
        auto it = conns_.find(url);
        if (it != conns_.end()) {
            ConnectionPtr conn = it->second;
            // There is a connection for this URL already. Check if it is idle.
            if (conn->isTransactionOngoing()) {
                // Connection is busy, so let's queue the request.
                queue_[url].push(RequestDescriptor(request, response,
                                                   request_timeout,
                                                   request_callback,
                                                   connect_callback));

            } else {
                // Connection is idle, so we can start the transaction.
                conn->doTransaction(request, response, request_timeout,
                                    request_callback, connect_callback);
            }

        } else {
            // There is no connection with this destination yet. Let's create
            // it and start the transaction.
            ConnectionPtr conn(new Connection(io_service_, shared_from_this(),
                                              url));
            conn->doTransaction(request, response, request_timeout, request_callback,
                                connect_callback);
            conns_[url] = conn;
        }
    }

    void closeConnection(const Url& url) {
        // Close connection for the specified URL.
        auto conns_it = conns_.find(url);
        if (conns_it != conns_.end()) {
            conns_it->second->close();
            conns_.erase(conns_it);
        }

        // Remove requests from the queue.
        auto queue_it = queue_.find(url);
        if (queue_it != queue_.end()) {
            queue_.erase(queue_it);
        }
    }

    void closeAll() {
        for (auto conns_it = conns_.begin(); conns_it != conns_.end();
             ++conns_it) {
            conns_it->second->close();
        }

        conns_.clear();
        queue_.clear();
    }

private:

    IOService& io_service_;

    std::map<Url, ConnectionPtr> conns_;

    struct RequestDescriptor {
        RequestDescriptor(const HttpRequestPtr& request,
                          const HttpResponsePtr& response,
                          const long request_timeout,
                          const HttpClient::RequestHandler& callback,
                          const HttpClient::ConnectHandler& connect_callback)
            : request_(request), response_(response),
              request_timeout_(request_timeout),
              callback_(callback),
              connect_callback_(connect_callback) {
        }

        HttpRequestPtr request_;
        HttpResponsePtr response_;
        long request_timeout_;
        HttpClient::RequestHandler callback_;
        HttpClient::ConnectHandler connect_callback_;
    };

    std::map<Url, std::queue<RequestDescriptor> > queue_;
};

Connection::Connection(IOService& io_service,
                       const ConnectionPoolPtr& conn_pool,
                       const Url& url)
    : conn_pool_(conn_pool), url_(url), socket_(io_service), timer_(io_service),
      current_request_(), current_response_(), parser_(), current_callback_(),
      buf_(), input_buf_() {
}

Connection::~Connection() {
    close();
}

void
Connection::resetState() {
    current_request_.reset();
    current_response_.reset();
    parser_.reset();
    current_callback_ = HttpClient::RequestHandler();
}

void
Connection::doTransaction(const HttpRequestPtr& request,
                          const HttpResponsePtr& response,
                          const long request_timeout,
                          const HttpClient::RequestHandler& callback,
                          const HttpClient::ConnectHandler& connect_callback) {
    try {
        current_request_ = request;
        current_response_ = response;
        parser_.reset(new HttpResponseParser(*current_response_));
        parser_->initModel();
        current_callback_ = callback;

        buf_ = request->toString();

        // If the socket is open we check if it is possible to transmit the data
        // over this socket by reading from it with message peeking. If the socket
        // is not usable, we close it and then re-open it. There is a narrow window of
        // time between checking the socket usability and actually transmitting the
        // data over this socket, when the peer may close the connection. In this
        // case we'll need to re-transmit but we don't handle it here.
        if (socket_.getASIOSocket().is_open() && !socket_.isUsable()) {
            socket_.close();
        }

        LOG_DEBUG(http_logger, isc::log::DBGLVL_TRACE_DETAIL,
                  HTTP_CLIENT_REQUEST_SEND)
            .arg(request->toBriefString())
            .arg(url_.toText());

        LOG_DEBUG(http_logger, isc::log::DBGLVL_TRACE_DETAIL_DATA,
                  HTTP_CLIENT_REQUEST_SEND_DETAILS)
            .arg(url_.toText())
            .arg(HttpMessageParserBase::logFormatHttpMessage(request->toString(),
                                                             MAX_LOGGED_MESSAGE_SIZE));

        // Setup request timer.
        scheduleTimer(request_timeout);

        TCPEndpoint endpoint(url_.getStrippedHostname(),
                             static_cast<unsigned short>(url_.getPort()));
        SocketCallback socket_cb(boost::bind(&Connection::connectCallback, shared_from_this(),
                                             connect_callback, _1));

        // Establish new connection or use existing connection.
        socket_.open(&endpoint, socket_cb);

    } catch (const std::exception& ex) {
        // Re-throw with the expected exception type.
        isc_throw(HttpClientError, ex.what());
    }
}

void
Connection::close() {
    timer_.cancel();
    socket_.close();
    resetState();
}

bool
Connection::isTransactionOngoing() const {
    return (static_cast<bool>(current_request_));
}

void
Connection::terminate(const boost::system::error_code& ec,
                      const std::string& parsing_error) {

    timer_.cancel();
    socket_.cancel();

    HttpResponsePtr response;

    if (!ec && current_response_->isFinalized()) {
        response = current_response_;

        LOG_DEBUG(http_logger, isc::log::DBGLVL_TRACE_BASIC,
                  HTTP_SERVER_RESPONSE_RECEIVED)
            .arg(url_.toText());

        LOG_DEBUG(http_logger, isc::log::DBGLVL_TRACE_BASIC_DATA,
                  HTTP_SERVER_RESPONSE_RECEIVED_DETAILS)
            .arg(url_.toText())
            .arg(parser_->getBufferAsString(MAX_LOGGED_MESSAGE_SIZE));

    } else {
        std::string err = parsing_error.empty() ? ec.message() : parsing_error;

        LOG_DEBUG(http_logger, isc::log::DBGLVL_TRACE_BASIC,
                  HTTP_BAD_SERVER_RESPONSE_RECEIVED)
            .arg(url_.toText())
            .arg(err);

        // Only log the details if we have received anything and tried
        // to parse it.
        if (!parsing_error.empty()) {
            LOG_DEBUG(http_logger, isc::log::DBGLVL_TRACE_BASIC_DATA,
                      HTTP_BAD_SERVER_RESPONSE_RECEIVED_DETAILS)
                .arg(url_.toText())
                .arg(parser_->getBufferAsString());
        }

    }

    try {
        // The callback should take care of its own exceptions but one
        // never knows.
        current_callback_(ec, response, parsing_error);

    } catch (...) {
    }

    // If we're not requesting connection persistence, we should close the socket.
    // We're going to reconnect for the next transaction.
    if (!current_request_->isPersistent()) {
        close();
    }

    resetState();

    // Check if there are any requests queued for this connection and start
    // another transaction if there is at least one.
    HttpRequestPtr request;
    long request_timeout;
    HttpClient::RequestHandler callback;
    HttpClient::ConnectHandler connect_callback;
    ConnectionPoolPtr conn_pool = conn_pool_.lock();
    if (conn_pool && conn_pool->getNextRequest(url_, request, response, request_timeout,
                                               callback, connect_callback)) {
        doTransaction(request, response, request_timeout, callback, connect_callback);
    }
}

void
Connection::scheduleTimer(const long request_timeout) {
    if (request_timeout > 0) {
        timer_.setup(boost::bind(&Connection::timerCallback, this), request_timeout,
                     IntervalTimer::ONE_SHOT);
    }
}

void
Connection::doSend() {
    SocketCallback socket_cb(boost::bind(&Connection::sendCallback, shared_from_this(),
                                         _1, _2));
    try {
        socket_.asyncSend(&buf_[0], buf_.size(), socket_cb);

    } catch (...) {
        terminate(boost::asio::error::not_connected);
    }
}

void
Connection::doReceive() {
    TCPEndpoint endpoint;
    SocketCallback socket_cb(boost::bind(&Connection::receiveCallback, shared_from_this(),
                                         _1, _2));

    try {
        socket_.asyncReceive(static_cast<void*>(input_buf_.data()), input_buf_.size(), 0,
                             &endpoint, socket_cb);
    } catch (...) {
        terminate(boost::asio::error::not_connected);
    }
}

void
Connection::connectCallback(HttpClient::ConnectHandler connect_callback,
                            const boost::system::error_code& ec) {
    // Run user defined connect callback if specified.
    if (connect_callback) {
        // If the user defined callback indicates that the connection
        // should not be continued.
        if (!connect_callback(ec)) {
            return;
        }
    }

    // In some cases the "in progress" status code may be returned. It doesn't
    // indicate an error. Sending the request over the socket is expected to
    // be successful. Getting such status appears to be highly dependent on
    // the operating system.
    if (ec &&
        (ec.value() != boost::asio::error::in_progress) &&
        (ec.value() != boost::asio::error::already_connected)) {
        terminate(ec);

    } else {
        // Start sending the request asynchronously.
        doSend();
    }
}

void
Connection::sendCallback(const boost::system::error_code& ec, size_t length) {
    if (ec) {
        // EAGAIN and EWOULDBLOCK don't really indicate an error. The length
        // should be 0 in this case but let's be sure.
        if ((ec.value() == boost::asio::error::would_block) ||
            (ec.value() == boost::asio::error::try_again)) {
            length = 0;

        } else {
            // Any other error should cause the transaction to terminate.
            terminate(ec);
            return;
        }
    }

    // Sending is in progress, so push back the timeout.
    scheduleTimer(timer_.getInterval());

    // If any data have been sent, remove it from the buffer and only leave the
    // portion that still has to be sent.
    if (length > 0) {
        buf_.erase(0, length);
    }

    // If there is no more data to be sent, start receiving a response. Otherwise,
    // continue sending.
    if (buf_.empty()) {
        doReceive();

    } else {
        doSend();
    }
}

void
Connection::receiveCallback(const boost::system::error_code& ec, size_t length) {
    if (ec) {
        // EAGAIN and EWOULDBLOCK don't indicate an error in this case. All
        // other errors should terminate the transaction.
        if ((ec.value() != boost::asio::error::try_again) &&
            (ec.value() != boost::asio::error::would_block)) {
            terminate(ec);
            return;

        } else {
            // For EAGAIN and EWOULDBLOCK the length should be 0 anyway, but let's
            // make sure.
            length = 0;
        }
    }

    // Receiving is in progress, so push back the timeout.
    scheduleTimer(timer_.getInterval());

    // If we have received any data, let's feed the parser with it.
    if (length != 0) {
        parser_->postBuffer(static_cast<void*>(input_buf_.data()), length);
        parser_->poll();
    }

    // If the parser still needs data, let's schedule another receive.
    if (parser_->needData()) {
        doReceive();

    } else if (parser_->httpParseOk()) {
        // No more data needed and parsing has been successful so far. Let's
        // try to finalize the response parsing.
        try {
            current_response_->finalize();
            terminate(ec);

        } catch (const std::exception& ex) {
            // If there is an error here, we need to return the error message.
            terminate(ec, ex.what());
        }

    } else {
        // Parsing was unsuccessul. Let's pass the error message held in the
        // parser.
        terminate(ec, parser_->getErrorMessage());
    }
}

void
Connection::timerCallback() {
    // Request timeout occured.
    terminate(boost::asio::error::timed_out);
}

}

namespace isc {
namespace http {

class HttpClientImpl {
public:

    HttpClientImpl(IOService& io_service)
        : conn_pool_(new ConnectionPool(io_service)) {
    }

    ConnectionPoolPtr conn_pool_;

};

HttpClient::HttpClient(IOService& io_service)
    : impl_(new HttpClientImpl(io_service)) {
}

void
HttpClient::asyncSendRequest(const Url& url, const HttpRequestPtr& request,
                             const HttpResponsePtr& response,
                             const HttpClient::RequestHandler& request_callback,
                             const HttpClient::RequestTimeout& request_timeout,
                             const HttpClient::ConnectHandler& connect_callback) {
    if (!url.isValid()) {
        isc_throw(HttpClientError, "invalid URL specified for the HTTP client");
    }

    if (!request) {
        isc_throw(HttpClientError, "HTTP request must not be null");
    }

    if (!response) {
        isc_throw(HttpClientError, "HTTP response must not be null");
    }

    if (!request_callback) {
        isc_throw(HttpClientError, "callback for HTTP transaction must not be null");
    }

    impl_->conn_pool_->queueRequest(url, request, response, request_timeout.value_,
                                    request_callback, connect_callback);
}

void
HttpClient::stop() {
    impl_->conn_pool_->closeAll();
}

} // end of namespace isc::http
} // end of namespace isc