[llvm] [Debuginfod] Add HTTP Server to Debuginfod library.

This provides a minimal HTTP server interface and an implementation wrapping [[ https://github.com/yhirose/cpp-httplib | cpp-httplib ]] in the Debuginfod library. If the Curl HTTP client is available (D112753) the server is tested by pinging it with the client.

Reviewed By: dblaikie

Differential Revision: https://reviews.llvm.org/D114415
This commit is contained in:
Noah Shutty 2022-07-06 18:50:55 +00:00
parent 8cb5c82ad2
commit 8366e21ef1
5 changed files with 628 additions and 0 deletions

View File

@ -0,0 +1,123 @@
//===-- llvm/Debuginfod/HTTPServer.h - HTTP server library ------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file contains the declarations of the HTTPServer and HTTPServerRequest
/// classes, the HTTPResponse, and StreamingHTTPResponse structs, and the
/// streamFile function.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_SUPPORT_HTTP_SERVER_H
#define LLVM_SUPPORT_HTTP_SERVER_H
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Error.h"
#ifdef LLVM_ENABLE_HTTPLIB
// forward declarations
namespace httplib {
class Request;
class Response;
class Server;
} // namespace httplib
#endif
namespace llvm {
struct HTTPResponse;
struct StreamingHTTPResponse;
class HTTPServer;
class HTTPServerRequest {
friend HTTPServer;
#ifdef LLVM_ENABLE_HTTPLIB
private:
HTTPServerRequest(const httplib::Request &HTTPLibRequest,
httplib::Response &HTTPLibResponse);
httplib::Response &HTTPLibResponse;
#endif
public:
std::string UrlPath;
/// The elements correspond to match groups in the url path matching regex.
SmallVector<std::string, 1> UrlPathMatches;
// TODO bring in HTTP headers
void setResponse(StreamingHTTPResponse Response);
void setResponse(HTTPResponse Response);
};
struct HTTPResponse {
unsigned Code;
const char *ContentType;
StringRef Body;
};
typedef std::function<void(HTTPServerRequest &)> HTTPRequestHandler;
/// An HTTPContentProvider is called by the HTTPServer to obtain chunks of the
/// streaming response body. The returned chunk should be located at Offset
/// bytes and have Length bytes.
typedef std::function<StringRef(size_t /*Offset*/, size_t /*Length*/)>
HTTPContentProvider;
/// Wraps the content provider with HTTP Status code and headers.
struct StreamingHTTPResponse {
unsigned Code;
const char *ContentType;
size_t ContentLength;
HTTPContentProvider Provider;
/// Called after the response transfer is complete with the success value of
/// the transfer.
std::function<void(bool)> CompletionHandler = [](bool Success) {};
};
/// Sets the response to stream the file at FilePath, if available, and
/// otherwise an HTTP 404 error response.
bool streamFile(HTTPServerRequest &Request, StringRef FilePath);
/// An HTTP server which can listen on a single TCP/IP port for HTTP
/// requests and delgate them to the appropriate registered handler.
class HTTPServer {
#ifdef LLVM_ENABLE_HTTPLIB
std::unique_ptr<httplib::Server> Server;
unsigned Port = 0;
#endif
public:
HTTPServer();
~HTTPServer();
/// Returns true only if LLVM has been compiled with a working HTTPServer.
static bool isAvailable();
/// Registers a URL pattern routing rule. When the server is listening, each
/// request is dispatched to the first registered handler whose UrlPathPattern
/// matches the UrlPath.
Error get(StringRef UrlPathPattern, HTTPRequestHandler Handler);
/// Attempts to assign the requested port and interface, returning an Error
/// upon failure.
Error bind(unsigned Port, const char *HostInterface = "0.0.0.0");
/// Attempts to assign any available port and interface, returning either the
/// port number or an Error upon failure.
Expected<unsigned> bind(const char *HostInterface = "0.0.0.0");
/// Attempts to listen for requests on the bound port. Returns an Error if
/// called before binding a port.
Error listen();
/// If the server is listening, stop and unbind the socket.
void stop();
};
} // end namespace llvm
#endif // LLVM_SUPPORT_HTTP_SERVER_H

View File

@ -3,12 +3,18 @@ if (LLVM_ENABLE_CURL)
set(imported_libs CURL::libcurl)
endif()
# Link cpp-httplib if the user wants it
if (LLVM_ENABLE_HTTPLIB)
set(imported_libs ${imported_libs} httplib::httplib)
endif()
# Note: This isn't a component, since that could potentially add a libcurl
# dependency to libLLVM.
add_llvm_library(LLVMDebuginfod
Debuginfod.cpp
DIFetcher.cpp
HTTPClient.cpp
HTTPServer.cpp
ADDITIONAL_HEADER_DIRS
${LLVM_MAIN_INCLUDE_DIR}/llvm/Debuginfod

View File

@ -0,0 +1,189 @@
//===-- llvm/Debuginfod/HTTPServer.cpp - HTTP server library -----*- C++-*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
///
/// This file defines the methods of the HTTPServer class and the streamFile
/// function.
///
//===----------------------------------------------------------------------===//
#include "llvm/Debuginfod/HTTPServer.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Regex.h"
#ifdef LLVM_ENABLE_HTTPLIB
#include "httplib.h"
#endif
using namespace llvm;
bool llvm::streamFile(HTTPServerRequest &Request, StringRef FilePath) {
Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(FilePath);
if (Error Err = FDOrErr.takeError()) {
consumeError(std::move(Err));
Request.setResponse({404u, "text/plain", "Could not open file to read.\n"});
return false;
}
ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr =
MemoryBuffer::getOpenFile(*FDOrErr, FilePath,
/*FileSize=*/-1,
/*RequiresNullTerminator=*/false);
sys::fs::closeFile(*FDOrErr);
if (Error Err = errorCodeToError(MBOrErr.getError())) {
consumeError(std::move(Err));
Request.setResponse({404u, "text/plain", "Could not memory-map file.\n"});
return false;
}
// Lambdas are copied on conversion to to std::function, preventing use of
// smart pointers.
MemoryBuffer *MB = MBOrErr->release();
Request.setResponse({200u, "application/octet-stream", MB->getBufferSize(),
[=](size_t Offset, size_t Length) -> StringRef {
return MB->getBuffer().substr(Offset, Length);
},
[=](bool Success) { delete MB; }});
return true;
}
#ifdef LLVM_ENABLE_HTTPLIB
bool HTTPServer::isAvailable() { return true; }
HTTPServer::HTTPServer() { Server = std::make_unique<httplib::Server>(); }
HTTPServer::~HTTPServer() { stop(); }
static void expandUrlPathMatches(const std::smatch &Matches,
HTTPServerRequest &Request) {
bool UrlPathSet = false;
for (const auto &it : Matches) {
if (UrlPathSet)
Request.UrlPathMatches.push_back(it);
else {
Request.UrlPath = it;
UrlPathSet = true;
}
}
}
HTTPServerRequest::HTTPServerRequest(const httplib::Request &HTTPLibRequest,
httplib::Response &HTTPLibResponse)
: HTTPLibResponse(HTTPLibResponse) {
expandUrlPathMatches(HTTPLibRequest.matches, *this);
}
void HTTPServerRequest::setResponse(HTTPResponse Response) {
HTTPLibResponse.set_content(Response.Body.begin(), Response.Body.size(),
Response.ContentType);
HTTPLibResponse.status = Response.Code;
}
void HTTPServerRequest::setResponse(StreamingHTTPResponse Response) {
HTTPLibResponse.set_content_provider(
Response.ContentLength, Response.ContentType,
[=](size_t Offset, size_t Length, httplib::DataSink &Sink) {
if (Offset < Response.ContentLength) {
StringRef Chunk = Response.Provider(Offset, Length);
Sink.write(Chunk.begin(), Chunk.size());
}
return true;
},
[=](bool Success) { Response.CompletionHandler(Success); });
HTTPLibResponse.status = Response.Code;
}
Error HTTPServer::get(StringRef UrlPathPattern, HTTPRequestHandler Handler) {
std::string ErrorMessage;
if (!Regex(UrlPathPattern).isValid(ErrorMessage))
return createStringError(errc::argument_out_of_domain, ErrorMessage);
Server->Get(std::string(UrlPathPattern),
[Handler](const httplib::Request &HTTPLibRequest,
httplib::Response &HTTPLibResponse) {
HTTPServerRequest Request(HTTPLibRequest, HTTPLibResponse);
Handler(Request);
});
return Error::success();
}
Error HTTPServer::bind(unsigned ListenPort, const char *HostInterface) {
if (!Server->bind_to_port(HostInterface, ListenPort))
return createStringError(errc::io_error,
"Could not assign requested address.");
Port = ListenPort;
return Error::success();
}
Expected<unsigned> HTTPServer::bind(const char *HostInterface) {
int ListenPort = Server->bind_to_any_port(HostInterface);
if (ListenPort < 0)
return createStringError(errc::io_error,
"Could not assign any port on requested address.");
return Port = ListenPort;
}
Error HTTPServer::listen() {
if (!Port)
return createStringError(errc::io_error,
"Cannot listen without first binding to a port.");
if (!Server->listen_after_bind())
return createStringError(
errc::io_error,
"An unknown error occurred when cpp-httplib attempted to listen.");
return Error::success();
}
void HTTPServer::stop() {
Server->stop();
Port = 0;
}
#else
// TODO: Implement barebones standalone HTTP server implementation.
bool HTTPServer::isAvailable() { return false; }
HTTPServer::HTTPServer() = default;
HTTPServer::~HTTPServer() = default;
void HTTPServerRequest::setResponse(HTTPResponse Response) {
llvm_unreachable("No HTTP server implementation available");
}
void HTTPServerRequest::setResponse(StreamingHTTPResponse Response) {
llvm_unreachable("No HTTP server implementation available");
}
Error HTTPServer::get(StringRef UrlPathPattern, HTTPRequestHandler Handler) {
llvm_unreachable("No HTTP server implementation available");
}
Error HTTPServer::bind(unsigned ListenPort, const char *HostInterface) {
llvm_unreachable("No HTTP server implementation available");
}
Expected<unsigned> HTTPServer::bind(const char *HostInterface) {
llvm_unreachable("No HTTP server implementation available");
}
Error HTTPServer::listen() {
llvm_unreachable("No HTTP server implementation available");
}
void HTTPServer::stop() {
llvm_unreachable("No HTTP server implementation available");
}
#endif // LLVM_ENABLE_HTTPLIB

View File

@ -1,4 +1,5 @@
add_llvm_unittest(DebuginfodTests
HTTPServerTests.cpp
DebuginfodTests.cpp
)

View File

@ -0,0 +1,309 @@
//===-- llvm/unittest/Support/HTTPServer.cpp - unit tests -------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "llvm/Debuginfod/HTTPClient.h"
#include "llvm/Debuginfod/HTTPServer.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ThreadPool.h"
#include "llvm/Testing/Support/Error.h"
#include "gmock/gmock.h"
#include "gtest/gtest.h"
using namespace llvm;
#ifdef LLVM_ENABLE_HTTPLIB
TEST(HTTPServer, IsAvailable) { EXPECT_TRUE(HTTPServer::isAvailable()); }
HTTPResponse Response = {200u, "text/plain", "hello, world\n"};
std::string UrlPathPattern = R"(/(.*))";
std::string InvalidUrlPathPattern = R"(/(.*)";
HTTPRequestHandler Handler = [](HTTPServerRequest &Request) {
Request.setResponse(Response);
};
HTTPRequestHandler DelayHandler = [](HTTPServerRequest &Request) {
std::this_thread::sleep_for(std::chrono::milliseconds(50));
Request.setResponse(Response);
};
HTTPRequestHandler StreamingHandler = [](HTTPServerRequest &Request) {
Request.setResponse({200, "text/plain", Response.Body.size(),
[=](size_t Offset, size_t Length) -> StringRef {
return Response.Body.substr(Offset, Length);
}});
};
TEST(HTTPServer, InvalidUrlPath) {
// test that we can bind to any address
HTTPServer Server;
EXPECT_THAT_ERROR(Server.get(InvalidUrlPathPattern, Handler),
Failed<StringError>());
EXPECT_THAT_EXPECTED(Server.bind(), Succeeded());
}
TEST(HTTPServer, bind) {
// test that we can bind to any address
HTTPServer Server;
EXPECT_THAT_ERROR(Server.get(UrlPathPattern, Handler), Succeeded());
EXPECT_THAT_EXPECTED(Server.bind(), Succeeded());
}
TEST(HTTPServer, ListenBeforeBind) {
// test that we can bind to any address
HTTPServer Server;
EXPECT_THAT_ERROR(Server.get(UrlPathPattern, Handler), Succeeded());
EXPECT_THAT_ERROR(Server.listen(), Failed<StringError>());
}
#ifdef LLVM_ENABLE_CURL
// Test the client and server against each other.
// Test fixture to initialize and teardown the HTTP client for each
// client-server test
class HTTPClientServerTest : public ::testing::Test {
protected:
void SetUp() override { HTTPClient::initialize(); }
void TearDown() override { HTTPClient::cleanup(); }
};
/// A simple handler which writes returned data to a string.
struct StringHTTPResponseHandler final : public HTTPResponseHandler {
std::string ResponseBody = "";
/// These callbacks store the body and status code in an HTTPResponseBuffer
/// allocated based on Content-Length. The Content-Length header must be
/// handled by handleHeaderLine before any calls to handleBodyChunk.
Error handleBodyChunk(StringRef BodyChunk) override {
ResponseBody = ResponseBody + BodyChunk.str();
return Error::success();
}
};
TEST_F(HTTPClientServerTest, Hello) {
HTTPServer Server;
EXPECT_THAT_ERROR(Server.get(UrlPathPattern, Handler), Succeeded());
Expected<unsigned> PortOrErr = Server.bind();
EXPECT_THAT_EXPECTED(PortOrErr, Succeeded());
unsigned Port = *PortOrErr;
ThreadPool Pool(hardware_concurrency(1));
Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); });
std::string Url = "http://localhost:" + utostr(Port);
HTTPRequest Request(Url);
StringHTTPResponseHandler Handler;
HTTPClient Client;
EXPECT_THAT_ERROR(Client.perform(Request, Handler), Succeeded());
EXPECT_EQ(Handler.ResponseBody, Response.Body);
EXPECT_EQ(Client.responseCode(), Response.Code);
Server.stop();
}
TEST_F(HTTPClientServerTest, LambdaHandlerHello) {
HTTPServer Server;
HTTPResponse LambdaResponse = {200u, "text/plain",
"hello, world from a lambda\n"};
EXPECT_THAT_ERROR(Server.get(UrlPathPattern,
[LambdaResponse](HTTPServerRequest &Request) {
Request.setResponse(LambdaResponse);
}),
Succeeded());
Expected<unsigned> PortOrErr = Server.bind();
EXPECT_THAT_EXPECTED(PortOrErr, Succeeded());
unsigned Port = *PortOrErr;
ThreadPool Pool(hardware_concurrency(1));
Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); });
std::string Url = "http://localhost:" + utostr(Port);
HTTPRequest Request(Url);
StringHTTPResponseHandler Handler;
HTTPClient Client;
EXPECT_THAT_ERROR(Client.perform(Request, Handler), Succeeded());
EXPECT_EQ(Handler.ResponseBody, LambdaResponse.Body);
EXPECT_EQ(Client.responseCode(), LambdaResponse.Code);
Server.stop();
}
// Test the streaming response.
TEST_F(HTTPClientServerTest, StreamingHello) {
HTTPServer Server;
EXPECT_THAT_ERROR(Server.get(UrlPathPattern, StreamingHandler), Succeeded());
Expected<unsigned> PortOrErr = Server.bind();
EXPECT_THAT_EXPECTED(PortOrErr, Succeeded());
unsigned Port = *PortOrErr;
ThreadPool Pool(hardware_concurrency(1));
Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); });
std::string Url = "http://localhost:" + utostr(Port);
HTTPRequest Request(Url);
StringHTTPResponseHandler Handler;
HTTPClient Client;
EXPECT_THAT_ERROR(Client.perform(Request, Handler), Succeeded());
EXPECT_EQ(Handler.ResponseBody, Response.Body);
EXPECT_EQ(Client.responseCode(), Response.Code);
Server.stop();
}
// Writes a temporary file and streams it back using streamFile.
HTTPRequestHandler TempFileStreamingHandler = [](HTTPServerRequest Request) {
int FD;
SmallString<64> TempFilePath;
sys::fs::createTemporaryFile("http-stream-file-test", "temp", FD,
TempFilePath);
raw_fd_ostream OS(FD, true, /*unbuffered=*/true);
OS << Response.Body;
OS.close();
streamFile(Request, TempFilePath);
};
// Test streaming back chunks of a file.
TEST_F(HTTPClientServerTest, StreamingFileResponse) {
HTTPServer Server;
EXPECT_THAT_ERROR(Server.get(UrlPathPattern, TempFileStreamingHandler),
Succeeded());
Expected<unsigned> PortOrErr = Server.bind();
EXPECT_THAT_EXPECTED(PortOrErr, Succeeded());
unsigned Port = *PortOrErr;
ThreadPool Pool(hardware_concurrency(1));
Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); });
std::string Url = "http://localhost:" + utostr(Port);
HTTPRequest Request(Url);
StringHTTPResponseHandler Handler;
HTTPClient Client;
EXPECT_THAT_ERROR(Client.perform(Request, Handler), Succeeded());
EXPECT_EQ(Handler.ResponseBody, Response.Body);
EXPECT_EQ(Client.responseCode(), Response.Code);
Server.stop();
}
// Deletes the temporary file before streaming it back, should give a 404 not
// found status code.
HTTPRequestHandler MissingTempFileStreamingHandler =
[](HTTPServerRequest Request) {
int FD;
SmallString<64> TempFilePath;
sys::fs::createTemporaryFile("http-stream-file-test", "temp", FD,
TempFilePath);
raw_fd_ostream OS(FD, true, /*unbuffered=*/true);
OS << Response.Body;
OS.close();
// delete the file
sys::fs::remove(TempFilePath);
streamFile(Request, TempFilePath);
};
// Streaming a missing file should give a 404.
TEST_F(HTTPClientServerTest, StreamingMissingFileResponse) {
HTTPServer Server;
EXPECT_THAT_ERROR(Server.get(UrlPathPattern, MissingTempFileStreamingHandler),
Succeeded());
Expected<unsigned> PortOrErr = Server.bind();
EXPECT_THAT_EXPECTED(PortOrErr, Succeeded());
unsigned Port = *PortOrErr;
ThreadPool Pool(hardware_concurrency(1));
Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); });
std::string Url = "http://localhost:" + utostr(Port);
HTTPRequest Request(Url);
StringHTTPResponseHandler Handler;
HTTPClient Client;
EXPECT_THAT_ERROR(Client.perform(Request, Handler), Succeeded());
EXPECT_EQ(Client.responseCode(), 404u);
Server.stop();
}
TEST_F(HTTPClientServerTest, ClientTimeout) {
HTTPServer Server;
EXPECT_THAT_ERROR(Server.get(UrlPathPattern, DelayHandler), Succeeded());
Expected<unsigned> PortOrErr = Server.bind();
EXPECT_THAT_EXPECTED(PortOrErr, Succeeded());
unsigned Port = *PortOrErr;
ThreadPool Pool(hardware_concurrency(1));
Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); });
std::string Url = "http://localhost:" + utostr(Port);
HTTPClient Client;
// Timeout below 50ms, request should fail
Client.setTimeout(std::chrono::milliseconds(40));
HTTPRequest Request(Url);
StringHTTPResponseHandler Handler;
EXPECT_THAT_ERROR(Client.perform(Request, Handler), Failed<StringError>());
Server.stop();
}
// Check that Url paths are dispatched to the first matching handler and provide
// the correct path pattern match components.
TEST_F(HTTPClientServerTest, PathMatching) {
HTTPServer Server;
EXPECT_THAT_ERROR(
Server.get(R"(/abc/(.*)/(.*))",
[&](HTTPServerRequest &Request) {
EXPECT_EQ(Request.UrlPath, "/abc/1/2");
ASSERT_THAT(Request.UrlPathMatches,
testing::ElementsAre("1", "2"));
Request.setResponse({200u, "text/plain", Request.UrlPath});
}),
Succeeded());
EXPECT_THAT_ERROR(Server.get(UrlPathPattern,
[&](HTTPServerRequest &Request) {
llvm_unreachable(
"Should not reach this handler");
Handler(Request);
}),
Succeeded());
Expected<unsigned> PortOrErr = Server.bind();
EXPECT_THAT_EXPECTED(PortOrErr, Succeeded());
unsigned Port = *PortOrErr;
ThreadPool Pool(hardware_concurrency(1));
Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); });
std::string Url = "http://localhost:" + utostr(Port) + "/abc/1/2";
HTTPRequest Request(Url);
StringHTTPResponseHandler Handler;
HTTPClient Client;
EXPECT_THAT_ERROR(Client.perform(Request, Handler), Succeeded());
EXPECT_EQ(Handler.ResponseBody, "/abc/1/2");
EXPECT_EQ(Client.responseCode(), 200u);
Server.stop();
}
TEST_F(HTTPClientServerTest, FirstPathMatched) {
HTTPServer Server;
EXPECT_THAT_ERROR(
Server.get(UrlPathPattern,
[&](HTTPServerRequest Request) { Handler(Request); }),
Succeeded());
EXPECT_THAT_ERROR(
Server.get(R"(/abc/(.*)/(.*))",
[&](HTTPServerRequest Request) {
EXPECT_EQ(Request.UrlPathMatches.size(), 2u);
llvm_unreachable("Should not reach this handler");
Request.setResponse({200u, "text/plain", Request.UrlPath});
}),
Succeeded());
Expected<unsigned> PortOrErr = Server.bind();
EXPECT_THAT_EXPECTED(PortOrErr, Succeeded());
unsigned Port = *PortOrErr;
ThreadPool Pool(hardware_concurrency(1));
Pool.async([&]() { EXPECT_THAT_ERROR(Server.listen(), Succeeded()); });
std::string Url = "http://localhost:" + utostr(Port) + "/abc/1/2";
HTTPRequest Request(Url);
StringHTTPResponseHandler Handler;
HTTPClient Client;
EXPECT_THAT_ERROR(Client.perform(Request, Handler), Succeeded());
EXPECT_EQ(Handler.ResponseBody, Response.Body);
EXPECT_EQ(Client.responseCode(), Response.Code);
Server.stop();
}
#endif
#else
TEST(HTTPServer, IsAvailable) { EXPECT_FALSE(HTTPServer::isAvailable()); }
#endif // LLVM_ENABLE_HTTPLIB