diff --git a/docs/README.md b/docs/README.md index 011c574..aba3ad6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,62 +19,68 @@ LOAD http_client; #### GET ```sql D WITH __input AS ( - SELECT - http_get( + SELECT + http_get( 'https://httpbin.org/delay/0' - ) AS data - ), - __features AS ( - SELECT - unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') ) - AS features - FROM - __input - ) + ) AS res + ), + __response AS ( SELECT - __features.Host AS host, + (res->>'status')::INT AS status, + (res->>'reason') AS reason, + unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features FROM - __features - ; -┌─────────────┐ -│ host │ -│ varchar │ -├─────────────┤ -│ httpbin.org │ -└─────────────┘ + __input + ) + SELECT + __response.status, + __response.reason, + __response.Host AS host, + FROM + __response + ; +┌────────┬─────────┬─────────────┐ +│ status │ reason │ host │ +│ int32 │ varchar │ varchar │ +├────────┼─────────┼─────────────┤ +│ 200 │ OK │ httpbin.org │ +└────────┴─────────┴─────────────┘ ``` #### POST ```sql -WITH __input AS ( - SELECT - http_post( +D WITH __input AS ( + SELECT + http_post( 'https://httpbin.org/delay/0', headers => MAP { 'accept': 'application/json', }, params => MAP { } - ) AS data - ), - __features AS ( - SELECT - unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') ) - AS features - FROM - __input - ) + ) AS res + ), + __response AS ( SELECT - __features.Host AS host, + (res->>'status')::INT AS status, + (res->>'reason') AS reason, + unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features FROM - __features - ; -┌─────────────┐ -│ host │ -│ varchar │ -├─────────────┤ -│ httpbin.org │ -└─────────────┘ + __input + ) + SELECT + __response.status, + __response.reason, + __response.Host AS host, + FROM + __response + ; +┌────────┬─────────┬─────────────┐ +│ status │ reason │ host │ +│ int32 │ varchar │ varchar │ +├────────┼─────────┼─────────────┤ +│ 200 │ OK │ httpbin.org │ +└────────┴─────────┴─────────────┘ ``` #### Full Example w/ spatial data @@ -88,11 +94,11 @@ D WITH __input AS ( SELECT http_get( 'https://earth-search.aws.element84.com/v0/search') - AS data + AS res ), __features AS ( SELECT - unnest( from_json((data::JSON)->'features', '["json"]') ) + unnest( from_json(((res->>'body')::JSON)->'features', '["json"]') ) AS features FROM __input diff --git a/src/http_client_extension.cpp b/src/http_client_extension.cpp index c71c5e5..46109ce 100644 --- a/src/http_client_extension.cpp +++ b/src/http_client_extension.cpp @@ -44,7 +44,44 @@ static std::pair SetupHttpClient(co return std::make_pair(std::move(client), path); } -static void HandleHttpError(const duckdb_httplib_openssl::Result &res, const std::string &request_type) { +// Helper function to escape chars of a string representing a JSON object +std::string escape_json(const std::string &input) { + std::ostringstream output; + + for (auto c = input.cbegin(); c != input.cend(); c++) { + switch (*c) { + case '"' : output << "\\\""; break; + case '\\': output << "\\\\"; break; + case '\b': output << "\\b"; break; + case '\f': output << "\\f"; break; + case '\n': output << "\\n"; break; + case '\r': output << "\\r"; break; + case '\t': output << "\\t"; break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + output << "\\u" + << std::hex << std::setw(4) << std::setfill('0') << static_cast(*c); + } else { + output << *c; + } + } + } + return output.str(); +} + +// Helper function to create a Response object as a string +static std::string GetJsonResponse(int status, const std::string &reason, const std::string &body) { + std::string response = StringUtil::Format( + "{ \"status\": %i, \"reason\": \"%s\", \"body\": \"%s\" }", + status, + escape_json(reason), + escape_json(body) + ); + return response; +} + +// Helper function to return the description of one HTTP error. +static std::string GetHttpErrorMessage(const duckdb_httplib_openssl::Result &res, const std::string &request_type) { std::string err_message = "HTTP " + request_type + " request failed. "; switch (res.error()) { @@ -85,7 +122,7 @@ static void HandleHttpError(const duckdb_httplib_openssl::Result &res, const std err_message += "Unknown error."; break; } - throw std::runtime_error(err_message); + return err_message; } @@ -103,17 +140,12 @@ static void HTTPGetRequestFunction(DataChunk &args, ExpressionState &state, Vect // Make the GET request auto res = client.Get(path.c_str()); if (res) { - if (res->status == 200) { - return StringVector::AddString(result, res->body); - } else { - throw std::runtime_error("HTTP GET error: " + std::to_string(res->status) + " - " + res->reason); - } + std::string response = GetJsonResponse(res->status, res->reason, res->body); + return StringVector::AddString(result, response); } else { - // Handle errors - HandleHttpError(res, "GET"); + std::string response = GetJsonResponse(-1, GetHttpErrorMessage(res, "POST"), ""); + return StringVector::AddString(result, response); } - // Ensure a return value in case of an error - return string_t(); }); } @@ -159,30 +191,25 @@ static void HTTPPostRequestFunction(DataChunk &args, ExpressionState &state, Vec // Make the POST request with headers and body auto res = client.Post(path.c_str(), header_map, body.val.GetString(), "application/json"); if (res) { - if (res->status == 200) { - return StringVector::AddString(result, res->body); - } else { - throw std::runtime_error("HTTP POST error: " + std::to_string(res->status) + " - " + res->reason); - } + std::string response = GetJsonResponse(res->status, res->reason, res->body); + return StringVector::AddString(result, response); } else { - // Handle errors - HandleHttpError(res, "POST"); + std::string response = GetJsonResponse(-1, GetHttpErrorMessage(res, "POST"), ""); + return StringVector::AddString(result, response); } - // Ensure a return value in case of an error - return string_t(); }); } static void LoadInternal(DatabaseInstance &instance) { ScalarFunctionSet http_get("http_get"); - http_get.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::VARCHAR, HTTPGetRequestFunction)); + http_get.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::JSON(), HTTPGetRequestFunction)); ExtensionUtil::RegisterFunction(instance, http_get); ScalarFunctionSet http_post("http_post"); http_post.AddFunction(ScalarFunction( {LogicalType::VARCHAR, LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR), LogicalType::JSON()}, - LogicalType::VARCHAR, HTTPPostRequestFunction)); + LogicalType::JSON(), HTTPPostRequestFunction)); ExtensionUtil::RegisterFunction(instance, http_post); } diff --git a/test/sql/httpclient.test b/test/sql/httpclient.test index 69049ab..c658552 100644 --- a/test/sql/httpclient.test +++ b/test/sql/httpclient.test @@ -14,55 +14,61 @@ require http_client require json # Confirm the GET extension works -query I +query III WITH __input AS ( - SELECT - http_get( - 'https://httpbin.org/delay/0' - ) AS data - ), - __features AS ( - SELECT - unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') ) - AS features - FROM - __input - ) - SELECT - __features.Host AS host, - FROM - __features - ; + SELECT + http_get( + 'https://httpbin.org/delay/0' + ) AS res +), +__response AS ( + SELECT + (res->>'status')::INT AS status, + (res->>'reason') AS reason, + unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features + FROM + __input +) +SELECT + __response.status, + __response.reason, + __response.Host AS host +FROM + __response +; ---- -httpbin.org +200 OK httpbin.org # Confirm the POST extension works -query I +query III WITH __input AS ( - SELECT - http_post( - 'https://httpbin.org/delay/0', - headers => MAP { - 'accept': 'application/json', - }, - params => MAP { - } - ) AS data - ), - __features AS ( - SELECT - unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') ) - AS features - FROM - __input - ) - SELECT - __features.Host AS host, - FROM - __features - ; + SELECT + http_post( + 'https://httpbin.org/delay/0', + headers => MAP { + 'accept': 'application/json', + }, + params => MAP { + } + ) AS res +), +__response AS ( + SELECT + (res->>'status')::INT AS status, + (res->>'reason') AS reason, + unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features + FROM + __input +) +SELECT + __response.status, + __response.reason, + __response.Host AS host +FROM + __response +; ---- -httpbin.org +200 OK httpbin.org # Confirm the POST extension works with headers and params query I @@ -81,19 +87,18 @@ WITH __input AS ( 'datetime': '2021-09-30/2021-09-30', 'limit': 10 } - ) AS data - ), - __features AS ( - SELECT - unnest( from_json((data::JSON)->'features', '["json"]') ) - AS features - FROM - __input - ) + ) AS res +), +__response AS ( SELECT - features->>'id' AS id + unnest( from_json(((res->>'body')::JSON)->'features', '["json"]') ) AS features FROM - __features - ; + __input +) +SELECT + features->>'id' AS id +FROM + __response +; ---- S2A_56LPN_20210930_0_L2A