Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Funtions return response data as a JSON instead of the body #4

Merged
merged 1 commit into from
Oct 23, 2024
Merged

Funtions return response data as a JSON instead of the body #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 49 additions & 43 deletions docs/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,62 +19,68 @@ LOAD http_client;
#### GET
```sql
D WITH __input AS (
SELECT
http_get(
SELECT
http_get(
'https://httpbin.org/delay/0'
) AS data
),
__features AS (
SELECT
unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') )
AS features
FROM
__input
)
) AS res
),
__response AS (
SELECT
__features.Host AS host,
(res->>'status')::INT AS status,
(res->>'reason') AS reason,
unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features
FROM
__features
;
┌─────────────┐
│ host │
│ varchar │
├─────────────┤
│ httpbin.org │
└─────────────┘
__input
)
SELECT
__response.status,
__response.reason,
__response.Host AS host,
FROM
__response
;
┌────────┬─────────┬─────────────┐
│ status │ reason │ host │
│ int32 │ varchar │ varchar │
├────────┼─────────┼─────────────┤
│ 200 │ OK │ httpbin.org │
└────────┴─────────┴─────────────┘
```

#### POST
```sql
WITH __input AS (
SELECT
http_post(
D WITH __input AS (
SELECT
http_post(
'https://httpbin.org/delay/0',
headers => MAP {
'accept': 'application/json',
},
params => MAP {
}
) AS data
),
__features AS (
SELECT
unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') )
AS features
FROM
__input
)
) AS res
),
__response AS (
SELECT
__features.Host AS host,
(res->>'status')::INT AS status,
(res->>'reason') AS reason,
unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features
FROM
__features
;
┌─────────────┐
│ host │
│ varchar │
├─────────────┤
│ httpbin.org │
└─────────────┘
__input
)
SELECT
__response.status,
__response.reason,
__response.Host AS host,
FROM
__response
;
┌────────┬─────────┬─────────────┐
│ status │ reason │ host │
│ int32 │ varchar │ varchar │
├────────┼─────────┼─────────────┤
│ 200 │ OK │ httpbin.org │
└────────┴─────────┴─────────────┘
```

#### Full Example w/ spatial data
Expand All @@ -88,11 +94,11 @@ D WITH __input AS (
SELECT
http_get(
'https://earth-search.aws.element84.com/v0/search')
AS data
AS res
),
__features AS (
SELECT
unnest( from_json((data::JSON)->'features', '["json"]') )
unnest( from_json(((res->>'body')::JSON)->'features', '["json"]') )
AS features
FROM
__input
Expand Down
71 changes: 49 additions & 22 deletions src/http_client_extension.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,44 @@ static std::pair<duckdb_httplib_openssl::Client, std::string> SetupHttpClient(co
return std::make_pair(std::move(client), path);
}

static void HandleHttpError(const duckdb_httplib_openssl::Result &res, const std::string &request_type) {
// Helper function to escape chars of a string representing a JSON object
std::string escape_json(const std::string &input) {
std::ostringstream output;

for (auto c = input.cbegin(); c != input.cend(); c++) {
switch (*c) {
case '"' : output << "\\\""; break;
case '\\': output << "\\\\"; break;
case '\b': output << "\\b"; break;
case '\f': output << "\\f"; break;
case '\n': output << "\\n"; break;
case '\r': output << "\\r"; break;
case '\t': output << "\\t"; break;
default:
if ('\x00' <= *c && *c <= '\x1f') {
output << "\\u"
<< std::hex << std::setw(4) << std::setfill('0') << static_cast<int>(*c);
} else {
output << *c;
}
}
}
return output.str();
}

// Helper function to create a Response object as a string
static std::string GetJsonResponse(int status, const std::string &reason, const std::string &body) {
std::string response = StringUtil::Format(
"{ \"status\": %i, \"reason\": \"%s\", \"body\": \"%s\" }",
status,
escape_json(reason),
escape_json(body)
);
return response;
}

// Helper function to return the description of one HTTP error.
static std::string GetHttpErrorMessage(const duckdb_httplib_openssl::Result &res, const std::string &request_type) {
std::string err_message = "HTTP " + request_type + " request failed. ";

switch (res.error()) {
Expand Down Expand Up @@ -85,7 +122,7 @@ static void HandleHttpError(const duckdb_httplib_openssl::Result &res, const std
err_message += "Unknown error.";
break;
}
throw std::runtime_error(err_message);
return err_message;
}


Expand All @@ -103,17 +140,12 @@ static void HTTPGetRequestFunction(DataChunk &args, ExpressionState &state, Vect
// Make the GET request
auto res = client.Get(path.c_str());
if (res) {
if (res->status == 200) {
return StringVector::AddString(result, res->body);
} else {
throw std::runtime_error("HTTP GET error: " + std::to_string(res->status) + " - " + res->reason);
}
std::string response = GetJsonResponse(res->status, res->reason, res->body);
return StringVector::AddString(result, response);
} else {
// Handle errors
HandleHttpError(res, "GET");
std::string response = GetJsonResponse(-1, GetHttpErrorMessage(res, "POST"), "");
return StringVector::AddString(result, response);
}
// Ensure a return value in case of an error
return string_t();
});
}

Expand Down Expand Up @@ -159,30 +191,25 @@ static void HTTPPostRequestFunction(DataChunk &args, ExpressionState &state, Vec
// Make the POST request with headers and body
auto res = client.Post(path.c_str(), header_map, body.val.GetString(), "application/json");
if (res) {
if (res->status == 200) {
return StringVector::AddString(result, res->body);
} else {
throw std::runtime_error("HTTP POST error: " + std::to_string(res->status) + " - " + res->reason);
}
std::string response = GetJsonResponse(res->status, res->reason, res->body);
return StringVector::AddString(result, response);
} else {
// Handle errors
HandleHttpError(res, "POST");
std::string response = GetJsonResponse(-1, GetHttpErrorMessage(res, "POST"), "");
return StringVector::AddString(result, response);
}
// Ensure a return value in case of an error
return string_t();
});
}


static void LoadInternal(DatabaseInstance &instance) {
ScalarFunctionSet http_get("http_get");
http_get.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::VARCHAR, HTTPGetRequestFunction));
http_get.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::JSON(), HTTPGetRequestFunction));
ExtensionUtil::RegisterFunction(instance, http_get);

ScalarFunctionSet http_post("http_post");
http_post.AddFunction(ScalarFunction(
{LogicalType::VARCHAR, LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR), LogicalType::JSON()},
LogicalType::VARCHAR, HTTPPostRequestFunction));
LogicalType::JSON(), HTTPPostRequestFunction));
ExtensionUtil::RegisterFunction(instance, http_post);
}

Expand Down
115 changes: 60 additions & 55 deletions test/sql/httpclient.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,55 +14,61 @@ require http_client
require json

# Confirm the GET extension works
query I
query III
WITH __input AS (
SELECT
http_get(
'https://httpbin.org/delay/0'
) AS data
),
__features AS (
SELECT
unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') )
AS features
FROM
__input
)
SELECT
__features.Host AS host,
FROM
__features
;
SELECT
http_get(
'https://httpbin.org/delay/0'
) AS res
),
__response AS (
SELECT
(res->>'status')::INT AS status,
(res->>'reason') AS reason,
unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features
FROM
__input
)
SELECT
__response.status,
__response.reason,
__response.Host AS host
FROM
__response
;
----
httpbin.org
200 OK httpbin.org

# Confirm the POST extension works
query I
query III
WITH __input AS (
SELECT
http_post(
'https://httpbin.org/delay/0',
headers => MAP {
'accept': 'application/json',
},
params => MAP {
}
) AS data
),
__features AS (
SELECT
unnest( from_json((data::JSON)->'headers', '{"Host": "VARCHAR"}') )
AS features
FROM
__input
)
SELECT
__features.Host AS host,
FROM
__features
;
SELECT
http_post(
'https://httpbin.org/delay/0',
headers => MAP {
'accept': 'application/json',
},
params => MAP {
}
) AS res
),
__response AS (
SELECT
(res->>'status')::INT AS status,
(res->>'reason') AS reason,
unnest( from_json(((res->>'body')::JSON)->'headers', '{"Host": "VARCHAR"}') ) AS features
FROM
__input
)
SELECT
__response.status,
__response.reason,
__response.Host AS host
FROM
__response
;
----
httpbin.org
200 OK httpbin.org

# Confirm the POST extension works with headers and params
query I
Expand All @@ -81,19 +87,18 @@ WITH __input AS (
'datetime': '2021-09-30/2021-09-30',
'limit': 10
}
) AS data
),
__features AS (
SELECT
unnest( from_json((data::JSON)->'features', '["json"]') )
AS features
FROM
__input
)
) AS res
),
__response AS (
SELECT
features->>'id' AS id
unnest( from_json(((res->>'body')::JSON)->'features', '["json"]') ) AS features
FROM
__features
;
__input
)
SELECT
features->>'id' AS id
FROM
__response
;
----
S2A_56LPN_20210930_0_L2A
Loading