diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index b4c46d6..f6325d3 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -1,6 +1,6 @@ name: Makefile CI -on: +on: push: branches: - main @@ -35,23 +35,28 @@ jobs: ~/.cache/mops ~/mops - - uses: aviate-labs/setup-dfx@v0.2.3 - with: - dfx-version: 0.15.0 + - name: Install dfx + uses: dfinity/setup-dfx@main + - name: Confirm successful installation + run: dfx --version + + - name: Install dfx cache + run: dfx cache install - name: Install mops & mocv run: | - npm -g i mocv - npm --yes -g i ic-mops - mops i + npm --yes -g i ic-mops + mops i + mops toolchain init - - name: install wasmtime - run: | - curl https://wasmtime.dev/install.sh -sSf | bash - echo "$HOME/.wasmtime/bin" >> $GITHUB_PATH + # set moc path for dfx to use + echo "DFX_MOC_PATH=$(mops toolchain bin moc)" >> $GITHUB_ENV - name: Detect warnings run: make check + - name: Install zx to run scripts + run: npm install -g zx + - name: Run Tests - run: make test \ No newline at end of file + run: make canister-tests diff --git a/bench/lib.bench.mo b/bench/lib.bench.mo index 850757d..a5da882 100644 --- a/bench/lib.bench.mo +++ b/bench/lib.bench.mo @@ -1,9 +1,9 @@ import Text "mo:base/Text"; import Iter "mo:base/Iter"; import Debug "mo:base/Debug"; -import Prelude "mo:base/Prelude"; import Nat16 "mo:base/Nat16"; import Buffer "mo:base/Buffer"; +import Sha256 "mo:sha2/Sha256"; import Bench "mo:bench"; import Fuzz "mo:fuzz"; @@ -13,7 +13,7 @@ import HttpTypes "mo:http-types"; module { - func random_endpoint(fuzz : Fuzz.Fuzzer) : (CertifiedAssets.EndpointRecord, Blob) { + func random_endpoint(fuzz : Fuzz.Fuzzer, sha256 : Sha256.Digest) : (CertifiedAssets.EndpointRecord, Blob) { let status_codes = [200, 201, 202, 203, 204, 205, 206, 207, 208, 226, 300, 301, 302, 303, 304, 305, 306, 307, 308, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 421, 422, 423, 424, 425, 426, 428, 429, 431, 451, 500, 501, 502, 503, 504, 505, 506, 507, 508, 510, 511]; @@ -57,10 +57,14 @@ module { let include_random_blob = fuzz.nat.randomRange(0, 10) > 5; let include_request_certification = fuzz.nat.randomRange(0, 10) > 10; - let endpoint = CertifiedAssets.Endpoint(path, null); - let body = if (include_random_blob) { blob } else { hello_world_blob }; + sha256.writeBlob(body); + let hash = sha256.sum(); + sha256.reset(); + + let endpoint = CertifiedAssets.Endpoint(path, null).hash(hash); + ignore endpoint.body(body); if (include_status) { @@ -109,14 +113,15 @@ module { ]); let fuzz = Fuzz.Fuzz(); + let sha256 = Sha256.Digest(#sha256); let limit = 1000; let cert_store = CertifiedAssets.init_stable_store(); - let certs = CertifiedAssets.CertifiedAssets(?cert_store); + let certs = CertifiedAssets.CertifiedAssets(cert_store); let endpoint_records = Buffer.Buffer<(CertifiedAssets.EndpointRecord)>(limit); let endpoint_bodies = Buffer.Buffer<(Blob)>(limit); for (_ in Iter.range(0, limit)) { - let (endpoint_record, blob) = random_endpoint(fuzz); + let (endpoint_record, blob) = random_endpoint(fuzz, sha256); endpoint_records.add(endpoint_record); endpoint_bodies.add(blob); }; diff --git a/demo/main.mo b/demo/main.mo index 9ef60d0..4a1aa5a 100644 --- a/demo/main.mo +++ b/demo/main.mo @@ -1,5 +1,4 @@ import Array "mo:base/Array"; -import Buffer "mo:base/Buffer"; import Debug "mo:base/Debug"; import Iter "mo:base/Iter"; import Text "mo:base/Text"; @@ -18,7 +17,7 @@ actor { type Vector = Vector.Vector; stable let st_certs = CertifiedAssets.init_stable_store(); - let certs = CertifiedAssets.CertifiedAssets(?st_certs); + let certs = CertifiedAssets.CertifiedAssets(st_certs); let { thash } = Map; @@ -120,7 +119,7 @@ actor { ).status(200).is_fallback_path(); certs.certify(homepage_endpoint); - let homepage_endpoint2 = CertifiedAssets.Endpoint("/home", ?Text.encodeUtf8(_homepage)).no_request_certification().response_header( + let homepage_endpoint2 = CertifiedAssets.Endpoint("/home", ?Text.encodeUtf8(_homepage)).response_header( "Content-Type", "text/html", ).response_header( @@ -192,6 +191,13 @@ actor { }; certify_endpoints_page(); + + let hello_page = CertifiedAssets.Endpoint("/hello", ?Text.encodeUtf8("👋 Hello World!")).no_request_certification().response_header( + "Content-Type", + "text/plain", + ).status(200); + + certs.certify(hello_page); }; func endpoints_json() : Text { @@ -279,7 +285,7 @@ actor { assert req.body == ""; assert req.method == "GET"; - Debug.print("url: " # url.path.original); + Debug.print("http_request: " # debug_show req); let response : HttpTypes.Response = switch (url.path.original, url.queryObj.get("size")) { case ("/endpoints", _) { @@ -291,6 +297,15 @@ actor { upgrade = null; }; }; + case ("/hello", _) { + { + status_code = 200; + body = Text.encodeUtf8("👋 Hello World!"); + headers = [("Content-Type", "text/plain")]; + streaming_strategy = null; + upgrade = null; + }; + }; case ("" or "/" or "/home" or "/home/" or "/h o m e" or "/home/index.html", _) { let _homepage = homepage(); @@ -313,9 +328,13 @@ actor { case (_) Debug.trap("Invalid size: " # text_size); }; + let teams = teams_json(?size); + + Debug.print("teams: " # teams); + { status_code = 200; - body = Text.encodeUtf8(teams_json(?size)); + body = Text.encodeUtf8(teams); headers = [("Content-Type", "application/json")]; streaming_strategy = null; upgrade = null; @@ -332,10 +351,18 @@ actor { }; case (_) { // path -> '/v1/teams/:team' + var response : HttpTypes.Response = { + status_code = 404; + body = Text.encodeUtf8("Not Found"); + headers = []; + streaming_strategy = null; + upgrade = null; + }; + if (Text.startsWith(url.path.original, #text "/v1/teams/")) { let team = url.path.array[2]; if (Map.has(teams, thash, team)) { - return { + response := { status_code = 200; body = Text.encodeUtf8(single_team_json(team)); headers = [("Content-Type", "application/json")]; @@ -344,13 +371,9 @@ actor { }; }; }; - return { - status_code = 404; - body = Text.encodeUtf8("Not Found"); - headers = []; - streaming_strategy = null; - upgrade = null; - }; + + response; + }; }; diff --git a/dfx.json b/dfx.json index 155b4bf..12fc9c3 100644 --- a/dfx.json +++ b/dfx.json @@ -11,14 +11,14 @@ "type": "motoko", "main": "./demo/main.mo" }, - "actor_test": { + "canister-tests": { "type": "motoko", - "main": "./tests/CertifiedAssets.Test.mo" + "main": "./tests/CertifiedAssets.CanisterTests.mo" } }, "networks": { "local": { - "bind": "127.0.0.1:8000", + "bind": "127.0.0.1:4943", "type": "ephemeral" } } diff --git a/makefile b/makefile index c4be368..38de329 100644 --- a/makefile +++ b/makefile @@ -20,4 +20,7 @@ docs: set-moc-version $(MocvPath)/mo-doc --format plain bench: set-dfx-moc-path - mops bench \ No newline at end of file + mops bench + +canister-tests: + zx -i ./z-scripts/canister-tests.mjs \ No newline at end of file diff --git a/mops.toml b/mops.toml index 01452da..86d4cdd 100644 --- a/mops.toml +++ b/mops.toml @@ -4,10 +4,10 @@ description = "A library for certifying assets served via HTTP, ensuring the sec repository = "https://github.com/NatLabs/certified-assets" version = "0.3.5" license = "MIT" -keywords = ["assets", "http", "certification"] +keywords = [ "assets", "http", "certification" ] [dependencies] -base = "0.12.1" +base = "0.13.3" ic-certification = "0.1.3" sha2 = "0.1.0" http-parser = "0.3.3" @@ -15,12 +15,14 @@ http-types = "1.0.1" encoding = "https://github.com/aviate-labs/encoding.mo#v0.4.1@2711d18727e954b11afc0d37945608512b5fbce2" serde = "3.1.0" map = "9.0.1" -vector = "0.4.0" +vector = "0.4.1" [dev-dependencies] test = "2.0.0" bench = "1.0.0" fuzz = "0.2.1" +itertools = "0.2.2" [toolchain] wasmtime = "24.0.0" +moc = "0.13.2" diff --git a/readme.md b/readme.md index ed51da8..7a28c83 100644 --- a/readme.md +++ b/readme.md @@ -1,8 +1,10 @@ # Certified Assets -A library designed to certify assets served via HTTP on the Internet Computer. -It implements the [Response Verification Standard](https://github.com/dfinity/interface-spec/blob/master/spec/http-gateway-protocol-spec.md#response-verification) and works by certifying data and their endpoints during update calls. -Once certified, the certificates are returned as headers in an HTTP response, ensuring the security and integrity of the data. +A library designed to certify assets served via HTTP on the Internet Computer. This library only stores the certificates, not the assets themselves. It implements the [Response Verification Standard](https://github.com/dfinity/interface-spec/blob/master/spec/http-gateway-protocol-spec.md#response-verification) and works by certifying data and their endpoints during update calls. Once certified, the certificates are returned as headers in an HTTP response, ensuring the security and integrity of the data. + +> Note that this library does not store the assets themselves, only the certificates. Either use the ic-assets library to store the assets or define your own storage mechanism. + +## Motivation ## Getting Started @@ -28,30 +30,24 @@ import CertifiedAssets "mo:certified-assets"; - stable version ```motoko -import CertifiedAssets "mo:certified-assets/CertifiedAssets"; +import CertifiedAssets "mo:certified-assets/Stable"; ``` #### Create a new instance -- `Heap` - Creates a new instance that will be cleared during canister upgrades. - - ```motoko - let certs = CertifiedAssets.CertifiedAssets(null); - ``` +- Creates a persistent instance that remains stable through canister upgrades -- `Stable Heap` - For creating a persistent instance that remains stable through canister upgrades - - ```motoko - stable let cert_store = CertifiedAssets.init_stable_store(); - let certs = CertifiedAssets.CertifiedAssets(?cert_store); - ``` - - > Note: For stable instances, it's recommended to `clear()` all certified endpoints and re-certify them if the data has changed during a canister upgrade. +```motoko + stable let cert_store = CertifiedAssets.init_stable_store(); + let certs = CertifiedAssets.CertifiedAssets(?cert_store); +``` #### Certify an Asset Define an `Endpoint` with the URL where the asset will be hosted, the data for certification, and optionally, details about the HTTP request and response. +> An Endpoint is a combination of the url path to the asset, and the http request and response details that are associated with the served asset. + ```motoko let endpoint = CertifiedAssets.Endpoint("/hello.txt", ?"Hello, World!"); certs.certify(endpoint); @@ -60,8 +56,8 @@ Define an `Endpoint` with the URL where the asset will be hosted, the data for c The above method creates a new sha256 hash of the data, if you already have the hash, you can pass it in via the `hash()` method to avoid recomputing it. ```motoko - let endpoint = CertifiedAssets.Endpoint("/hello.txt", ?"Hello, World!") - .hash(); + let endpoint = CertifiedAssets.Endpoint("/hello.txt", null) + .hash(hello_world_sha256_hash); certs.certify(endpoint); ``` @@ -71,13 +67,13 @@ These additional parameters include: - **Flags**: - `no_certification()`: if called, none of the data will be certified - - `no_request_certification()`: if called, only the response will be certified + - `no_request_certification()`: if called, only the response will be certified. Set by default, as request certification is not supported - **Request methods**: - - `method()`: the request method + - `method()`: the request method, defaults to 'GET' if not set - `query_param()`: the query parameters - `request_headers()`: the request headers - **Response methods**: - - `status()`: the response status code + - `status()`: the response status code, defaults to 200 if not set - `response_headers()`: the response headers When certifying assets, it's crucial to consider not just the content but also the context in which it's served, including HTTP headers, status code and query parameters. @@ -97,7 +93,7 @@ When certifying assets, it's crucial to consider not just the content but also t #### Update Certified Assets -A unique hash is generated for each endpoint, so any change to the data will require re-certification. To re-certify an asset, you need to `remove()` the existing one and `certify()` the new one. +A unique hash is generated for each endpoint, so any change to the data will require re-certification. To re-certify an asset, you need to `remove()` the old one and `certify()` the new one. ```motoko let old_endpoint = CertifiedAssets.Endpoint("/hello.txt", ?"Hello, World!"); @@ -109,14 +105,18 @@ A unique hash is generated for each endpoint, so any change to the data will req #### Serving A Certified Asset -To serve a certified asset, call the `get_certified_response()` function with the request and response, ensuring they match the defined endpoint. If they don't match, the function will return an error. +Serving a certified asset it is as easy as adding the two headers (`IC-Certificate` and `IC-CertificateExpression`) with the certificates in your HTTP response. +Using `get_certificate()` allows you to retrieve those two headers for the given endpoint. +While `get_certified_response()` returns an updated version of your response with the header certificates added to them. +These two functions will search the internal store for the certificates and return it if the search was successful. If not, the function will return an error. To prevent an error ensure that the Http Request and response match the details in the endpoint originally used to certify the asset. ```motoko import Debug "mo:base/Debug"; import Text "mo:base/Text"; import HttpTypes "mo:http-types"; // -> https://mops.one/http-types + import CertifiedAssets "mo:certified-assets"; - public func http_request(req : HttpTypes.Request) : HttpTypes.Response { + public func http_request(req : CertifiedAssets.HttpRequest) : CertifiedAssets.HttpResponse { assert req.url == "/hello.html"; let res : HttpTypes.Response = { @@ -141,7 +141,73 @@ Once again, you can include the hash of the data when retrieving the certified r let result = certs.get_certified_response(req, res, ?sha256_of_html_page); ``` -### Credits & References +### Fallback index.html files + +Asset certification V2 allows you to fallback to a default index.html file if the requested file. Fallbacks only work if the index.html directory path is a prefix of the requested file path. For example, if the requested file is `/path/to/file.txt`, the fallback index.html file could be stored at either `/path/to/index.html`, `/path/index.html` or `/index.html`. + +- Certify a fallback index.html file + +```motoko + let fallback = CertifiedAssets + .Endpoint("/path/to/index.html", ?"Hello, World!") + .status(200); + + certs.certify(endpoint); +``` + +- Request a missing file and fallback to the index.html file + +```motoko + + public query func http_request(req: CertifiedAssets.HttpRequest) : CertifiedAssets.HttpResponse { + // suppose this request was sent to your canister + assert req == { + method = "GET"; + url = "/path/to/unknown/file.txt"; + headers = []; + }; + + // search your file store for the requested file + // if the file is not found, create a response with the index.html file + + let ?fallback_path = certs.get_fallback_path(req.url); + + let res : HttpTypes.Response = { + status_code = 200; + headers = []; + body = Text.encodeUtf8("Hello, World!"); + streaming_strategy = null; + upgrade = null; + }; + + // replace the url in the request with the index.html file's path + let req_with_fallback_url = { req with url = "/path/to/index.html" }; + + let result = certs.get_certified_response(req_with_fallback_url, res, null); + + let #ok(certified_response) = result else return Debug.trap(debug_show result); + + return certified_response; + + }; + +``` + +## Unit Testing + +- Install zx with `npm install -g zx` +- Run the following commands: + +```bash + dfx start --background + zx -i ./z-scripts/canister-tests.mjs +``` + +## Limitations + +- This implementation does not support request certification. + +## Credits & References - [Response Verification Standard](https://github.com/dfinity/interface-spec/blob/master/spec/http-gateway-protocol-spec.md#response-verification) - Libraries: [ic-certification](https://github.com/nomeata/ic-certification), [certified-http](https://github.com/infu/certified-http) diff --git a/src/Endpoint.mo b/src/Endpoint.mo index 78ad07f..39eca00 100644 --- a/src/Endpoint.mo +++ b/src/Endpoint.mo @@ -7,10 +7,9 @@ import SHA256 "mo:sha2/Sha256"; import HttpParser "mo:http-parser"; import HttpTypes "mo:http-types"; -import Utils "Utils"; - module { + /// An immutable record that contains all the information needed to certify a given endpoint. public type EndpointRecord = { url : Text; hash : Blob; @@ -24,10 +23,11 @@ module { is_fallback_path : Bool; }; - /// A class that contains all the information needed to certify a given endpoint. - /// Recieves a URL endpoint and the data to be certified. - /// Only the path of the URL is used for certification. - /// If you need to certify the query parameters, use either the [`query_param()`](#query_param) + /// An Endpoint is a combination of the url path to the asset, and the http request and response details that are associated with the served asset. + /// It contains all the information needed to certify a given endpoint. + /// + /// > Note, that the first parameter only consumes the path to the asset and not the full URL. + /// If you need to certify the URL's query parameters, use either the [`query_param()`](#query_param) /// function or the [`query_params()`](#query_params) function. public class Endpoint(url_text : Text, opt_body : ?Blob) = self { @@ -55,6 +55,7 @@ module { let _url = HttpParser.URL(url_text, HttpParser.Headers([])); // clashing feature: removes ending slash if present let _queries = Buffer.Buffer(8); + /// Hashes the given blob and sets it as the hash for the endpoint. public func body(blob : Blob) : Endpoint { sha256.writeBlob(blob); _hash := sha256.sum(); @@ -62,11 +63,13 @@ module { return self; }; + /// Sets given value as the hash for the endpoint. public func hash(hash : Blob) : Endpoint { _hash := hash; return self; }; + /// Hashes the given blob chunks and sets it as the hash for the endpoint. public func chunks(chunks : [Blob]) : Endpoint { for (chunk in chunks.vals()) { sha256.writeBlob(chunk); @@ -78,16 +81,19 @@ module { return self; }; + /// Sets the method for the endpoint. public func method(method : Text) : Endpoint { _method := method; return self; }; + /// Adds a request header to the endpoint. public func request_header(field : Text, value : Text) : Endpoint { _request_headers.add((field, value)); return self; }; + /// Adds multiple request headers to the endpoint. public func request_headers(params : [(Text, Text)]) : Endpoint { for ((field, value) in params.vals()) { _request_headers.add((field, value)); @@ -95,11 +101,13 @@ module { return self; }; + /// Adds a query parameter to the endpoint. public func query_param(field : Text, value : Text) : Endpoint { _queries.add((field, value)); return self; }; + /// Adds multiple query parameters to the endpoint. public func query_params(params : [(Text, Text)]) : Endpoint { for ((field, value) in params.vals()) { _queries.add((field, value)); @@ -107,16 +115,19 @@ module { return self; }; + /// Sets the status code for the endpoint. public func status(code : Nat16) : Endpoint { _status := code; return self; }; + /// Adds a response header to the endpoint. public func response_header(field : Text, value : Text) : Endpoint { _response_headers.add((field, value)); return self; }; + /// Adds multiple response headers to the endpoint. public func response_headers(params : [(Text, Text)]) : Endpoint { for ((field, value) in params.vals()) { _response_headers.add((field, value)); @@ -125,16 +136,19 @@ module { return self; }; - public func is_fallback_path() : Endpoint { - _is_fallback_path := true; + /// Sets the endpoint as a fallback path. + public func is_fallback_path(val : Bool) : Endpoint { + _is_fallback_path := val; return self; }; + /// Disables certification for the method, query parameters and request headers. public func no_request_certification() : Endpoint { _no_request_certification := true; return self; }; + /// Disables request certification and certification for the status and response headers. public func no_certification() : Endpoint { _no_certification := true; return self; @@ -146,6 +160,8 @@ module { _queries.add(entry); }; + /// Constructs an `EndpointRecord` from the current state of the `Endpoint`. + /// > The endpoint can still be modified after calling this function and used to create more records. public func build() : EndpointRecord { let record : EndpointRecord = { diff --git a/src/Stable.mo b/src/Stable.mo index 81c1b3d..ca2d36e 100644 --- a/src/Stable.mo +++ b/src/Stable.mo @@ -17,6 +17,7 @@ import Map "mo:map/Map"; import RepIndyHash "mo:rep-indy-hash"; import Vector "mo:vector"; import Itertools "mo:itertools/Iter"; +import RevIter "mo:itertools/RevIter"; import Base64 "mo:encoding/Base64"; @@ -52,16 +53,25 @@ module Module { public type Endpoint = EndpointModule.Endpoint; public type EndpointRecord = EndpointModule.EndpointRecord; + public type HttpRequest = HttpTypes.Request; + public type HttpResponse = HttpTypes.Response; + public type Header = HttpTypes.Header; + public type CertifiedTree = { certificate : Blob; tree : Blob; }; - let IC_CERTIFICATE_EXPRESSION = "ic-certificateexpression"; - let IC_CERT_BODY = ":ic-cert-body"; - let IC_CERT_METHOD = ":ic-cert-method"; - let IC_CERT_QUERY = ":ic-cert-query"; - let IC_CERT_STATUS = ":ic-cert-status"; + public let IC_CERTIFICATE_EXPRESSION = "ic-certificateexpression"; + public let IC_CERT_BODY = ":ic-cert-body"; + public let IC_CERT_METHOD = ":ic-cert-method"; + public let IC_CERT_QUERY = ":ic-cert-query"; + public let IC_CERT_STATUS = ":ic-cert-status"; + + public type CertifiedAssetErrors = { + #GetCertifiedDataFailed : Text; + #NoMatchingEndpointFound : Text; + }; public func init_stable_store() : StableStore { { @@ -75,29 +85,34 @@ module Module { certify_record(ct, endpoint_record); }; - public func certify_record(ct : StableStore, endpoint_record : EndpointRecord) { - // Debug.print("certifying endpoint: " # debug_show (endpoint_record.url)); - MerkleTreeOps.put(ct, ["http_assets", Text.encodeUtf8(endpoint_record.url)], endpoint_record.hash); + func url_to_encoded_expr_path(endpoint_record : EndpointRecord) : ([Blob], Blob) { - let paths = if (endpoint_record.url == "")[""] else Iter.toArray( + let paths = Iter.toArray( Text.tokens(endpoint_record.url, #text "/") ); - // Debug.print("url: " # debug_show endpoint_record.url); - // Debug.print("paths: " # debug_show paths); + let ends_with_index_dot_html = Text.endsWith(endpoint_record.url, #text("index.html")); + let is_fallback = endpoint_record.is_fallback_path; + let text_expr_path = Array.tabulate( - paths.size() + 2, + paths.size() + (if (ends_with_index_dot_html) 1 else 2), func(i : Nat) : Text { if (i == 0) return "http_expr"; - if (i < paths.size() + 1) return paths[i - 1]; - - // if (Text.endsWith(endpoint_record.url, #text ".html")) return "<$>"; + if (i < paths.size() + (if (ends_with_index_dot_html) 0 else 1)) return paths[i - 1]; - return if (endpoint_record.is_fallback_path) "<*>" else "<$>"; + if (is_fallback or ends_with_index_dot_html) { + "<*>"; + } else { + "<$>"; + }; }, ); - // encode the segments to cbor for the expr_path field for the certificate + // Debug.print("text_expr_path: " # debug_show text_expr_path); + // Debug.print("is_fallback: " # debug_show is_fallback); + + let expr_path = Array.map(text_expr_path, Text.encodeUtf8); + let candid_record_expr_path = #Array( Array.map(text_expr_path, func(t : Text) : Serde.Candid = #Text(t)) ); @@ -108,7 +123,15 @@ module Module { case (#err(errMsg)) Debug.trap("Internal Error: Report bug in NatLabs/certified-assets repo.\n\t" # errMsg); }; - let expr_path = Array.map(text_expr_path, Text.encodeUtf8); + (expr_path, encoded_expr_path); + + }; + + public func certify_record(ct : StableStore, endpoint_record : EndpointRecord) { + // v1 certification + MerkleTreeOps.put(ct, ["http_assets", Text.encodeUtf8(endpoint_record.url)], endpoint_record.hash); + + let (expr_path, encoded_expr_path) = url_to_encoded_expr_path(endpoint_record); let extract_field = func((field, _) : (Text, Text)) : Text = field; let certified_query_params = endpoint_record.query_params; @@ -121,8 +144,6 @@ module Module { let request_headers_fields = Array.map(endpoint_record.request_headers, extract_field); let response_headers_fields = Array.map(endpoint_record.response_headers, extract_field); - let fields = Buffer.Buffer(8); - var ic_certificate_expression = switch (no_certification, no_request_certification) { case (true, _) { " @@ -172,7 +193,7 @@ module Module { ic_certificate_expression := Text.join(" ", Text.tokens(ic_certificate_expression, #predicate(func(c : Char) : Bool = c == ' ' or c == '\n'))); - let expr_hash = SHA256.fromBlob(#sha256, Text.encodeUtf8(ic_certificate_expression)); + let ic_certificate_expression_hash = SHA256.fromBlob(#sha256, Text.encodeUtf8(ic_certificate_expression)); var request_hash : Blob = ""; @@ -243,10 +264,11 @@ module Module { assert (not no_certification) or (no_certification and ((request_hash == "") and (response_hash == ""))); - let full_expr_path = Array.append(expr_path, [expr_hash, request_hash, response_hash]); + let full_expr_path = Array.append(expr_path, [ic_certificate_expression_hash, request_hash, response_hash]); + // Debug.print("full_expr_path: " # debug_show full_expr_path); + // v2 certification MerkleTreeOps.put(ct, full_expr_path, ""); - MerkleTreeOps.setCertifiedData(ct); let metadata : Metadata = { @@ -261,15 +283,14 @@ module Module { // this is not an official field, but it is used internally to uniquely identify the http request buffer.add((IC_CERT_BODY, #Blob(endpoint_record.hash))); - if (not no_request_certification and not no_certification) { - buffer.add((IC_CERT_METHOD, #Text(endpoint_record.method))); - }; - if (not no_certification) { buffer.add((IC_CERT_STATUS, #Nat(Nat16.toNat(endpoint_record.status)))); }; - // Debug.print("buffer for unique_http_hash: " # debug_show Buffer.toArray(buffer)); + if (not no_request_certification and not no_certification) { + buffer.add((IC_CERT_METHOD, #Text(endpoint_record.method))); + }; + let unique_http_hash = Blob.fromArray(RepIndyHash.hash_val(#Map(Buffer.toArray(buffer)))); let opt_nested_map = Map.get(ct.metadata_map, thash, endpoint_record.url); @@ -368,6 +389,24 @@ module Module { ); }; + public func endpoints_by_url(ct : StableStore, url : Text) : Iter { + let ?nested_map = Map.get(ct.metadata_map, thash, url) else return [].vals(); + + Itertools.flatten( + Iter.map( + Map.vals(nested_map), + func(vector : Vector) : Iter { + Iter.map( + Vector.vals(vector), + func(metadata : Metadata) : EndpointRecord { + metadata.endpoint; + }, + ); + }, + ) + ); + }; + /// Clear all certified endpoints. public func clear(ct : StableStore) { MerkleTreeOps.delete(ct, ["http_assets"]); @@ -390,6 +429,33 @@ module Module { if (req.certificate_version == ?2) v2(ct, req, res, response_hash) else v1(ct, req); }; + /// Gets the closest fallback path for the given path that has a certificate associated with it. + public func get_fallback_path(ct : StableStore, path : Text) : ?Text { + + let paths = Iter.toArray(Text.split(path, #text("/"))); + + for (i in RevIter.range(0, paths.size()).rev()) { + let slice = Itertools.fromArraySlice(paths, 0, i + 1); + let possible_fallback_prefix = Text.join(("/"), slice); + let possible_fallback_key = possible_fallback_prefix # "/index.html"; + + switch (Map.get(ct.metadata_map, thash, possible_fallback_key)) { + case (?_) return ?possible_fallback_key; + case (_) {}; + }; + }; + + null; + + }; + + public func get_fallback_certificate(ct : StableStore, req : HttpTypes.Request, fallback_path : Text, res : HttpTypes.Response, response_hash : ?Blob) : Result<[HttpTypes.Header], Text> { + + let fallback_req = { req with url = fallback_path }; + + if (fallback_req.certificate_version == ?2) v2(ct, fallback_req, res, response_hash) else v1(ct, fallback_req); + }; + // /// Get the sha256 hash of the given data. // public func get_hash(ct: StableStore, endpoint: Endpoint): ?Blob { @@ -458,6 +524,8 @@ module Module { return #err("CertifiedData.getCertificate failed. Call this as a query call!"); }; + // Debug.print("encoded_witness: " # debug_show encoded_witness); + let ic_certificate_fields = [ "certificate=:" # base64(certificate) # ":", "tree=:" # base64(encoded_witness) # ":", @@ -590,12 +658,6 @@ module Module { }; public func get_metadata_index_from_vector(endpoint_record : EndpointRecord, metadata_array : Vector) : ?(Vector, Nat) { - // Debug.print("endpoint query_params: " # debug_show (endpoint_record.query_params)); - // Debug.print("endpoint response_headers: " # debug_show (endpoint_record.response_headers)); - // Debug.print("endpoint request_headers: " # debug_show (endpoint_record.request_headers)); - - // Debug.print("metadata_array: " # debug_show (metadata_array)); - var i = 0; for (metadata in Vector.vals(metadata_array)) { var check = true; @@ -622,7 +684,7 @@ module Module { utf8; }; - module MerkleTreeOps { + public module MerkleTreeOps { type Path = MerkleTree.Path; type Value = MerkleTree.Value; type Key = MerkleTree.Key; diff --git a/src/Utils.mo b/src/Utils.mo index e46930c..742b476 100644 --- a/src/Utils.mo +++ b/src/Utils.mo @@ -10,7 +10,7 @@ import Hex "mo:encoding/Hex"; module { type Result = Result.Result; - public func send_error(res: Result): Result{ + public func send_error(res : Result) : Result { switch (res) { case (#ok(_)) Prelude.unreachable(); case (#err(errorMsg)) #err(errorMsg); @@ -50,6 +50,7 @@ module { let n32 = Char.toNat32(char); let n = Nat32.toNat(n32); let n8 = Nat8.fromNat(n); + n8; }; public func percent_decoding(t : Text) : Text { @@ -98,4 +99,4 @@ module { }; encoded; }; -}; \ No newline at end of file +}; diff --git a/src/lib.mo b/src/lib.mo index 4b113c6..1193aee 100644 --- a/src/lib.mo +++ b/src/lib.mo @@ -4,14 +4,10 @@ import Iter "mo:base/Iter"; import Result "mo:base/Result"; import Text "mo:base/Text"; -import HttpParser "mo:http-parser"; import HttpTypes "mo:http-types"; -import { CBOR } "mo:serde"; import Map "mo:map/Map"; -import RepIndyHash "mo:rep-indy-hash"; import Vector "mo:vector"; -import Utils "Utils"; import EndpointModule "Endpoint"; import Stable "Stable"; @@ -22,13 +18,12 @@ module { type Result = Result.Result; type Vector = Vector.Vector; - public type HttpRequest = HttpTypes.Request; - public type HttpResponse = HttpTypes.Response; + public type HttpRequest = Stable.HttpRequest; + public type HttpResponse = Stable.HttpResponse; + public type Header = Stable.Header; public type StableStore = Stable.StableStore; - let { thash; bhash } = Map; - public type MetadataMap = Stable.MetadataMap; public let Endpoint = EndpointModule.Endpoint; @@ -37,47 +32,44 @@ module { public type CertifiedTree = Stable.CertifiedTree; + public let IC_CERTIFICATE_EXPRESSION = Stable.IC_CERTIFICATE_EXPRESSION; + public let IC_CERT_BODY = Stable.IC_CERT_BODY; + public let IC_CERT_METHOD = Stable.IC_CERT_METHOD; + public let IC_CERT_QUERY = Stable.IC_CERT_QUERY; + public let IC_CERT_STATUS = Stable.IC_CERT_STATUS; + /// Create a new stable CertifiedAssets instance on the heap. /// This instance is stable and will not be cleared on canister upgrade. /// /// ```motoko /// let cert_store = CertifiedAssets.init_stable_store(); - /// let certs = CertifiedAssets.CertifiedAssets(?cert_store); + /// let certs = CertifiedAssets.CertifiedAssets(cert_store); /// ``` - public func init_stable_store() : StableStore = Stable.init_stable_store(); /// The implementation of the IC's Response Verification version 2. /// /// The module provides a way to store the certified data on the heap or in stable persistent memory. - /// - heap - creates a new instance of the class that will be cleared on canister upgrade. - /// ```motoko - /// let certs = CertifiedAssets.CertifiedAssets(null); - /// ``` - /// /// - stable heap - creates a new stable instance of the class that will persist on canister upgrade. /// ```motoko /// let cert_store = CertifiedAssets.init_stable_store(); - /// let certs = CertifiedAssets.CertifiedAssets(?cert_store); + /// let certs = CertifiedAssets.CertifiedAssets(cert_store); /// ``` /// - /// If your instance is stable, it is advised to `clear()` all the certified endpoints and - /// re-certify them on canister upgrade if the data has changed. - /// - public class CertifiedAssets(stable_store : ?StableStore) = self { + public class CertifiedAssets(stable_store : StableStore) = self { - let internal : StableStore = switch (stable_store) { - case (?(stable_store)) stable_store; - case (null) Stable.init_stable_store(); - }; + let internal : StableStore = stable_store; + /// Certify a given endpoint. public func certify(endpoint : Endpoint) = Stable.certify(internal, endpoint); + /// Certify a given endpoint record. public func certify_record(endpoint_record : EndpointRecord) = Stable.certify_record(internal, endpoint_record); /// Remove a certified EndpointModule. public func remove(endpoint : Endpoint) = Stable.remove(internal, endpoint); + /// Remove a certified EndpointRecord. public func remove_record(endpoint_record : EndpointRecord) = Stable.remove_record(internal, endpoint_record); /// Removes all the certified endpoints that match the given URL. @@ -122,5 +114,22 @@ module { public func get_certified_tree(keys : ?[Text]) : Result { Stable.get_certified_tree(internal, keys); }; + + /// Gets the fallback certificate for an endpoint that has not been certified. + /// + /// #### Input parameters: + /// - **req** - The request object. + /// - **path** - The path to the fallback certificate. Can use the `get_fallback_path` function. + /// - **res** - The http response for the fallback asset containing its headers, status and body. + /// - **response_hash** - Optional hash of the response body. + /// + public func get_fallback_certificate(req : HttpTypes.Request, fallback_path : Text, res : HttpTypes.Response, response_hash : ?Blob) : Result<[HttpTypes.Header], Text> { + Stable.get_fallback_certificate(internal, req, fallback_path, res, response_hash); + }; + + /// Gets the fallback index.html file with the closest matching prefix for the given path that has a certificate associated with it. + public func get_fallback_path(path : Text) : ?Text { + Stable.get_fallback_path(internal, path); + }; }; }; diff --git a/tests/CanisterTests/lib.mo b/tests/CanisterTests/lib.mo new file mode 100644 index 0000000..a919fd1 --- /dev/null +++ b/tests/CanisterTests/lib.mo @@ -0,0 +1,230 @@ +import Debug "mo:base/Debug"; +import Buffer "mo:base/Buffer"; +import Result "mo:base/Result"; +import Text "mo:base/Text"; + +import Map "mo:map/Map"; +import Serde "mo:serde" + +module { + type Result = Result.Result; + + let { thash } = Map; + + public type TestTools = { + ts_assert : AssertFn; + ts_print : PrintFn; + ts_assert_or_print : AssertOrPrintFn; + }; + + public type AssertFn = (Bool) -> (); + public type PrintFn = (Text) -> (); + public type AssertOrPrintFn = (Bool, Text) -> (); + + public type QueryTestFn = (TestTools) -> (); + public type UpdateTestFn = (TestTools) -> async (); + + public type TestFn = { + #Query : QueryTestFn; + #Update : UpdateTestFn; + }; + + public type Test = { + test_fn : TestFn; + var result : ?Bool; + print_log : Buffer.Buffer; + }; + + public type TestResult = { + name : Text; + result : Bool; + print_log : [Text]; + }; + + public type TestDetails = { + name : Text; + is_query : Bool; + }; + + public class Suite() { + + let tests = Map.new(); + + public func add(name : Text, test_fn : UpdateTestFn) : () { + ignore Map.put( + tests, + thash, + name, + { + var result = null; + test_fn = #Update(test_fn); + print_log = Buffer.Buffer(8); + }, + ); + }; + + public func add_query(name : Text, test_fn : QueryTestFn) : () { + ignore Map.put( + tests, + thash, + name, + { + var result = null; + test_fn = #Query(test_fn); + print_log = Buffer.Buffer(8); + }, + ); + }; + + func get_test_tools(test : Test) : TestTools { + let test_tools = { + ts_assert = func(result : Bool) = switch (test.result) { + case (null) test.result := ?result; + case (?old_result) test.result := ?(old_result and result); + }; + ts_print = func(msg : Text) { + test.print_log.add(Text.replace(msg, #char('\"'), ("\\\""))); + }; + ts_assert_or_print = func(result : Bool, msg : Text) { + switch (test.result) { + case (null) test.result := ?result; + case (?old_result) test.result := ?(old_result and result); + }; + + if (not result) { + test_tools.ts_print(msg); + }; + }; + + }; + + test_tools; + }; + + public func run(test_name : Text) : async (TestResult, Text) { + + let test = switch (Map.get(tests, thash, test_name)) { + case (?test) test; + case (null) Debug.trap("Test '" # test_name # "'' not found"); + }; + + switch (test.test_fn) { + case (#Update(test_fn)) { await test_fn(get_test_tools(test)) }; + case (#Query(test_fn)) { + Debug.trap("Test '" # test_name # "'' is a query test, use run_query instead"); + }; + }; + + let test_result = { + name = test_name; + result = switch (test.result) { + case (?result) result; + case (null) true; + }; + print_log = Buffer.toArray(test.print_log); + }; + + let test_result_candid = to_candid (test_result); + let #ok(test_result_in_json) = Serde.JSON.toText(test_result_candid, ["result", "name", "print_log"], null); + (test_result, test_result_in_json); + }; + + public func run_query(test_name : Text) : (TestResult, Text) { + let test = switch (Map.get(tests, thash, test_name)) { + case (?test) { test }; + case (null) Debug.trap("Test '" # test_name # "'' not found"); + }; + + switch (test.test_fn) { + case (#Update(test_fn)) { + Debug.trap("Test '" # test_name # "'' is an update test, use run instead"); + }; + case (#Query(test_fn)) { test_fn(get_test_tools(test)) }; + }; + + let test_result = { + name = test_name; + result = switch (test.result) { + case (?result) result; + case (null) true; + }; + print_log = Buffer.toArray(test.print_log); + }; + + let test_result_candid = to_candid (test_result); + let #ok(test_result_in_json) = Serde.JSON.toText(test_result_candid, ["result", "name", "print_log"], null); + (test_result, test_result_in_json); + }; + + public func get_test_result(test_name : Text) : (TestResult, Text) { + let test = switch (Map.get(tests, thash, test_name)) { + case (?test) { test }; + case (null) Debug.trap("Test '" # test_name # "'' not found"); + }; + + let test_result = { + name = test_name; + result = switch (test.result) { + case (?result) result; + case (null) true; + }; + print_log = Buffer.toArray(test.print_log); + }; + + let test_result_candid = to_candid (test_result); + let #ok(test_result_in_json) = Serde.JSON.toText(test_result_candid, ["result", "name", "print_log"], null); + (test_result, test_result_in_json); + }; + + public func get_finished_test_results() : ([TestResult], Text) { + let test_results = Buffer.Buffer(Map.size(tests)); + + for ((name, test) in Map.entries(tests)) { + test_results.add({ + name; + result = switch (test.result) { + case (?result) result; + case (null) true; + }; + print_log = Buffer.toArray(test.print_log); + }); + }; + + let test_results_array = Buffer.toArray(test_results); + let test_results_candid = to_candid (test_results_array); + let #ok(test_results_in_json) = Serde.JSON.toText(test_results_candid, ["result", "name", "print_log"], null); + + (test_results_array, test_results_in_json); + + }; + + public func get_test_details() : ([TestDetails], Text) { + let test_details = Buffer.Buffer(Map.size(tests)); + + for ((name, test) in Map.entries(tests)) { + test_details.add({ + name; + is_query = switch (test.test_fn) { + case (#Query(_)) true; + case (#Update(_)) false; + }; + }); + }; + + let test_details_array = Buffer.toArray(test_details); + let test_details_candid = to_candid (test_details_array); + let #ok(test_details_in_json) = Serde.JSON.toText(test_details_candid, ["name", "is_query"], null); + + (test_details_array, test_details_in_json); + + }; + + }; + + public func exists_in(array : [A], equal : (A, A) -> Bool, value : A) : Bool { + for (element in array.vals()) { + if (equal(element, value)) { return true }; + }; + false; + }; +}; diff --git a/tests/CertifiedAssets.ActorTest.mo b/tests/CertifiedAssets.ActorTest.mo deleted file mode 100644 index 8f572fe..0000000 --- a/tests/CertifiedAssets.ActorTest.mo +++ /dev/null @@ -1,60 +0,0 @@ -// @testmode wasi -import Debug "mo:base/Debug"; - -import CertifiedAssets "../src"; - -actor { - stable let sstore = CertifiedAssets.init_stable_store(); - let certs = CertifiedAssets.CertifiedAssets(?sstore); - - public func test_certify() : async () { - await test_cerify_path("/symbols/ç˙∆å¨∆´ˆ˚ߨçß.pdf"); - }; - - public query func test_exists() : async () { - test_get_certificate_path("/symbols/ç˙∆å¨∆´ˆ˚ߨçß.pdf"); - }; - - public func test_cerify_path(path : Text) : async () { - let endpoint = CertifiedAssets.Endpoint( - path, - ?"Hello, World!", - ).status( - 200 - ).no_request_certification(); - Debug.print("about to certify"); - - certs.certify(endpoint); - Debug.print("endpoint certified"); - - }; - - func test_get_certificate_path(path : Text) : () { - let req : CertifiedAssets.HttpRequest = { - method = "GET"; - url = path; - headers = []; - body = ""; - certificate_version = ?2; - }; - - let res : CertifiedAssets.HttpResponse = { - status_code = 200; - headers = []; - body = "Hello, World!"; - streaming_strategy = null; - upgrade = null; - }; - - switch (certs.get_certificate(req, res, null)) { - case (#ok(certificate_headers)) Debug.print("certificate headers: " # debug_show certificate_headers); - case (#err(error)) Debug.trap("error: " # error); - }; - - }; - - func test_delete_path(path : Text) : async () { - certs.remove_all(path); - }; - -}; diff --git a/tests/CertifiedAssets.CanisterTests.mo b/tests/CertifiedAssets.CanisterTests.mo new file mode 100644 index 0000000..8be1bc4 --- /dev/null +++ b/tests/CertifiedAssets.CanisterTests.mo @@ -0,0 +1,598 @@ +// @testmode wasi +import Array "mo:base/Array"; +import Debug "mo:base/Debug"; +import Option "mo:base/Option"; +import Iter "mo:base/Iter"; +import Text "mo:base/Text"; +import Result "mo:base/Result"; +import Buffer "mo:base/Buffer"; +import CertifiedData "mo:base/CertifiedData"; +import Blob "mo:base/Blob"; +import Nat16 "mo:base/Nat16"; + +import SHA256 "mo:sha2/Sha256"; +import Itertools "mo:itertools/Iter"; +import Serde "mo:serde"; +import Base64 "mo:encoding/Base64"; +import RepIndyHash "mo:rep-indy-hash"; + +import CertifiedAssets "../src"; +import {MerkleTreeOps; IC_CERT_METHOD; IC_CERT_QUERY; IC_CERTIFICATE_EXPRESSION; IC_CERT_STATUS} "../src/Stable"; +import CanisterTests "CanisterTests"; + +actor { + + let suite = CanisterTests.Suite(); + + // for running tests + public query func run_query_test(test_name: Text) : async CanisterTests.TestResult { suite.run_query(test_name).0; }; + + public func run_test(test_name: Text) : async CanisterTests.TestResult { (await suite.run(test_name)).0; }; + + public func get_test_details() : async [CanisterTests.TestDetails] { suite.get_test_details().0; }; + + public func get_test_result(test_name: Text) : async CanisterTests.TestResult { suite.get_test_result(test_name).0; }; + + public func get_finished_test_results() : async [CanisterTests.TestResult] { suite.get_finished_test_results().0 }; + + + type Result = Result.Result; + + stable let sstore = CertifiedAssets.init_stable_store(); + let certs = CertifiedAssets.CertifiedAssets(sstore); + + func strip_start(t: Text, prefix: Text) : Text = Option.get( + Text.stripStart(t, #text(prefix)), + t, + ); + + type CertificateDetails = { + certificate: Text; + tree: Text; + version: Text; + expr_path: Text; + }; + + func split_certificate(ic_certificate: Text) : CertificateDetails { + let split_certificate = Iter.toArray(Text.split(ic_certificate, #text(", "))); + + let certificate = strip_start(split_certificate[0], ("certificate=")); + let tree = strip_start(split_certificate[1], ("tree=")); + let version = strip_start(split_certificate[2], ("version=")); + let expr_path = strip_start(split_certificate[3], ("expr_path=")); + + return { + certificate = certificate; + tree = tree; + version = version; + expr_path = expr_path; + }; + }; + + func to_cbor(paths: [Text]) : Blob { + + let candid_record_expr_path = #Array( + Array.map(paths, func(t : Text) : Serde.Candid = #Text(t)) + ); + + let cbor_res = Serde.CBOR.fromCandid(candid_record_expr_path, Serde.defaultOptions); + let encoded_expr_path = switch (cbor_res) { + case (#ok(encoded_expr_path)) encoded_expr_path; + case (#err(errMsg)) Debug.trap("Internal Error: Report bug in NatLabs/certified-assets repo.\n\t" # errMsg); + }; + + encoded_expr_path; + + }; + + + + func to_base64(blob: Blob) : Text { + let res = Base64.StdEncoding.encode(Blob.toArray(blob)); + let ?utf8 = Text.decodeUtf8(Blob.fromArray(res)) else Debug.trap("base64 encoding failed"); + utf8; + }; + + func encode_expr_path(paths: [Text]) : Text { + let cbor = to_cbor(paths); + let base64 = to_base64(cbor); + ":" # base64 # ":"; + }; + + func get_witness(full_expr_path: [Blob]) : Text { + let witness = MerkleTreeOps.reveal(sstore, full_expr_path); + let encoded_witness = MerkleTreeOps.encodeWitness(witness); + + ":" # to_base64(encoded_witness) # ":"; + }; + + func get_request_hash(method: Text, certified_request_headers: [(Text, Text)], certified_query_params: [(Text, Text)]): Blob { + let buffer = Buffer.Buffer<(Text, RepIndyHash.Value)>(8); + + for ((name, value) in certified_request_headers.vals()) { + if (value.size() != 0) { + buffer.add((Text.toLowercase(name), #Text(value))); + }; + }; + + buffer.add((IC_CERT_METHOD, #Text(method))); + + let query_params = Array.tabulate( + certified_query_params.size(), + func(i : Nat) : Text { + let (name, value) = certified_query_params[i]; + (name # "=" # value); + }, + ); + + let concatenated_query_params = Text.join("&", query_params.vals()); + // Debug.print("concatenated_query_params: " # debug_show concatenated_query_params); + let hashed_query_params = SHA256.fromBlob(#sha256, Text.encodeUtf8(concatenated_query_params)); + buffer.add((IC_CERT_QUERY, #Blob(hashed_query_params))); + + let rep_val = #Map(Buffer.toArray(buffer)); + let request_header_hash = RepIndyHash.hash_val(rep_val); + + let request_body_hash : Blob = SHA256.fromBlob(#sha256, ""); // the body is empty because this is expected to be either a GET, HEAD or OPTIONS requests + // Debug.print("request rep val: " # debug_show rep_val); + SHA256.fromArray(#sha256, Array.append(request_header_hash, Blob.toArray(request_body_hash))); + }; + + func get_respoonse_hash(status: Nat16, certified_response_headers: [(Text, Text)], body_hash: Blob, ic_certificate_expression: Text): Blob { + let buffer = Buffer.Buffer<(Text, RepIndyHash.Value)>(8); + + for ((name, value) in certified_response_headers.vals()) { + if (value.size() != 0 and Text.toLowercase(name) != "ic-certificate") { + buffer.add((Text.toLowercase(name), #Text(value))); + }; + }; + + buffer.add(IC_CERTIFICATE_EXPRESSION, #Text(ic_certificate_expression)); + + buffer.add((IC_CERT_STATUS, #Nat(Nat16.toNat(status)))); + + let rep_val = #Map(Buffer.toArray(buffer)); + let response_headers_hash = RepIndyHash.hash_val(rep_val); + + let headers_and_body_hash = Array.append( + response_headers_hash, + Blob.toArray(body_hash), + ); + + SHA256.fromArray(#sha256, headers_and_body_hash); + }; + + + + suite.add( + "verify certificate headers - upload hello file", + func({ ts_assert; ts_print; ts_assert_or_print } : CanisterTests.TestTools) : async () { + + let hello_endpoint = CertifiedAssets.Endpoint( + "/hello", + ?Text.encodeUtf8("👋 Hello, World!"), + ).status( + 200 + ).response_header( + "Content-Type", + "text/plain", + ).no_request_certification(); + + certs.certify(hello_endpoint); + + }, + ); + + suite.add_query( + "verify certificate headers - headers should match expected values", + func({ ts_assert; ts_print; ts_assert_or_print } : CanisterTests.TestTools): (){ + + let req : CertifiedAssets.HttpRequest = { + method = "GET"; + url = "/hello"; + headers = []; + body = ""; + certificate_version = ?2; + }; + + let res : CertifiedAssets.HttpResponse = { + status_code = 200; + headers = [("Content-Type", "text/plain")]; + body = "👋 Hello, World!"; + streaming_strategy = null; + upgrade = null; + }; + + let #ok(certificate_headers) = certs.get_certificate(req, res, null) else return assert false; + + let (ic_certificate, ic_certificate_expression) = if (certificate_headers[0].0 == "ic-certificate") { + (certificate_headers[0].1, certificate_headers[1].1) + } else { + (certificate_headers[1].1, certificate_headers[0].1) + }; + + + let {certificate; tree; version; expr_path} = split_certificate(ic_certificate); + + ts_assert_or_print( + certificate == ":" # to_base64(Option.get(CertifiedData.getCertificate(), "": Blob)) # ":", + "certificate does not match expected value" + ); + + ts_assert_or_print( + version == "2", + "version does not match expected value" + ); + + ts_assert_or_print( + expr_path == encode_expr_path(["http_expr", "hello", "<$>"]), + "expr_path does not match expected value" + ); + + ts_assert_or_print( + ic_certificate_expression == "default_certification ( ValidationArgs { certification: Certification { no_request_certification: Empty { }, response_certification: ResponseCertification { certified_response_headers: ResponseHeaderList { headers: [\"Content-Type\"] } } } } )", + "ic_certificate_expression does not match expected value" + ); + + let ic_certificate_expression_hash = SHA256.fromBlob(#sha256, Text.encodeUtf8(ic_certificate_expression)); + let request_hash : Blob = ""; // no_request_certification + let response_hash = get_respoonse_hash(200, [("Content-Type", "text/plain")], SHA256.fromBlob(#sha256, "👋 Hello, World!"), ic_certificate_expression); + let blob_http_expr_path = Array.map(["http_expr", "hello", "<$>"], func(t : Text) : Blob { Text.encodeUtf8(t) }); + let full_expr_path = Array.append(blob_http_expr_path, [ic_certificate_expression_hash, request_hash, response_hash]); + let tree_witness = get_witness(full_expr_path); + + ts_assert_or_print( + tree == tree_witness, + "tree does not match expected value " + ); + + } + ); + + suite.add( + "Certify '/hello_world.txt' asset", + func({ ts_assert; ts_print; ts_assert_or_print } : CanisterTests.TestTools) : async () { + let endpoint = CertifiedAssets.Endpoint( + "/hello_world.txt", + ?Text.encodeUtf8("Hello, World!"), + ).status( + 200 + ).response_header( + "Content-Type", + "text/plain", + ).no_request_certification(); + + certs.certify(endpoint); + + ts_assert( + Itertools.any( + certs.endpoints(), + func(endpoint_record : CertifiedAssets.EndpointRecord) : Bool { + endpoint_record.url == "/hello_world.txt" and endpoint_record.status == 200 and endpoint_record.response_headers == [("Content-Type", "text/plain")] and endpoint_record.hash == SHA256.fromBlob(#sha256, "Hello, World!") and endpoint_record.no_request_certification + }, + ) + ); + }, + ); + + suite.add_query( + "Retrieve '/hello_world.txt' asset", + func({ ts_assert; ts_print; ts_assert_or_print } : CanisterTests.TestTools) { + let req : CertifiedAssets.HttpRequest = { + method = "GET"; + url = "/hello_world.txt"; + headers = []; + body = ""; + certificate_version = ?2; + }; + + let res : CertifiedAssets.HttpResponse = { + status_code = 200; + headers = [("Content-Type", "text/plain")]; + body = "Hello, World!"; + streaming_strategy = null; + upgrade = null; + }; + + let response_with_additional_headers = { + res with headers = [("Content-Type", "text/plain"), ("X-Test", "Test")]; + }; + + let should_succeed = [ + res, + response_with_additional_headers, + ]; + + for (res in should_succeed.vals()) switch (certs.get_certificate(req, res, null)) { + case (#ok(_)) {}; + case (#err(_)) ts_assert(false); + }; + + let response_with_no_header = { + res with headers = []; + }; + + let response_with_incorrect_status_code = { + res with status_code : Nat16 = 404; + }; + + let response_with_incorrect_body = { + res with body : Blob = "Goodbye, World!"; + }; + + let response_with_incorrect_content_type = { + res with headers = [("Content-Type", "text/html")]; + }; + + let should_fail = [ + response_with_no_header, + response_with_incorrect_status_code, + response_with_incorrect_body, + response_with_incorrect_content_type, + ]; + + for (res in should_fail.vals()) switch (certs.get_certificate(req, res, null)) { + case (#ok(_)) ts_assert(false); + case (#err(_)) {}; + }; + }, + ); + + suite.add( + "Certify '/hello_world.txt' asset with different headers and status code", + func({ ts_assert; ts_print; ts_assert_or_print } : CanisterTests.TestTools) : async () { + + certs.certify( + CertifiedAssets.Endpoint( + "/hello_world.txt", + ?Text.encodeUtf8("Hello, World!"), + ).status( + 200 + ).response_header( + "Content-Type", + "gzip", + ).response_header( + "X-Test", + "Test", + ).no_request_certification() + ); + + ts_assert( + Itertools.any( + certs.endpoints(), + func(endpoint_record : CertifiedAssets.EndpointRecord) : Bool { + endpoint_record.url == "/hello_world.txt" and endpoint_record.status == 200 and endpoint_record.response_headers == [("Content-Type", "gzip"), ("X-Test", "Test")] and endpoint_record.hash == SHA256.fromBlob(#sha256, "Hello, World!") and endpoint_record.no_request_certification + }, + ) + ); + + certs.certify( + CertifiedAssets.Endpoint( + "/hello_world.txt", + ?Text.encodeUtf8("Hello, World!"), + ).status( + 304 + ).response_header( + "Content-Type", + "text/plain", + ).response_header( + "X-Test", + "Test", + ).no_request_certification() + ); + + ts_assert( + Itertools.any( + certs.endpoints(), + func(endpoint_record : CertifiedAssets.EndpointRecord) : Bool { + endpoint_record.url == "/hello_world.txt" and endpoint_record.status == 304 and endpoint_record.response_headers == [("Content-Type", "text/plain"), ("X-Test", "Test")] and endpoint_record.hash == SHA256.fromBlob(#sha256, "Hello, World!") and endpoint_record.no_request_certification + }, + ) + ); + + }, + ); + + suite.add_query( + "Retrieve 'hello_world.txt' asset with different headers and status code", + func({ ts_assert; ts_print; ts_assert_or_print } : CanisterTests.TestTools) { + let req : CertifiedAssets.HttpRequest = { + method = "GET"; + url = "/hello_world.txt"; + headers = []; + body = ""; + certificate_version = ?2; + }; + + let res_200 : CertifiedAssets.HttpResponse = { + status_code = 200; + headers = [("Content-Type", "gzip"), ("X-Test", "Test")]; + body = "Hello, World!"; + streaming_strategy = null; + upgrade = null; + }; + + let res_304 : CertifiedAssets.HttpResponse = { + status_code = 304; + headers = [("Content-Type", "text/plain"), ("X-Test", "Test")]; + body = "Hello, World!"; + streaming_strategy = null; + upgrade = null; + }; + + let should_succeed = [ + res_200, + res_304, + ]; + + for (res in should_succeed.vals()) switch (certs.get_certificate(req, res, null)) { + case (#ok(_)) {}; + case (#err(_)) ts_assert(false); + }; + + let should_fail = [ + { + res_200 with status_code : Nat16 = 404; + }, + { + res_200 with body : Blob = "Goodbye, World!"; + }, + { + res_200 with headers = [("Content-Type", "text/html")]; + }, + { + res_304 with status_code : Nat16 = 404; + }, + { + res_304 with body : Blob = "Goodbye, World!"; + }, + { + res_304 with headers = [("Content-Type", "text/html")]; + }, + ]; + + for (res in should_fail.vals()) switch (certs.get_certificate(req, res, null)) { + case (#ok(_)) ts_assert(false); + case (#err(_)) {}; + }; + }, + ); + + suite.add( + "Certify '/assets/delete-me.txt'", + func({ ts_assert; ts_print; ts_assert_or_print } : CanisterTests.TestTools) : async () { + certs.certify( + CertifiedAssets.Endpoint( + "/assets/delete-me.txt", + ?Text.encodeUtf8("Delete me!"), + ).status( + 200 + ).response_header( + "Content-Type", + "text/plain", + ).no_request_certification() + ); + + ts_assert( + Itertools.any( + certs.endpoints(), + func(endpoint_record : CertifiedAssets.EndpointRecord) : Bool { + endpoint_record.url == "/assets/delete-me.txt" and endpoint_record.status == 200 and endpoint_record.response_headers == [("Content-Type", "text/plain")] and endpoint_record.hash == SHA256.fromBlob(#sha256, "Delete me!") and endpoint_record.no_request_certification + }, + ) + ); + }, + ); + + suite.add( + "delete certified endpoint", + func({ ts_assert; ts_print; ts_assert_or_print } : CanisterTests.TestTools) : async () { + certs.remove( + CertifiedAssets.Endpoint( + "/assets/delete-me.txt", + ?Text.encodeUtf8("Delete me!"), + ).status( + 200 + ).response_header( + "Content-Type", + "text/plain", + ).no_request_certification() + ); + + ts_assert( + not Itertools.all( + certs.endpoints(), + func(endpoint_record : CertifiedAssets.EndpointRecord) : Bool { + endpoint_record.url == "/assets/delete-me.txt" and endpoint_record.status == 200 and endpoint_record.response_headers == [("Content-Type", "text/plain")] and endpoint_record.hash == SHA256.fromBlob(#sha256, "Delete me!") and endpoint_record.no_request_certification + }, + ) + ); + }, + ); + + suite.add( + "certify fallback path", + func({ ts_assert; ts_print; ts_assert_or_print } : CanisterTests.TestTools) : async () { + certs.certify( + CertifiedAssets.Endpoint( + "/fallback/index.html", + ?Text.encodeUtf8("Fallback!"), + ).status( + 200 + ).response_header( + "Content-Type", + "text/plain", + ).no_request_certification() + ); + + }, + ); + + suite.add_query( + "get fallback", + func({ ts_assert; ts_print; ts_assert_or_print } : CanisterTests.TestTools): () { + let req : CertifiedAssets.HttpRequest = { + method = "GET"; + url = "/fallback/missing_file.txt"; + headers = []; + body = ""; + certificate_version = ?2; + }; + + let fallback_res : CertifiedAssets.HttpResponse = { + status_code = 200; + headers = [("Content-Type", "text/plain")]; + body = "Fallback!"; + streaming_strategy = null; + upgrade = null; + }; + + let certificates = switch (certs.get_fallback_certificate(req, "/fallback/index.html", fallback_res, null)) { + case (#ok(certificates)) certificates; + case (#err(msg)) return ts_assert_or_print(false, "Failed to retrieve fallback certificate: " # msg); + }; + + let (ic_certificate, ic_certificate_expression) = if (certificates[0].0 == "ic-certificate") { + (certificates[0].1, certificates[1].1) + } else { + (certificates[1].1, certificates[0].1) + }; + + let {certificate; tree; version; expr_path} = split_certificate(ic_certificate); + + ts_assert_or_print( + certificate == ":" # to_base64(Option.get(CertifiedData.getCertificate(), "": Blob)) # ":", + "certificate does not match expected value" + ); + + ts_assert_or_print( + version == "2", + "version does not match expected value" + ); + + ts_assert_or_print( + expr_path == encode_expr_path(["http_expr", "fallback", "<*>"]), + "expr_path does not match expected value " + ); + + ts_assert_or_print( + ic_certificate_expression == "default_certification ( ValidationArgs { certification: Certification { no_request_certification: Empty { }, response_certification: ResponseCertification { certified_response_headers: ResponseHeaderList { headers: [\"Content-Type\"] } } } } )", + "ic_certificate_expression does not match expected value" + ); + + let ic_certificate_expression_hash = SHA256.fromBlob(#sha256, Text.encodeUtf8(ic_certificate_expression)); + let request_hash : Blob = ""; // no_request_certification + let response_hash = get_respoonse_hash(200, [("Content-Type", "text/plain")], SHA256.fromBlob(#sha256, "Fallback!"), ic_certificate_expression); + let blob_http_expr_path = Array.map(["http_expr", "fallback", "<*>"], func(t : Text) : Blob { Text.encodeUtf8(t) }); + let full_expr_path = Array.append(blob_http_expr_path, [ic_certificate_expression_hash, request_hash, response_hash]); + let tree_witness = get_witness(full_expr_path); + + ts_assert_or_print( + tree == tree_witness, + "tree does not match expected value " + ); + } + ); + + +}; diff --git a/z-scripts/canister-tests.mjs b/z-scripts/canister-tests.mjs new file mode 100644 index 0000000..7b5a890 --- /dev/null +++ b/z-scripts/canister-tests.mjs @@ -0,0 +1,70 @@ +#!/usr/bin/env zx +import { parse_json, print_test_result, entitle } from "./utils.mjs"; +import chalk from "chalk"; + +$.verbose = true; // set to true so that we can see the deployment logs and respond if ther are input prompts +const is_deploy = "deploy" in argv ? argv.deploy : true; + +if (is_deploy) { + entitle("Deploying canister-tests canister..."); + await $`dfx deploy canister-tests`; +} + +await $`dfx ledger fabricate-cycles --canister canister-tests`; + +entitle("Adding canister-tests canister as a controller..."); +// Add the canister-tests canister as a controller of itself +await $`dfx canister update-settings canister-tests --add-controller $(dfx canister id canister-tests)`; + +$.verbose = false; +entitle("Retrieving test details..."); +const test_details_raw_test = + await $`dfx canister call canister-tests get_test_details | idl2json -c`; + +const test_details = parse_json(test_details_raw_test.stdout); + +let passed = 0; +let failed = 0; + +const test_prefix = argv?.test || ""; + +entitle("\nRunning tests...\n"); + +for (let test of test_details) { + if (!is_deploy && !test.name.startsWith(test_prefix)) { + continue; + } + + let args = `("${test.name}")`; + + try { + let raw_test = test.is_query + ? await $`dfx canister call canister-tests run_query_test ${args} | idl2json -c` + : await $`dfx canister call canister-tests run_test ${args} | idl2json -c`; + + test = { ...test, ...parse_json(raw_test.stdout) }; + } catch (e) { + let raw_test = + await $`dfx canister call canister-tests get_test_result ${args} | idl2json -c`; + + test = { ...test, ...parse_json(raw_test.stdout) }; + test.result = false; + test.print_log.push(`${chalk.red("[Canister Error] ")} ${e.stderr}`); + } + + print_test_result(test); + + if (test.result) { + passed += 1; + } else { + failed += 1; + } +} + +console.log("\n" + `Passed: ${passed}, Failed: ${failed}`); + +if (failed === 0) { + console.log(chalk.bold.green("All tests passed!")); +} else { + process.exit(1); +} diff --git a/z-scripts/utils.mjs b/z-scripts/utils.mjs new file mode 100644 index 0000000..47538e7 --- /dev/null +++ b/z-scripts/utils.mjs @@ -0,0 +1,47 @@ +import chalk from "chalk"; + +export function extract_text_from_brackets(str, first, last) { + const start = str.indexOf(first) + first.length; + const end = str.lastIndexOf(last) + 1 - last.length; + + if (start === -1 || end === -1) { + return str; + } + + return str.slice(start, end); +} + +export function trim_end(str, char) { + return str.replace(new RegExp(char + "*$"), ""); +} + +export function parse_json(str) { + let json = {}; + + try { + json = JSON.parse(str); + } catch (e) { + console.log({ str }); + throw new Error("Failed to parse JSON " + e); + } + + return json; +} + +export const entitle = (str) => + console.log("\n" + chalk.bold.underline(str) + "\n"); + +export const print_test_result = (test) => { + let chalk_grey = chalk.rgb(210, 220, 220); + + const func_type = chalk_grey(test.is_query ? "[query] " : "[update]"); + const passed_or_failed_test = test.result + ? ` ✅ ${chalk.green(test.name)}` + : ` ${chalk.bold.red("❌")} ${chalk.bold.red(test.name)}`; + + console.log(func_type + " " + passed_or_failed_test); + + for (const print_statement of test.print_log) { + console.log("\t" + chalk_grey("➥ " + print_statement)); + } +};