/// An HTTP/1.1 client for fetching what a URL points to (ADR 0079): one /// request (a GET, or any method with a body: `request`, ADR 0079 /// Revision 1), over TCP or TLS, guarded against server-side request /// forgery. /// /// use tls_config <- result.try (tls.config (tls.defaults ())) // once, and share it /// let opts = Options(..fetch.options tls_config, max_body: 2097152) /// use page <- result.try (url.parse "https://example.com/") /// match fetch.get page [("User-Agent", "toki")] opts /// { Ok(Response(200, headers, body, _)) -> ... /// , Ok(Response(status, ..)) -> ... // 3xx too: redirects are not followed /// , Error(e) -> ... /// } /// /// What `get` does: /// - only `http` and `https`; a header name that is not a token, or a /// value with a CR, LF or NUL, is refused; /// - resolves the host and checks **every** address it names against the /// policy (`PublicOnly` refuses any private one, see `fetch/address`), /// then connects to those addresses in order, never resolving again; /// - over `https`, verifies the certificate against the host name (also /// sent as SNI) with the caller's TLS configuration; /// - sends `Host`, `Connection: close` and `Accept-Encoding: identity` /// (there is no decompression), then the caller's headers (theirs for /// `Host`, `Connection`, `Accept-Encoding`, `Content-Length`, /// `Transfer-Encoding`, `TE` and `Upgrade` are dropped); /// - skips `1xx` responses; reads the body by `Content-Length`, chunked, /// or until the server closes (a TLS close without `close_notify` ends /// such a body too); keeps at most `max_body` bytes of it and says when /// it cut it (`truncated`); /// - never follows redirects: a 3xx is returned as it is, and the caller /// resolves `Location` with `url.resolve` and calls `get` again, which /// checks the new host; /// - runs the whole exchange in an actor of its own, and kills it at the /// deadline (`TimedOut`), wherever it waits: resolving, connecting, the /// handshake or a read. /// /// `request_streaming` does the same, but returns once the head has /// arrived, with the body to read as it comes (`next`, `cancel`; ADR 0079 /// Revision 2); `fetch/sse` parses server-sent events from it. /// /// **Needs engineering review** (ADR 0079): this is an SSRF surface. import address import response. {type Dechunker, type Framing, type Head, Chunked, Length, NoBody, UntilClose} import std/bytes import std/ffi.{type Resource} import std/list import std/net.{type TcpStream} import std/process.{type Down, type Pid, type Subject, type proc, Down} import std/result import std/string import std/time.{type clock} import std/tls.{type TlsConfig, type TlsError, type TlsStream, Truncated} import url.{type Url, Url} /// Which addresses a fetch may connect to. @derive Eq Show Debug pub type Policy { /// Only public ones: every address of the host must be public. PublicOnly /// Any: for tests against `127.0.0.1`, and for a deployment that knows /// its network. , AnyAddress } /// How a fetch runs: the TLS configuration for `https`, the address /// policy, the most body bytes kept, the largest response head, and the /// deadline (a `time.monotonic` instant in milliseconds; see /// [`deadline_in`]). pub type Options { Options( tls: Resource , policy: Policy , max_body: Int , max_head: Int , deadline: Int ) } /// `PublicOnly`, bodies kept to 1 MiB, heads up to 64 KiB, and a deadline /// 10 seconds from now. pub fn options :: config: Resource -> Options / { Options(config, PublicOnly, 1048576, 65536, deadline_in 10000) } /// The deadline `ms` milliseconds from now. One deadline can cover /// several fetches (a redirect chain). pub fn deadline_in :: ms: Int -> Int / { now_ms () + ms } /// A response: its status, its headers in order, its body (at most /// `max_body` bytes), and whether the body was cut there. @derive Eq Debug pub type Response { Response( status: Int , headers: List<(String, String)> , body: Bytes , truncated: Bool ) } impl Show { fn show :: resp { debug resp } } /// The value of the response's header `name` (any case), if present. pub fn header :: resp: Response name: String -> Option { response.header resp.headers name } /// Why a fetch failed. @derive Eq Show Debug pub type Error { /// Not an `http` or `https` URL with a valid host. BadUrl(String) /// A request header that cannot be sent; its name. , BadHeader(String) /// The host names an address the policy refuses. , Refused(host: String, ip: String) /// The host names no address. , NoAddress(String) /// No address of the host accepted a connection, or the connection /// failed while sending or reading. , Connect(String) /// TLS failed (the certificate, the handshake). , Tls(TlsError) /// The response is not HTTP/1.1 as this client reads it. , Malformed(String) /// The deadline passed. , TimedOut } /// GET `target` with `headers` (see the module's documentation). pub fn get :: target: Url headers: List<(String, String)> opts: Options -> Result / { request "GET" target headers (bytes.empty ()) opts } /// `method` (`POST`, `PUT`, …) to `target` with `headers` and `body`, /// under the same guards as `get`. A body is sent with its /// `Content-Length` (one of the caller's is dropped, as the other framing /// headers are); the caller names its type (`Content-Type`). /// /// fetch.request "POST" api [("Content-Type", "application/json")] (bytes.from_string json) opts pub fn request :: method: String target: Url headers: List<(String, String)> body: Bytes opts: Options -> Result / { use checked <- result.try (valid_url target) use () <- result.try (valid_headers headers) use () <- result.try (valid_method method) let left = opts.deadline - now_ms () if left <= 0 { Error(TimedOut) } else { let reply = process.subject () let (pid, down) = process.spawn_monitor fn { process.send reply (exchange method checked headers body opts) } let selector = process.new_selector () |> process.selecting reply Got |> process.selecting down Stopped match process.select selector left { Some(Got(answer)) -> { process.demonitor_flush pid down answer } , Some(Stopped(Down(_, reason))) -> Error(Malformed("the fetch stopped: {show reason}")) , None -> { process.kill pid process.demonitor_flush pid down // A reply sent just before the kill would stay in the mailbox. let _ = process.receive reply 0 Error(TimedOut) } } } } type Outcome { Got(Result), Stopped(Down) } fn now_ms :: -> Int / { time.in_milliseconds (time.monotonic ()) } // ---- streamed responses --------------------------------------------------------- /// A response whose body is read as it arrives (ADR 0079, Revision 2): /// its status, its headers, and the body to read with [`next`]. pub type Streamed { Streamed(status: Int, headers: List<(String, String)>, body: BodyStream) } /// A body still arriving. Only the actor that made the request can read /// it; one that stops reading early calls [`cancel`]. pub opaque type BodyStream { BodyStream( pid: Pid , down: Subject , events: Subject , deadline: Int ) } /// A piece of a streamed body: some bytes, or the end (and whether the /// body was cut at `max_body`). @derive Eq Debug pub type Piece { Data(Bytes), End(truncated: Bool) } impl Show { fn show :: piece { debug piece } } /// What the exchange's actor sends a streaming caller, in order: the head, /// body bytes, then the end or a failure. type StreamEvent { Opened(Int, List<(String, String)>) , Received(Bytes) , Finished(Bool) , Failed(Error) } type Arrival { Arrived(StreamEvent), Ended(Down) } /// [`request`], but returning once the response head has arrived, with /// the body to read as it comes ([`next`]): for server-sent events and /// other long answers. Everything `request` checks and does applies, and /// `opts.deadline` covers the whole exchange, the body included: give a /// long answer a long deadline. At most `max_body` bytes are delivered, /// so a reader that falls behind holds at most that much. /// /// match fetch.request_streaming "POST" api headers body opts /// { Ok(Streamed(200, _, stream)) -> read stream // fetch.next until End /// , Ok(Streamed(status, _, stream)) -> fetch.cancel stream /// , Error(e) -> ... /// } pub fn request_streaming :: method: String target: Url headers: List<(String, String)> body: Bytes opts: Options -> Result / { use checked <- result.try (valid_url target) use () <- result.try (valid_headers headers) use () <- result.try (valid_method method) if opts.deadline - now_ms () <= 0 { Error(TimedOut) } else { let events = process.subject () let (pid, down) = process.spawn_monitor fn { stream_exchange method checked headers body opts events } let stream = BodyStream(pid, down, events, opts.deadline) match wait stream { Ok(Opened(status, head)) -> Ok(Streamed(status, head, stream)) , Ok(Failed(e)) -> { process.demonitor_flush pid down Error(e) } , Ok(_) -> { cancel stream Error(Malformed("the body came before the head")) } , Error(e) -> Error(e) } } } /// The next piece of `stream`: bytes as they arrived (a chunked body /// already decoded), then `End`. After `End` or an `Error` the stream is /// over: its connection is closed. `TimedOut` at the request's deadline. pub fn next :: stream: BodyStream -> Result / { match wait stream { Ok(Received(data)) -> Ok(Data(data)) , Ok(Finished(truncated)) -> { process.demonitor_flush stream.pid stream.down Ok(End(truncated)) } , Ok(Failed(e)) -> { process.demonitor_flush stream.pid stream.down Error(e) } , Ok(Opened(_, _)) -> { cancel stream Error(Malformed("a second response head")) } , Error(e) -> Error(e) } } /// Stops reading `stream`: closes its connection (the server sees the /// client leave) and drops what arrived unread. Harmless after the end. pub fn cancel :: stream: BodyStream -> () / { process.kill stream.pid process.demonitor_flush stream.pid stream.down drain stream.events } /// The next event, until the deadline. On a timeout or a crash, the /// exchange is stopped and its leftovers dropped. fn wait :: stream: BodyStream -> Result / { let left = stream.deadline - now_ms () let selector = process.new_selector () |> process.selecting stream.events Arrived |> process.selecting stream.down Ended match process.select selector (if left < 0 { 0 } else { left }) { Some(Arrived(event)) -> Ok(event) , Some(Ended(Down(_, reason))) -> { drain stream.events Error(Malformed("the fetch stopped: {show reason}")) } , None -> { cancel stream Error(TimedOut) } } } fn drain :: events: Subject -> () / { match process.receive events 0 { Some(_) -> drain events, None -> () } } // ---- checks -------------------------------------------------------------------- /// The URL parsed again from its text (a `Url` can be built by hand), /// and only `http` or `https`. fn valid_url :: target: Url -> Result { use parsed <- result.try (url.parse (url.to_string target) |> result.map_error BadUrl) if parsed.scheme == "http" || parsed.scheme == "https" { Ok(parsed) } else { Error(BadUrl("only http and https can be fetched")) } } /// A method is a token (RFC 9110), as a header name is. fn valid_method :: method: String -> Result<(), Error> { if is_token method { Ok(()) } else { Error(BadHeader(method)) } } fn valid_headers :: headers: List<(String, String)> -> Result<(), Error> { match list.find headers fn h { !is_token h.0 || has_line_break h.1 } { Some((name, _)) -> Error(BadHeader(name)), None -> Ok(()) } } /// RFC 9110's `token`: digits, letters and ``!#$%&'*+-.^_`|~``. fn is_token :: name: String -> Bool { let symbols = bytes.to_list (bytes.from_string "!#$%&'*+-.^_`|~") name != "" && list.all (bytes.to_list (bytes.from_string name)) fn b { b >= 48 && b <= 57 || b >= 65 && b <= 90 || b >= 97 && b <= 122 || list.any symbols fn s { s == b } } } fn has_line_break :: value: String -> Bool { list.any (bytes.to_list (bytes.from_string value)) fn b { b == 13 || b == 10 || b == 0 } } // ---- the exchange (in its own actor) ------------------------------------------- /// A connection, plain or over TLS. type Conn { Plain(Resource), Secure(Resource) } fn exchange :: method: String target: Url headers: List<(String, String)> body: Bytes opts: Options -> Result / { use conn <- result.try (connected target opts) let answer = round_trip conn method target headers body opts close conn answer } /// A connection to `target`'s host, under the policy: resolved once, /// every address checked, then the first that accepts, secured for /// `https`. fn connected :: target: Url opts: Options -> Result / { let host = host_name target.host let port = match url.port_or_default target { Some(p) -> p, None -> 80 } use ips <- result.try (net.resolve host |> result.map_error fn _ { NoAddress(host) }) use ips <- result.try (allowed opts.policy host ips) use tcp <- result.try (connect_any ips port "") secured target.scheme tcp host opts.tls } /// The host without an IPv6 address's brackets: what is resolved, and /// the TLS server name. fn host_name :: host: String -> String { if string.starts_with host "[" { string.slice host 1 (string.length host - 1) } else { host } } /// Every address, if the policy allows them all. fn allowed :: policy: Policy host: String ips: List -> Result, Error> { match (policy, ips) { (_, []) -> Error(NoAddress(host)) , (AnyAddress, _) -> Ok(ips) , (PublicOnly, _) -> match list.find ips fn ip { !address.is_public ip } { Some(ip) -> Error(Refused(host, ip)), None -> Ok(ips) } } } /// A connection to the first of `ips` that accepts. fn connect_any :: ips: List port: Int last: String -> Result, Error> / { match ips { [] -> Error(Connect(last)) , [ip, ..rest] -> match net.connect ip port { Ok(conn) -> Ok(conn), Error(e) -> connect_any rest port e } } } fn secured :: scheme: String tcp: Resource host: String config: Resource -> Result / { if scheme == "https" { match tls.connect tcp host config { Ok(stream) -> Ok(Secure(stream)) , Error(e) -> { net.close tcp Error(Tls(e)) } } } else { Ok(Plain(tcp)) } } fn round_trip :: conn: Conn method: String target: Url headers: List<(String, String)> body: Bytes opts: Options -> Result / { use (head, framing, rest) <- result.try (send_and_read_head conn method target headers body opts) use (body, truncated) <- result.try (read_body conn framing rest opts.max_body) Ok(Response(head.status, head.headers, body, truncated)) } /// Sends the request, then reads the final head: it, the body's framing, /// and the body bytes read with it. fn send_and_read_head :: conn: Conn method: String target: Url headers: List<(String, String)> body: Bytes opts: Options -> Result<(Head, Framing, Bytes), Error> / { use () <- result.try (write conn (bytes.concat (request_head method target headers body) body)) use (head, rest) <- result.try (final_head conn opts.max_head (bytes.empty ())) use framing <- result.try (response.framing head |> result.map_error Malformed) Ok((head, framing, rest)) } fn request_head :: method: String target: Url headers: List<(String, String)> body: Bytes -> Bytes { let default = url.port_or_default Url(..target, port: None) let port = match target.port { Some(p) if Some(p) != default -> ":" <> show p, _ -> "" } let own = list.filter headers fn h { !is_framing h.0 } // A body says its length; so does a POST or PUT without one, which // some servers otherwise wait on. let length = if bytes.length body > 0 || method == "POST" || method == "PUT" { ["Content-Length: " <> show (bytes.length body)] } else { [] } let lines = [ method <> " " <> url.request_target target <> " HTTP/1.1" , "Host: " <> target.host <> port , "Connection: close" , "Accept-Encoding: identity" , ..list.append length (list.map own fn h { h.0 <> ": " <> h.1 }) ] bytes.from_string (string.join lines "\r\n" <> "\r\n\r\n") } fn is_framing :: name: String -> Bool { list.any [ "host" , "connection" , "accept-encoding" , "content-length" , "transfer-encoding" , "te" , "upgrade" ] fn n { string.eq_ignore_ascii_case n name } } // ---- reading ------------------------------------------------------------------- /// The first head that is not `1xx`, and the bytes read after it. fn final_head :: conn: Conn max_head: Int buffer: Bytes -> Result<(Head, Bytes), Error> / { match response.head buffer max_head { Error(e) -> Error(Malformed(e)) , Ok(Some(head)) -> { let rest = bytes.slice buffer head.size (bytes.length buffer) if head.status < 200 { final_head conn max_head rest } else { Ok((head, rest)) } } , Ok(None) -> more_head conn max_head buffer } } /// Reads until the head's blank line has arrived (only the new bytes /// and the 3 before them are searched), or until there is too much. fn more_head :: conn: Conn max_head: Int buffer: Bytes -> Result<(Head, Bytes), Error> / { use more <- result.try (read conn) if bytes.length more == 0 { Error(Malformed("the connection closed before the response head")) } else { let n = bytes.length buffer let from = if n > 3 { n - 3 } else { 0 } let buffer = bytes.concat buffer more if has_blank_line buffer from || bytes.length buffer > max_head { final_head conn max_head buffer } else { more_head conn max_head buffer } } } fn has_blank_line :: data: Bytes at: Int -> Bool { if at + 4 > bytes.length data { False } else if bytes.slice data at (at + 4) == bytes.from_string "\r\n\r\n" { True } else { has_blank_line data (at + 1) } } fn read_body :: conn: Conn framing: Framing rest: Bytes max: Int -> Result<(Bytes, Bool), Error> / { match framing { NoBody -> Ok((bytes.empty (), False)) , Length(n) -> if n > max { exactly conn rest max False |> result.map fn b { (b, True) } } else { exactly conn rest n False |> result.map fn b { (b, False) } } , UntilClose -> until_close conn (bytes.append (bytes.builder ()) rest) (bytes.length rest) max , Chunked -> match response.feed (response.dechunker max) rest { Ok(d) -> chunks conn d, Error(e) -> Error(Malformed(e)) } } } /// The first `n` bytes, starting with `have`; `closed` once a read found /// the end. fn exactly :: conn: Conn have: Bytes n: Int closed: Bool -> Result / { if bytes.length have >= n { Ok(bytes.slice have 0 n) } else if closed { Error(Malformed("the connection closed before the end of the body")) } else { use more <- result.try (read conn) exactly conn (bytes.concat have more) n (bytes.length more == 0) } } fn until_close :: conn: Conn body: bytes.Builder size: Int max: Int -> Result<(Bytes, Bool), Error> / { if size > max { Ok((bytes.slice (bytes.build body) 0 max, True)) } else { use more <- result.try (read conn) if bytes.length more == 0 { Ok((bytes.build body, False)) } else { until_close conn (bytes.append body more) (size + bytes.length more) max } } } fn chunks :: conn: Conn d: Dechunker -> Result<(Bytes, Bool), Error> / { if response.is_done d { Ok(response.body d) } else { use more <- result.try (read conn) if bytes.length more == 0 { Error(Malformed("the connection closed inside a chunked body")) } else { match response.feed d more { Ok(d) -> chunks conn d, Error(e) -> Error(Malformed(e)) } } } } // ---- the streamed exchange (in its own actor) ---------------------------------- fn stream_exchange :: method: String target: Url headers: List<(String, String)> body: Bytes opts: Options to: Subject -> () / { let outcome = match connected target opts { Ok(conn) -> { let streamed = stream_round_trip conn method target headers body opts to close conn streamed } , Error(e) -> Error(e) } match outcome { Ok(truncated) -> process.send to (Finished(truncated)) , Error(e) -> process.send to (Failed(e)) } } /// Sends the head once it is read, then the body as it arrives; whether /// the body was cut at `max_body`. fn stream_round_trip :: conn: Conn method: String target: Url headers: List<(String, String)> body: Bytes opts: Options to: Subject -> Result / { use (head, framing, rest) <- result.try (send_and_read_head conn method target headers body opts) process.send to (Opened(head.status, head.headers)) let max = opts.max_body match framing { NoBody -> Ok(False) , Length ( n ) -> if n > max { stream_exactly conn rest max True to } else { stream_exactly conn rest n False to } , UntilClose -> stream_until_close conn rest 0 max to , Chunked -> match response.feed (response.dechunker max) rest { Ok(d) -> stream_chunks conn d to, Error(e) -> Error(Malformed(e)) } } } fn emit :: to: Subject data: Bytes -> () / { if bytes.length data > 0 { process.send to (Received(data)) } } /// Delivers the next `left` bytes, starting with `have`; `cut` is the /// answer once they are all delivered. fn stream_exactly :: conn: Conn have: Bytes left: Int cut: Bool to: Subject -> Result / { let piece = bytes.slice have 0 left emit to piece let left = left - bytes.length piece if left == 0 { Ok(cut) } else { use more <- result.try (read conn) if bytes.length more == 0 { Error(Malformed("the connection closed before the end of the body")) } else { stream_exactly conn more left cut to } } } /// Delivers what arrives until the server closes, `size` bytes so far. fn stream_until_close :: conn: Conn have: Bytes size: Int max: Int to: Subject -> Result / { let room = max - size if bytes.length have > room { emit to (bytes.slice have 0 room) Ok(True) } else { emit to have use more <- result.try (read conn) if bytes.length more == 0 { Ok(False) } else { stream_until_close conn more (size + bytes.length have) max to } } } fn stream_chunks :: conn: Conn d: Dechunker to: Subject -> Result / { let (data, d) = response.take d emit to data if response.is_done d { Ok((response.body d).1) } else { use more <- result.try (read conn) if bytes.length more == 0 { Error(Malformed("the connection closed inside a chunked body")) } else { match response.feed d more { Ok(d) -> stream_chunks conn d to, Error(e) -> Error(Malformed(e)) } } } } // ---- the connection ------------------------------------------------------------ /// Up to 64 KiB; empty at the end. A TLS stream that ends without /// `close_notify` (`Truncated`) is an end too: the framing says whether /// the body was whole. fn read :: conn: Conn -> Result / { match conn { Plain(tcp) -> net.read tcp 65536 |> result.map_error Connect , Secure(stream) -> match tls.read stream 65536 { Ok(b) -> Ok(b) , Error(Truncated) -> Ok(bytes.empty ()) , Error(e) -> Error(Tls(e)) } } } fn write :: conn: Conn data: Bytes -> Result<(), Error> / { match conn { Plain(tcp) -> net.write tcp data |> result.map_error Connect , Secure(stream) -> tls.write stream data |> result.map_error Tls } } fn close :: conn: Conn -> () / { match conn { Plain(tcp) -> net.close tcp, Secure(stream) -> tls.close stream } }