tobogganSe connecter

packages

The Louise packages, each released on its own

Clone avec HTTPS, un jeton personnel en mot de passe, ou avec SSH, une clé ajoutée dans vos paramètres.

1070 lignes · 28687 octetsBrut · Historique
1/// An HTTP/1.1 client for fetching what a URL points to (ADR 0079): one
2/// request (a GET, or any method with a body: `request`, ADR 0079
3/// Revision 1), over TCP or TLS, guarded against server-side request
4/// forgery.
5///
6/// use tls_config <- result.try (tls.config (tls.defaults ())) // once, and share it
7/// let opts = Options(..fetch.options tls_config, max_body: 2097152)
8/// use page <- result.try (url.parse "https://example.com/")
9/// match fetch.get page [("User-Agent", "toki")] opts
10/// { Ok(Response(200, headers, body, _)) -> ...
11/// , Ok(Response(status, ..)) -> ... // 3xx too: redirects are not followed
12/// , Error(e) -> ...
13/// }
14///
15/// What `get` does:
16/// - only `http` and `https`; a header name that is not a token, or a
17/// value with a CR, LF or NUL, is refused;
18/// - resolves the host and checks **every** address it names against the
19/// policy (`PublicOnly` refuses any private one, see `fetch/address`),
20/// then connects to those addresses in order, never resolving again;
21/// - over `https`, verifies the certificate against the host name (also
22/// sent as SNI) with the caller's TLS configuration;
23/// - sends `Host`, `Connection: close` and `Accept-Encoding: identity`
24/// (there is no decompression), then the caller's headers (theirs for
25/// `Host`, `Connection`, `Accept-Encoding`, `Content-Length`,
26/// `Transfer-Encoding`, `TE` and `Upgrade` are dropped);
27/// - skips `1xx` responses; reads the body by `Content-Length`, chunked,
28/// or until the server closes (a TLS close without `close_notify` ends
29/// such a body too); keeps at most `max_body` bytes of it and says when
30/// it cut it (`truncated`);
31/// - never follows redirects: a 3xx is returned as it is, and the caller
32/// resolves `Location` with `url.resolve` and calls `get` again, which
33/// checks the new host;
34/// - runs the whole exchange in an actor of its own, and kills it at the
35/// deadline (`TimedOut`), wherever it waits: resolving, connecting, the
36/// handshake or a read.
37///
38/// `request_streaming` does the same, but returns once the head has
39/// arrived, with the body to read as it comes (`next`, `cancel`; ADR 0079
40/// Revision 2); `fetch/sse` parses server-sent events from it.
41///
42/// **Needs engineering review** (ADR 0079): this is an SSRF surface.
43
44import address
45import response.
46{type Dechunker, type Framing, type Head, Chunked, Length, NoBody, UntilClose}
47import std/bytes
48import std/ffi.{type Resource}
49import std/list
50import std/net.{type TcpStream}
51import std/process.{type Down, type Pid, type Subject, type proc, Down, Killed}
52import std/result
53import std/string
54import std/time.{type clock}
55import std/tls.{type TlsConfig, type TlsError, type TlsStream, Truncated}
56import url.{type Url, Url}
57
58/// Which addresses a fetch may connect to.
59@derive Eq Show Debug
60pub type Policy
61{ /// Only public ones: every address of the host must be public.
62 PublicOnly
63 /// Any: for tests against `127.0.0.1`, and for a deployment that knows
64 /// its network.
65, AnyAddress
66}
67
68/// How a fetch runs: the TLS configuration for `https`, the address
69/// policy, the most body bytes kept, the largest response head, and the
70/// deadline (a `time.monotonic` instant in milliseconds; see
71/// [`deadline_in`]).
72pub type Options
73{ Options(
74 tls: Resource<TlsConfig>
75 , policy: Policy
76 , max_body: Int
77 , max_head: Int
78 , deadline: Int
79 )
80}
81
82/// `PublicOnly`, bodies kept to 1 MiB, heads up to 64 KiB, and a deadline
83/// 10 seconds from now.
84pub fn options :: config: Resource<TlsConfig> -> Options / <clock>
85{
86 Options(config, PublicOnly, 1048576, 65536, deadline_in 10000)
87}
88
89/// The deadline `ms` milliseconds from now. One deadline can cover
90/// several fetches (a redirect chain).
91pub fn deadline_in :: ms: Int -> Int / <clock>
92{
93 now_ms () + ms
94}
95
96/// A response: its status, its headers in order, its body (at most
97/// `max_body` bytes), and whether the body was cut there.
98@derive Eq Debug
99pub type Response
100{ Response(
101 status: Int
102 , headers: List<(String, String)>
103 , body: Bytes
104 , truncated: Bool
105 )
106}
107
108impl Show<Response>
109{
110 fn show :: resp
111 {
112 debug resp
113 }
114}
115
116/// The value of the response's header `name` (any case), if present.
117pub fn header :: resp: Response name: String -> Option<String>
118{
119 response.header resp.headers name
120}
121
122/// Why a fetch failed.
123@derive Eq Show Debug
124pub type Error
125{ /// Not an `http` or `https` URL with a valid host.
126 BadUrl(String)
127 /// A request header that cannot be sent; its name.
128, BadHeader(String)
129 /// The host names an address the policy refuses.
130, Refused(host: String, ip: String)
131 /// The host names no address.
132, NoAddress(String)
133 /// No address of the host accepted a connection, or the connection
134 /// failed while sending or reading.
135, Connect(String)
136 /// TLS failed (the certificate, the handshake).
137, Tls(TlsError)
138 /// The response is not HTTP/1.1 as this client reads it.
139, Malformed(String)
140 /// The deadline passed.
141, TimedOut
142}
143
144/// GET `target` with `headers` (see the module's documentation).
145pub fn get ::
146 target: Url
147 headers: List<(String, String)>
148 opts: Options
149 -> Result<Response, Error> / <proc, clock>
150{
151 request "GET" target headers (bytes.empty ()) opts
152}
153
154/// `method` (`POST`, `PUT`, …) to `target` with `headers` and `body`,
155/// under the same guards as `get`. A body is sent with its
156/// `Content-Length` (one of the caller's is dropped, as the other framing
157/// headers are); the caller names its type (`Content-Type`).
158///
159/// fetch.request "POST" api [("Content-Type", "application/json")] (bytes.from_string json) opts
160pub fn request ::
161 method: String
162 target: Url
163 headers: List<(String, String)>
164 body: Bytes
165 opts: Options
166 -> Result<Response, Error> / <proc, clock>
167{
168 use checked <- result.try (valid_url target)
169 use () <- result.try (valid_headers headers)
170 use () <- result.try (valid_method method)
171 let left = opts.deadline - now_ms ()
172 if left <= 0
173 {
174 Error(TimedOut)
175 }
176 else
177 {
178 let reply = process.subject ()
179 let (pid, down) =
180 process.spawn_monitor
181 fn { process.send reply (exchange method checked headers body opts) }
182 let selector =
183 process.new_selector ()
184 |> process.selecting reply Got
185 |> process.selecting down Stopped
186 match process.select selector left
187 { Some(Got(answer)) ->
188 {
189 process.demonitor_flush pid down
190 answer
191 }
192 , Some(Stopped(Down(_, reason))) ->
193 Error(Malformed("the fetch stopped: {show reason}"))
194 , None ->
195 {
196 process.kill pid
197 process.demonitor_flush pid down
198 // A reply sent just before the kill would stay in the mailbox.
199 let _ = process.receive reply 0
200 Error(TimedOut)
201 }
202 }
203 }
204}
205
206/// A collected request whose exchange is running in its own actor. The
207/// actor that called [`start_request`] must call [`await_request`] exactly
208/// once, including after cancellation, to consume the reply and monitor.
209/// Any actor holding this handle may call [`cancel_request`].
210pub opaque type PendingRequest
211{ PendingRequest(
212 pid: Pid
213 , down: Subject<Down>
214 , reply: Subject<Result<Response, Error>>
215 , deadline: Int
216 )
217}
218
219/// The end of a pending request: its ordinary response or failure, or an
220/// explicit cancellation. A reply completed before a cancellation may still
221/// be `Completed`.
222@derive Eq Show Debug
223pub type RequestOutcome { Completed(Result<Response, Error>), Cancelled }
224
225/// Start [`request`] without waiting for its response. The same checks and
226/// guards apply. The returned handle affects only this exchange, never other
227/// requests to the same URL or host.
228pub fn start_request ::
229 method: String
230 target: Url
231 headers: List<(String, String)>
232 body: Bytes
233 opts: Options
234 -> Result<PendingRequest, Error> / <proc, clock>
235{
236 use checked <- result.try (valid_url target)
237 use () <- result.try (valid_headers headers)
238 use () <- result.try (valid_method method)
239 if opts.deadline - now_ms () <= 0
240 {
241 Error(TimedOut)
242 }
243 else
244 {
245 let reply = process.subject ()
246 let (pid, down) =
247 process.spawn_monitor
248 fn { process.send reply (exchange method checked headers body opts) }
249 Ok(PendingRequest(pid, down, reply, opts.deadline))
250 }
251}
252
253/// Wait for a started request's whole response, body included, or its
254/// deadline. Only the actor that called [`start_request`] may wait. After
255/// [`cancel_request`] this answers `Cancelled` unless the reply completed
256/// first. Do not wait on the same handle twice.
257///
258/// As with [`request`]'s deadline, a cancellation that lands while the actor
259/// waits in `net.resolve` takes effect when the lookup returns; the caller
260/// does not wait for that. Every other pending reply is discarded here.
261pub fn await_request ::
262 pending: PendingRequest
263 -> RequestOutcome / <proc, clock>
264{
265 let left = pending.deadline - now_ms ()
266 let selector =
267 process.new_selector ()
268 |> process.selecting pending.reply Got
269 |> process.selecting pending.down Stopped
270 match process.select selector (if left < 0 { 0 } else { left })
271 { Some(Got(answer)) ->
272 {
273 process.demonitor_flush pending.pid pending.down
274 Completed(answer)
275 }
276 , Some(Stopped(Down(_, Killed))) ->
277 {
278 discard_request pending
279 Cancelled
280 }
281 , Some(Stopped(Down(_, reason))) ->
282 {
283 discard_request pending
284 Completed(Error(Malformed("the fetch stopped: {show reason}")))
285 }
286 , None ->
287 {
288 process.kill pending.pid
289 discard_request pending
290 Completed(Error(TimedOut))
291 }
292 }
293}
294
295/// Stop only this pending request, closing its connection when its actor
296/// unwinds. Harmless before waiting, while waiting, and after completion;
297/// repeat calls are harmless. Any actor may cancel, and the starting actor's
298/// [`await_request`] then performs the final mailbox cleanup.
299pub fn cancel_request :: pending: PendingRequest -> () / <proc>
300{
301 process.kill pending.pid
302}
303
304fn discard_request :: pending: PendingRequest -> () / <proc>
305{
306 process.demonitor_flush pending.pid pending.down
307 // A reply sent just before a kill would otherwise stay in the mailbox.
308 let _ = process.receive pending.reply 0
309 ()
310}
311
312type Outcome { Got(Result<Response, Error>), Stopped(Down) }
313
314fn now_ms :: -> Int / <clock>
315{
316 time.in_milliseconds (time.monotonic ())
317}
318
319// ---- streamed responses ---------------------------------------------------------
320
321/// A response whose body is read as it arrives (ADR 0079, Revision 2):
322/// its status, its headers, and the body to read with [`next`].
323pub type Streamed
324{ Streamed(status: Int, headers: List<(String, String)>, body: BodyStream) }
325
326/// A body still arriving. Only the actor that made the request can read
327/// it; one that stops reading early calls [`cancel`].
328pub opaque type BodyStream
329{ BodyStream(
330 pid: Pid
331 , down: Subject<Down>
332 , events: Subject<StreamEvent>
333 , deadline: Int
334 )
335}
336
337/// A piece of a streamed body: some bytes, or the end (and whether the
338/// body was cut at `max_body`).
339@derive Eq Debug
340pub type Piece { Data(Bytes), End(truncated: Bool) }
341
342impl Show<Piece>
343{
344 fn show :: piece
345 {
346 debug piece
347 }
348}
349
350/// What the exchange's actor sends a streaming caller, in order: the head,
351/// body bytes, then the end or a failure.
352type StreamEvent
353{ Opened(Int, List<(String, String)>)
354, Received(Bytes)
355, Finished(Bool)
356, Failed(Error)
357}
358
359type Arrival { Arrived(StreamEvent), Ended(Down) }
360
361/// [`request`], but returning once the response head has arrived, with
362/// the body to read as it comes ([`next`]): for server-sent events and
363/// other long answers. Everything `request` checks and does applies, and
364/// `opts.deadline` covers the whole exchange, the body included: give a
365/// long answer a long deadline. At most `max_body` bytes are delivered,
366/// so a reader that falls behind holds at most that much.
367///
368/// match fetch.request_streaming "POST" api headers body opts
369/// { Ok(Streamed(200, _, stream)) -> read stream // fetch.next until End
370/// , Ok(Streamed(status, _, stream)) -> fetch.cancel stream
371/// , Error(e) -> ...
372/// }
373pub fn request_streaming ::
374 method: String
375 target: Url
376 headers: List<(String, String)>
377 body: Bytes
378 opts: Options
379 -> Result<Streamed, Error> / <proc, clock>
380{
381 use checked <- result.try (valid_url target)
382 use () <- result.try (valid_headers headers)
383 use () <- result.try (valid_method method)
384 if opts.deadline - now_ms () <= 0
385 {
386 Error(TimedOut)
387 }
388 else
389 {
390 let events = process.subject ()
391 let (pid, down) =
392 process.spawn_monitor
393 fn { stream_exchange method checked headers body opts events }
394 let stream = BodyStream(pid, down, events, opts.deadline)
395 match wait stream
396 { Ok(Opened(status, head)) -> Ok(Streamed(status, head, stream))
397 , Ok(Failed(e)) ->
398 {
399 process.demonitor_flush pid down
400 Error(e)
401 }
402 , Ok(_) ->
403 {
404 cancel stream
405 Error(Malformed("the body came before the head"))
406 }
407 , Error(e) -> Error(e)
408 }
409 }
410}
411
412/// The next piece of `stream`: bytes as they arrived (a chunked body
413/// already decoded), then `End`. After `End` or an `Error` the stream is
414/// over: its connection is closed. `TimedOut` at the request's deadline.
415pub fn next :: stream: BodyStream -> Result<Piece, Error> / <proc, clock>
416{
417 match wait stream
418 { Ok(Received(data)) -> Ok(Data(data))
419 , Ok(Finished(truncated)) ->
420 {
421 process.demonitor_flush stream.pid stream.down
422 Ok(End(truncated))
423 }
424 , Ok(Failed(e)) ->
425 {
426 process.demonitor_flush stream.pid stream.down
427 Error(e)
428 }
429 , Ok(Opened(_, _)) ->
430 {
431 cancel stream
432 Error(Malformed("a second response head"))
433 }
434 , Error(e) -> Error(e)
435 }
436}
437
438/// Stops reading `stream`: closes its connection (the server sees the
439/// client leave) and drops what arrived unread. Harmless after the end.
440pub fn cancel :: stream: BodyStream -> () / <proc>
441{
442 process.kill stream.pid
443 process.demonitor_flush stream.pid stream.down
444 drain stream.events
445}
446
447/// The next event, until the deadline. On a timeout or a crash, the
448/// exchange is stopped and its leftovers dropped.
449fn wait :: stream: BodyStream -> Result<StreamEvent, Error> / <proc, clock>
450{
451 let left = stream.deadline - now_ms ()
452 let selector =
453 process.new_selector ()
454 |> process.selecting stream.events Arrived
455 |> process.selecting stream.down Ended
456 match process.select selector (if left < 0 { 0 } else { left })
457 { Some(Arrived(event)) -> Ok(event)
458 , Some(Ended(Down(_, reason))) ->
459 {
460 drain stream.events
461 Error(Malformed("the fetch stopped: {show reason}"))
462 }
463 , None ->
464 {
465 cancel stream
466 Error(TimedOut)
467 }
468 }
469}
470
471fn drain :: events: Subject<StreamEvent> -> () / <proc>
472{
473 match process.receive events 0 { Some(_) -> drain events, None -> () }
474}
475
476// ---- checks --------------------------------------------------------------------
477
478/// The URL parsed again from its text (a `Url` can be built by hand),
479/// and only `http` or `https`.
480fn valid_url :: target: Url -> Result<Url, Error>
481{
482 use parsed <-
483 result.try (url.parse (url.to_string target) |> result.map_error BadUrl)
484 if parsed.scheme == "http" || parsed.scheme == "https"
485 {
486 Ok(parsed)
487 }
488 else
489 {
490 Error(BadUrl("only http and https can be fetched"))
491 }
492}
493
494/// A method is a token (RFC 9110), as a header name is.
495fn valid_method :: method: String -> Result<(), Error>
496{
497 if is_token method
498 {
499 Ok(())
500 }
501 else
502 {
503 Error(BadHeader(method))
504 }
505}
506
507fn valid_headers :: headers: List<(String, String)> -> Result<(), Error>
508{
509 match list.find headers fn h { !is_token h.0 || has_line_break h.1 }
510 { Some((name, _)) -> Error(BadHeader(name)), None -> Ok(()) }
511}
512
513/// RFC 9110's `token`: digits, letters and ``!#$%&'*+-.^_`|~``.
514fn is_token :: name: String -> Bool
515{
516 let symbols = bytes.to_list (bytes.from_string "!#$%&'*+-.^_`|~")
517 name != ""
518 && list.all
519 (bytes.to_list (bytes.from_string name))
520 fn b
521 {
522 b >= 48 && b <= 57 || b >= 65 && b <= 90 || b >= 97 && b <= 122
523 || list.any symbols fn s { s == b }
524 }
525}
526
527fn has_line_break :: value: String -> Bool
528{
529 list.any
530 (bytes.to_list (bytes.from_string value))
531 fn b { b == 13 || b == 10 || b == 0 }
532}
533
534// ---- the exchange (in its own actor) -------------------------------------------
535
536/// A connection, plain or over TLS.
537type Conn { Plain(Resource<TcpStream>), Secure(Resource<TlsStream>) }
538
539fn exchange ::
540 method: String
541 target: Url
542 headers: List<(String, String)>
543 body: Bytes
544 opts: Options
545 -> Result<Response, Error> / <io>
546{
547 use conn <- result.try (connected target opts)
548 let answer = round_trip conn method target headers body opts
549 close conn
550 answer
551}
552
553/// A connection to `target`'s host, under the policy: resolved once,
554/// every address checked, then the first that accepts, secured for
555/// `https`.
556fn connected :: target: Url opts: Options -> Result<Conn, Error> / <io>
557{
558 let host = host_name target.host
559 let port = match url.port_or_default target { Some(p) -> p, None -> 80 }
560 use ips <-
561 result.try (net.resolve host |> result.map_error fn _ { NoAddress(host) })
562 use ips <- result.try (allowed opts.policy host ips)
563 use tcp <- result.try (connect_any ips port "")
564 secured target.scheme tcp host opts.tls
565}
566
567/// The host without an IPv6 address's brackets: what is resolved, and
568/// the TLS server name.
569fn host_name :: host: String -> String
570{
571 if string.starts_with host "["
572 {
573 string.slice host 1 (string.length host - 1)
574 }
575 else
576 {
577 host
578 }
579}
580
581/// Every address, if the policy allows them all.
582fn allowed ::
583 policy: Policy
584 host: String
585 ips: List<String>
586 -> Result<List<String>, Error>
587{
588 match (policy, ips)
589 { (_, []) -> Error(NoAddress(host))
590 , (AnyAddress, _) -> Ok(ips)
591 , (PublicOnly, _) -> match list.find ips fn ip { !address.is_public ip }
592 { Some(ip) -> Error(Refused(host, ip)), None -> Ok(ips) }
593 }
594}
595
596/// A connection to the first of `ips` that accepts.
597fn connect_any ::
598 ips: List<String>
599 port: Int
600 last: String
601 -> Result<Resource<TcpStream>, Error> / <io>
602{
603 match ips
604 { [] -> Error(Connect(last))
605 , [ip, ..rest] -> match net.connect ip port
606 { Ok(conn) -> Ok(conn), Error(e) -> connect_any rest port e }
607 }
608}
609
610fn secured ::
611 scheme: String
612 tcp: Resource<TcpStream>
613 host: String
614 config: Resource<TlsConfig>
615 -> Result<Conn, Error> / <io>
616{
617 if scheme == "https"
618 {
619 match tls.connect tcp host config
620 { Ok(stream) -> Ok(Secure(stream))
621 , Error(e) ->
622 {
623 net.close tcp
624 Error(Tls(e))
625 }
626 }
627 }
628 else
629 {
630 Ok(Plain(tcp))
631 }
632}
633
634fn round_trip ::
635 conn: Conn
636 method: String
637 target: Url
638 headers: List<(String, String)>
639 body: Bytes
640 opts: Options
641 -> Result<Response, Error> / <io>
642{
643 use (head, framing, rest) <-
644 result.try (send_and_read_head conn method target headers body opts)
645 use (body, truncated) <-
646 result.try (read_body conn framing rest opts.max_body)
647 Ok(Response(head.status, head.headers, body, truncated))
648}
649
650/// Sends the request, then reads the final head: it, the body's framing,
651/// and the body bytes read with it.
652fn send_and_read_head ::
653 conn: Conn
654 method: String
655 target: Url
656 headers: List<(String, String)>
657 body: Bytes
658 opts: Options
659 -> Result<(Head, Framing, Bytes), Error> / <io>
660{
661 use () <-
662 result.try
663 (write conn (bytes.concat (request_head method target headers body) body))
664 use (head, rest) <-
665 result.try (final_head conn opts.max_head (bytes.empty ()))
666 use framing <-
667 result.try (response.framing head |> result.map_error Malformed)
668 Ok((head, framing, rest))
669}
670
671fn request_head ::
672 method: String
673 target: Url
674 headers: List<(String, String)>
675 body: Bytes
676 -> Bytes
677{
678 let default = url.port_or_default Url(..target, port: None)
679 let port = match target.port
680 { Some(p) if Some(p) != default -> ":" <> show p, _ -> "" }
681 let own = list.filter headers fn h { !is_framing h.0 }
682 // A body says its length; so does a POST or PUT without one, which
683 // some servers otherwise wait on.
684 let length = if bytes.length body > 0 || method == "POST" || method == "PUT"
685 {
686 ["Content-Length: " <> show (bytes.length body)]
687 }
688 else
689 {
690 []
691 }
692 let lines =
693 [ method <> " " <> url.request_target target <> " HTTP/1.1"
694 , "Host: " <> target.host <> port
695 , "Connection: close"
696 , "Accept-Encoding: identity"
697 , ..list.append length (list.map own fn h { h.0 <> ": " <> h.1 })
698 ]
699 bytes.from_string (string.join lines "\r\n" <> "\r\n\r\n")
700}
701
702fn is_framing :: name: String -> Bool
703{
704 list.any
705 [ "host"
706 , "connection"
707 , "accept-encoding"
708 , "content-length"
709 , "transfer-encoding"
710 , "te"
711 , "upgrade"
712 ]
713 fn n { string.eq_ignore_ascii_case n name }
714}
715
716// ---- reading -------------------------------------------------------------------
717
718/// The first head that is not `1xx`, and the bytes read after it.
719fn final_head ::
720 conn: Conn
721 max_head: Int
722 buffer: Bytes
723 -> Result<(Head, Bytes), Error> / <io>
724{
725 match response.head buffer max_head
726 { Error(e) -> Error(Malformed(e))
727 , Ok(Some(head)) ->
728 {
729 let rest = bytes.slice buffer head.size (bytes.length buffer)
730 if head.status < 200
731 {
732 final_head conn max_head rest
733 }
734 else
735 {
736 Ok((head, rest))
737 }
738 }
739 , Ok(None) -> more_head conn max_head buffer
740 }
741}
742
743/// Reads until the head's blank line has arrived (only the new bytes
744/// and the 3 before them are searched), or until there is too much.
745fn more_head ::
746 conn: Conn
747 max_head: Int
748 buffer: Bytes
749 -> Result<(Head, Bytes), Error> / <io>
750{
751 use more <- result.try (read conn)
752 if bytes.length more == 0
753 {
754 Error(Malformed("the connection closed before the response head"))
755 }
756 else
757 {
758 let n = bytes.length buffer
759 let from = if n > 3 { n - 3 } else { 0 }
760 let buffer = bytes.concat buffer more
761 if has_blank_line buffer from || bytes.length buffer > max_head
762 {
763 final_head conn max_head buffer
764 }
765 else
766 {
767 more_head conn max_head buffer
768 }
769 }
770}
771
772fn has_blank_line :: data: Bytes at: Int -> Bool
773{
774 if at + 4 > bytes.length data
775 {
776 False
777 }
778 else if bytes.slice data at (at + 4) == bytes.from_string "\r\n\r\n"
779 {
780 True
781 }
782 else
783 {
784 has_blank_line data (at + 1)
785 }
786}
787
788fn read_body ::
789 conn: Conn
790 framing: Framing
791 rest: Bytes
792 max: Int
793 -> Result<(Bytes, Bool), Error> / <io>
794{
795 match framing
796 { NoBody -> Ok((bytes.empty (), False))
797 , Length(n) -> if n > max
798 {
799 exactly conn rest max False |> result.map fn b { (b, True) }
800 }
801 else
802 {
803 exactly conn rest n False |> result.map fn b { (b, False) }
804 }
805 , UntilClose ->
806 until_close
807 conn
808 (bytes.append (bytes.builder ()) rest)
809 (bytes.length rest)
810 max
811 , Chunked -> match response.feed (response.dechunker max) rest
812 { Ok(d) -> chunks conn d, Error(e) -> Error(Malformed(e)) }
813 }
814}
815
816/// The first `n` bytes, starting with `have`; `closed` once a read found
817/// the end.
818fn exactly ::
819 conn: Conn
820 have: Bytes
821 n: Int
822 closed: Bool
823 -> Result<Bytes, Error> / <io>
824{
825 if bytes.length have >= n
826 {
827 Ok(bytes.slice have 0 n)
828 }
829 else if closed
830 {
831 Error(Malformed("the connection closed before the end of the body"))
832 }
833 else
834 {
835 use more <- result.try (read conn)
836 exactly conn (bytes.concat have more) n (bytes.length more == 0)
837 }
838}
839
840fn until_close ::
841 conn: Conn
842 body: bytes.Builder
843 size: Int
844 max: Int
845 -> Result<(Bytes, Bool), Error> / <io>
846{
847 if size > max
848 {
849 Ok((bytes.slice (bytes.build body) 0 max, True))
850 }
851 else
852 {
853 use more <- result.try (read conn)
854 if bytes.length more == 0
855 {
856 Ok((bytes.build body, False))
857 }
858 else
859 {
860 until_close conn (bytes.append body more) (size + bytes.length more) max
861 }
862 }
863}
864
865fn chunks :: conn: Conn d: Dechunker -> Result<(Bytes, Bool), Error> / <io>
866{
867 if response.is_done d
868 {
869 Ok(response.body d)
870 }
871 else
872 {
873 use more <- result.try (read conn)
874 if bytes.length more == 0
875 {
876 Error(Malformed("the connection closed inside a chunked body"))
877 }
878 else
879 {
880 match response.feed d more
881 { Ok(d) -> chunks conn d, Error(e) -> Error(Malformed(e)) }
882 }
883 }
884}
885
886// ---- the streamed exchange (in its own actor) ----------------------------------
887
888fn stream_exchange ::
889 method: String
890 target: Url
891 headers: List<(String, String)>
892 body: Bytes
893 opts: Options
894 to: Subject<StreamEvent>
895 -> () / <io, proc>
896{
897 let outcome = match connected target opts
898 { Ok(conn) ->
899 {
900 let streamed = stream_round_trip conn method target headers body opts to
901 close conn
902 streamed
903 }
904 , Error(e) -> Error(e)
905 }
906 match outcome
907 { Ok(truncated) -> process.send to (Finished(truncated))
908 , Error(e) -> process.send to (Failed(e))
909 }
910}
911
912/// Sends the head once it is read, then the body as it arrives; whether
913/// the body was cut at `max_body`.
914fn stream_round_trip ::
915 conn: Conn
916 method: String
917 target: Url
918 headers: List<(String, String)>
919 body: Bytes
920 opts: Options
921 to: Subject<StreamEvent>
922 -> Result<Bool, Error> / <io, proc>
923{
924 use (head, framing, rest) <-
925 result.try (send_and_read_head conn method target headers body opts)
926 process.send to (Opened(head.status, head.headers))
927 let max = opts.max_body
928 match framing
929 { NoBody -> Ok(False)
930 , Length
931 ( n
932 ) -> if n > max
933 {
934 stream_exactly conn rest max True to
935 }
936 else
937 {
938 stream_exactly conn rest n False to
939 }
940 , UntilClose -> stream_until_close conn rest 0 max to
941 , Chunked -> match response.feed (response.dechunker max) rest
942 { Ok(d) -> stream_chunks conn d to, Error(e) -> Error(Malformed(e)) }
943 }
944}
945
946fn emit :: to: Subject<StreamEvent> data: Bytes -> () / <proc>
947{
948 if bytes.length data > 0
949 {
950 process.send to (Received(data))
951 }
952}
953
954/// Delivers the next `left` bytes, starting with `have`; `cut` is the
955/// answer once they are all delivered.
956fn stream_exactly ::
957 conn: Conn
958 have: Bytes
959 left: Int
960 cut: Bool
961 to: Subject<StreamEvent>
962 -> Result<Bool, Error> / <io, proc>
963{
964 let piece = bytes.slice have 0 left
965 emit to piece
966 let left = left - bytes.length piece
967 if left == 0
968 {
969 Ok(cut)
970 }
971 else
972 {
973 use more <- result.try (read conn)
974 if bytes.length more == 0
975 {
976 Error(Malformed("the connection closed before the end of the body"))
977 }
978 else
979 {
980 stream_exactly conn more left cut to
981 }
982 }
983}
984
985/// Delivers what arrives until the server closes, `size` bytes so far.
986fn stream_until_close ::
987 conn: Conn
988 have: Bytes
989 size: Int
990 max: Int
991 to: Subject<StreamEvent>
992 -> Result<Bool, Error> / <io, proc>
993{
994 let room = max - size
995 if bytes.length have > room
996 {
997 emit to (bytes.slice have 0 room)
998 Ok(True)
999 }
1000 else
1001 {
1002 emit to have
1003 use more <- result.try (read conn)
1004 if bytes.length more == 0
1005 {
1006 Ok(False)
1007 }
1008 else
1009 {
1010 stream_until_close conn more (size + bytes.length have) max to
1011 }
1012 }
1013}
1014
1015fn stream_chunks ::
1016 conn: Conn
1017 d: Dechunker
1018 to: Subject<StreamEvent>
1019 -> Result<Bool, Error> / <io, proc>
1020{
1021 let (data, d) = response.take d
1022 emit to data
1023 if response.is_done d
1024 {
1025 Ok((response.body d).1)
1026 }
1027 else
1028 {
1029 use more <- result.try (read conn)
1030 if bytes.length more == 0
1031 {
1032 Error(Malformed("the connection closed inside a chunked body"))
1033 }
1034 else
1035 {
1036 match response.feed d more
1037 { Ok(d) -> stream_chunks conn d to, Error(e) -> Error(Malformed(e)) }
1038 }
1039 }
1040}
1041
1042// ---- the connection ------------------------------------------------------------
1043
1044/// Up to 64 KiB; empty at the end. A TLS stream that ends without
1045/// `close_notify` (`Truncated`) is an end too: the framing says whether
1046/// the body was whole.
1047fn read :: conn: Conn -> Result<Bytes, Error> / <io>
1048{
1049 match conn
1050 { Plain(tcp) -> net.read tcp 65536 |> result.map_error Connect
1051 , Secure(stream) -> match tls.read stream 65536
1052 { Ok(b) -> Ok(b)
1053 , Error(Truncated) -> Ok(bytes.empty ())
1054 , Error(e) -> Error(Tls(e))
1055 }
1056 }
1057}
1058
1059fn write :: conn: Conn data: Bytes -> Result<(), Error> / <io>
1060{
1061 match conn
1062 { Plain(tcp) -> net.write tcp data |> result.map_error Connect
1063 , Secure(stream) -> tls.write stream data |> result.map_error Tls
1064 }
1065}
1066
1067fn close :: conn: Conn -> () / <io>
1068{
1069 match conn { Plain(tcp) -> net.close tcp, Secure(stream) -> tls.close stream }
1070}