
louise
Clone avec HTTPS, un jeton personnel en mot de passe, ou avec SSH, une clé ajoutée dans vos paramètres.
Dernier runRéussi#1197 · main · 10 min 35 s
1Étoiles
0Forks
0Issues
| 1 | /// Supervisors: restart children that crash ("let it crash", ADR 0004), |
| 2 | /// within a restart intensity, and stop them in order (ADR 0060). |
| 3 | |
| 4 | import std/file.{type fs} |
| 5 | import std/list |
| 6 | import std/net.{type net} |
| 7 | import std/os.{type Signal} |
| 8 | import std/process. |
| 9 | { type ExitSignal |
| 10 | , type Pid |
| 11 | , type Subject |
| 12 | , type proc |
| 13 | , Crashed |
| 14 | , ExitSignal |
| 15 | , Normal |
| 16 | , Shutdown |
| 17 | } |
| 18 | import std/random.{type random} |
| 19 | import std/time.{type clock} |
| 20 | |
| 21 | pub type Strategy |
| 22 | { /// Restart only the child that crashed. |
| 23 | OneForOne |
| 24 | /// Restart every child when one crashes. |
| 25 | , OneForAll |
| 26 | /// Restart the child that crashed and every child started after it |
| 27 | /// (they may depend on it); the ones before it keep running. |
| 28 | , RestForOne |
| 29 | } |
| 30 | |
| 31 | /// When a child is restarted (OTP's child kinds). |
| 32 | @derive Eq Show |
| 33 | pub type Restart |
| 34 | { /// Always: after it crashes, and after it returns. |
| 35 | Permanent |
| 36 | /// After it crashes only: a child that returns (or is stopped with |
| 37 | /// `Shutdown`) is done. |
| 38 | , Transient |
| 39 | /// Never. |
| 40 | , Temporary |
| 41 | } |
| 42 | |
| 43 | /// How to start a child. The child is linked to its supervisor. |
| 44 | pub type Child |
| 45 | { /// Restarted after a crash only (`Transient`), given 5 seconds to stop. |
| 46 | Child( |
| 47 | name: String |
| 48 | , start: send fn() -> () / <proc, clock, random, fs, net, io, ffi> |
| 49 | ) |
| 50 | /// Restarted as `restart` says, given `shutdown_timeout` milliseconds to |
| 51 | /// stop when its supervisor shuts down (then killed). |
| 52 | , ChildWith( |
| 53 | name: String |
| 54 | , start: send fn() -> () / <proc, clock, random, fs, net, io, ffi> |
| 55 | , restart: Restart |
| 56 | , shutdown_timeout: Int |
| 57 | ) |
| 58 | } |
| 59 | |
| 60 | /// A supervisor: its children, started in order, its strategy, and its |
| 61 | /// restart intensity: more than `intensity` restarts within `period_ms` |
| 62 | /// milliseconds make it give up (crash). A negative `period_ms` counts |
| 63 | /// restarts over the supervisor's whole life. |
| 64 | pub type Spec |
| 65 | { Spec( |
| 66 | strategy: Strategy |
| 67 | , children: List<Child> |
| 68 | , intensity: Int |
| 69 | , period_ms: Int |
| 70 | ) |
| 71 | } |
| 72 | |
| 73 | /// Run a supervisor in the current actor: start the children, restart |
| 74 | /// them as `strategy` and their kinds say, and crash itself after |
| 75 | /// `max_restarts` restarts (over its whole life; `run` takes a window). |
| 76 | /// Returns when every child is done. |
| 77 | pub fn supervise :: |
| 78 | strategy: Strategy |
| 79 | children: List<Child> |
| 80 | max_restarts: Int |
| 81 | -> () / <proc, io> |
| 82 | { |
| 83 | // Over its whole life: no clock needed. |
| 84 | run_with Spec(strategy, children, max_restarts, -1) fn { 0 } |
| 85 | } |
| 86 | |
| 87 | /// Run the supervisor `spec` in the current actor (see `Spec`). It traps |
| 88 | /// exits: told to stop (`process.stop`, a `Shutdown` exit signal), it |
| 89 | /// stops its children in reverse start order, each with `Shutdown` and |
| 90 | /// killed after its `shutdown_timeout`, then returns. Returns too when |
| 91 | /// every child is done. |
| 92 | pub fn run :: spec: Spec -> () / <proc, io, clock> |
| 93 | { |
| 94 | run_with spec now_ms |
| 95 | } |
| 96 | |
| 97 | /// The monotonic clock, in milliseconds. |
| 98 | fn now_ms :: -> Int / <clock> |
| 99 | { |
| 100 | time.in_milliseconds (time.monotonic ()) |
| 101 | } |
| 102 | |
| 103 | /// `run`, reading the time from `now` (for the restart window). |
| 104 | fn run_with :: spec: Spec now: fn() -> Int / <e> -> () / <proc, io | e> |
| 105 | { |
| 106 | process.trap_exits True |
| 107 | let exits = process.exits () |
| 108 | watch spec exits (list.map spec.children start_child) [] now |
| 109 | } |
| 110 | |
| 111 | type Running = List<(Pid, Child)> |
| 112 | |
| 113 | fn start_child :: child: Child -> (Pid, Child) / <proc> |
| 114 | { |
| 115 | (process.spawn_link (start_of child), child) |
| 116 | } |
| 117 | |
| 118 | fn start_of :: |
| 119 | child: Child |
| 120 | -> send fn() -> () / <proc, clock, random, fs, net, io, ffi> |
| 121 | { |
| 122 | match child { Child(_, start) -> start, ChildWith(_, start, _, _) -> start } |
| 123 | } |
| 124 | |
| 125 | fn restart_of :: child: Child -> Restart |
| 126 | { |
| 127 | match child |
| 128 | { Child(_, _) -> Transient, ChildWith(_, _, restart, _) -> restart } |
| 129 | } |
| 130 | |
| 131 | fn shutdown_timeout_of :: child: Child -> Int |
| 132 | { |
| 133 | match child { Child(_, _) -> 5000, ChildWith(_, _, _, timeout) -> timeout } |
| 134 | } |
| 135 | |
| 136 | /// `restarts`: when the restarts within the window happened (`now`'s |
| 137 | /// milliseconds, the latest first). |
| 138 | fn watch :: |
| 139 | spec: Spec |
| 140 | exits: Subject<ExitSignal> |
| 141 | running: Running |
| 142 | restarts: List<Int> |
| 143 | now: fn() -> Int / <e> |
| 144 | -> () / <proc, io | e> |
| 145 | { |
| 146 | match running |
| 147 | { [] -> () |
| 148 | , _ -> |
| 149 | { |
| 150 | let ExitSignal(pid, reason) = process.receive_forever exits |
| 151 | if !is_running running pid |
| 152 | { |
| 153 | if reason == Shutdown |
| 154 | { |
| 155 | // Told to stop, by whoever started it. |
| 156 | shutdown running exits |
| 157 | } |
| 158 | else |
| 159 | { |
| 160 | // A child replaced or killed by a restart: already handled. |
| 161 | watch spec exits running restarts now |
| 162 | } |
| 163 | } |
| 164 | else |
| 165 | { |
| 166 | let child = child_of running pid |
| 167 | let done = match (restart_of child, reason) |
| 168 | { (Temporary, _) -> True |
| 169 | , (Transient, Normal) -> True |
| 170 | , (Transient, Shutdown) -> True |
| 171 | , _ -> False |
| 172 | } |
| 173 | if done |
| 174 | { |
| 175 | watch spec exits (remove running pid) restarts now |
| 176 | } |
| 177 | else |
| 178 | { |
| 179 | let restarts = counted restarts spec.period_ms (now ()) |
| 180 | if list.length restarts > spec.intensity |
| 181 | { |
| 182 | // Stop the others (the crashed child is gone already). |
| 183 | shutdown (remove running pid) exits |
| 184 | panic "supervisor: too many restarts" |
| 185 | } |
| 186 | else |
| 187 | { |
| 188 | let restarted = match spec.strategy |
| 189 | { OneForOne -> restart_one running pid |
| 190 | , OneForAll -> restart_all running |
| 191 | , RestForOne -> restart_rest running pid |
| 192 | } |
| 193 | watch spec exits restarted restarts now |
| 194 | } |
| 195 | } |
| 196 | } |
| 197 | } |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | /// `restarts` with one more `now`, forgetting those older than |
| 202 | /// `period_ms` (none if it is negative). |
| 203 | fn counted :: restarts: List<Int> period_ms: Int now: Int -> List<Int> |
| 204 | { |
| 205 | let recent = if period_ms < 0 |
| 206 | { |
| 207 | restarts |
| 208 | } |
| 209 | else |
| 210 | { |
| 211 | list.filter restarts fn t { now - t < period_ms } |
| 212 | } |
| 213 | [now, ..recent] |
| 214 | } |
| 215 | |
| 216 | fn is_running :: running: Running pid: Pid -> Bool |
| 217 | { |
| 218 | use found, entry <- list.fold running False |
| 219 | found || entry.0 == pid |
| 220 | } |
| 221 | |
| 222 | fn child_of :: running: Running pid: Pid -> Child |
| 223 | { |
| 224 | match running |
| 225 | { [] -> panic "supervisor: not a child" |
| 226 | , [entry, ..rest] -> if entry.0 == pid { entry.1 } else { child_of rest pid } |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | fn remove :: running: Running pid: Pid -> Running |
| 231 | { |
| 232 | use entry <- list.filter running |
| 233 | entry.0 != pid |
| 234 | } |
| 235 | |
| 236 | fn restart_one :: running: Running pid: Pid -> Running / <proc> |
| 237 | { |
| 238 | use entry <- list.map running |
| 239 | if entry.0 == pid |
| 240 | { |
| 241 | start_child entry.1 |
| 242 | } |
| 243 | else |
| 244 | { |
| 245 | entry |
| 246 | } |
| 247 | } |
| 248 | |
| 249 | fn restart_all :: running: Running -> Running / <proc> |
| 250 | { |
| 251 | list.each running fn entry { process.kill entry.0 } |
| 252 | restart_kept running |
| 253 | } |
| 254 | |
| 255 | /// Restart `pid` and the children after it, in order; the later ones are |
| 256 | /// stopped first. |
| 257 | fn restart_rest :: running: Running pid: Pid -> Running / <proc> |
| 258 | { |
| 259 | match running |
| 260 | { [] -> [] |
| 261 | , [ entry |
| 262 | , ..rest |
| 263 | ] -> if entry.0 == pid |
| 264 | { |
| 265 | // Stop the later children last-started first, as OTP does. |
| 266 | list.each (list.reverse rest) fn later { process.kill later.0 } |
| 267 | restart_kept [entry, ..rest] |
| 268 | } |
| 269 | else |
| 270 | { |
| 271 | [entry, ..restart_rest rest pid] |
| 272 | } |
| 273 | } |
| 274 | } |
| 275 | |
| 276 | /// Start `running`'s children again, but the temporary ones, which are |
| 277 | /// done. |
| 278 | fn restart_kept :: running: Running -> Running / <proc> |
| 279 | { |
| 280 | let kept = list.filter running fn entry { restart_of entry.1 != Temporary } |
| 281 | list.map kept fn entry { start_child entry.1 } |
| 282 | } |
| 283 | |
| 284 | /// Stop every child, last-started first: each with `Shutdown`, killed if |
| 285 | /// it is not done within its `shutdown_timeout`. |
| 286 | fn shutdown :: running: Running exits: Subject<ExitSignal> -> () / <proc> |
| 287 | { |
| 288 | let _ = |
| 289 | list.fold |
| 290 | (list.reverse running) |
| 291 | [] |
| 292 | fn gone entry |
| 293 | { |
| 294 | stop_child entry.0 (shutdown_timeout_of entry.1) exits gone |
| 295 | } |
| 296 | () |
| 297 | } |
| 298 | |
| 299 | /// Stop `pid`, waiting up to `timeout` milliseconds for its exit signal, |
| 300 | /// then kill it and wait for it. `gone`: the children whose exit signals |
| 301 | /// arrived meanwhile (each arrives once); returns it updated. |
| 302 | fn stop_child :: |
| 303 | pid: Pid |
| 304 | timeout: Int |
| 305 | exits: Subject<ExitSignal> |
| 306 | gone: List<Pid> |
| 307 | -> List<Pid> / <proc> |
| 308 | { |
| 309 | if list.any gone fn p { p == pid } |
| 310 | { |
| 311 | gone |
| 312 | } |
| 313 | else |
| 314 | { |
| 315 | process.stop pid |
| 316 | match wait_exit pid timeout exits gone |
| 317 | { (True, gone) -> gone |
| 318 | , (False, gone) -> |
| 319 | { |
| 320 | process.kill pid |
| 321 | (wait_exit pid (-1) exits gone).1 |
| 322 | } |
| 323 | } |
| 324 | } |
| 325 | } |
| 326 | |
| 327 | /// Wait up to `timeout` milliseconds (negative: forever) for `pid`'s exit |
| 328 | /// signal: whether it came, and the other children that exited meanwhile. |
| 329 | fn wait_exit :: |
| 330 | pid: Pid |
| 331 | timeout: Int |
| 332 | exits: Subject<ExitSignal> |
| 333 | gone: List<Pid> |
| 334 | -> (Bool, List<Pid>) / <proc> |
| 335 | { |
| 336 | match process.receive exits timeout |
| 337 | { None -> (False, gone) |
| 338 | , Some(ExitSignal(from, _)) -> if from == pid |
| 339 | { |
| 340 | (True, gone) |
| 341 | } |
| 342 | else |
| 343 | { |
| 344 | wait_exit pid timeout exits [from, ..gone] |
| 345 | } |
| 346 | } |
| 347 | } |
| 348 | |
| 349 | /// How the runtime supervises a program's `main` (ADR 0060): the |
| 350 | /// `[application]` section of `louise.toml`. `main` is restarted as |
| 351 | /// `restart` says after it crashes, at most `intensity` times within |
| 352 | /// `period_ms` milliseconds; on one of `signals`, it is stopped |
| 353 | /// (`Shutdown`), and killed if it is not done within `shutdown_timeout_ms`. |
| 354 | pub type Application |
| 355 | { Application( |
| 356 | restart: Restart |
| 357 | , intensity: Int |
| 358 | , period_ms: Int |
| 359 | , shutdown_timeout_ms: Int |
| 360 | , signals: List<Signal> |
| 361 | ) |
| 362 | } |
| 363 | |
| 364 | /// Run `main` under `app` (the program's root when `louise.toml` has an |
| 365 | /// `[application]` section; the compiler generates the call). Returns, |
| 366 | /// and the program exits 0, when `main` returns, or once it stopped after |
| 367 | /// a signal; a crash beyond the restart intensity (or any crash, for a |
| 368 | /// `Temporary` main) ends the program with that panic. |
| 369 | pub fn application :: |
| 370 | app: Application |
| 371 | main: send fn() -> () / <proc, clock, random, fs, net, io, ffi> |
| 372 | -> () / <proc, io, clock> |
| 373 | { |
| 374 | process.trap_exits True |
| 375 | let exits = process.exits () |
| 376 | let signals = os.signals app.signals |
| 377 | let events = |
| 378 | process.new_selector () |
| 379 | |> process.selecting exits Exited |
| 380 | |> process.selecting signals Signalled |
| 381 | supervise_main app main events (process.spawn_link main) [] |
| 382 | } |
| 383 | |
| 384 | /// What `application` waits for. |
| 385 | type MainEvent { Exited(ExitSignal), Signalled(Signal) } |
| 386 | |
| 387 | fn supervise_main :: |
| 388 | app: Application |
| 389 | main: send fn() -> () / <proc, clock, random, fs, net, io, ffi> |
| 390 | events: process.Selector<MainEvent> |
| 391 | pid: Pid |
| 392 | restarts: List<Int> |
| 393 | -> () / <proc, io, clock> |
| 394 | { |
| 395 | match process.select_forever events |
| 396 | { Signalled(_) -> stop_main pid app.shutdown_timeout_ms events |
| 397 | , Exited |
| 398 | ( ExitSignal(from, reason) |
| 399 | ) -> if from != pid |
| 400 | { |
| 401 | // Another linked actor: not `main`'s business here. |
| 402 | supervise_main app main events pid restarts |
| 403 | } |
| 404 | else |
| 405 | { |
| 406 | match (app.restart, reason) |
| 407 | { (_, Normal) -> () |
| 408 | , (_, Shutdown) -> () |
| 409 | , (Temporary, _) -> panic (crash_message reason) |
| 410 | , _ -> |
| 411 | { |
| 412 | let restarts = counted restarts app.period_ms (now_ms ()) |
| 413 | if list.length restarts > app.intensity |
| 414 | { |
| 415 | panic (crash_message reason) |
| 416 | } |
| 417 | else |
| 418 | { |
| 419 | supervise_main app main events (process.spawn_link main) restarts |
| 420 | } |
| 421 | } |
| 422 | } |
| 423 | } |
| 424 | } |
| 425 | } |
| 426 | |
| 427 | fn crash_message :: reason: process.ExitReason -> String |
| 428 | { |
| 429 | match reason { Crashed(message) -> message, _ -> "main was killed" } |
| 430 | } |
| 431 | |
| 432 | /// Stop `main` (`Shutdown`), waiting up to `timeout` milliseconds for it, |
| 433 | /// then kill it. Further signals meanwhile are ignored. |
| 434 | fn stop_main :: |
| 435 | pid: Pid |
| 436 | timeout: Int |
| 437 | events: process.Selector<MainEvent> |
| 438 | -> () / <proc> |
| 439 | { |
| 440 | process.stop pid |
| 441 | if !main_exited pid timeout events |
| 442 | { |
| 443 | process.kill pid |
| 444 | let _ = main_exited pid (-1) events |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | /// Whether `main` (`pid`) exited within `timeout` milliseconds (negative: |
| 449 | /// forever). |
| 450 | fn main_exited :: |
| 451 | pid: Pid |
| 452 | timeout: Int |
| 453 | events: process.Selector<MainEvent> |
| 454 | -> Bool / <proc> |
| 455 | { |
| 456 | match process.select events timeout |
| 457 | { None -> False |
| 458 | , Some |
| 459 | ( Exited(ExitSignal(from, _)) |
| 460 | ) -> if from == pid { True } else { main_exited pid timeout events } |
| 461 | , Some(Signalled(_)) -> main_exited pid timeout events |
| 462 | } |
| 463 | } |