tawasuyu

tawasuyu/brahman

Rust 463TOML 180Markdown 49JSON 176353 Definitions

⎇ main crates/modules/shuma/docs/RECIPES.md Markdown · 279 lines · 5.6 KB History raw

refactor(monorepo): reorganización lógica + renames + SDDs + split CHANGELOG 550c98f27 · sergio ·

1# Recetas de specs
2
3Specs TOML para casos comunes. Todas asumen que `<WS>` ya existe (creado con `shipote workspace create <ws-spec>`).
4
5## Workspaces
6
7### Workspace mínimo
8```toml
9label = "demo"
10on_exit = "reap"
11```
12
13### Workspace con TTL (auto-stop tras N ms)
14```toml
15label = "ephemeral"
16on_exit = "reap"
17ttl = 30000  # 30 s
18```
19
20### Workspace con rlimits (sólo accounting)
21```toml
22label = "bounded"
23on_exit = "reap"
24
25[soma.rlimits]
26mem_bytes = 10485760    # 10 MiB — visible en `shipote workspace quota`
27nproc = 4
28```
29
30### Workspace con enforcement automático
31```toml
32label = "strict"
33on_exit = "reap"
34
35[soma.rlimits]
36mem_bytes = 5242880
37nproc = 2
38
39[quota_enforce]
40mem = "kill"            # None | Log | Kill
41nproc = "kill"
42```
43
44### Workspace con cgroup delegado (kernel enforces)
45```toml
46label = "cgroup-enforced"
47on_exit = "reap"
48
49[soma.rlimits]
50mem_bytes = 10485760
51nproc = 4
52
53[soma.cgroup]
54path = "shipote/bounded"   # bajo $cgroup_actual/shipote/bounded
55```
56> Requiere `cgroup_delegated: true` en `shipote caps`. Sino el accounting funciona pero el kernel no enforces.
57
58### Workspace con namespacing real
59```toml
60label = "isolated"
61on_exit = "reap"
62
63[soma.namespaces]
64user = true
65pid = true
66mount = true
67net = false
68uts = false
69ipc = false
70cgroup = false
71
72[soma.rlimits]
73mem_bytes = 0
74nproc = 0
75nofile = 0
76
77[soma.cgroup]
78path = ""
79```
80> Requiere `user_ns: Allowed` (o `cap_sys_admin: true`).
81
82## Pipelines
83
84### Pipeline lineal con tap (data plane)
85```toml
86label = "echo-cat"
87workspace = "<WS>"
88discern = { sample_bytes = 4096, enrich_producer = true }
89
90[[nodes]]
91label = "producer"
92payload.Native = { exec = "/bin/echo", argv = ['{"hello": 1}'], envp = [] }
93
94[[nodes]]
95label = "consumer"
96payload.Native = { exec = "/bin/cat", argv = [], envp = [] }
97
98[[edges]]
99from = 0
100from_output = "stdout"
101to = 1
102to_input = "stdin"
103```
104
105Run:
106```sh
107shipote pipeline run echo-cat.toml --tap
108# imprime: edge ty=json mime=application/json conf=0.95
109```
110
111### Pipeline con fan-out (1 → N)
112```toml
113label = "broadcast"
114workspace = "<WS>"
115discern = { sample_bytes = 4096, enrich_producer = true }
116
117[[nodes]]
118label = "src"
119payload.Native = { exec = "/bin/echo", argv = ["mensaje compartido"], envp = [] }
120
121[[nodes]]
122label = "wc-c"
123payload.Native = { exec = "/usr/bin/wc", argv = ["-c"], envp = [] }
124
125[[nodes]]
126label = "wc-l"
127payload.Native = { exec = "/usr/bin/wc", argv = ["-l"], envp = [] }
128
129[[edges]]
130from = 0
131from_output = "stdout"
132to = 1
133to_input = "stdin"
134
135[[edges]]
136from = 0
137from_output = "stdout"
138to = 2
139to_input = "stdin"
140```
141
142### Pipeline con fan-in (N → 1)
143```toml
144label = "merge"
145workspace = "<WS>"
146
147[[nodes]]
148label = "p1"
149payload.Native = { exec = "/bin/echo", argv = ["from-p1"], envp = [] }
150
151[[nodes]]
152label = "p2"
153payload.Native = { exec = "/bin/echo", argv = ["from-p2"], envp = [] }
154
155[[nodes]]
156label = "merge-sink"
157payload.Native = { exec = "/bin/cat", argv = [], envp = [] }
158
159[[edges]]
160from = 0
161from_output = "stdout"
162to = 2
163to_input = "stdin"
164
165[[edges]]
166from = 1
167from_output = "stdout"
168to = 2
169to_input = "stdin"
170```
171
172### Pipeline con replay y rate-limit
173```toml
174label = "throttled"
175workspace = "<WS>"
176
177[discern]
178sample_bytes = 4096
179enrich_producer = true
180replay_chunks = 32        # default
181replay_bytes = 65536      # cap adicional por bytes (0 = sólo chunks)
182max_bytes_per_sec = 1024  # token-bucket con burst de 1s
183
184[[nodes]]
185label = "fast"
186payload.Native = { exec = "/bin/sh", argv = ["-c", "for i in 1 2 3 4 5; do echo line-$i; done"], envp = [] }
187
188[[nodes]]
189label = "sink"
190payload.Native = { exec = "/bin/cat", argv = [], envp = [] }
191
192[[edges]]
193from = 0
194from_output = "stdout"
195to = 1
196to_input = "stdin"
197```
198
199### Pipeline supervisado (restart on failure con backoff)
200```toml
201label = "supervised"
202workspace = "<WS>"
203restart_on_failure = true
204restart_backoff_ms = 200       # inicial; escala x2
205restart_max_backoff_ms = 30000 # cap
206restart_max = 5                # 0 = infinito
207
208[[nodes]]
209label = "flaky"
210payload.Native = { exec = "/bin/false", argv = [], envp = [] }
211```
212
213Después de 5 restarts (`/bin/false` siempre exit=1), el daemon loguea `restart_max reached — giving up` y el supervisor se descarta.
214
215### Pipeline con templating
216Spec con placeholders:
217```toml
218label = "tmpl-${VARIANT}"
219workspace = "<WS>"
220discern = { sample_bytes = 4096, enrich_producer = true }
221
222[[nodes]]
223label = "gen-${VARIANT}"
224payload.Native = { exec = "/bin/echo", argv = ["greeting from ${VARIANT}"], envp = [] }
225
226[[nodes]]
227label = "sink"
228payload.Native = { exec = "/bin/cat", argv = [], envp = [] }
229
230[[edges]]
231from = 0
232from_output = "stdout"
233to = 1
234to_input = "stdin"
235```
236
237Run con vars:
238```sh
239shipote pipeline run tmpl.toml --var VARIANT=alpha
240shipote pipeline run tmpl.toml --var VARIANT=beta
241```
242
243Variables sin match quedan intactas (útil para detectar olvidos).
244
245## Subscribers externos
246
247### Tail directo a un flow socket
248```sh
249shipote pipeline run mypipe.toml --tap &
250sleep 0.3
251SOCK=$(shipote flow list | grep shipote-flow | xargs)
252shipote flow tail "$SOCK"
253```
254
255Si conectás tarde, el replay buffer te entrega los últimos N chunks (según `replay_chunks` y `replay_bytes` del spec).
256
257### Modo live-tail integrado
258```sh
259shipote pipeline run mypipe.toml --tail
260# vuelca el primer flow_socket a stdout hasta que el productor termine.
261```
262
263## Combinatorias útiles
264
265### Workspace con cleanup automático
266```toml
267label = "burst-and-die"
268on_exit = "reap"
269ttl = 10000  # auto-stop a los 10s
270
271[soma.rlimits]
272mem_bytes = 5242880
273```
274
275### Pipeline JSON-aware con discern enriched
276- Producer escribe JSON.
277- Discern detecta `application/json` con confidence 0.95.
278- Card efímera anunciada al broker (si está corriendo): `shipote.flow.<id>.<from>.<output>.json`.
279- Subscribers downstream pueden filtrar por TypeRef en el broker.

RECIPES.md

Recetas de specs

Specs TOML para casos comunes. Todas asumen que <WS> ya existe (creado con shipote workspace create <ws-spec>).

Workspaces

Workspace mínimo

label = "demo"
on_exit = "reap"

Workspace con TTL (auto-stop tras N ms)

label = "ephemeral"
on_exit = "reap"
ttl = 30000  # 30 s

Workspace con rlimits (sólo accounting)

label = "bounded"
on_exit = "reap"

[soma.rlimits]
mem_bytes = 10485760    # 10 MiB — visible en `shipote workspace quota`
nproc = 4

Workspace con enforcement automático

label = "strict"
on_exit = "reap"

[soma.rlimits]
mem_bytes = 5242880
nproc = 2

[quota_enforce]
mem = "kill"            # None | Log | Kill
nproc = "kill"

Workspace con cgroup delegado (kernel enforces)

label = "cgroup-enforced"
on_exit = "reap"

[soma.rlimits]
mem_bytes = 10485760
nproc = 4

[soma.cgroup]
path = "shipote/bounded"   # bajo $cgroup_actual/shipote/bounded

Requiere cgroup_delegated: true en shipote caps. Sino el accounting funciona pero el kernel no enforces.

Workspace con namespacing real

label = "isolated"
on_exit = "reap"

[soma.namespaces]
user = true
pid = true
mount = true
net = false
uts = false
ipc = false
cgroup = false

[soma.rlimits]
mem_bytes = 0
nproc = 0
nofile = 0

[soma.cgroup]
path = ""

Requiere user_ns: Allowed (o cap_sys_admin: true).

Pipelines

Pipeline lineal con tap (data plane)

label = "echo-cat"
workspace = "<WS>"
discern = { sample_bytes = 4096, enrich_producer = true }

[[nodes]]
label = "producer"
payload.Native = { exec = "/bin/echo", argv = ['{"hello": 1}'], envp = [] }

[[nodes]]
label = "consumer"
payload.Native = { exec = "/bin/cat", argv = [], envp = [] }

[[edges]]
from = 0
from_output = "stdout"
to = 1
to_input = "stdin"

Run:

shipote pipeline run echo-cat.toml --tap
# imprime: edge ty=json mime=application/json conf=0.95

Pipeline con fan-out (1 → N)

label = "broadcast"
workspace = "<WS>"
discern = { sample_bytes = 4096, enrich_producer = true }

[[nodes]]
label = "src"
payload.Native = { exec = "/bin/echo", argv = ["mensaje compartido"], envp = [] }

[[nodes]]
label = "wc-c"
payload.Native = { exec = "/usr/bin/wc", argv = ["-c"], envp = [] }

[[nodes]]
label = "wc-l"
payload.Native = { exec = "/usr/bin/wc", argv = ["-l"], envp = [] }

[[edges]]
from = 0
from_output = "stdout"
to = 1
to_input = "stdin"

[[edges]]
from = 0
from_output = "stdout"
to = 2
to_input = "stdin"

Pipeline con fan-in (N → 1)

label = "merge"
workspace = "<WS>"

[[nodes]]
label = "p1"
payload.Native = { exec = "/bin/echo", argv = ["from-p1"], envp = [] }

[[nodes]]
label = "p2"
payload.Native = { exec = "/bin/echo", argv = ["from-p2"], envp = [] }

[[nodes]]
label = "merge-sink"
payload.Native = { exec = "/bin/cat", argv = [], envp = [] }

[[edges]]
from = 0
from_output = "stdout"
to = 2
to_input = "stdin"

[[edges]]
from = 1
from_output = "stdout"
to = 2
to_input = "stdin"

Pipeline con replay y rate-limit

label = "throttled"
workspace = "<WS>"

[discern]
sample_bytes = 4096
enrich_producer = true
replay_chunks = 32        # default
replay_bytes = 65536      # cap adicional por bytes (0 = sólo chunks)
max_bytes_per_sec = 1024  # token-bucket con burst de 1s

[[nodes]]
label = "fast"
payload.Native = { exec = "/bin/sh", argv = ["-c", "for i in 1 2 3 4 5; do echo line-$i; done"], envp = [] }

[[nodes]]
label = "sink"
payload.Native = { exec = "/bin/cat", argv = [], envp = [] }

[[edges]]
from = 0
from_output = "stdout"
to = 1
to_input = "stdin"

Pipeline supervisado (restart on failure con backoff)

label = "supervised"
workspace = "<WS>"
restart_on_failure = true
restart_backoff_ms = 200       # inicial; escala x2
restart_max_backoff_ms = 30000 # cap
restart_max = 5                # 0 = infinito

[[nodes]]
label = "flaky"
payload.Native = { exec = "/bin/false", argv = [], envp = [] }

Después de 5 restarts (/bin/false siempre exit=1), el daemon loguea restart_max reached — giving up y el supervisor se descarta.

Pipeline con templating

Spec con placeholders:

label = "tmpl-${VARIANT}"
workspace = "<WS>"
discern = { sample_bytes = 4096, enrich_producer = true }

[[nodes]]
label = "gen-${VARIANT}"
payload.Native = { exec = "/bin/echo", argv = ["greeting from ${VARIANT}"], envp = [] }

[[nodes]]
label = "sink"
payload.Native = { exec = "/bin/cat", argv = [], envp = [] }

[[edges]]
from = 0
from_output = "stdout"
to = 1
to_input = "stdin"

Run con vars:

shipote pipeline run tmpl.toml --var VARIANT=alpha
shipote pipeline run tmpl.toml --var VARIANT=beta

Variables sin match quedan intactas (útil para detectar olvidos).

Subscribers externos

Tail directo a un flow socket

shipote pipeline run mypipe.toml --tap &
sleep 0.3
SOCK=$(shipote flow list | grep shipote-flow | xargs)
shipote flow tail "$SOCK"

Si conectás tarde, el replay buffer te entrega los últimos N chunks (según replay_chunks y replay_bytes del spec).

Modo live-tail integrado

shipote pipeline run mypipe.toml --tail
# vuelca el primer flow_socket a stdout hasta que el productor termine.

Combinatorias útiles

Workspace con cleanup automático

label = "burst-and-die"
on_exit = "reap"
ttl = 10000  # auto-stop a los 10s

[soma.rlimits]
mem_bytes = 5242880

Pipeline JSON-aware con discern enriched