Add EC2 source-code discovery and reliability triage
Second discovery pass over the Scrivas EC2 estate (716468089330, us-east-2, 7 instances), prompted by the client reporting reliability issues and by their lack of access to source code held under contract by the incumbent vendor. Source code recovery - All 10 application repositories exist as complete git checkouts on Scrivas-owned instances, with full history rather than deployed artifacts: 4 app repos on Scrivas_dev_env, 6 ML repos on ML_dev. - Every remote points at git@git.devteam.space (the contractor's self-hosted GitLab), which Scrivas does not control. The on-instance checkouts are the client's only independent leverage over their own source. - Gap: both /var/www frontends are build output with no .git, so frontend source is not recoverable from EC2. - Time-sensitive: scrivas_backend received a commit on the assessment date. Reliability triage - Production runs 23 containers on a single 15 GiB host, including 3 Postgres instances, Kafka and OpenSearch, at 73% memory at rest with no per-container memory limits and no swap on any of the 7 instances. - Kafka, OpenSearch and search-api carry restart policy `no`, so a host reboot yields a partially-recovered stack. - Recorded as a structural exposure, not an observed root cause: no OOM event is present in retained logs and RestartCount is 0 on every prod container. Confirming the hypothesis needs CloudWatch history the boxes do not retain. Contents - scripts/ec2_code_discovery.py EC2 inventory (describe + user data) - scripts/ec2_code_inspect.py read-only SSM probe set, reviewable in PROBES - findings/ec2_code_discovery_report.md narrative writeup - findings/code_dashboard.html client-facing dashboard - findings/ec2_code_inspect*.json raw probe output - index.html links the new dashboard and evidence All access was read-only: no writes, restarts or config changes on any instance. Probe output was scanned for credentials before commit; git metadata was read as the owning user rather than by writing a safe.directory entry. Claude-Session: https://claude.ai/code/session_01YMxVaHXJsqpqKwncNQ9b1e
This commit is contained in:
Alvaro Del Valle
@@ -0,0 +1,318 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Scrivas — Source Code & Reliability</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg:#eef0f3; --panel:#ffffff; --panel-2:#f7f8fa; --ink:#171d27;
|
||||
--ink-soft:#4a5563; --ink-faint:#79828f; --line:#dde1e7; --line-soft:#e8ebef;
|
||||
--accent:#0d6e78; --accent-ink:#0a565e;
|
||||
--good:#2f8657; --warn:#b5832a; --crit:#bd463b;
|
||||
--crit-bg:#fbeceb; --good-bg:#eaf4ee; --warn-bg:#f8f1e2;
|
||||
--mono: ui-monospace,"SF Mono","SFMono-Regular",Menlo,Consolas,monospace;
|
||||
--sans: ui-sans-serif,system-ui,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;
|
||||
--shadow:0 1px 2px rgba(20,26,38,.04),0 6px 20px -12px rgba(20,26,38,.18);
|
||||
}
|
||||
@media (prefers-color-scheme:dark){:root{
|
||||
--bg:#0d1117;--panel:#161c26;--panel-2:#1b222e;--ink:#e8ecf1;--ink-soft:#aab3c0;
|
||||
--ink-faint:#7c8695;--line:#28303c;--line-soft:#212934;--accent:#3bb4bf;--accent-ink:#8fdfe6;
|
||||
--good:#54b681;--warn:#d6a94a;--crit:#e26b60;--crit-bg:#2a1917;--good-bg:#132419;--warn-bg:#251d10;
|
||||
--shadow:0 1px 2px rgba(0,0,0,.3),0 8px 24px -14px rgba(0,0,0,.6);}}
|
||||
:root[data-theme="dark"]{
|
||||
--bg:#0d1117;--panel:#161c26;--panel-2:#1b222e;--ink:#e8ecf1;--ink-soft:#aab3c0;
|
||||
--ink-faint:#7c8695;--line:#28303c;--line-soft:#212934;--accent:#3bb4bf;--accent-ink:#8fdfe6;
|
||||
--good:#54b681;--warn:#d6a94a;--crit:#e26b60;--crit-bg:#2a1917;--good-bg:#132419;--warn-bg:#251d10;
|
||||
--shadow:0 1px 2px rgba(0,0,0,.3),0 8px 24px -14px rgba(0,0,0,.6);}
|
||||
:root[data-theme="light"]{
|
||||
--bg:#eef0f3;--panel:#ffffff;--panel-2:#f7f8fa;--ink:#171d27;--ink-soft:#4a5563;
|
||||
--ink-faint:#79828f;--line:#dde1e7;--line-soft:#e8ebef;--accent:#0d6e78;--accent-ink:#0a565e;
|
||||
--good:#2f8657;--warn:#b5832a;--crit:#bd463b;--crit-bg:#fbeceb;--good-bg:#eaf4ee;--warn-bg:#f8f1e2;
|
||||
--shadow:0 1px 2px rgba(20,26,38,.04),0 6px 20px -12px rgba(20,26,38,.18);}
|
||||
|
||||
*{box-sizing:border-box;}
|
||||
body{margin:0;background:var(--bg);color:var(--ink);font-family:var(--sans);line-height:1.5;-webkit-font-smoothing:antialiased;}
|
||||
.wrap{max-width:1120px;margin:0 auto;padding:32px 24px 64px;}
|
||||
header{display:flex;flex-wrap:wrap;align-items:flex-end;justify-content:space-between;gap:16px;padding-bottom:20px;border-bottom:2px solid var(--ink);}
|
||||
.eyebrow{font-size:11px;letter-spacing:.16em;text-transform:uppercase;color:var(--accent);font-weight:700;}
|
||||
h1{font-size:clamp(24px,4vw,34px);margin:6px 0 4px;letter-spacing:-.02em;text-wrap:balance;font-weight:750;}
|
||||
.sub{color:var(--ink-soft);font-size:14px;}
|
||||
.idcard{font-family:var(--mono);font-size:12px;color:var(--ink-soft);text-align:right;line-height:1.7;}
|
||||
.idcard b{color:var(--ink);}
|
||||
|
||||
.kpis{display:grid;grid-template-columns:repeat(4,1fr);gap:14px;margin:22px 0 12px;}
|
||||
@media (max-width:760px){.kpis{grid-template-columns:repeat(2,1fr);}}
|
||||
.kpi{background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:16px;box-shadow:var(--shadow);position:relative;overflow:hidden;}
|
||||
.kpi::before{content:"";position:absolute;left:0;top:0;bottom:0;width:3px;background:var(--accent);}
|
||||
.kpi.crit::before{background:var(--crit);}.kpi.warn::before{background:var(--warn);}.kpi.good::before{background:var(--good);}
|
||||
.kpi .n{font-size:28px;font-weight:750;letter-spacing:-.02em;font-variant-numeric:tabular-nums;}
|
||||
.kpi.crit .n{color:var(--crit);}.kpi.warn .n{color:var(--warn);}.kpi.good .n{color:var(--good);}
|
||||
.kpi .lbl{font-size:12px;color:var(--ink-faint);margin-top:2px;}
|
||||
.kpi .meta{font-size:11px;color:var(--ink-soft);margin-top:8px;}
|
||||
|
||||
section{background:var(--panel);border:1px solid var(--line);border-radius:14px;padding:22px 24px;margin-bottom:18px;box-shadow:var(--shadow);}
|
||||
.sec-head{display:flex;align-items:baseline;gap:10px;margin-bottom:16px;flex-wrap:wrap;}
|
||||
.sec-head .tag{font-family:var(--mono);font-size:12px;color:var(--accent);font-weight:700;}
|
||||
.sec-head h2{font-size:17px;margin:0;letter-spacing:-.01em;}
|
||||
.sec-head .note{margin-left:auto;font-size:12px;color:var(--ink-faint);}
|
||||
.grid2{display:grid;grid-template-columns:1fr 1fr;gap:18px;}
|
||||
@media (max-width:820px){.grid2{grid-template-columns:1fr;}}
|
||||
|
||||
table{width:100%;border-collapse:collapse;font-size:13px;}
|
||||
th{text-align:left;font-size:11px;text-transform:uppercase;letter-spacing:.07em;color:var(--ink-faint);
|
||||
font-weight:700;padding:0 10px 8px 0;border-bottom:1px solid var(--line);}
|
||||
td{padding:9px 10px 9px 0;border-bottom:1px solid var(--line-soft);vertical-align:top;color:var(--ink-soft);}
|
||||
td b{color:var(--ink);}
|
||||
tbody tr:last-child td{border-bottom:none;}
|
||||
.num{font-family:var(--mono);font-variant-numeric:tabular-nums;text-align:right;color:var(--ink);}
|
||||
code{font-family:var(--mono);font-size:12px;color:var(--ink-soft);}
|
||||
.scroll{overflow-x:auto;}
|
||||
|
||||
.pill{display:inline-flex;align-items:center;gap:6px;font-size:11px;font-weight:700;padding:3px 9px;border-radius:999px;white-space:nowrap;}
|
||||
.pill.crit{background:var(--crit-bg);color:var(--crit);}
|
||||
.pill.good{background:var(--good-bg);color:var(--good);}
|
||||
.pill.warn{background:var(--warn-bg);color:var(--warn);}
|
||||
|
||||
.callout{border-radius:10px;padding:14px 16px;font-size:13px;color:var(--ink-soft);margin:0 0 16px;
|
||||
background:var(--warn-bg);border:1px solid color-mix(in srgb,var(--warn) 30%,transparent);}
|
||||
.callout.crit{background:var(--crit-bg);border-color:color-mix(in srgb,var(--crit) 30%,transparent);}
|
||||
.callout.good{background:var(--good-bg);border-color:color-mix(in srgb,var(--good) 30%,transparent);}
|
||||
.callout b{color:var(--ink);}
|
||||
.callout p{margin:8px 0 0;}
|
||||
|
||||
.memtrack{height:26px;background:var(--panel-2);border:1px solid var(--line);border-radius:7px;overflow:hidden;display:flex;margin:6px 0 4px;}
|
||||
.memtrack .used{background:var(--crit);height:100%;}
|
||||
.memtrack .cache{background:color-mix(in srgb,var(--warn) 55%,transparent);height:100%;}
|
||||
.memlegend{display:flex;gap:16px;font-size:11.5px;color:var(--ink-faint);flex-wrap:wrap;}
|
||||
.memlegend i{display:inline-block;width:9px;height:9px;border-radius:2px;margin-right:5px;vertical-align:baseline;}
|
||||
|
||||
ol.steps{margin:0;padding-left:20px;font-size:13px;color:var(--ink-soft);}
|
||||
ol.steps li{margin-bottom:9px;}
|
||||
ol.steps b{color:var(--ink);}
|
||||
|
||||
footer{display:flex;flex-wrap:wrap;gap:18px;margin-top:24px;font-size:12px;color:var(--ink-faint);font-family:var(--mono);}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
|
||||
<header>
|
||||
<div>
|
||||
<div class="eyebrow">Dasnuve · Cloud Discovery</div>
|
||||
<h1>Source Code Recovery & Reliability Triage</h1>
|
||||
<div class="sub">What is running on the Scrivas EC2 estate, where the source lives, and why production is fragile.</div>
|
||||
</div>
|
||||
<div class="idcard">
|
||||
Account <b>716468089330</b><br>
|
||||
Region <b>us-east-2</b> · 7 instances<br>
|
||||
2026-08-28 · <b>CONFIDENTIAL</b>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="kpis">
|
||||
<div class="kpi good">
|
||||
<div class="n">10</div>
|
||||
<div class="lbl">Repos recoverable on EC2</div>
|
||||
<div class="meta">Full git history, not just build output</div>
|
||||
</div>
|
||||
<div class="kpi warn">
|
||||
<div class="n">2</div>
|
||||
<div class="lbl">Frontends <u>not</u> recoverable</div>
|
||||
<div class="meta">Deployed build only — no <code>.git</code></div>
|
||||
</div>
|
||||
<div class="kpi crit">
|
||||
<div class="n">23</div>
|
||||
<div class="lbl">Prod containers, one host</div>
|
||||
<div class="meta">Incl. 3 × Postgres, Kafka, OpenSearch</div>
|
||||
</div>
|
||||
<div class="kpi crit">
|
||||
<div class="n">0</div>
|
||||
<div class="lbl">Memory limits set</div>
|
||||
<div class="meta">And zero swap, on all 7 instances</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section>
|
||||
<div class="sec-head">
|
||||
<span class="tag">01</span><h2>The source code is recoverable — and that is time-sensitive</h2>
|
||||
<span class="note">Scrivas_dev_env · ML_dev</span>
|
||||
</div>
|
||||
|
||||
<div class="callout good">
|
||||
<b>All 10 application repositories exist as complete git checkouts on Scrivas-owned instances</b>, with full
|
||||
commit history — not merely deployed artifacts. Scrivas can recover its codebase from its own AWS account today.
|
||||
<p>Every repository points at a single external remote: <code>git@git.devteam.space:scrivas/<repo>.git</code> —
|
||||
the incumbent contractor's self-hosted GitLab, which Scrivas does <b>not</b> control. The on-instance checkouts are
|
||||
therefore the client's only independent leverage over their own source.</p>
|
||||
</div>
|
||||
|
||||
<div class="grid2">
|
||||
<div>
|
||||
<div class="sec-head"><span class="tag">APP</span><h2>Scrivas_dev_env <code>/home/admin/</code></h2></div>
|
||||
<div class="scroll"><table>
|
||||
<thead><tr><th>Repository</th><th class="num">Commits</th><th>Last commit</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td><b>scrivas_backend</b></td><td class="num">1380</td><td>2026-08-28 · Vasilii</td></tr>
|
||||
<tr><td><b>scrivas_gate</b></td><td class="num">183</td><td>2026-08-14 · Vasilii</td></tr>
|
||||
<tr><td><b>scrivas_search</b></td><td class="num">96</td><td>2026-06-24 · Azamat</td></tr>
|
||||
<tr><td><b>patient</b></td><td class="num">59</td><td>2026-08-13 · vturtugeshev</td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="sec-head"><span class="tag">ML</span><h2>ML_dev <code>/srv/</code></h2></div>
|
||||
<div class="scroll"><table>
|
||||
<thead><tr><th>Repository</th><th class="num">Commits</th><th>Last commit</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td><b>post_processor</b></td><td class="num">137</td><td>2026-06-30</td></tr>
|
||||
<tr><td><b>sai_suggestions</b></td><td class="num">93</td><td>2026-06-16</td></tr>
|
||||
<tr><td><b>ml_monitoring</b></td><td class="num">62</td><td>2026-07-20</td></tr>
|
||||
<tr><td><b>patient-summary-service</b></td><td class="num">49</td><td>2026-06-12</td></tr>
|
||||
<tr><td><b>patient_document_parser</b></td><td class="num">41</td><td>2026-06-16</td></tr>
|
||||
<tr><td><b>soniox_transcriber</b></td><td class="num">25</td><td>2026-06-16</td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="callout" style="margin:18px 0 0;">
|
||||
<b>The gap:</b> <code>/var/www/scrivas_frontend</code> and <code>/var/www/Scrivas_admin_frontend</code> are
|
||||
<b>deployed build output with no <code>.git</code></b>. Frontend source cannot be recovered from EC2 and must be
|
||||
obtained from the contractor's GitLab.
|
||||
<p><b>Why this is urgent:</b> <code>scrivas_backend</code> received a commit <b>on the day of this assessment</b>.
|
||||
Development is active on infrastructure the contractor controls. Mirror all 10 repos
|
||||
(<code>git clone --mirror</code>) to Scrivas-controlled storage <i>before</i> any contract conversation changes
|
||||
access posture.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="sec-head">
|
||||
<span class="tag">02</span><h2>Two tiers, two engineering standards</h2>
|
||||
<span class="note">the better pattern already exists in-house</span>
|
||||
</div>
|
||||
<div class="grid2">
|
||||
<div>
|
||||
<div class="sec-head"><span class="tag">APP</span><h2>Application tier <span class="pill crit">fragile</span></h2></div>
|
||||
<p class="sub" style="font-size:13px;">23 containers on a <b>single host</b> — <code>encounter_api</code>,
|
||||
<code>gate_api</code>, <code>patient_api</code>, 8 Celery workers, <code>celery_beat</code>,
|
||||
<b>3 × Postgres 17.5</b>, Redis, Kafka, Debezium CDC, OpenSearch, autoheal.</p>
|
||||
<p class="sub" style="font-size:13px;">Deployed by <code>docker-compose</code> directly from git checkouts.
|
||||
No orchestration, no image registry, no immutable artifact.</p>
|
||||
</div>
|
||||
<div>
|
||||
<div class="sec-head"><span class="tag">ML</span><h2>ML tier <span class="pill good">mature</span></h2></div>
|
||||
<p class="sub" style="font-size:13px;">Deployed from <b>ECR</b>
|
||||
(<code>…dkr.ecr.us-east-2…/scrivas/*</code>) with <b>commit-SHA image tags</b> and
|
||||
<b>blue/green slots</b> (<code>dev-green-*</code>, <code>stage-blue-*</code>).</p>
|
||||
<p class="sub" style="font-size:13px;">Health checks defined and passing across every service.
|
||||
This is a materially stronger delivery pipeline — and a template the application tier can adopt.</p>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="sec-head">
|
||||
<span class="tag">03</span><h2>Reliability findings</h2>
|
||||
<span class="note">ranked by likely contribution to reported instability</span>
|
||||
</div>
|
||||
<div class="scroll"><table>
|
||||
<thead><tr><th>#</th><th>Finding</th><th>Evidence</th><th>Risk</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="num">1</td><td><b>No memory limit on any prod container</b></td><td><code>HostConfig.Memory=0</code> on all 23</td><td><span class="pill crit">High</span></td></tr>
|
||||
<tr><td class="num">2</td><td><b>No swap on any instance</b></td><td><code>swapon --show</code> empty, all 7</td><td><span class="pill crit">High</span></td></tr>
|
||||
<tr><td class="num">3</td><td><b>Prod at 73% memory at rest</b></td><td>11 GiB used of 15 GiB</td><td><span class="pill crit">High</span></td></tr>
|
||||
<tr><td class="num">4</td><td><b>Whole stack on one host, incl. 3 databases</b></td><td><code>docker ps</code> — 23 containers</td><td><span class="pill crit">High</span></td></tr>
|
||||
<tr><td class="num">5</td><td><b>3 prod containers set to never restart</b></td><td>Kafka, OpenSearch, search-api — policy <code>no</code></td><td><span class="pill crit">High</span></td></tr>
|
||||
<tr><td class="num">6</td><td>Prod OS differs from dev/stage</td><td>Ubuntu <b>26.04</b> vs <b>24.04</b> elsewhere</td><td><span class="pill warn">Medium</span></td></tr>
|
||||
<tr><td class="num">7</td><td><code>autoheal</code> deployed in production</td><td><code>willfarrell/autoheal:1.2.0</code></td><td><span class="pill warn">Medium</span></td></tr>
|
||||
<tr><td class="num">8</td><td>Uncommitted config drift in prod checkouts</td><td><code>docker-compose.yml.bkp</code>, <code>.env.save</code></td><td><span class="pill warn">Medium</span></td></tr>
|
||||
<tr><td class="num">9</td><td>47 interactive login sessions on prod</td><td><code>uptime</code> (32 on Ml_prod)</td><td><span class="pill warn">Medium</span></td></tr>
|
||||
<tr><td class="num">10</td><td><code>sai-suggestions-redis.service</code> failed</td><td>ML_stage <code>systemctl --state=failed</code></td><td><span class="pill good">Low</span></td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="sec-head">
|
||||
<span class="tag">04</span><h2>Primary hypothesis: unbounded memory on a shared host</h2>
|
||||
</div>
|
||||
|
||||
<p class="sub" style="font-size:13px;margin-top:0;">Production memory, at rest — before any workload spike:</p>
|
||||
<div class="memtrack" role="img" aria-label="Production memory: 11 GiB used, 3.5 GiB cache, of 15 GiB total">
|
||||
<div class="used" style="width:73%"></div><div class="cache" style="width:23%"></div>
|
||||
</div>
|
||||
<div class="memlegend">
|
||||
<span><i style="background:var(--crit)"></i>11 GiB used (73%)</span>
|
||||
<span><i style="background:color-mix(in srgb,var(--warn) 55%,transparent)"></i>3.5 GiB cache</span>
|
||||
<span><i style="background:var(--panel-2);border:1px solid var(--line)"></i>441 MiB free</span>
|
||||
<span><b style="color:var(--crit)">0 B swap</b></span>
|
||||
</div>
|
||||
|
||||
<div class="callout crit" style="margin-top:18px;">
|
||||
Findings 1–4 compound into a single failure mode. Production runs 23 containers — including three Postgres
|
||||
instances, Kafka and OpenSearch — on one 15 GiB host already <b>73% consumed at rest</b>, with
|
||||
<b>no per-container memory limits</b> and <b>no swap</b>. A single Celery worker handling an oversized document
|
||||
or audio chunk can exhaust host memory. With no limits, the kernel OOM killer selects by resident size — so it
|
||||
will typically kill a <b>database</b>, not the worker that caused the problem.
|
||||
<p>That yields precisely the profile a client reports as "random reliability issues": unpredictable, uncorrelated
|
||||
with the triggering workload, and difficult to reproduce.</p>
|
||||
<p><b>Finding 5 compounds recovery.</b> Kafka, OpenSearch and search-api carry restart policy <code>no</code> — if
|
||||
they die or the host reboots, they <b>stay down</b> until someone intervenes. Everything else is
|
||||
<code>unless-stopped</code>, so a reboot yields a partially-recovered stack that looks healthy from the outside.</p>
|
||||
<p><b>Finding 7 corroborates.</b> Someone deliberately deployed <code>autoheal</code>, a daemon whose only job is
|
||||
restarting containers that fail health checks. That treats a symptom — and indicates health-check failures were
|
||||
frequent enough to be worth automating around.</p>
|
||||
</div>
|
||||
|
||||
<div class="callout">
|
||||
<b>What the evidence does not show — stated plainly, because it bounds the conclusion.</b>
|
||||
<p>· <b>No OOM kill was captured.</b> <code>dmesg</code> and <code>journalctl</code> on prod show no OOM events in
|
||||
the retained window; <code>State.OOMKilled</code> is <code>false</code> on all 23 containers.</p>
|
||||
<p>· <b>No crash-looping.</b> <code>RestartCount</code> is <b>0</b> on every prod container. The Celery workers
|
||||
showing "Up 6 hours" were <b>redeployed</b> at 13:49 that day, not restarted by failure.</p>
|
||||
<p>· <code>last</code> is not installed on prod, so reboot history could not be read there.</p>
|
||||
<p>Findings 1–5 are therefore a demonstrated <b>structural exposure</b>, not an observed root cause. Confirming
|
||||
the hypothesis requires history these instances do not currently retain — which is itself a finding.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="sec-head"><span class="tag">05</span><h2>Recommended next steps</h2></div>
|
||||
<div class="grid2">
|
||||
<div>
|
||||
<div class="sec-head"><span class="tag">NOW</span><h2>Immediate</h2></div>
|
||||
<ol class="steps">
|
||||
<li><b>Mirror all 10 repos</b> to Scrivas-controlled storage. Highest value, lowest effort, time-sensitive.</li>
|
||||
<li><b>Set <code>unless-stopped</code></b> on Kafka, OpenSearch and search-api — removes the silent partial-recovery failure.</li>
|
||||
</ol>
|
||||
<div class="sec-head" style="margin-top:18px;"><span class="tag">PROVE</span><h2>Confirm the hypothesis</h2></div>
|
||||
<ol class="steps" start="3">
|
||||
<li><b>Enable per-container memory metrics</b> in CloudWatch — <code>EC2-CloudWatchAgent-Role</code> is already attached.</li>
|
||||
<li><b>Add <code>mem_limit</code> per service</b>, sized from observed peaks; add swap as an OOM buffer.</li>
|
||||
<li><b>Query historic <code>mem_used_percent</code></b> to date the onset of instability.</li>
|
||||
</ol>
|
||||
</div>
|
||||
<div>
|
||||
<div class="sec-head"><span class="tag">FIX</span><h2>Structural</h2></div>
|
||||
<ol class="steps" start="6">
|
||||
<li><b>Move the three Postgres instances</b> off the shared application host.</li>
|
||||
<li><b>Align prod OS/AMI with stage</b> — 26.04 vs 24.04 makes staging an unreliable rehearsal for production.</li>
|
||||
<li><b>Extend the ML tier's pattern</b> — ECR, commit-SHA tags, blue/green — to the application tier. The better pattern already exists in-house.</li>
|
||||
<li><b>Reconcile config drift</b> (<code>.env.save</code>, <code>docker-compose.yml.bkp</code>) into version control.</li>
|
||||
<li><b>Audit the 47 interactive prod sessions.</b></li>
|
||||
</ol>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<footer>
|
||||
<span>Read-only · SSM probes via <code>dasnuve-scrivas-louis-impersonation</code></span>
|
||||
<span>Reproduce: <code>scripts/ec2_code_inspect.py</code></span>
|
||||
<span>Raw: <code>findings/ec2_code_inspect.json</code></span>
|
||||
<span>Narrative: <code>findings/ec2_code_discovery_report.md</code></span>
|
||||
</footer>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,142 @@
|
||||
# Scrivas — EC2 Code Discovery & Reliability Triage
|
||||
|
||||
**Prepared by:** Dasnuve · **Date:** 2026-08-28
|
||||
**Access:** `dasnuve-scrivas-louis-impersonation` → `arn:aws:iam::716468089330:user/louis`
|
||||
**Method:** Read-only EC2 describe + SSM `AWS-RunShellScript` probes (no writes, no restarts, no config changes).
|
||||
**Scripts:** `scripts/ec2_code_discovery.py` (inventory), `scripts/ec2_code_inspect.py` (probes)
|
||||
**Raw:** `findings/ec2_code_inspect.json`, `findings/ec2_code_inspect_prod.json`
|
||||
|
||||
---
|
||||
|
||||
## 1. Headline: the source code is recoverable
|
||||
|
||||
**All 10 application repositories exist as full git checkouts on Scrivas-owned EC2 instances**, with complete
|
||||
history — not just deployed artifacts. The client can recover their codebase from their own account today.
|
||||
|
||||
All repos point to a single external remote:
|
||||
|
||||
> `git@git.devteam.space:scrivas/<repo>.git`
|
||||
|
||||
`git.devteam.space` is the **incumbent contractor's self-hosted GitLab** (DevTeam.Space; commit authors are
|
||||
`@devteamspace.com`). Scrivas does not control this host. The checkouts on EC2 are therefore the client's
|
||||
only leverage over their own source.
|
||||
|
||||
### Application repos — `Scrivas_dev_env` (i-010066e6c9027aa6e), `/home/admin/`
|
||||
|
||||
| Repo | Commits | Branch | Last commit |
|
||||
|---|---:|---|---|
|
||||
| `scrivas_backend` | **1380** | `dev` | 2026-08-28 — Vasilii |
|
||||
| `scrivas_gate` | 183 | `dev` | 2026-08-14 — Vasilii |
|
||||
| `scrivas_search` | 96 | `dev` | 2026-06-24 — Azamat |
|
||||
| `patient` | 59 | `dev` | 2026-08-13 — vturtugeshev@webiomed.ru |
|
||||
|
||||
### ML repos — `ML_dev` (i-095bd68aff22b103b), `/srv/`
|
||||
|
||||
| Repo | Commits | Last commit |
|
||||
|---|---:|---|
|
||||
| `post_processor` | 137 | 2026-06-30 — Yegor Kovalev |
|
||||
| `sai_suggestions` | 93 | 2026-06-16 |
|
||||
| `ml_monitoring` | 62 | 2026-07-20 |
|
||||
| `patient-summary-service` | 49 | 2026-06-12 |
|
||||
| `patient_document_parser` | 41 | 2026-06-16 |
|
||||
| `soniox_transcriber` | 25 | 2026-06-16 |
|
||||
|
||||
**Not in git:** `/var/www/scrivas_frontend` and `/var/www/Scrivas_admin_frontend` on the dev box are
|
||||
**deployed build output only — no `.git`**. Frontend source is *not* recoverable from EC2 and must come
|
||||
from the contractor's GitLab. This is the one genuine gap.
|
||||
|
||||
### Recommended immediate action
|
||||
Mirror all 10 repos (`git clone --mirror`) off the instances to Scrivas-controlled storage **before** any
|
||||
contract conversation changes access posture. `scrivas_backend` received a commit *today* — the contractor
|
||||
is actively developing, so the on-box checkouts are current.
|
||||
|
||||
---
|
||||
|
||||
## 2. Architecture as actually deployed
|
||||
|
||||
Two clearly different engineering standards are in play.
|
||||
|
||||
**Application tier — `scrivas_prod_env` (i-073154fb4fa773bbd), Ubuntu 26.04, m6a.xlarge, 15 GiB**
|
||||
23 containers on a **single host**: `encounter_api`, `gate_api`, `patient_api`, 8 Celery workers,
|
||||
`celery_beat`, **3× Postgres 17.5**, Redis 8.2.1, Kafka 4.1.1, Debezium CDC 2.7.3, OpenSearch, autoheal.
|
||||
Deployed via `docker-compose` from git checkouts. No orchestration, no registry.
|
||||
|
||||
**ML tier — `Ml_prod` / `ML_stage` / `ML_dev`, 7.6 GiB**
|
||||
Deployed from **ECR** (`716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/*`) with commit-SHA image
|
||||
tags and **blue/green slots** (`dev-green-*`, `stage-blue-*`). Health checks present and passing.
|
||||
This tier is materially more mature than the application tier.
|
||||
|
||||
**`Netbird_scrivas` (i-02754e7ae419cd5db)** — NetBird VPN mesh (server, dashboard, Traefik v3.6).
|
||||
|
||||
---
|
||||
|
||||
## 3. Reliability findings
|
||||
|
||||
Ranked by likely contribution to the reported instability.
|
||||
|
||||
| # | Finding | Evidence | Risk |
|
||||
|---|---|---|---|
|
||||
| 1 | **No memory limit on any prod container** | `HostConfig.Memory=0` on all 23 | **High** |
|
||||
| 2 | **No swap on any instance** | `swapon --show` empty, all 7 | **High** |
|
||||
| 3 | **Prod at 11 GiB / 15 GiB (73%) steady-state** | `free -h` | **High** |
|
||||
| 4 | **Entire prod stack on one host — incl. 3 databases** | `docker ps` | **High** |
|
||||
| 5 | **Three prod containers have restart policy `no`** | `scrivas-kafka`, `scrivas-opensearch`, `scrivas_search-search-api-1` | **High** |
|
||||
| 6 | **Prod OS differs from dev/stage** | prod Ubuntu 26.04 vs 24.04 elsewhere | Medium |
|
||||
| 7 | **`autoheal` deployed in prod** | `willfarrell/autoheal:1.2.0` | Medium |
|
||||
| 8 | **Uncommitted config drift in prod checkouts** | `docker-compose.yml.bkp`, `.bkpwithlogs`, `src/gate/.env.save` | Medium |
|
||||
| 9 | **47 interactive login sessions on prod** | `uptime` (32 on `Ml_prod`) | Medium |
|
||||
| 10 | `sai-suggestions-redis.service` failed on ML_stage | `systemctl --state=failed` | Low |
|
||||
|
||||
### The primary hypothesis: unbounded memory on a shared host
|
||||
|
||||
Findings 1–4 compound into one failure mode. Prod runs 23 containers — including three Postgres
|
||||
instances, Kafka, and OpenSearch — on a single 15 GiB box that is already **73% consumed at rest**,
|
||||
with **no per-container memory limits** and **no swap**. Any single Celery worker processing an
|
||||
oversized document or audio chunk can exhaust host memory. With no limits, the kernel OOM killer
|
||||
chooses the victim — and it will typically pick the largest RSS process, which is a **database**, not
|
||||
the worker that caused the problem.
|
||||
|
||||
That produces exactly the symptom profile a client describes as "random reliability issues":
|
||||
unpredictable, uncorrelated with the triggering workload, and hard to reproduce.
|
||||
|
||||
Finding 5 makes it worse. Kafka, OpenSearch, and the search API have restart policy `no` — if they
|
||||
die or the host reboots, **they stay down** until someone intervenes manually. Everything else is
|
||||
`unless-stopped`, so a reboot yields a partially-recovered stack.
|
||||
|
||||
The presence of `autoheal` (finding 7) is corroborating: someone deliberately installed a daemon whose
|
||||
only job is restarting containers that fail health checks. That is treatment of a symptom, and it tells
|
||||
us health-check failures were frequent enough to be worth automating around.
|
||||
|
||||
### What the evidence does *not* show
|
||||
|
||||
Stated plainly, because it constrains the conclusion:
|
||||
|
||||
- **No OOM kill was captured in the current logs.** `dmesg` and `journalctl` on prod show no OOM events
|
||||
in the retained window. `State.OOMKilled` is `false` on all 23 containers.
|
||||
- **No crash-looping.** `RestartCount` is **0** on every prod container. The Celery workers showing
|
||||
"Up 6 hours" were **redeployed** at 13:49 today, not restarted by failure.
|
||||
- `last` is not installed on prod, so reboot history could not be read there.
|
||||
|
||||
So findings 1–5 are a demonstrated **structural exposure**, not an observed root cause. Confirming the
|
||||
hypothesis requires history the boxes do not currently retain.
|
||||
|
||||
---
|
||||
|
||||
## 4. Recommended next steps
|
||||
|
||||
**Immediately**
|
||||
1. Mirror all 10 repos to Scrivas-controlled storage. Highest-value, lowest-effort, time-sensitive.
|
||||
2. Set restart policy to `unless-stopped` on Kafka, OpenSearch, and search-api.
|
||||
|
||||
**To confirm the memory hypothesis** — the current boxes cannot answer this from retained logs
|
||||
3. Enable CloudWatch per-container memory metrics (the `EC2-CloudWatchAgent-Role` profile is already attached).
|
||||
4. Add `mem_limit` per service in compose, sized from observed peaks; add swap as an OOM buffer.
|
||||
5. Query CloudWatch for historic `mem_used_percent` on prod to date the onset of instability.
|
||||
|
||||
**Structural**
|
||||
6. Move the three Postgres instances off the shared application host.
|
||||
7. Align prod OS/AMI with stage (26.04 vs 24.04 makes staging an unreliable rehearsal).
|
||||
8. Extend the ML tier's ECR + commit-SHA + blue/green pattern to the application tier — the better
|
||||
pattern already exists in-house.
|
||||
9. Reconcile the uncommitted prod config drift into version control.
|
||||
10. Audit the 47 interactive prod sessions.
|
||||
@@ -0,0 +1,302 @@
|
||||
{
|
||||
"generated": "2026-08-28T15:49:39.403488-04:00",
|
||||
"region": "us-east-2",
|
||||
"probes": [
|
||||
"os_release",
|
||||
"uptime_load",
|
||||
"disk",
|
||||
"memory",
|
||||
"code_dirs",
|
||||
"git_checkouts",
|
||||
"git_remotes",
|
||||
"manifests",
|
||||
"processes",
|
||||
"listening",
|
||||
"systemd_units",
|
||||
"systemd_failed",
|
||||
"docker",
|
||||
"docker_images",
|
||||
"oom_kills",
|
||||
"svc_restarts",
|
||||
"reboots",
|
||||
"cron"
|
||||
],
|
||||
"results": {
|
||||
"i-095bd68aff22b103b": {
|
||||
"os_release": {
|
||||
"status": "Success",
|
||||
"stdout": "PRETTY_NAME=\"Ubuntu 24.04.4 LTS\"\nNAME=\"Ubuntu\"\nVERSION_ID=\"24.04\"\nVERSION=\"24.04.4 LTS (Noble Numbat)\"\nVERSION_CODENAME=noble\nID=ubuntu\nID_LIKE=debian\nHOME_URL=\"https://www.ubuntu.com/\"\nSUPPORT_URL=\"https://help.ubuntu.com/\"\nBUG_REPORT_URL=\"https://bugs.launchpad.net/ubuntu/\"\nPRIVACY_POLICY_URL=\"https://www.ubuntu.com/legal/terms-and-policies/privacy-policy\"\nUBUNTU_CODENAME=noble\nLOGO=ubuntu-logo\nLinux ip-172-31-26-26 7.0.0-1010-aws #10~24.04.1-Ubuntu SMP PREEMPT Mon Jul 27 17:41:33 UTC 2026 x86_64 x86_64 x86_64 GNU/Linux",
|
||||
"stderr": ""
|
||||
},
|
||||
"uptime_load": {
|
||||
"status": "Success",
|
||||
"stdout": " 19:49:42 up 15 days, 13:34, 0 user, load average: 0.07, 0.11, 0.09\n0.07 0.11 0.09 1/556 3708993",
|
||||
"stderr": ""
|
||||
},
|
||||
"disk": {
|
||||
"status": "Success",
|
||||
"stdout": "Filesystem Size Used Avail Use% Mounted on\n/dev/root 193G 46G 148G 24% /\ntmpfs 3.8G 0 3.8G 0% /dev/shm\ntmpfs 1.6G 2.2M 1.6G 1% /run\ntmpfs 5.0M 0 5.0M 0% /run/lock\nefivarfs 128K 3.8K 120K 4% /sys/firmware/efi/efivars\n/dev/nvme0n1p16 881M 202M 617M 25% /boot\n/dev/nvme0n1p15 105M 6.2M 99M 6% /boot/efi\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/e44ad8ada982413e134a0f808f5258611b96a2be7e5e78186d8098aef0ef5e42\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/d991d98f4a596c46d465054646dbf70ed26353b02a78c1c7c9752fe31650373c\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/f6063b3345a8dc4cb95c1142a63c2492122ed481aaf4120267445e9eabe926ac\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/0698fd968fa90c2803cc9168d05554a2af6f1f25f0280bae4ae0b5fbd84136aa\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/ac62d3eaf65e4733f6f83f85f409553724b99a0b87494ebfc3d607b0295cd84d\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/a8f1465efc695cbe55fc1b0a2a3352ad0532104847f062065bd3dc7e7075a5aa\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/15f495363998b7c196826202b6aa584976bf8b4a37ece873c3d3869caa973efe\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/e14d551d7fb67c76ff6533c116b0a70ba63f6523034278e508aa891c7fb4cc6e\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/d1bb2ab90222d9a4a1fc77c300dd48907993721c1b95ed4bead27653a20d3a07\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/e1312221d3f3e67b195ca1d561cce9984cd71cde22d7f04bf04b13558a8160b3\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/dcc956a69830a91dd3c16703e5653dfa277c007c2e8ceb6a516171ca0367cefa\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/83dcd9e8930be916501aef45855ffbf3b4eeba87ccc4ececf30198be162c5cd5\noverlay 193G 46G 148G 24% /var/lib/docker/rootfs/overlayfs/58d8d5bb29c0eae51f0fd8601bf8c1ef5163bae1ad41b2be27ce04e0ccdc052c\n--- inodes\nFilesystem Inodes IUsed IFree IUse% Mounted on\n/dev/root 26083328 763326 25320002 3% /\ntmpfs 989732 2 989730 1% /dev/shm\ntmpfs 819200 1143 818057 1% /run\ntmpfs 989732 3 989729 1% /run/lock\nefivarfs 0 0 0 - /sys/firmware/efi/efivars\n/dev/nvme0n1p16 58496 605 57891 2% /boot\n/dev/nvme0n1p15 0 0 0 - /boot/efi\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/e44ad8ada982413e134a0f808f5258611b96a2be7e5e78186d8098aef0ef5e42\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/d991d98f4a596c46d465054646dbf70ed26353b02a78c1c7c9752fe31650373c\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/f6063b3345a8dc4cb95c1142a63c2492122ed481aaf4120267445e9eabe926ac\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/0698fd968fa90c2803cc9168d05554a2af6f1f25f0280bae4ae0b5fbd84136aa\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/ac62d3eaf65e4733f6f83f85f409553724b99a0b87494ebfc3d607b0295cd84d\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/a8f1465efc695cbe55fc1b0a2a3352ad0532104847f062065bd3dc7e7075a5aa\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/15f495363998b7c196826202b6aa584976bf8b4a37ece873c3d3869caa973efe\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/e14d551d7fb67c76ff6533c116b0a70ba63f6523034278e508aa891c7fb4cc6e\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/d1bb2ab90222d9a4a1fc77c300dd48907993721c1b95ed4bead27653a20d3a07\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/e1312221d3f3e67b195ca1d561cce9984cd71cde22d7f04bf04b13558a8160b3\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/dcc956a69830a91dd3c16703e5653dfa277c007c2e8ceb6a516171ca0367cefa\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/83dcd9e8930be916501aef45855ffbf3b4eeba87ccc4ececf30198be162c5cd5\noverlay 26083328 763326 25320002 3% /var/lib/docker/rootfs/overlayfs/58d8d5bb29c0eae51f0fd8601bf8c1ef5163bae1ad41b2be27ce04e0ccdc052c",
|
||||
"stderr": ""
|
||||
},
|
||||
"memory": {
|
||||
"status": "Success",
|
||||
"stdout": " total used free shared buff/cache available\nMem: 7.6Gi 2.3Gi 2.5Gi 4.4Mi 3.1Gi 5.3Gi\nSwap: 0B 0B 0B\n--- swap",
|
||||
"stderr": ""
|
||||
},
|
||||
"code_dirs": {
|
||||
"status": "Success",
|
||||
"stdout": "/home:\ntotal 20\ndrwxr-xr-x 5 root root 4096 Jun 23 16:13 .\ndrwxr-xr-x 23 root root 4096 Aug 13 06:14 ..\ndrwxr-x--- 5 deploy deploy 4096 Jul 13 12:56 deploy\ndrwxr-x--- 2 gitlab-runner gitlab-runner 4096 Jun 23 16:13 gitlab-runner\ndrwxr-x--- 12 ubuntu ubuntu 4096 Jun 18 06:32 ubuntu\n\n/opt:\ntotal 40\ndrwxr-xr-x 6 root root 4096 Jul 15 13:42 .\ndrwxr-xr-x 23 root root 4096 Aug 13 06:14 ..\n-rw-rw-r-- 1 deploy deploy 797 Jul 9 18:26 .env\n-rw-r--r-- 1 deploy deploy 689 Jun 16 04:10 .env.bak.20260616-041003\ndrwxr-xr-x 5 root root 4096 Aug 26 00:53 aws\ndrwx--x--x 4 root root 4096 Mar 5 15:13 containerd\n-rw-r--r-- 1 root root 2412 Nov 30 2023 default\n-rw-r--r-- 1 root root 1659 Jun 23 11:01 ml-dev\ndrwxr-xr-x 11 root root 4096 Jun 18 10:06 patient-summary-service\ndrwxr-xr-x 14 root root 4096 Jun 16 10:13 post_processor\n\n/srv:\ntotal 56\ndrwxr-xr-x 10 ubuntu ubuntu 4096 Jun 24 09:22 .\ndrwxr-xr-x 23 root root 4096 Aug 13 06:14 ..\n-rw-rw-r-- 1 ubuntu ubuntu 15164 Mar 31 19:00 AGENTS.md\ndrwxr-xr-x 9 deploy deploy 4096 Jul 20 19:37 ml_monitoring\ndrwxr-xr-x 11 ubuntu ubuntu 4096 Jun 16 15:05 patient-summary-service\ndrwxr-xr-x 12 ubuntu ubuntu 4096 Jun 16 15:04 patient_document_parser\ndrwxr-xr-x 14 deploy deploy 4096 Jun 30 15:39 post_processor\ndrwxr-xr-x 2 root root 4096 Jun 24 09:22 redis-tls\ndrwxr-xr-x 4 root root 4096 Jun 24 08:55 redis_db\ndrwxr-xr-x 23 ubuntu ubuntu 4096 Jun 16 03:33 sai_suggestions\ndrwxr-xr-x 11 ubuntu ubuntu 4096 Jun 16 03:33 soniox_transcriber\n\n/var/www:\ntotal 12\ndrwxr-xr-x 3 root root 4096 May 1 12:04 .\ndrwxr-xr-x 14 root root 4096 May 1 12:04 ..\ndrwxr-xr-x 2 root root 4096 May 1 12:04 html",
|
||||
"stderr": ""
|
||||
},
|
||||
"git_checkouts": {
|
||||
"status": "Success",
|
||||
"stdout": "/srv/sai_suggestions/.git\n/srv/patient-summary-service/.git\n/srv/patient_document_parser/.git\n/srv/ml_monitoring/.git\n/srv/post_processor/.git\n/srv/soniox_transcriber/.git\n/opt/patient-summary-service/.git\n/opt/post_processor/.git\n/home/ubuntu/.codex/.tmp/plugins/.git",
|
||||
"stderr": ""
|
||||
},
|
||||
"git_remotes": {
|
||||
"status": "Success",
|
||||
"stdout": "== /srv/sai_suggestions\n== /srv/patient-summary-service\n== /srv/patient_document_parser\n== /srv/ml_monitoring\n== /srv/post_processor\n== /srv/soniox_transcriber\n== /opt/patient-summary-service\norigin\tgit@git.devteam.space:scrivas/patient-summary-service.git (fetch)\norigin\tgit@git.devteam.space:scrivas/patient-summary-service.git (push)\n7178a26575e261dbf889f40c974f607bb3c91ca7 Fri Jun 12 03:43:55 2026 +0500 Yegor Kovalev phi_logging: never crash the service when the audit log is unwritable (SOC 2 deploy fix)\n## dev...origin/dev\n== /opt/post_processor\norigin\tgit@git.devteam.space:scrivas/post_processor.git (fetch)\norigin\tgit@git.devteam.space:scrivas/post_processor.git (push)\n2f5cd0ada1c96f10616f378668c9b8cb99f73fc9 Tue Jun 16 08:29:18 2026 +0500 Yegor Kovalev SOC 2: Redis auth+TLS + loopback bind (host-net)\n## dev...origin/dev\n== /home/ubuntu/.codex/.tmp/plugins",
|
||||
"stderr": ""
|
||||
},
|
||||
"manifests": {
|
||||
"status": "Success",
|
||||
"stdout": "/usr/share/doc/python3-botocore/requirements.txt\n/srv/sai_suggestions/requirements.txt\n/srv/sai_suggestions/tests/e2e/docker-compose.yml\n/srv/sai_suggestions/pyproject.toml\n/srv/sai_suggestions/Dockerfile\n/srv/sai_suggestions/docker-compose.yml\n/srv/patient-summary-service/requirements.txt\n/srv/patient-summary-service/pyproject.toml\n/srv/patient-summary-service/Dockerfile\n/srv/patient-summary-service/docker-compose.yml\n/srv/patient_document_parser/requirements.txt\n/srv/patient_document_parser/Dockerfile\n/srv/patient_document_parser/docker-compose.yml\n/srv/ml_monitoring/tests/mocks/Dockerfile\n/srv/ml_monitoring/pyproject.toml\n/srv/ml_monitoring/docker-compose.yml\n/srv/post_processor/requirements.txt\n/srv/post_processor/pyproject.toml\n/srv/post_processor/docker/Dockerfile\n/srv/soniox_transcriber/requirements.txt\n/srv/soniox_transcriber/tests/e2e/docker-compose.yml\n/srv/soniox_transcriber/pyproject.toml\n/srv/soniox_transcriber/docker/Dockerfile\n/srv/soniox_transcriber/docker/docker-compose.yml\n/opt/patient-summary-service/requirements.txt\n/opt/patient-summary-service/pyproject.toml\n/opt/patient-summary-service/Dockerfile\n/opt/patient-summary-service/docker-compose.yml\n/opt/post_processor/requirements.txt\n/opt/post_processor/pyproject.toml\n/opt/post_processor/docker/Dockerfile\n/opt/post_processor/docker/docker-compose.yml\n/home/ubuntu/.vscode-server/extensions/github.copilot-chat-0.36.2/package.json\n/home/ubuntu/.vscode-server/extensions/openai.chatgpt-26.609.30741-linux-x64/package.json\n/home/ubuntu/.vscode-server/extensions/openai.chatgpt-26.527.60818-linux-x64/package.json\n/home/ubuntu/.vscode-server/extensions/github.copilot-1.388.0/package.json\n/home/ubuntu/.vscode-server/extensions/github.vscode-pull-request-github-0.126.0/package.json",
|
||||
"stderr": ""
|
||||
},
|
||||
"processes": {
|
||||
"status": "Success",
|
||||
"stdout": "USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND\nubuntu 3541290 0.2 3.8 547092 305344 ? Ssl Aug25 11:07 /usr/local/bin/python3.11 /usr/local/bin/uvicorn main:app --host 0.0.0.0 --port 8000\nroot 3536532 0.2 3.7 541604 299376 ? Ssl Aug25 12:31 /usr/local/bin/python3.11 /usr/local/bin/uvicorn sai_suggestions.api.main:app --host 0.0.0.0 --port 8000\nroot 3538327 0.2 3.2 978320 258268 ? Ssl Aug25 10:33 /usr/local/bin/python3.11 /usr/local/bin/uvicorn app.main:app --host 0.0.0.0 --port 8000\nroot 2202636 0.2 3.2 643376 254172 ? Ssl Aug27 4:02 python -m app.main\nubuntu 3538482 0.2 3.0 502032 238000 ? Ssl Aug25 9:46 /usr/local/bin/python3.11 /usr/local/bin/uvicorn app:app --host 0.0.0.0 --port 8000\n472 1652 0.0 1.9 1530556 150580 ? Ssl Aug13 20:31 grafana server --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini --packaging=docker cfg:default.log.mode=console cfg:default.paths.data=/var/lib/grafana cfg:default.paths.logs=/var/log/grafana cfg:default.paths.plugins=/var/lib/grafana/plugins cfg:default.paths.provisioning=/etc/grafana/provisioning\nroot 1653 0.1 1.2 1402200 95976 ? Ssl Aug13 23:36 traefik traefik --providers.docker=true --providers.docker.exposedbydefault=false --providers.docker.network=traefik-net --entrypoints.web.address=:8090 --entrypoints.internal.address=:8091\nroot 815 0.5 1.0 2673500 80284 ? Ssl Aug13 114:41 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock\nnobody 1720 0.1 0.8 1752732 67052 ? Ssl Aug13 40:37 /bin/prometheus --web.listen-address=0.0.0.0:9090 --config.file=/tmp/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=7d\nroot 651 0.3 0.8 2573492 63364 ? Ssl Aug13 80:21 /usr/bin/containerd\nroot 30117 0.1 0.7 1068508 55696 ? Ssl Aug13 38:38 /opt/aws/amazon-guardduty-agent/bin/amazon-guardduty-agent --worker-threads 8\nroot 2429504 0.3 0.6 1380784 52960 ? Sl Aug24 21:03 /snap/netbird/242/bin/netbird service run\nroot 3006517 0.0 0.6 83352 47916 ? S<s 06:53 0:01 /usr/lib/systemd/systemd-journald\nnobody 1664 0.0 0.4 1254824 35428 ? Ssl Aug13 15:01 /bin/alertmanager --config.file=/tmp/alertmanager.yml --web.listen-address=0.0.0.0:9093 --storage.path=/alertmanager\nroot 565 0.0 0.4 1366160 34096 ? Ssl Aug13 9:02 /usr/bin/gitlab-runner run --config /etc/gitlab-runner/config.toml --working-directory /home/gitlab-runner --service gitlab-runner --user gitlab-runner\nroot 2585 0.0 0.4 2212664 33724 ? Sl Aug13 5:26 /snap/amazon-ssm-agent/13349/ssm-agent-worker\nroot 581 0.0 0.3 1926012 29320 ? Ssl Aug13 0:48 /snap/snapd/current/usr/lib/snapd/snapd\nroot 3709299 1.7 0.3 1620160 28360 ? Sl 19:49 0:00 /snap/amazon-ssm-agent/13349/ssm-document-worker 30b02320-117d-466a-8755-44b8cd816992\nroot 3709350 50.0 0.3 1620416 28192 ? Sl 19:50 0:00 /snap/amazon-ssm-agent/13349/ssm-document-worker 5759e243-ffd3-4f6f-a765-41414513143e\nroot 183 0.0 0.3 288960 26532 ? SLsl Aug13 1:24 /sbin/multipathd -d -s\nroot 698 0.0 0.2 110060 21896 ? Ssl Aug13 0:00 /usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal\nroot 570 0.0 0.2 32564 20684 ? Ss Aug13 0:03 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startup-triggers\nroot 577 0.0 0.2 1832820 19344 ? Ssl Aug13 1:08 /snap/amazon-ssm-agent/13349/amazon-ssm-agent\nroot 2681 0.0 0.1 1268532 15380 ? Sl Aug13 11:57 /usr/bin/containerd-shim-runc-v2 -namespace moby -id a8f1465efc695cbe55fc1b0a2a3352ad0532104847f062065bd3dc7e7075a5aa -address /run/containerd/containerd.sock\nroot 2700 0.0 0.1 1268532 15168 ? Sl Aug13 11:57 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 15f495363998b7c196826202b6aa584976bf8b4a37ece873c3d3869caa973efe -address /run/containerd/containerd.sock\n59000 1649 0.0 0.1 1233880 14316 ? Ssl Aug13 11:16 /redis_exporter --web.listen-address=0.0.0.0:9121 --tls-ca-cert-file=/tls/ca.crt\nroot 1 0.0 0.1 22824 14228 ? Ss Aug13 0:57 /usr/lib/systemd/systemd --system --deserialize=87\nroot 1623733 0.0 0.1 469160 13664 ? Ssl Aug27 0:03 /usr/libexec/udisks2/udisksd\nsystemd+ 3006514 0.0 0.1 21476 13300 ? Ss 06:53 0:03 /usr/lib/systemd/systemd-resolved",
|
||||
"stderr": ""
|
||||
},
|
||||
"listening": {
|
||||
"status": "Success",
|
||||
"stdout": "udp UNCONN 0 0 0.0.0.0:51820 0.0.0.0:* \nudp UNCONN 0 0 127.0.0.1:3128 0.0.0.0:* users:((\"netbird\",pid=2429504,fd=16)) \nudp UNCONN 0 0 127.0.0.54:53 0.0.0.0:* users:((\"systemd-resolve\",pid=3006514,fd=16)) \nudp UNCONN 0 0 127.0.0.53%lo:53 0.0.0.0:* users:((\"systemd-resolve\",pid=3006514,fd=14)) \nudp UNCONN 0 0 100.112.95.128:53 0.0.0.0:* users:((\"netbird\",pid=2429504,fd=28)) \nudp UNCONN 0 0 172.31.26.26%ens5:68 0.0.0.0:* users:((\"systemd-network\",pid=3006513,fd=23)) \nudp UNCONN 0 0 127.0.0.1:323 0.0.0.0:* users:((\"chronyd\",pid=3006120,fd=5)) \nudp UNCONN 0 0 [::]:51820 [::]:* \nudp UNCONN 0 0 [::1]:323 [::]:* users:((\"chronyd\",pid=3006120,fd=6)) \ntcp LISTEN 0 4096 127.0.0.1:3000 0.0.0.0:* users:((\"docker-proxy\",pid=2176,fd=8)) \ntcp LISTEN 0 4096 127.0.0.1:6380 0.0.0.0:* users:((\"docker-proxy\",pid=2819,fd=8)) \ntcp LISTEN 0 4096 127.0.0.1:8090 0.0.0.0:* users:((\"docker-proxy\",pid=1818,fd=8)) \ntcp LISTEN 0 4096 127.0.0.1:8091 0.0.0.0:* users:((\"docker-proxy\",pid=1927,fd=8)) \ntcp LISTEN 0 4096 0.0.0.0:22 0.0.0.0:* users:((\"sshd\",pid=3006515,fd=3),(\"systemd\",pid=1,fd=242)) \ntcp LISTEN 0 4096 127.0.0.1:9093 0.0.0.0:* users:((\"docker-proxy\",pid=2063,fd=8)) \ntcp LISTEN 0 4096 127.0.0.1:9090 0.0.0.0:* users:((\"docker-proxy\",pid=2134,fd=8)) \ntcp LISTEN 0 511 0.0.0.0:443 0.0.0.0:* users:((\"nginx\",pid=1623744,fd=7),(\"nginx\",pid=1623743,fd=7),(\"nginx\",pid=1623742,fd=7))\ntcp LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:((\"systemd-resolve\",pid=3006514,fd=15)) \ntcp LISTEN 0 4096 127.0.0.54:53 0.0.0.0:* users:((\"systemd-resolve\",pid=3006514,fd=17)) \ntcp LISTEN 0 4096 100.112.95.128:53 0.0.0.0:* users:((\"netbird\",pid=2429504,fd=27)) \ntcp LISTEN 0 4096 [::]:22 [::]:* users:((\"sshd\",pid=3006515,fd=4),(\"systemd\",pid=1,fd=247))",
|
||||
"stderr": ""
|
||||
},
|
||||
"systemd_units": {
|
||||
"status": "Success",
|
||||
"stdout": " acpid.service loaded active running ACPI event daemon\n amazon-guardduty-agent.service loaded active running Amazon GuardDuty Agent\n chrony.service loaded active running chrony, an NTP client/server\n containerd.service loaded active running containerd container runtime\n cron.service loaded active running Regular background program processing daemon\n dbus.service loaded active running D-Bus System Message Bus\n docker.service loaded active running Docker Application Container Engine\n getty@tty1.service loaded active running Getty on tty1\n gitlab-runner.service loaded active running GitLab Runner\n irqbalance.service loaded active running irqbalance daemon\n ModemManager.service loaded active running Modem Manager\n multipathd.service loaded active running Device-Mapper Multipath Device Controller\n networkd-dispatcher.service loaded active running Dispatcher daemon for systemd-networkd\n nginx.service loaded active running A high performance web server and a reverse proxy server\n polkit.service loaded active running Authorization Manager\n rsyslog.service loaded active running System Logging Service\n serial-getty@ttyS0.service loaded active running Serial Getty on ttyS0\n snap.amazon-ssm-agent.amazon-ssm-agent.service loaded active running Service for snap application amazon-ssm-agent.amazon-ssm-agent\n snap.netbird.service-run.service loaded active running Service for snap application netbird.service-run\n snapd.service loaded active running Snap Daemon\n ssh.service loaded active running OpenBSD Secure Shell server\n systemd-journald.service loaded active running Journal Service\n systemd-logind.service loaded active running User Login Management\n systemd-networkd.service loaded active running Network Configuration\n systemd-resolved.service loaded active running Network Name Resolution\n systemd-udevd.service loaded active running Rule-based Manager for Device Events and Files\n udisks2.service loaded active running Disk Manager\n unattended-upgrades.service loaded active running Unattended Upgrades Shutdown",
|
||||
"stderr": ""
|
||||
},
|
||||
"systemd_failed": {
|
||||
"status": "Success",
|
||||
"stdout": "",
|
||||
"stderr": ""
|
||||
},
|
||||
"docker": {
|
||||
"status": "Success",
|
||||
"stdout": "dev-green-postprocess-1\t716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/postprocessor:33504e02\tUp 27 hours (healthy)\ndev-green-redis-1\tredis:7-alpine\tUp 27 hours (healthy)\ndev-green-soniox-1\t716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/sonioxtranscriber:7393c940\tUp 3 days (healthy)\ndev-green-patient-document-parser-1\t716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/patientdocumentparser:53586cc8\tUp 3 days (healthy)\ndev-blue-svc-1\t716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/patientsummaryservice:a41e2105\tUp 3 days (healthy)\ndev-green-saisuggestions-1\t716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/saisuggestions:02c3dd66\tUp 3 days (healthy)\nsai_suggestions_redis\tredis:7-alpine\tUp 2 weeks (healthy)\nsoniox_transcriber_redis\tredis:7-alpine\tUp 2 weeks (healthy)\nml_monitoring-grafana-1\tgrafana/grafana:11.2.0\tUp 2 weeks (healthy)\nml_monitoring-prometheus-1\tprom/prometheus:v2.54.1\tUp 2 weeks (healthy)\nml_monitoring-alertmanager-1\tprom/alertmanager:v0.28.1\tUp 2 weeks (healthy)\nml_monitoring-redis_exporter-1\toliver006/redis_exporter:v1.62.0\tUp 2 weeks\ntraefik-traefik-1\ttraefik:v3.7\tUp 2 weeks\npatient_document_parser-patient-document-parser-1\tpatient_document_parser-patient-document-parser\tExited (0) 2 months ago\nsoniox_transcriber-soniox-transcriber-service-1\tsoniox_transcriber-soniox-transcriber-service\tExited (137) 2 months ago\nsoniox_transcriber-redis-1\tredis:7-alpine\tExited (0) 2 months ago\nsai_suggestions_v2\tsai_suggestions-sai_suggestions\tExited (0) 2 months ago\npatient-summary-service-svc-1\tpatient-summary-service-svc\tExited (0) 2 months ago\ncompetent_lichterman\t4bfe3adfd150\tExited (0) 5 months ago",
|
||||
"stderr": ""
|
||||
},
|
||||
"docker_images": {
|
||||
"status": "Success",
|
||||
"stdout": "716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/postprocessor:33504e02\t2026-08-27 16:45:03 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/saisuggestions:02c3dd66\t2026-08-25 12:19:58 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/postprocessor:239d587d\t2026-08-25 12:19:11 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/sonioxtranscriber:7393c940\t2026-08-25 12:17:54 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/patientdocumentparser:53586cc8\t2026-08-25 12:16:59 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/patientsummaryservice:a41e2105\t2026-08-25 12:15:25 +0000 UTC\npython:3.10-slim\t2026-08-25 01:21:04 +0000 UTC\npython:3.11-slim\t2026-08-25 01:20:06 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/saisuggestions:d8fffe5a\t2026-08-24 14:22:21 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/patientdocumentparser:737ff087\t2026-08-21 19:06:25 +0000 UTC\nregistry.gitlab.com/gitlab-org/gitlab-runner/gitlab-runner-helper:x86_64-v19.2.2\t2026-08-13 08:40:04 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/sonioxtranscriber:6f14e0cf\t2026-08-10 09:10:33 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/patientsummaryservice:69147b24\t2026-08-10 09:09:29 +0000 UTC\npatient_document_parser-patient-document-parser:latest\t2026-06-16 03:40:32 +0000 UTC\nsoniox_transcriber-soniox-transcriber-service:latest\t2026-06-16 03:39:43 +0000 UTC\nsai_suggestions-sai_suggestions:latest\t2026-06-16 03:35:54 +0000 UTC\npatient-summary-service-svc:latest\t2026-06-11 22:49:51 +0000 UTC\ntraefik:v3.7\t2026-06-10 17:56:26 +0000 UTC\nredis:7-alpine\t2026-02-23 17:35:24 +0000 UTC\nprom/alertmanager:v0.28.1\t2025-03-07 15:10:29 +0000 UTC",
|
||||
"stderr": ""
|
||||
},
|
||||
"oom_kills": {
|
||||
"status": "Success",
|
||||
"stdout": "",
|
||||
"stderr": ""
|
||||
},
|
||||
"svc_restarts": {
|
||||
"status": "Success",
|
||||
"stdout": "",
|
||||
"stderr": ""
|
||||
},
|
||||
"reboots": {
|
||||
"status": "Success",
|
||||
"stdout": "reboot system boot 7.0.0-1010-aws Thu Aug 13 06:14 still running\nreboot system boot 6.17.0-1013-aws Mon May 11 12:01 - 06:14 (93+18:13)\nreboot system boot 6.17.0-1009-aws Mon Mar 16 14:20 - 11:59 (55+21:38)\nreboot system boot 6.14.0-1018-aws Thu Mar 5 15:07 - 11:59 (66+20:51)\n\nwtmp begins Thu Mar 5 15:07:06 2026",
|
||||
"stderr": ""
|
||||
},
|
||||
"cron": {
|
||||
"status": "Success",
|
||||
"stdout": "total 32\ndrwxr-xr-x 2 root root 4096 May 4 09:16 .\ndrwxr-xr-x 117 root root 12288 Aug 28 06:53 ..\n-rw-r--r-- 1 root root 102 Mar 31 2024 .placeholder\n-rw-r--r-- 1 root root 802 Apr 16 2023 certbot\n-rw-r--r-- 1 root root 201 Apr 8 2024 e2scrub_all\n-rw-r--r-- 1 root root 396 Jan 9 2024 sysstat\n# Edit this file to introduce tasks to be run by cron.\n# \n# Each task to run has to be defined through a single line\n# indicating with different fields when the task will be run\n# and what command to run for the task\n# \n# To define the time you can provide concrete values for\n# minute (m), hour (h), day of month (dom), month (mon),\n# and day of week (dow) or use '*' in these fields (for 'any').\n# \n# Notice that tasks will be started based on the cron's system\n# daemon's notion of time and timezones.\n# \n# Output of the crontab jobs (including errors) is sent through\n# email to the user the crontab file belongs to (unless redirected).\n# \n# For example, you can run a backup of all your user accounts\n# at 5 a.m every week with:\n# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/\n# \n# For more information see the manual pages of crontab(5) and cron(8)\n# \n# m h dom mon dow command\n0 3 * * * /home/ubuntu/docker_clean.sh >> /var/log/docker_clean.log 2>&1",
|
||||
"stderr": ""
|
||||
}
|
||||
},
|
||||
"i-046e7fc4677eaa796": {
|
||||
"os_release": {
|
||||
"status": "Success",
|
||||
"stdout": "PRETTY_NAME=\"Ubuntu 24.04.4 LTS\"\nNAME=\"Ubuntu\"\nVERSION_ID=\"24.04\"\nVERSION=\"24.04.4 LTS (Noble Numbat)\"\nVERSION_CODENAME=noble\nID=ubuntu\nID_LIKE=debian\nHOME_URL=\"https://www.ubuntu.com/\"\nSUPPORT_URL=\"https://help.ubuntu.com/\"\nBUG_REPORT_URL=\"https://bugs.launchpad.net/ubuntu/\"\nPRIVACY_POLICY_URL=\"https://www.ubuntu.com/legal/terms-and-policies/privacy-policy\"\nUBUNTU_CODENAME=noble\nLOGO=ubuntu-logo\nLinux ip-172-31-15-39 7.0.0-1010-aws #10~24.04.1-Ubuntu SMP PREEMPT Mon Jul 27 17:41:33 UTC 2026 x86_64 x86_64 x86_64 GNU/Linux",
|
||||
"stderr": ""
|
||||
},
|
||||
"uptime_load": {
|
||||
"status": "Success",
|
||||
"stdout": " 19:50:28 up 15 days, 13:15, 0 user, load average: 0.09, 0.14, 0.15\n0.09 0.14 0.15 1/567 3544357",
|
||||
"stderr": ""
|
||||
},
|
||||
"disk": {
|
||||
"status": "Success",
|
||||
"stdout": "Filesystem Size Used Avail Use% Mounted on\n/dev/root 193G 16G 177G 9% /\ntmpfs 3.8G 0 3.8G 0% /dev/shm\ntmpfs 1.6G 2.1M 1.6G 1% /run\ntmpfs 5.0M 0 5.0M 0% /run/lock\nefivarfs 128K 3.8K 120K 4% /sys/firmware/efi/efivars\n/dev/nvme0n1p16 881M 202M 617M 25% /boot\n/dev/nvme0n1p15 105M 6.2M 99M 6% /boot/efi\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/dc4f0eea66d0ef7f68fdbce545dde313a70dc31cb7ebc974151456fc00e32c9b\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/593c1f008ef4b3435f5c87dba7aaf3e0e7626474afd85723bf20f39641b17ebd\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/09d2bcc0019a83128cefc04bd3fe7b285139300fb719785c3eaff8f25a455049\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/8bf98332778bea88cef93a3a7f2f3aae75adf266af13595c32097f84d1455cc3\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/d3082c17054f4df1d0f4153c227394c5c81a21cdddc77dcd780522d299d38a9b\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/70b3522388f403c2ca8efc798a3e70d6ede66cba286abb5f9dbb983eb7c4d029\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/585d70b786a8639b2ef434cfb94b3e671b7b43eeef79939c425e8141486479b0\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/f789b113f1043cffd988aaecaf20b779671d7b9147c2c74ea51ef20780f21919\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/02bfe27dc799c3db9b2ee96fa2b4dc580091cc1cf6339a2ef606d332da728d37\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/0a15f48f5e0ff2766db09d7424a7422d15f039604897123ab18bde5967306b7f\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/18171bc17d5e150474540790ed497f21043316b3787e247c5b3f47ea78679f17\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/d6e65594744f415245bd590bb829cba8cf2e472fb3760eaa48643245fc93bdea\noverlay 193G 16G 177G 9% /var/lib/docker/rootfs/overlayfs/212e6f24ca3a19a26aa952f2efd504b980ed4af6b1e33908ea7eb52a9e0118c9\n--- inodes\nFilesystem Inodes IUsed IFree IUse% Mounted on\n/dev/root 26083328 406304 25677024 2% /\ntmpfs 989732 2 989730 1% /dev/shm\ntmpfs 819200 1110 818090 1% /run\ntmpfs 989732 3 989729 1% /run/lock\nefivarfs 0 0 0 - /sys/firmware/efi/efivars\n/dev/nvme0n1p16 58496 605 57891 2% /boot\n/dev/nvme0n1p15 0 0 0 - /boot/efi\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/dc4f0eea66d0ef7f68fdbce545dde313a70dc31cb7ebc974151456fc00e32c9b\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/593c1f008ef4b3435f5c87dba7aaf3e0e7626474afd85723bf20f39641b17ebd\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/09d2bcc0019a83128cefc04bd3fe7b285139300fb719785c3eaff8f25a455049\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/8bf98332778bea88cef93a3a7f2f3aae75adf266af13595c32097f84d1455cc3\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/d3082c17054f4df1d0f4153c227394c5c81a21cdddc77dcd780522d299d38a9b\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/70b3522388f403c2ca8efc798a3e70d6ede66cba286abb5f9dbb983eb7c4d029\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/585d70b786a8639b2ef434cfb94b3e671b7b43eeef79939c425e8141486479b0\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/f789b113f1043cffd988aaecaf20b779671d7b9147c2c74ea51ef20780f21919\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/02bfe27dc799c3db9b2ee96fa2b4dc580091cc1cf6339a2ef606d332da728d37\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/0a15f48f5e0ff2766db09d7424a7422d15f039604897123ab18bde5967306b7f\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/18171bc17d5e150474540790ed497f21043316b3787e247c5b3f47ea78679f17\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/d6e65594744f415245bd590bb829cba8cf2e472fb3760eaa48643245fc93bdea\noverlay 26083328 406304 25677024 2% /var/lib/docker/rootfs/overlayfs/212e6f24ca3a19a26aa952f2efd504b980ed4af6b1e33908ea7eb52a9e0118c9",
|
||||
"stderr": ""
|
||||
},
|
||||
"memory": {
|
||||
"status": "Success",
|
||||
"stdout": " total used free shared buff/cache available\nMem: 7.6Gi 2.3Gi 834Mi 3.9Mi 4.7Gi 5.2Gi\nSwap: 0B 0B 0B\n--- swap",
|
||||
"stderr": ""
|
||||
},
|
||||
"code_dirs": {
|
||||
"status": "Success",
|
||||
"stdout": "/home:\ntotal 20\ndrwxr-xr-x 5 root root 4096 Jul 1 10:18 .\ndrwxr-xr-x 22 root root 4096 Aug 13 06:35 ..\ndrwxr-x--- 5 deploy deploy 4096 Jul 1 12:11 deploy\ndrwxr-x--- 2 gitlab-runner gitlab-runner 4096 Jul 1 10:18 gitlab-runner\ndrwxr-x--- 11 ubuntu ubuntu 4096 Jul 14 07:05 ubuntu\n\n/opt:\ntotal 32\ndrwxr-xr-x 5 root root 4096 Jul 15 15:14 .\ndrwxr-xr-x 22 root root 4096 Aug 13 06:35 ..\n-rw-rw-r-- 1 deploy deploy 663 Jun 24 11:34 .env\ndrwxr-xr-x 5 root root 4096 Aug 26 00:53 aws\ndrwx--x--x 4 root root 4096 Mar 5 14:10 containerd\n-rw-r--r-- 1 root root 4325 May 6 11:05 default\ndrwxr-xr-x 3 root root 4096 Jul 14 06:57 redis-tls\n\n/srv:\ntotal 56\ndrwxr-xr-x 10 ubuntu ubuntu 4096 Jul 14 07:01 .\ndrwxr-xr-x 22 root root 4096 Aug 13 06:35 ..\n-rw-rw-r-- 1 ubuntu ubuntu 13833 Mar 23 13:13 AGENTS.md\ndrwxr-xr-x 7 deploy deploy 4096 Jul 20 19:38 ml_monitoring\ndrwxr-xr-x 10 ubuntu ubuntu 4096 Jun 29 20:13 patient-summary-service\ndrwxr-xr-x 9 ubuntu ubuntu 4096 Jun 29 20:13 patient_document_parser\ndrwxr-xr-x 13 ubuntu ubuntu 4096 Jun 29 20:13 post_processor\ndrwxr-xr-x 2 root root 4096 Jul 15 12:47 redis-tls\ndrwxr-xr-x 4 root root 4096 Jul 1 18:06 redis_db\ndrwxr-xr-x 23 ubuntu ubuntu 4096 Jun 29 20:13 sai_suggestions\ndrwxr-xr-x 10 ubuntu ubuntu 4096 Jun 29 20:13 soniox_transcriber\n\n/var/www:\ntotal 12\ndrwxr-xr-x 3 root root 4096 May 5 07:11 .\ndrwxr-xr-x 14 root root 4096 May 5 07:11 ..\ndrwxr-xr-x 2 root root 4096 May 5 07:11 html",
|
||||
"stderr": ""
|
||||
},
|
||||
"git_checkouts": {
|
||||
"status": "Success",
|
||||
"stdout": "/srv/sai_suggestions/.git\n/srv/patient-summary-service/.git\n/srv/patient_document_parser/.git\n/srv/ml_monitoring/.git\n/srv/post_processor/.git\n/srv/soniox_transcriber/.git\n/home/ubuntu/.codex/.tmp/plugins/.git",
|
||||
"stderr": ""
|
||||
},
|
||||
"git_remotes": {
|
||||
"status": "Success",
|
||||
"stdout": "== /srv/sai_suggestions\n== /srv/patient-summary-service\n== /srv/patient_document_parser\n== /srv/ml_monitoring\n== /srv/post_processor\n== /srv/soniox_transcriber\n== /home/ubuntu/.codex/.tmp/plugins",
|
||||
"stderr": ""
|
||||
},
|
||||
"manifests": {
|
||||
"status": "Success",
|
||||
"stdout": "/usr/share/doc/python3-botocore/requirements.txt\n/srv/sai_suggestions/requirements.txt\n/srv/sai_suggestions/tests/e2e/docker-compose.yml\n/srv/sai_suggestions/pyproject.toml\n/srv/sai_suggestions/Dockerfile\n/srv/sai_suggestions/docker-compose.yml\n/srv/patient-summary-service/requirements.txt\n/srv/patient-summary-service/pyproject.toml\n/srv/patient-summary-service/Dockerfile\n/srv/patient-summary-service/docker-compose.yml\n/srv/patient_document_parser/requirements.txt\n/srv/patient_document_parser/Dockerfile\n/srv/patient_document_parser/docker-compose.yml\n/srv/ml_monitoring/tests/mocks/Dockerfile\n/srv/ml_monitoring/pyproject.toml\n/srv/ml_monitoring/docker-compose.yml\n/srv/post_processor/requirements.txt\n/srv/post_processor/pyproject.toml\n/srv/post_processor/docker/Dockerfile\n/srv/post_processor/docker/docker-compose.yml\n/srv/soniox_transcriber/requirements.txt\n/srv/soniox_transcriber/tests/e2e/docker-compose.yml\n/srv/soniox_transcriber/pyproject.toml\n/srv/soniox_transcriber/docker/Dockerfile\n/srv/soniox_transcriber/docker/docker-compose.yml\n/home/ubuntu/.vscode-server/extensions/github.copilot-chat-0.36.2/package.json\n/home/ubuntu/.vscode-server/extensions/openai.chatgpt-26.506.31421-linux-x64/package.json\n/home/ubuntu/.vscode-server/extensions/openai.chatgpt-26.429.30905-linux-x64/package.json\n/home/ubuntu/.vscode-server/extensions/github.copilot-1.388.0/package.json\n/home/ubuntu/.vscode-server/extensions/github.vscode-pull-request-github-0.126.0/package.json",
|
||||
"stderr": ""
|
||||
},
|
||||
"processes": {
|
||||
"status": "Success",
|
||||
"stdout": "USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND\nroot 1791 0.2 4.0 1110456 322956 ? Ssl Aug13 50:34 /usr/local/bin/python3.11 /usr/local/bin/uvicorn app.main:app --host 0.0.0.0 --port 8000\nubuntu 1859 0.2 3.9 550928 312692 ? Ssl Aug13 53:11 /usr/local/bin/python3.11 /usr/local/bin/uvicorn main:app --host 0.0.0.0 --port 8000\nroot 1857 0.2 3.8 684340 301360 ? Ssl Aug13 58:17 /usr/local/bin/python3.11 /usr/local/bin/uvicorn sai_suggestions.api.main:app --host 0.0.0.0 --port 8000\nroot 2091197 0.2 3.2 644384 259620 ? Ssl Aug27 4:12 python -m app.main\nubuntu 2803851 0.2 3.0 501016 243864 ? Ssl Aug21 22:05 /usr/local/bin/python3.11 /usr/local/bin/uvicorn app:app --host 0.0.0.0 --port 8000\n472 1887 0.0 2.0 1538692 165088 ? Ssl Aug13 22:21 grafana server --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini --packaging=docker cfg:default.log.mode=console cfg:default.paths.data=/var/lib/grafana cfg:default.paths.logs=/var/log/grafana cfg:default.paths.plugins=/var/lib/grafana/plugins cfg:default.paths.provisioning=/etc/grafana/provisioning\nroot 850 0.5 1.4 2692076 114856 ? Ssl Aug13 115:49 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock\nnobody 1788 0.1 1.3 1757612 110120 ? Ssl Aug13 42:20 /bin/prometheus --web.listen-address=0.0.0.0:9090 --config.file=/tmp/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=7d\nroot 1778 0.1 1.2 1402200 95220 ? Ssl Aug13 22:40 traefik traefik --providers.docker=true --providers.docker.exposedbydefault=false --providers.docker.network=traefik-net --entrypoints.web.address=:8090 --entrypoints.internal.address=:8091\nroot 548 0.0 0.9 1365648 77652 ? Ssl Aug13 9:36 /usr/bin/gitlab-runner run --config /etc/gitlab-runner/config.toml --working-directory /home/gitlab-runner --service gitlab-runner --user gitlab-runner\nroot 551 0.4 0.8 1282016 67988 ? Ssl Aug13 94:50 /usr/bin/netbird service run --log-level info --daemon-addr unix:///var/run/netbird.sock --log-file /var/log/netbird/client.log\nroot 618 0.3 0.8 2514128 66892 ? Ssl Aug13 75:44 /usr/bin/containerd\nroot 11880 0.1 0.7 998760 58708 ? Ssl Aug13 39:01 /opt/aws/amazon-guardduty-agent/bin/amazon-guardduty-agent --worker-threads 8\nroot 1485944 0.0 0.6 107768 51588 ? S<s Aug27 0:03 /usr/lib/systemd/systemd-journald\nroot 3458301 0.0 0.5 1925500 42632 ? Ssl Aug15 0:36 /snap/snapd/current/usr/lib/snapd/snapd\nnobody 1779 0.0 0.4 1255080 38724 ? Ssl Aug13 18:26 /bin/alertmanager --config.file=/tmp/alertmanager.yml --web.listen-address=0.0.0.0:9093 --storage.path=/alertmanager\nroot 2663 0.0 0.4 2212664 35296 ? Sl Aug13 5:57 /snap/amazon-ssm-agent/13349/ssm-agent-worker\nroot 3544776 75.0 0.3 1620160 28152 ? Sl 19:50 0:00 /snap/amazon-ssm-agent/13349/ssm-document-worker e4868bb4-b811-4733-9872-1aa517f39b08\nroot 3544764 1.8 0.3 1693892 28084 ? Sl 19:50 0:00 /snap/amazon-ssm-agent/13349/ssm-document-worker 27264072-49f5-4577-98ab-80b0dac9b69e\nroot 182 0.0 0.3 288960 26924 ? SLsl Aug13 1:35 /sbin/multipathd -d -s\nroot 684 0.0 0.2 110060 22756 ? Ssl Aug13 0:00 /usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal\nroot 556 0.0 0.2 1832820 21392 ? Ssl Aug13 1:13 /snap/amazon-ssm-agent/13349/amazon-ssm-agent\nroot 552 0.0 0.2 32564 20732 ? Ss Aug13 0:00 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startup-triggers\nroot 3204 0.0 0.1 1268276 15252 ? Sl Aug13 12:18 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 0a15f48f5e0ff2766db09d7424a7422d15f039604897123ab18bde5967306b7f -address /run/containerd/containerd.sock\nroot 3172 0.0 0.1 1268532 15116 ? Sl Aug13 12:02 /usr/bin/containerd-shim-runc-v2 -namespace moby -id 02bfe27dc799c3db9b2ee96fa2b4dc580091cc1cf6339a2ef606d332da728d37 -address /run/containerd/containerd.sock\n59000 1807 0.0 0.1 1233880 14956 ? Ssl Aug13 10:26 /redis_exporter --web.listen-address=0.0.0.0:9121 --tls-ca-cert-file=/tls/ca.crt\nroot 1385 0.0 0.1 1268276 14316 ? Sl Aug13 15:47 /usr/bin/containerd-shim-runc-v2 -namespace moby -id dc4f0eea66d0ef7f68fdbce545dde313a70dc31cb7ebc974151456fc00e32c9b -address /run/containerd/containerd.sock\nroot 1 0.0 0.1 22688 14172 ? Ss Aug13 0:37 /usr/lib/systemd/systemd --system --deserialize=92\nroot 1485947 0.0 0.1 469148 13960 ? Ssl Aug27 0:03 /usr/libexec/udisks2/udisksd",
|
||||
"stderr": ""
|
||||
},
|
||||
"listening": {
|
||||
"status": "Success",
|
||||
"stdout": "udp UNCONN 0 0 0.0.0.0:51820 0.0.0.0:* \nudp UNCONN 0 0 127.0.0.1:3128 0.0.0.0:* users:((\"netbird\",pid=551,fd=18)) \nudp UNCONN 0 0 127.0.0.54:53 0.0.0.0:* users:((\"systemd-resolve\",pid=1485946,fd=16)) \nudp UNCONN 0 0 127.0.0.53%lo:53 0.0.0.0:* users:((\"systemd-resolve\",pid=1485946,fd=14)) \nudp UNCONN 0 0 100.112.114.187:53 0.0.0.0:* users:((\"netbird\",pid=551,fd=26)) \nudp UNCONN 0 0 172.31.15.39%ens5:68 0.0.0.0:* users:((\"systemd-network\",pid=1485967,fd=23)) \nudp UNCONN 0 0 127.0.0.1:323 0.0.0.0:* users:((\"chronyd\",pid=637,fd=5)) \nudp UNCONN 0 0 [::]:51820 [::]:* \nudp UNCONN 0 0 [::1]:323 [::]:* users:((\"chronyd\",pid=637,fd=6)) \ntcp LISTEN 0 4096 127.0.0.1:6380 0.0.0.0:* users:((\"docker-proxy\",pid=3304,fd=8)) \ntcp LISTEN 0 4096 127.0.0.1:3000 0.0.0.0:* users:((\"docker-proxy\",pid=2397,fd=8)) \ntcp LISTEN 0 4096 0.0.0.0:22 0.0.0.0:* users:((\"sshd\",pid=1485948,fd=3),(\"systemd\",pid=1,fd=113)) \ntcp LISTEN 0 511 0.0.0.0:443 0.0.0.0:* users:((\"nginx\",pid=1485963,fd=7),(\"nginx\",pid=1485962,fd=7),(\"nginx\",pid=1485961,fd=7))\ntcp LISTEN 0 4096 127.0.0.1:8090 0.0.0.0:* users:((\"docker-proxy\",pid=2300,fd=8)) \ntcp LISTEN 0 4096 127.0.0.1:8091 0.0.0.0:* users:((\"docker-proxy\",pid=2326,fd=8)) \ntcp LISTEN 0 4096 100.112.114.187:53 0.0.0.0:* users:((\"netbird\",pid=551,fd=27)) \ntcp LISTEN 0 4096 127.0.0.54:53 0.0.0.0:* users:((\"systemd-resolve\",pid=1485946,fd=17)) \ntcp LISTEN 0 4096 127.0.0.1:9093 0.0.0.0:* users:((\"docker-proxy\",pid=2034,fd=8)) \ntcp LISTEN 0 4096 127.0.0.1:9090 0.0.0.0:* users:((\"docker-proxy\",pid=2364,fd=8)) \ntcp LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:((\"systemd-resolve\",pid=1485946,fd=15)) \ntcp LISTEN 0 4096 [::]:22 [::]:* users:((\"sshd\",pid=1485948,fd=4),(\"systemd\",pid=1,fd=114))",
|
||||
"stderr": ""
|
||||
},
|
||||
"systemd_units": {
|
||||
"status": "Success",
|
||||
"stdout": " acpid.service loaded active running ACPI event daemon\n amazon-guardduty-agent.service loaded active running Amazon GuardDuty Agent\n chrony.service loaded active running chrony, an NTP client/server\n containerd.service loaded active running containerd container runtime\n cron.service loaded active running Regular background program processing daemon\n dbus.service loaded active running D-Bus System Message Bus\n docker.service loaded active running Docker Application Container Engine\n getty@tty1.service loaded active running Getty on tty1\n gitlab-runner.service loaded active running GitLab Runner\n irqbalance.service loaded active running irqbalance daemon\n ModemManager.service loaded active running Modem Manager\n multipathd.service loaded active running Device-Mapper Multipath Device Controller\n netbird.service loaded active running NetBird mesh network client\n networkd-dispatcher.service loaded active running Dispatcher daemon for systemd-networkd\n nginx.service loaded active running A high performance web server and a reverse proxy server\n polkit.service loaded active running Authorization Manager\n rsyslog.service loaded active running System Logging Service\n serial-getty@ttyS0.service loaded active running Serial Getty on ttyS0\n snap.amazon-ssm-agent.amazon-ssm-agent.service loaded active running Service for snap application amazon-ssm-agent.amazon-ssm-agent\n snapd.service loaded active running Snap Daemon\n ssh.service loaded active running OpenBSD Secure Shell server\n systemd-journald.service loaded active running Journal Service\n systemd-logind.service loaded active running User Login Management\n systemd-networkd.service loaded active running Network Configuration\n systemd-resolved.service loaded active running Network Name Resolution\n systemd-udevd.service loaded active running Rule-based Manager for Device Events and Files\n udisks2.service loaded active running Disk Manager\n unattended-upgrades.service loaded active running Unattended Upgrades Shutdown",
|
||||
"stderr": ""
|
||||
},
|
||||
"systemd_failed": {
|
||||
"status": "Success",
|
||||
"stdout": "\u25cf sai-suggestions-redis.service loaded failed failed SAI Suggestions Redis",
|
||||
"stderr": ""
|
||||
},
|
||||
"docker": {
|
||||
"status": "Success",
|
||||
"stdout": "stage-blue-postprocess-1\t716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/postprocessor:33504e02\tUp 27 hours (healthy)\nstage-blue-redis-1\tredis:7-alpine\tUp 27 hours (healthy)\nstage-green-patient-document-parser-1\t716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/patientdocumentparser:737ff087\tUp 7 days (healthy)\nsai_suggestions_redis\tredis:7-alpine\tUp 2 weeks (healthy)\nsoniox_transcriber_redis\tredis:7-alpine\tUp 2 weeks (healthy)\nstage-blue-saisuggestions-1\t716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/saisuggestions:a188d92a\tUp 2 weeks (healthy)\nstage-green-soniox-1\t716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/sonioxtranscriber:6f14e0cf\tUp 2 weeks (healthy)\nstage-blue-svc-1\t716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/patientsummaryservice:69147b24\tUp 2 weeks (healthy)\nml_monitoring-grafana-1\tgrafana/grafana:11.2.0\tUp 2 weeks (healthy)\nml_monitoring-prometheus-1\tprom/prometheus:v2.54.1\tUp 2 weeks (healthy)\nml_monitoring-alertmanager-1\tprom/alertmanager:v0.28.1\tUp 2 weeks (healthy)\nml_monitoring-redis_exporter-1\toliver006/redis_exporter:v1.62.0\tUp 2 weeks\ntraefik-traefik-1\ttraefik:v3.7\tUp 2 weeks\npatient_document_parser-patient-document-parser-1\tpatient_document_parser-patient-document-parser\tExited (0) 7 weeks ago\npatient-summary-service-svc-1\tpatient-summary-service-svc\tExited (0) 7 weeks ago",
|
||||
"stderr": ""
|
||||
},
|
||||
"docker_images": {
|
||||
"status": "Success",
|
||||
"stdout": "716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/postprocessor:33504e02\t2026-08-27 17:00:05 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/postprocessor:1fa89c2d\t2026-08-21 20:08:56 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/patientdocumentparser:737ff087\t2026-08-21 20:05:01 +0000 UTC\nregistry.gitlab.com/gitlab-org/gitlab-runner/gitlab-runner-helper:x86_64-v19.2.2\t2026-08-17 12:48:38 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/saisuggestions:a188d92a\t2026-08-10 13:10:12 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/patientdocumentparser:717a2d89\t2026-08-10 13:06:24 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/sonioxtranscriber:6f14e0cf\t2026-08-10 13:06:15 +0000 UTC\n716468089330.dkr.ecr.us-east-2.amazonaws.com/scrivas/patientsummaryservice:69147b24\t2026-08-10 13:03:56 +0000 UTC\npatient_document_parser-patient-document-parser:latest\t2026-06-29 15:54:43 +0000 UTC\npatient-summary-service-svc:latest\t2026-06-29 15:41:59 +0000 UTC\ntraefik:v3.7\t2026-06-10 17:56:26 +0000 UTC\nredis:7-alpine\t2026-02-23 17:35:24 +0000 UTC\nprom/alertmanager:v0.28.1\t2025-03-07 15:10:29 +0000 UTC\ngrafana/grafana:11.2.0\t2024-08-27 13:56:08 +0000 UTC\nprom/prometheus:v2.54.1\t2024-08-27 11:20:46 +0000 UTC\noliver006/redis_exporter:v1.62.0\t2024-07-17 20:53:05 +0000 UTC\ndocker:24\t2024-06-19 00:17:01 +0000 UTC",
|
||||
"stderr": ""
|
||||
},
|
||||
"oom_kills": {
|
||||
"status": "Success",
|
||||
"stdout": "",
|
||||
"stderr": ""
|
||||
},
|
||||
"svc_restarts": {
|
||||
"status": "Success",
|
||||
"stdout": "",
|
||||
"stderr": ""
|
||||
},
|
||||
"reboots": {
|
||||
"status": "Success",
|
||||
"stdout": "reboot system boot 7.0.0-1010-aws Thu Aug 13 06:35 still running\nreboot system boot 6.17.0-1013-aws Thu May 14 10:34 - 06:35 (90+20:00)\nreboot system boot 6.14.0-1018-aws Thu Mar 5 14:04 - 10:33 (69+20:29)\n\nwtmp begins Thu Mar 5 14:04:12 2026",
|
||||
"stderr": ""
|
||||
},
|
||||
"cron": {
|
||||
"status": "Success",
|
||||
"stdout": "total 32\ndrwxr-xr-x 2 root root 4096 May 5 07:41 .\ndrwxr-xr-x 117 root root 12288 Aug 27 06:00 ..\n-rw-r--r-- 1 root root 102 Mar 31 2024 .placeholder\n-rw-r--r-- 1 root root 802 Apr 16 2023 certbot\n-rw-r--r-- 1 root root 201 Apr 8 2024 e2scrub_all\n-rw-r--r-- 1 root root 396 Jan 9 2024 sysstat\n# Edit this file to introduce tasks to be run by cron.\n# \n# Each task to run has to be defined through a single line\n# indicating with different fields when the task will be run\n# and what command to run for the task\n# \n# To define the time you can provide concrete values for\n# minute (m), hour (h), day of month (dom), month (mon),\n# and day of week (dow) or use '*' in these fields (for 'any').\n# \n# Notice that tasks will be started based on the cron's system\n# daemon's notion of time and timezones.\n# \n# Output of the crontab jobs (including errors) is sent through\n# email to the user the crontab file belongs to (unless redirected).\n# \n# For example, you can run a backup of all your user accounts\n# at 5 a.m every week with:\n# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/\n# \n# For more information see the manual pages of crontab(5) and cron(8)\n# \n# m h dom mon dow command\n0 3 * * * /home/ubuntu/docker_clean.sh >> /var/log/docker_clean.log 2>&1",
|
||||
"stderr": ""
|
||||
}
|
||||
},
|
||||
"i-02754e7ae419cd5db": {
|
||||
"os_release": {
|
||||
"status": "Success",
|
||||
"stdout": "PRETTY_NAME=\"Ubuntu 24.04.4 LTS\"\nNAME=\"Ubuntu\"\nVERSION_ID=\"24.04\"\nVERSION=\"24.04.4 LTS (Noble Numbat)\"\nVERSION_CODENAME=noble\nID=ubuntu\nID_LIKE=debian\nHOME_URL=\"https://www.ubuntu.com/\"\nSUPPORT_URL=\"https://help.ubuntu.com/\"\nBUG_REPORT_URL=\"https://bugs.launchpad.net/ubuntu/\"\nPRIVACY_POLICY_URL=\"https://www.ubuntu.com/legal/terms-and-policies/privacy-policy\"\nUBUNTU_CODENAME=noble\nLOGO=ubuntu-logo\nLinux ip-172-31-19-86 7.0.0-1010-aws #10~24.04.1-Ubuntu SMP PREEMPT Mon Jul 27 17:41:33 UTC 2026 x86_64 x86_64 x86_64 GNU/Linux",
|
||||
"stderr": ""
|
||||
},
|
||||
"uptime_load": {
|
||||
"status": "Success",
|
||||
"stdout": " 19:51:11 up 15 days, 13:25, 0 user, load average: 0.19, 0.12, 0.10\n0.19 0.12 0.10 2/328 105213",
|
||||
"stderr": ""
|
||||
},
|
||||
"disk": {
|
||||
"status": "Success",
|
||||
"stdout": "Filesystem Size Used Avail Use% Mounted on\n/dev/root 24G 6.4G 17G 28% /\ntmpfs 2.0G 0 2.0G 0% /dev/shm\ntmpfs 782M 1.2M 781M 1% /run\ntmpfs 5.0M 0 5.0M 0% /run/lock\n/dev/xvda16 881M 202M 617M 25% /boot\n/dev/xvda15 105M 6.2M 99M 6% /boot/efi\noverlay 24G 6.4G 17G 28% /var/lib/docker/rootfs/overlayfs/ec16e04fb310adc22427b12093cf855bbcb4bb10f23444feb6791372e27fc66d\noverlay 24G 6.4G 17G 28% /var/lib/docker/rootfs/overlayfs/efac20386cb64dd82f0dcd510beef519f21f25a1e7ebb621277334c8f8dc415f\noverlay 24G 6.4G 17G 28% /var/lib/docker/rootfs/overlayfs/b83f92da28a542a416d2524d0e2d12bef5f19d71a0302b8fe6ece3706e84066f\n--- inodes\nFilesystem Inodes IUsed IFree IUse% Mounted on\n/dev/root 3145728 163314 2982414 6% /\ntmpfs 500170 2 500168 1% /dev/shm\ntmpfs 819200 758 818442 1% /run\ntmpfs 500170 3 500167 1% /run/lock\n/dev/xvda16 58496 605 57891 2% /boot\n/dev/xvda15 0 0 0 - /boot/efi\noverlay 3145728 163314 2982414 6% /var/lib/docker/rootfs/overlayfs/ec16e04fb310adc22427b12093cf855bbcb4bb10f23444feb6791372e27fc66d\noverlay 3145728 163314 2982414 6% /var/lib/docker/rootfs/overlayfs/efac20386cb64dd82f0dcd510beef519f21f25a1e7ebb621277334c8f8dc415f\noverlay 3145728 163314 2982414 6% /var/lib/docker/rootfs/overlayfs/b83f92da28a542a416d2524d0e2d12bef5f19d71a0302b8fe6ece3706e84066f",
|
||||
"stderr": ""
|
||||
},
|
||||
"memory": {
|
||||
"status": "Success",
|
||||
"stdout": " total used free shared buff/cache available\nMem: 3.8Gi 859Mi 391Mi 2.1Mi 2.9Gi 3.0Gi\nSwap: 0B 0B 0B\n--- swap",
|
||||
"stderr": ""
|
||||
},
|
||||
"code_dirs": {
|
||||
"status": "Failed",
|
||||
"stdout": "/home:\ntotal 12\ndrwxr-xr-x 3 root root 4096 Mar 12 14:41 .\ndrwxr-xr-x 22 root root 4096 Aug 13 06:25 ..\ndrwxr-x--- 4 ubuntu ubuntu 4096 Mar 12 18:00 ubuntu\n\n/opt:\ntotal 16\ndrwxr-xr-x 4 root root 4096 Mar 17 07:41 .\ndrwxr-xr-x 22 root root 4096 Aug 13 06:25 ..\ndrwxr-xr-x 5 root root 4096 Aug 26 00:53 aws\ndrwx--x--x 4 root root 4096 Mar 17 07:41 containerd\n\n/srv:\ntotal 28\ndrwxr-xr-x 3 root root 4096 Jun 11 20:58 .\ndrwxr-xr-x 22 root root 4096 Aug 13 06:25 ..\ndrwxr-xr-x 3 root root 4096 Jun 11 20:58 backup\n-rw-r--r-- 1 root root 4687 Mar 17 08:01 compose.yml\n-rw-r--r-- 1 root root 850 Mar 17 08:01 config.yaml\n-rw-r--r-- 1 root root 515 Mar 17 08:01 dashboard.env",
|
||||
"stderr": "failed to run commands: exit status 2"
|
||||
},
|
||||
"git_checkouts": {
|
||||
"status": "Success",
|
||||
"stdout": "",
|
||||
"stderr": ""
|
||||
},
|
||||
"git_remotes": {
|
||||
"status": "Success",
|
||||
"stdout": "",
|
||||
"stderr": ""
|
||||
},
|
||||
"manifests": {
|
||||
"status": "Success",
|
||||
"stdout": "/usr/share/doc/python3-botocore/requirements.txt",
|
||||
"stderr": ""
|
||||
},
|
||||
"processes": {
|
||||
"status": "Success",
|
||||
"stdout": "USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND\nroot 1253 0.2 2.6 1404280 107588 ? Ssl Aug13 58:26 traefik traefik --log.level=INFO --accesslog=true --providers.docker=true --providers.docker.exposedbydefault=false --providers.docker.network=netbird --entrypoints.web.address=:80 --entrypoints.websecure.address=:443 --entrypoints.websecure.allowACMEByPass=true --entrypoints.websecure.transport.respondingTimeouts.readTimeout=0 --entrypoints.websecure.transport.respondingTimeouts.writeTimeout=0 --entrypoints.websecure.transport.respondingTimeouts.idleTimeout=0 --entrypoints.web.http.redirections.entrypoint.to=websecure --entrypoints.web.http.redirections.entrypoint.scheme=https --certificatesresolvers.letsencrypt.acme.email=dilbag@devteamspace.com --certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json --certificatesresolvers.letsencrypt.acme.tlschallenge=true --serverstransport.forwardingtimeouts.responseheadertimeout=0s --serverstransport.forwardingtimeouts.idleconntimeout=0s\nroot 783 0.0 2.2 2533408 90852 ? Ssl Aug13 17:17 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock\nroot 1252 0.1 1.9 2033336 77536 ? Ssl Aug13 36:44 /go/bin/netbird-server --config /etc/netbird/config.yaml\nroot 1932 0.2 1.5 998780 60872 ? Ssl Aug13 49:58 /opt/aws/amazon-guardduty-agent/bin/amazon-guardduty-agent --worker-threads 8\nroot 101706 0.0 1.2 87900 50444 ? S<s 06:19 0:01 /usr/lib/systemd/systemd-journald\nroot 607 0.0 1.1 1896676 46568 ? Ssl Aug13 20:42 /usr/bin/containerd\nroot 4621 0.0 1.0 1925756 42432 ? Ssl Aug13 0:49 /snap/snapd/current/usr/lib/snapd/snapd\nroot 1503 0.0 0.8 2212664 35504 ? Sl Aug13 6:37 /snap/amazon-ssm-agent/13349/ssm-agent-worker\nroot 105270 1.7 0.7 1620160 28376 ? Sl 19:51 0:00 /snap/amazon-ssm-agent/13349/ssm-document-worker ce03d6ec-1114-4ace-85dc-3cc96b817568\nroot 105281 60.0 0.7 1620160 28120 ? Sl 19:51 0:00 /snap/amazon-ssm-agent/13349/ssm-document-worker 5dbf20c5-44d1-4549-8097-c7f6e7d0eef8\nroot 191 0.0 0.6 288960 27292 ? SLsl Aug13 1:45 /sbin/multipathd -d -s\nroot 694 0.0 0.5 110008 22840 ? Ssl Aug13 0:00 /usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal\nroot 1249 0.0 0.5 24916 22540 ? Ss Aug13 4:03 /usr/bin/python3 /usr/bin/supervisord -c /etc/supervisord.conf\nroot 570 0.0 0.5 1832820 21308 ? Ssl Aug13 1:17 /snap/amazon-ssm-agent/13349/amazon-ssm-agent\nroot 566 0.0 0.5 32428 20784 ? Ss Aug13 0:00 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startup-triggers\nroot 89081 0.0 0.3 468960 13768 ? Ssl Aug26 0:04 /usr/libexec/udisks2/udisksd\nroot 1 0.0 0.3 22304 13740 ? Ss Aug13 0:38 /usr/lib/systemd/systemd --system --deserialize=75\nsystemd+ 101705 0.0 0.3 21476 13256 ? Ss 06:19 0:02 /usr/lib/systemd/systemd-resolved\nroot 698 0.0 0.3 318156 12812 ? Ssl Aug13 0:00 /usr/sbin/ModemManager\nroot 1180 0.0 0.2 1268276 11096 ? Sl Aug13 1:09 /usr/bin/containerd-shim-runc-v2 -namespace moby -id b83f92da28a542a416d2524d0e2d12bef5f19d71a0302b8fe6ece3706e84066f -address /run/containerd/containerd.sock\nroot 1183 0.0 0.2 1268020 10812 ? Sl Aug13 4:25 /usr/bin/containerd-shim-runc-v2 -namespace moby -id efac20386cb64dd82f0dcd510beef519f21f25a1e7ebb621277334c8f8dc415f -address /run/containerd/containerd.sock\nroot 1179 0.0 0.2 1268084 10704 ? Sl Aug13 1:13 /usr/bin/containerd-shim-runc-v2 -namespace moby -id ec16e04fb310adc22427b12093cf855bbcb4bb10f23444feb6791372e27fc66d -address /run/containerd/containerd.sock\npolkitd 567 0.0 0.2 383668 10012 ? Ssl Aug13 0:23 /usr/lib/polkit-1/polkitd --no-debug\nsystemd+ 101701 0.0 0.2 19004 9988 ? Ss 06:19 0:00 /usr/lib/systemd/systemd-networkd\nroot 574 0.0 0.2 18172 9052 ? Ss Aug13 0:03 /usr/lib/systemd/systemd-logind\nroot 89080 0.0 0.2 29364 8564 ? Ss Aug26 0:01 /usr/lib/systemd/systemd-udevd\nroot 101707 0.0 0.2 12028 8388 ? Ss 06:19 0:00 sshd: /usr/sbin/sshd -D -o AuthorizedKeysCommand /usr/share/ec2-instance-connect/eic_run_authorized_keys %u %f -o AuthorizedKeysCommandUser ec2-instance-connect [listener] 0 of 10-100 startups\nsyslog 738 0.0 0.1 222516 6688 ? Ssl Aug13 0:03 /usr/sbin/rsyslogd -n -iNONE\nmessage+ 559 0.0 0.1 9964 5868 ? Ss Aug13 0:25 @dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation --syslog-only",
|
||||
"stderr": ""
|
||||
},
|
||||
"listening": {
|
||||
"status": "Success",
|
||||
"stdout": "udp UNCONN 0 0 127.0.0.54:53 0.0.0.0:* users:((\"systemd-resolve\",pid=101705,fd=16)) \nudp UNCONN 0 0 127.0.0.53%lo:53 0.0.0.0:* users:((\"systemd-resolve\",pid=101705,fd=14)) \nudp UNCONN 0 0 172.31.19.86%enX0:68 0.0.0.0:* users:((\"systemd-network\",pid=101701,fd=23)) \nudp UNCONN 0 0 127.0.0.1:323 0.0.0.0:* users:((\"chronyd\",pid=102077,fd=5)) \nudp UNCONN 0 0 0.0.0.0:3478 0.0.0.0:* users:((\"docker-proxy\",pid=1414,fd=8)) \nudp UNCONN 0 0 [::1]:323 [::]:* users:((\"chronyd\",pid=102077,fd=6)) \nudp UNCONN 0 0 [::]:3478 [::]:* users:((\"docker-proxy\",pid=1419,fd=8)) \ntcp LISTEN 0 4096 127.0.0.54:53 0.0.0.0:* users:((\"systemd-resolve\",pid=101705,fd=17)) \ntcp LISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:((\"systemd-resolve\",pid=101705,fd=15)) \ntcp LISTEN 0 4096 0.0.0.0:443 0.0.0.0:* users:((\"docker-proxy\",pid=1378,fd=8)) \ntcp LISTEN 0 4096 0.0.0.0:80 0.0.0.0:* users:((\"docker-proxy\",pid=1333,fd=8)) \ntcp LISTEN 0 4096 0.0.0.0:22 0.0.0.0:* users:((\"sshd\",pid=101707,fd=3),(\"systemd\",pid=1,fd=114))\ntcp LISTEN 0 4096 [::]:443 [::]:* users:((\"docker-proxy\",pid=1383,fd=8)) \ntcp LISTEN 0 4096 [::]:80 [::]:* users:((\"docker-proxy\",pid=1352,fd=8)) \ntcp LISTEN 0 4096 [::]:22 [::]:* users:((\"sshd\",pid=101707,fd=4),(\"systemd\",pid=1,fd=118))",
|
||||
"stderr": ""
|
||||
},
|
||||
"systemd_units": {
|
||||
"status": "Success",
|
||||
"stdout": " acpid.service loaded active running ACPI event daemon\n amazon-guardduty-agent.service loaded active running Amazon GuardDuty Agent\n chrony.service loaded active running chrony, an NTP client/server\n containerd.service loaded active running containerd container runtime\n cron.service loaded active running Regular background program processing daemon\n dbus.service loaded active running D-Bus System Message Bus\n docker.service loaded active running Docker Application Container Engine\n getty@tty1.service loaded active running Getty on tty1\n irqbalance.service loaded active running irqbalance daemon\n ModemManager.service loaded active running Modem Manager\n multipathd.service loaded active running Device-Mapper Multipath Device Controller\n networkd-dispatcher.service loaded active running Dispatcher daemon for systemd-networkd\n polkit.service loaded active running Authorization Manager\n rsyslog.service loaded active running System Logging Service\n serial-getty@ttyS0.service loaded active running Serial Getty on ttyS0\n snap.amazon-ssm-agent.amazon-ssm-agent.service loaded active running Service for snap application amazon-ssm-agent.amazon-ssm-agent\n snapd.service loaded active running Snap Daemon\n ssh.service loaded active running OpenBSD Secure Shell server\n systemd-journald.service loaded active running Journal Service\n systemd-logind.service loaded active running User Login Management\n systemd-networkd.service loaded active running Network Configuration\n systemd-resolved.service loaded active running Network Name Resolution\n systemd-udevd.service loaded active running Rule-based Manager for Device Events and Files\n udisks2.service loaded active running Disk Manager\n unattended-upgrades.service loaded active running Unattended Upgrades Shutdown",
|
||||
"stderr": ""
|
||||
},
|
||||
"systemd_failed": {
|
||||
"status": "Success",
|
||||
"stdout": "",
|
||||
"stderr": ""
|
||||
},
|
||||
"docker": {
|
||||
"status": "Success",
|
||||
"stdout": "netbird-traefik\ttraefik:v3.6\tUp 2 weeks\nnetbird-server\tnetbirdio/netbird-server:latest\tUp 2 weeks\nnetbird-dashboard\tnetbirdio/dashboard:latest\tUp 2 weeks",
|
||||
"stderr": ""
|
||||
},
|
||||
"docker_images": {
|
||||
"status": "Success",
|
||||
"stdout": "netbirdio/netbird-server:latest\t2026-06-10 20:25:13 +0000 UTC\ntraefik:v3.6\t2026-06-10 17:57:07 +0000 UTC\nnetbirdio/dashboard:latest\t2026-06-05 15:14:36 +0000 UTC",
|
||||
"stderr": ""
|
||||
},
|
||||
"oom_kills": {
|
||||
"status": "Success",
|
||||
"stdout": "",
|
||||
"stderr": ""
|
||||
},
|
||||
"svc_restarts": {
|
||||
"status": "Success",
|
||||
"stdout": "",
|
||||
"stderr": ""
|
||||
},
|
||||
"reboots": {
|
||||
"status": "Success",
|
||||
"stdout": "reboot system boot 7.0.0-1010-aws Thu Aug 13 06:25 still running\nreboot system boot 6.14.0-1018-aws Thu Mar 12 14:41 - 06:24 (153+15:43)\n\nwtmp begins Thu Mar 12 14:41:10 2026",
|
||||
"stderr": ""
|
||||
},
|
||||
"cron": {
|
||||
"status": "Failed",
|
||||
"stdout": "total 28\ndrwxr-xr-x 2 root root 4096 Dec 12 2025 .\ndrwxr-xr-x 116 root root 12288 Aug 28 06:19 ..\n-rw-r--r-- 1 root root 102 Mar 31 2024 .placeholder\n-rw-r--r-- 1 root root 201 Apr 8 2024 e2scrub_all\n-rw-r--r-- 1 root root 396 Jan 9 2024 sysstat",
|
||||
"stderr": "failed to run commands: exit status 1"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
+11
-2
@@ -55,10 +55,16 @@
|
||||
<div class="eyebrow">Dasnuve · Cloud Discovery</div>
|
||||
<h1>Scrivas — AWS Discovery Reports</h1>
|
||||
<p class="lede">Read-only assessment of the Scrivas AWS Organization (<code>o-qfj0pvhhv7</code>) — footprint,
|
||||
security posture, cost, and Organizations governance. Prepared to inform a proposal.</p>
|
||||
<div class="meta">2 accounts · us-east-2 primary · generated 2026-08-19 · CONFIDENTIAL</div>
|
||||
security posture, cost, and Organizations governance — extended with an EC2 source-code recovery and production reliability triage. Prepared to inform a proposal.</p>
|
||||
<div class="meta">2 accounts · us-east-2 primary · generated 2026-08-19, EC2 code pass 2026-08-28 · CONFIDENTIAL</div>
|
||||
|
||||
<div class="cards">
|
||||
<a class="card full" href="findings/code_dashboard.html">
|
||||
<span class="k">CODE + RELIABILITY</span>
|
||||
<h2>Source Code & Reliability Triage <span class="badge">time-sensitive</span></h2>
|
||||
<p>All 10 application repos found as full git checkouts on Scrivas-owned EC2 — recoverable despite the contractor holding the GitLab. Plus the reliability triage: 23 prod containers on one host, no memory limits, no swap.</p>
|
||||
<span class="go">Open dashboard →</span>
|
||||
</a>
|
||||
<a class="card" href="findings/discovery_dashboard.html">
|
||||
<span class="k">FINDINGS</span>
|
||||
<h2>Discovery Findings</h2>
|
||||
@@ -83,6 +89,9 @@
|
||||
<h3>Written report & raw evidence</h3>
|
||||
<ul>
|
||||
<li><a href="findings/discovery_report.md">discovery_report.md</a> — narrative writeup <code>(renders on GitHub)</code></li>
|
||||
<li><a href="findings/ec2_code_discovery_report.md">ec2_code_discovery_report.md</a> — code recovery + reliability triage <code>(renders on GitHub)</code></li>
|
||||
<li><code>findings/ec2_code_inspect.json</code> — SSM probe output, all 7 instances</li>
|
||||
<li><code>findings/ec2_code_inspect_prod.json</code> — prod/stage probe snapshot</li>
|
||||
<li><code>findings/org_assessment_report.json</code> — org / trust / delegated-admin raw data</li>
|
||||
<li><code>findings/fast_discovery.json</code> — footprint, security, cost raw data</li>
|
||||
<li><code>findings/member_lazka_547868853286.json</code> — member account raw data</li>
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Scrivas — EC2 source-code discovery (stage 1: inventory)
|
||||
========================================================
|
||||
Read-only. Enumerates EC2 instances in active regions and collects the
|
||||
signals that indicate where application source code lives:
|
||||
|
||||
- instance identity, state, type, AMI, launch time, tags
|
||||
- IAM instance profile (what the box can reach)
|
||||
- SSM managed status (can we run a read-only inspection on it?)
|
||||
- EC2 user data (frequently contains the bootstrap/deploy script,
|
||||
git remote URLs, artifact bucket names, image registries)
|
||||
|
||||
Context: the client does not hold the source code for their own platform;
|
||||
it was built under contract by a third party. This inventories what is
|
||||
running in the client's own account to locate their code.
|
||||
|
||||
Usage: python3 scripts/ec2_code_discovery.py [--profile PROFILE]
|
||||
Output: findings/ec2_code_discovery.json
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, date
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
|
||||
DEFAULT_PROFILE = "dasnuve-scrivas-louis-impersonation"
|
||||
REGIONS = ["us-east-2", "us-east-1"]
|
||||
FINDINGS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "findings")
|
||||
|
||||
|
||||
def _default(o):
|
||||
return o.isoformat() if isinstance(o, (datetime, date)) else str(o)
|
||||
|
||||
|
||||
def safe(fn, *a, **k):
|
||||
try:
|
||||
return fn(*a, **k), None
|
||||
except (ClientError, BotoCoreError) as e:
|
||||
return None, str(e)
|
||||
|
||||
|
||||
def tag_map(tags):
|
||||
return {t["Key"]: t["Value"] for t in (tags or [])}
|
||||
|
||||
|
||||
def user_data(ec2, iid):
|
||||
resp, err = safe(ec2.describe_instance_attribute, InstanceId=iid, Attribute="userData")
|
||||
if err:
|
||||
return {"error": err}
|
||||
raw = (resp.get("UserData") or {}).get("Value")
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return {"text": base64.b64decode(raw).decode("utf-8", "replace")}
|
||||
except Exception as e: # noqa: BLE001
|
||||
return {"error": f"decode failed: {e}"}
|
||||
|
||||
|
||||
def ssm_managed(session, region):
|
||||
"""Instance IDs SSM can reach — these are inspectable without SSH keys."""
|
||||
ssm = session.client("ssm", region_name=region)
|
||||
out = {}
|
||||
try:
|
||||
for page in ssm.get_paginator("describe_instance_information").paginate():
|
||||
for i in page.get("InstanceInformationList", []):
|
||||
out[i.get("InstanceId")] = {
|
||||
"ping_status": i.get("PingStatus"),
|
||||
"platform": f'{i.get("PlatformName")} {i.get("PlatformVersion")}',
|
||||
"agent": i.get("AgentVersion"),
|
||||
"last_ping": i.get("LastPingDateTime"),
|
||||
}
|
||||
except (ClientError, BotoCoreError) as e:
|
||||
return {"_error": str(e)}
|
||||
return out
|
||||
|
||||
|
||||
def scan_region(session, region):
|
||||
ec2 = session.client("ec2", region_name=region)
|
||||
managed = ssm_managed(session, region)
|
||||
instances = []
|
||||
|
||||
pages, err = safe(lambda: list(ec2.get_paginator("describe_instances").paginate()))
|
||||
if err:
|
||||
return {"error": err}
|
||||
|
||||
for page in pages:
|
||||
for res in page.get("Reservations", []):
|
||||
for i in res.get("Instances", []):
|
||||
iid = i["InstanceId"]
|
||||
tags = tag_map(i.get("Tags"))
|
||||
instances.append({
|
||||
"instance_id": iid,
|
||||
"name": tags.get("Name"),
|
||||
"state": i["State"]["Name"],
|
||||
"type": i.get("InstanceType"),
|
||||
"image_id": i.get("ImageId"),
|
||||
"launch_time": i.get("LaunchTime"),
|
||||
"key_name": i.get("KeyName"),
|
||||
"private_ip": i.get("PrivateIpAddress"),
|
||||
"public_ip": i.get("PublicIpAddress"),
|
||||
"iam_instance_profile": (i.get("IamInstanceProfile") or {}).get("Arn"),
|
||||
"security_groups": [g["GroupId"] for g in i.get("SecurityGroups", [])],
|
||||
"volumes": [
|
||||
b["Ebs"]["VolumeId"]
|
||||
for b in i.get("BlockDeviceMappings", []) if b.get("Ebs")
|
||||
],
|
||||
"tags": tags,
|
||||
"ssm": managed.get(iid),
|
||||
"user_data": user_data(ec2, iid),
|
||||
})
|
||||
return {
|
||||
"region": region,
|
||||
"ssm_managed_count": len([k for k in managed if not k.startswith("_")]),
|
||||
"instance_count": len(instances),
|
||||
"instances": instances,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--profile", default=DEFAULT_PROFILE)
|
||||
ap.add_argument("--regions", nargs="*", default=REGIONS)
|
||||
args = ap.parse_args()
|
||||
|
||||
session = boto3.Session(profile_name=args.profile)
|
||||
ident = session.client("sts").get_caller_identity()
|
||||
|
||||
report = {
|
||||
"generated": datetime.now().astimezone().isoformat(),
|
||||
"account": ident["Account"],
|
||||
"principal": ident["Arn"],
|
||||
"regions": [scan_region(session, r) for r in args.regions],
|
||||
}
|
||||
|
||||
os.makedirs(FINDINGS, exist_ok=True)
|
||||
path = os.path.join(FINDINGS, "ec2_code_discovery.json")
|
||||
with open(path, "w") as fh:
|
||||
json.dump(report, fh, indent=2, default=_default)
|
||||
print(f"wrote {path}")
|
||||
|
||||
for r in report["regions"]:
|
||||
if r.get("error"):
|
||||
print(f'{r}')
|
||||
continue
|
||||
print(f'\n== {r["region"]}: {r["instance_count"]} instances, {r["ssm_managed_count"]} SSM-managed')
|
||||
for i in r["instances"]:
|
||||
ud = i["user_data"]
|
||||
ud_flag = "userdata" if (ud and ud.get("text")) else "-"
|
||||
ssm_flag = i["ssm"]["ping_status"] if i.get("ssm") else "-"
|
||||
print(f' {i["instance_id"]:22} {i["state"]:10} {i["type"]:14} '
|
||||
f'ssm={ssm_flag:8} {ud_flag:9} {i.get("name")}')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Scrivas — EC2 code & reliability inspection (stage 2)
|
||||
=====================================================
|
||||
Runs a fixed, read-only command set on SSM-managed instances to locate the
|
||||
deployed application code and collect reliability evidence.
|
||||
|
||||
Context: Scrivas does not hold the source for their own platform (built by a
|
||||
third-party contractor) and is reporting production reliability issues. This
|
||||
inspects the client's own instances, in the client's own account, to (a) find
|
||||
where the code lives and (b) gather triage evidence.
|
||||
|
||||
SCOPE NOTE: ssm:SendCommand executes on the host. The command set below is
|
||||
read-only by construction -- no writes, no restarts, no config changes, and
|
||||
no dumping of file *contents* beyond manifests and VCS metadata. Review
|
||||
PROBES before running. Requires explicit operator approval.
|
||||
|
||||
Usage:
|
||||
python3 scripts/ec2_code_inspect.py --list
|
||||
python3 scripts/ec2_code_inspect.py --instance i-073154fb4fa773bbd
|
||||
python3 scripts/ec2_code_inspect.py --all
|
||||
Output: findings/ec2_code_inspect.json
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, date
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
|
||||
DEFAULT_PROFILE = "dasnuve-scrivas-louis-impersonation"
|
||||
REGION = "us-east-2"
|
||||
FINDINGS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "findings")
|
||||
|
||||
# Read-only probes. Each is (label, shell). Keep every command non-mutating.
|
||||
PROBES = [
|
||||
("os_release", "cat /etc/os-release; uname -a"),
|
||||
("uptime_load", "uptime; cat /proc/loadavg"),
|
||||
("disk", "df -h; echo '--- inodes'; df -i"),
|
||||
("memory", "free -h; echo '--- swap'; swapon --show"),
|
||||
("code_dirs", "ls -la /opt /srv /var/www /home 2>/dev/null"),
|
||||
("git_checkouts", "find / -maxdepth 6 -name .git -type d "
|
||||
"-not -path '*/node_modules/*' 2>/dev/null | head -40"),
|
||||
("git_remotes", "for g in $(find / -maxdepth 6 -name .git -type d "
|
||||
"-not -path '*/node_modules/*' 2>/dev/null | head -20); do "
|
||||
"r=$(dirname $g); echo \"== $r\"; "
|
||||
"git -C $r remote -v 2>/dev/null; "
|
||||
"git -C $r log -1 --format='%H %ad %an %s' 2>/dev/null; "
|
||||
"git -C $r status -sb 2>/dev/null | head -5; done"),
|
||||
("manifests", "find / -maxdepth 6 \\( -name package.json -o -name requirements.txt "
|
||||
"-o -name pyproject.toml -o -name go.mod -o -name Dockerfile "
|
||||
"-o -name docker-compose.y*ml \\) -not -path '*/node_modules/*' "
|
||||
"2>/dev/null | head -40"),
|
||||
("processes", "ps auxww --sort=-%mem | head -30"),
|
||||
("listening", "ss -tulpnH 2>/dev/null | head -40"),
|
||||
("systemd_units", "systemctl list-units --type=service --state=running --no-pager --no-legend | head -40"),
|
||||
("systemd_failed", "systemctl list-units --state=failed --no-pager --no-legend"),
|
||||
("docker", "docker ps -a --format '{{.Names}}\t{{.Image}}\t{{.Status}}' 2>/dev/null | head -30"),
|
||||
("docker_images", "docker images --format '{{.Repository}}:{{.Tag}}\t{{.CreatedAt}}' 2>/dev/null | head -20"),
|
||||
("oom_kills", "sudo dmesg -T 2>/dev/null | grep -iE 'oom|killed process' | tail -20"),
|
||||
("svc_restarts", "sudo journalctl --since '7 days ago' --no-pager 2>/dev/null "
|
||||
"| grep -iE 'segfault|out of memory|failed with result|start-limit' | tail -40"),
|
||||
("reboots", "last -x reboot 2>/dev/null | head -10"),
|
||||
("cron", "ls -la /etc/cron.d 2>/dev/null; crontab -l 2>/dev/null"),
|
||||
]
|
||||
|
||||
|
||||
def _default(o):
|
||||
return o.isoformat() if isinstance(o, (datetime, date)) else str(o)
|
||||
|
||||
|
||||
def managed(session):
|
||||
ssm = session.client("ssm", region_name=REGION)
|
||||
out = []
|
||||
for page in ssm.get_paginator("describe_instance_information").paginate():
|
||||
out += [i for i in page.get("InstanceInformationList", [])
|
||||
if i.get("PingStatus") == "Online"]
|
||||
return out
|
||||
|
||||
|
||||
def run_probe(ssm, iid, label, shell, timeout=120):
|
||||
try:
|
||||
cmd = ssm.send_command(
|
||||
InstanceIds=[iid],
|
||||
DocumentName="AWS-RunShellScript",
|
||||
Comment=f"dasnuve-discovery:{label}"[:100],
|
||||
Parameters={"commands": [shell], "executionTimeout": [str(timeout)]},
|
||||
)["Command"]["CommandId"]
|
||||
except (ClientError, BotoCoreError) as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
for _ in range(int(timeout / 2)):
|
||||
time.sleep(2)
|
||||
try:
|
||||
r = ssm.get_command_invocation(CommandId=cmd, InstanceId=iid)
|
||||
except ClientError as e:
|
||||
if "InvocationDoesNotExist" in str(e):
|
||||
continue
|
||||
return {"error": str(e)}
|
||||
if r["Status"] in ("Pending", "InProgress", "Delayed"):
|
||||
continue
|
||||
return {
|
||||
"status": r["Status"],
|
||||
"stdout": r.get("StandardOutputContent", "").rstrip(),
|
||||
"stderr": r.get("StandardErrorContent", "").rstrip(),
|
||||
}
|
||||
return {"error": "timed out waiting for invocation"}
|
||||
|
||||
|
||||
def inspect(session, iid):
|
||||
ssm = session.client("ssm", region_name=REGION)
|
||||
print(f"\n=== {iid}")
|
||||
res = {}
|
||||
for label, shell in PROBES:
|
||||
print(f" .. {label}", flush=True)
|
||||
res[label] = run_probe(ssm, iid, label, shell)
|
||||
return res
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--profile", default=DEFAULT_PROFILE)
|
||||
ap.add_argument("--instance", action="append", dest="instances")
|
||||
ap.add_argument("--all", action="store_true")
|
||||
ap.add_argument("--list", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
session = boto3.Session(profile_name=args.profile)
|
||||
online = managed(session)
|
||||
|
||||
if args.list:
|
||||
for i in online:
|
||||
print(f'{i["InstanceId"]:22} {i["PingStatus"]:8} '
|
||||
f'{i.get("PlatformName")} {i.get("PlatformVersion")}')
|
||||
return
|
||||
|
||||
targets = args.instances or ([i["InstanceId"] for i in online] if args.all else [])
|
||||
if not targets:
|
||||
ap.error("pass --instance ID (repeatable), --all, or --list")
|
||||
|
||||
report = {
|
||||
"generated": datetime.now().astimezone().isoformat(),
|
||||
"region": REGION,
|
||||
"probes": [p[0] for p in PROBES],
|
||||
"results": {iid: inspect(session, iid) for iid in targets},
|
||||
}
|
||||
|
||||
os.makedirs(FINDINGS, exist_ok=True)
|
||||
path = os.path.join(FINDINGS, "ec2_code_inspect.json")
|
||||
with open(path, "w") as fh:
|
||||
json.dump(report, fh, indent=2, default=_default)
|
||||
print(f"\nwrote {path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user