-
Notifications
You must be signed in to change notification settings - Fork 9
878 lines (788 loc) · 38.9 KB
/
Copy pathprod.docs.plus.yml
File metadata and controls
878 lines (788 loc) · 38.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
# =============================================================================
# Production CI/CD Pipeline — Quality Gates + Blue-Green Deployment
# =============================================================================
#
# Pipeline:
# 1. Quality Gates (parallel): lint, typecheck, security
# 2. Build Verification: extension + Next.js builds (smoke test)
# 3. Deploy Gate: parses commit message with a strict regex (anti-footgun)
# 4. Deploy: docker compose build on the prod host → blue-green rollout
#
# Triggers:
# - push to main: full pipeline. Deploy chain gated by triage outputs
# (`app_deploy`, `has_observability`, `has_uptime_kuma`). Ordered:
# app → observability → uptime-kuma (fail-fast).
# - pull_request to main: quality gates + build verification only.
# - schedule (Sunday 00:00 UTC): security audit only.
# - workflow_dispatch: manual run with `force_deploy` / `skip_quality_gates`.
#
# Architectural notes (decided 2026-05, see commit history):
# - Images are built on the prod self-hosted runner (`prod.docs.plus`).
# Disk pressure is mitigated by a pre-build disk guard, not by pushing
# to a registry. If pressure resurfaces, revisit M3 (ghcr.io push).
# - Rollback uses an on-disk tag stash on the prod host:
# /opt/projects/prod.docs.plus/.deploy/last-good-tag.
# - All third-party actions (workflow + composite) are pinned to commit
# SHA. Renovate/Dependabot should bump them; never use floating tags.
# =============================================================================
name: CI/CD Production
on:
push:
branches: [main]
pull_request:
branches: [main]
schedule:
# Weekly security scan (Sunday 00:00 UTC)
- cron: '0 0 * * 0'
workflow_dispatch:
inputs:
skip_quality_gates:
description: 'Skip quality gates (emergency deploy)'
required: false
default: false
type: boolean
force_deploy:
description: 'Force deployment (bypass commit-message gate)'
required: false
default: false
type: boolean
dry_run_subject:
description: 'Simulated commit subject to observe trigger gating (no deploy on dispatch)'
required: false
default: ''
type: string
# Two concurrency groups:
# - quality-gates can be cancelled freely (cheap to redo)
# - deploy MUST finish or rollback (mid-deploy SIGTERM corrupts blue-green state)
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-quality
cancel-in-progress: true
# Default to bash so set -e/-o pipefail behavior is consistent across runners.
defaults:
run:
shell: bash
# Workflow-level least privilege; per-job overrides where needed.
permissions:
contents: read
env:
ENV_SOURCE: /opt/projects/prod.docs.plus/.env
ENV_FILE: .env.production
COMPOSE_FILE: docker-compose.prod.yml
DEPLOY_TAG: ${{ github.sha }}
# Where the prod host stashes the last successfully deployed SHA for rollback.
DEPLOY_STATE_DIR: /opt/projects/prod.docs.plus/.deploy
LAST_GOOD_TAG_FILE: /opt/projects/prod.docs.plus/.deploy/last-good-tag
jobs:
# ===========================================================================
# STAGE −1 — TRIAGE (runs parse-build-trigger.sh; exposes skip_app_ci output)
# ===========================================================================
# Skipped on `schedule` (weekly security scan has no commit message to parse).
# All app-CI entry jobs gate on `needs.triage.outputs.skip_app_ci != 'true'`,
# replacing the six per-job `startsWith(…, '(build): observability')` guards.
# ===========================================================================
triage:
name: 🧭 Triage build trigger
runs-on: ubuntu-latest
timeout-minutes: 2
if: github.event_name != 'schedule'
permissions:
contents: read
outputs:
triggered: ${{ steps.t.outputs.triggered }}
domains: ${{ steps.t.outputs.domains }}
has_back: ${{ steps.t.outputs.has_back }}
has_front: ${{ steps.t.outputs.has_front }}
has_observability: ${{ steps.t.outputs.has_observability }}
has_uptime_kuma: ${{ steps.t.outputs.has_uptime_kuma }}
app_deploy: ${{ steps.t.outputs.app_deploy }}
skip_app_ci: ${{ steps.t.outputs.skip_app_ci }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: 🧭 Parse trigger
id: t
env:
COMMIT_MSG: ${{ github.event.head_commit.message || inputs.dry_run_subject }}
run: bash .github/scripts/parse-build-trigger.sh
# ===========================================================================
# STAGE 0 — CHANGE DETECTION (cheap; gates the expensive extension suite)
# ===========================================================================
# The clean-room extension suite (~14 min) is the pipeline's long pole and is
# a hard deploy gate. It only needs to run when something that affects an
# extension build/test actually changed. This job emits a boolean the
# extension-tests job keys off; lint/typecheck/security stay always-on
# because they are fast and repo-global (typecheck still catches extension
# TYPE regressions on every push even when the Cypress suite is skipped).
# ===========================================================================
changes:
name: 🔎 Detect Changes
runs-on: ubuntu-latest
timeout-minutes: 5
needs: [triage]
# Skip on `(build): observability` infra-only commits (skip_app_ci=true from
# triage) or when there's no app CI to run (schedule runs security only).
if: |
github.event_name != 'schedule' &&
needs.triage.outputs.skip_app_ci != 'true'
permissions:
contents: read
outputs:
changed_extensions: ${{ steps.assemble.outputs.list }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- id: filter
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
with:
filters: .github/filters/extensions.yaml
- id: assemble
env:
# Fallback (NEW.md §7.4): no reliable diff base -> treat all five as changed.
# Covers manual dispatch AND a first push / history rewrite (before == all-zeros SHA).
FORCE_ALL: ${{ github.event_name == 'workflow_dispatch' || github.event.before == '0000000000000000000000000000000000000000' }}
R_INDENT: ${{ steps.filter.outputs.extension-indent }}
R_HYPERLINK: ${{ steps.filter.outputs.extension-hyperlink }}
R_HMM: ${{ steps.filter.outputs.extension-hypermultimedia }}
R_INLINE: ${{ steps.filter.outputs.extension-inline-code }}
R_PLACEHOLDER: ${{ steps.filter.outputs.extension-placeholder }}
run: |
set -euo pipefail
all='["extension-indent","extension-hyperlink","extension-hypermultimedia","extension-inline-code","extension-placeholder"]'
if [ "${FORCE_ALL}" = "true" ]; then echo "list=${all}" >> "$GITHUB_OUTPUT"; exit 0; fi
sel=()
[ "${R_INDENT}" = "true" ] && sel+=('"extension-indent"')
[ "${R_HYPERLINK}" = "true" ] && sel+=('"extension-hyperlink"')
[ "${R_HMM}" = "true" ] && sel+=('"extension-hypermultimedia"')
[ "${R_INLINE}" = "true" ] && sel+=('"extension-inline-code"')
[ "${R_PLACEHOLDER}" = "true" ] && sel+=('"extension-placeholder"')
if [ "${#sel[@]}" -eq 0 ]; then echo "list=[]" >> "$GITHUB_OUTPUT"; else
IFS=,; echo "list=[${sel[*]}]" >> "$GITHUB_OUTPUT"; fi
# ===========================================================================
# STAGE 1 — QUALITY GATES (parallel, fast feedback)
# ===========================================================================
lint:
needs: [triage]
# Skip on infra-only commits (skip_app_ci=true) or schedule.
if: |
github.event_name != 'schedule' &&
needs.triage.outputs.skip_app_ci != 'true'
uses: ./.github/workflows/quality-lint.yml
typecheck:
name: 📝 Type Check
runs-on: ubuntu-latest
timeout-minutes: 15
needs: [triage]
# Skip on infra-only commits (skip_app_ci=true) or schedule.
if: |
github.event_name != 'schedule' &&
needs.triage.outputs.skip_app_ci != 'true'
permissions:
contents: read
steps:
- name: 📦 Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: 🥟 Setup Environment
uses: ./.github/actions/setup-bun
- name: 🔧 Build Extensions (required for types)
uses: ./.github/actions/build-extensions
- name: 📝 Type Check All
run: bun run typecheck
security:
name: 🔒 Security Audit
runs-on: ubuntu-latest
timeout-minutes: 10
needs: [triage]
# Keeps its schedule escape (weekly scan) even when triage is skipped.
if: |
always() &&
(github.event_name == 'schedule' || needs.triage.outputs.skip_app_ci != 'true')
permissions:
contents: read
steps:
- name: 📦 Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: 🥟 Setup Environment
uses: ./.github/actions/setup-bun
with:
ignore-scripts: 'true'
- name: 🔍 Bun Audit
run: |
set -o pipefail
echo "🔍 Checking for known vulnerabilities..."
# Capture both human-readable and machine-readable output.
# `bun pm audit` exits non-zero when vulns exist; we always want both files.
bun pm audit 2>&1 | tee audit-results.txt || true
bun pm audit --json > audit-results.json 2>/dev/null || echo '{}' > audit-results.json
# Structured parsing — replaces brittle `grep -ci "critical"` which
# matched the summary header line and produced false positives.
# Expected shape: { "vulnerabilities": { "critical": N, "high": N, ... } }
CRITICAL=$(bun -e 'const a=JSON.parse(require("fs").readFileSync("audit-results.json","utf8"));process.stdout.write(String(a?.vulnerabilities?.critical||0))')
HIGH=$(bun -e 'const a=JSON.parse(require("fs").readFileSync("audit-results.json","utf8"));process.stdout.write(String(a?.vulnerabilities?.high||0))')
echo ""
echo "📊 Summary: critical=${CRITICAL}, high=${HIGH}"
# Defensive: if both are 0 AND the JSON looks empty, the audit
# shape may have changed (Bun has changed it before). Print the
# raw JSON head so a future regression doesn't silently neutralize
# this gate. Caps at 4 KB to keep logs tidy.
if [ "${CRITICAL}" -eq 0 ] && [ "${HIGH}" -eq 0 ]; then
BYTES=$(wc -c < audit-results.json | tr -d ' ')
if [ "${BYTES}" -lt 32 ]; then
echo "::warning::audit-results.json is suspiciously small (${BYTES} bytes). Bun audit JSON shape may have changed."
echo "--- audit-results.json (head 4KB) ---"
head -c 4096 audit-results.json || true
echo ""
echo "--- end ---"
fi
fi
if [ "${CRITICAL}" -gt 0 ] || [ "${HIGH}" -gt 0 ]; then
echo "::error::Critical/High vulnerabilities detected (critical=${CRITICAL}, high=${HIGH})"
exit 1
fi
echo "✅ No critical/high vulnerabilities"
- name: 📤 Upload Audit Results
if: always()
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: security-audit-${{ github.sha }}
path: |
audit-results.txt
audit-results.json
retention-days: 30
back-validation:
name: 🔧 Backend Validation
needs: [triage]
if: needs.triage.outputs.has_back == 'true'
uses: ./.github/workflows/backend-ci.yml
extension-tests:
name: 🧪 Ext (${{ matrix.ext }})
needs: [triage, changes, back-validation]
# Run for a `front` trigger OR a normal app-CI commit; only changed extensions.
if: |
always() &&
needs.triage.outputs.skip_app_ci != 'true' &&
needs.back-validation.result != 'failure' &&
needs.changes.outputs.changed_extensions != '[]' &&
(needs.triage.outputs.has_front == 'true' || needs.triage.outputs.triggered != 'true' || github.event_name == 'pull_request')
strategy:
fail-fast: false
matrix:
# `|| '[]'` guards the runtime throw if `changes` is skipped and the output is '' (actionlint won't catch this).
ext: ${{ fromJson(needs.changes.outputs.changed_extensions || '[]') }}
runs-on: ubuntu-latest
timeout-minutes: 25
permissions:
contents: read
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: ./.github/actions/setup-bun
- uses: ./.github/actions/build-extensions
with:
extensions: ${{ matrix.ext }}
- uses: ./.github/actions/setup-cypress
- name: 🧪 Clean-room suite (${{ matrix.ext }})
env:
EXTENSION_DIST_READY: '1'
EXT_ONLY: ${{ matrix.ext }}
run: bash scripts/run-tests.sh --extensions
- name: ✈️ Preflight (${{ matrix.ext }})
env:
EXT_ONLY: ${{ matrix.ext }}
run: bash scripts/extension-preflight.sh
webapp-unit-tests:
name: 🧪 Webapp Unit Tests
runs-on: ubuntu-latest
timeout-minutes: 10
needs: [triage, back-validation]
if: |
always() &&
github.event_name != 'schedule' &&
needs.triage.outputs.skip_app_ci != 'true' &&
needs.back-validation.result != 'failure' &&
(needs.triage.outputs.has_front == 'true' || needs.triage.outputs.triggered != 'true' || github.event_name == 'pull_request')
permissions:
contents: read
steps:
- name: 📦 Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: 🥟 Setup Environment
uses: ./.github/actions/setup-bun
# Webapp tests import built extension dist (e.g. @docs.plus/extension-hyperlink
# via useHyperlinkEditorForm); build them first or the suite fails to resolve.
- name: 🔧 Build Extensions (required for webapp imports)
uses: ./.github/actions/build-extensions
- name: 🧪 Jest (webapp)
# Some webapp modules construct a Supabase client at import time (utils
# barrel → ensureAnonymousSession), which throws without these. Dummy
# fallbacks mirror the build job; tests never make real Supabase calls.
env:
NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL || 'http://localhost:54321' }}
NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY || 'dummy-key' }}
run: bun run --filter @docs.plus/webapp test
# ===========================================================================
# STAGE 2 — BUILD VERIFICATION (smoke test, no artifacts produced)
# ===========================================================================
# Note: this job intentionally does NOT build Docker images. The deploy job
# rebuilds them on the prod host anyway (decided 2026-05); duplicating the
# docker build here would just slow the pipeline without sharing cache. The
# webapp/admin Next.js compile here catches type/build regressions early.
# ===========================================================================
build:
name: 🏗️ Build Verification
runs-on: ubuntu-latest
timeout-minutes: 35
needs:
[
triage,
changes,
lint,
typecheck,
security,
extension-tests,
webapp-unit-tests,
back-validation
]
if: |
always() &&
github.event_name != 'schedule' &&
needs.triage.outputs.skip_app_ci != 'true' &&
needs.changes.result == 'success' &&
(needs.lint.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
(needs.typecheck.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
(needs.security.result == 'success' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
(needs.back-validation.result == 'success' || needs.back-validation.result == 'skipped' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
(needs.extension-tests.result == 'success' || needs.extension-tests.result == 'skipped' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates)) &&
(needs.webapp-unit-tests.result == 'success' || needs.webapp-unit-tests.result == 'skipped' || (github.event_name == 'workflow_dispatch' && inputs.skip_quality_gates))
permissions:
contents: read
steps:
- name: 📦 Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: 🥟 Setup Environment
uses: ./.github/actions/setup-bun
- name: 🔧 Build TipTap Extensions
uses: ./.github/actions/build-extensions
- name: 🏗️ Build Webapp
run: bun run --filter @docs.plus/webapp build:ci
env:
NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL || 'http://localhost:54321' }}
NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY || 'dummy-key' }}
- name: 🏗️ Build Admin Dashboard
run: bun run --filter @docs.plus/admin-dashboard build:ci
env:
NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.NEXT_PUBLIC_SUPABASE_URL || 'http://localhost:54321' }}
NEXT_PUBLIC_SUPABASE_ANON_KEY: ${{ secrets.NEXT_PUBLIC_SUPABASE_ANON_KEY || 'dummy-key' }}
NEXT_PUBLIC_API_URL: ${{ secrets.NEXT_PUBLIC_API_URL || 'http://localhost:3003' }}
NEXT_PUBLIC_APP_URL: ${{ secrets.NEXT_PUBLIC_APP_URL || 'http://localhost:3000' }}
# ===========================================================================
# STAGE 3 — PRODUCTION DEPLOYMENT (deploy gate inlined — no cross-job output)
# ===========================================================================
# The deploy decision lives directly on this job's `if`. `app_deploy` already
# encodes `app_pipeline && !no_deploy`, so `(build): back front no-deploy` →
# app_deploy=false → no deploy. `always()` lets the `if` evaluate even when an
# upstream is skipped; `needs.build.result == 'success'` is the real fail-fast
# gate. (Inlined from a former deploy-gate job whose constant output did not
# propagate reliably through `needs.deploy-gate.outputs` under `always()`.)
# IMPORTANT: opts OUT of cancel-in-progress — a mid-deploy SIGTERM during
# `docker compose up --scale` can corrupt blue-green state.
# ===========================================================================
deploy:
name: 🚀 Deploy Production
runs-on: prod.docs.plus
timeout-minutes: 30
needs: [triage, build]
concurrency:
group: ${{ github.workflow }}-deploy
cancel-in-progress: false
if: |
always() &&
needs.build.result == 'success' &&
((github.event_name == 'push' && needs.triage.outputs.app_deploy == 'true') ||
(github.event_name == 'workflow_dispatch' && inputs.force_deploy))
environment:
name: production
url: https://docs.plus
permissions:
contents: read
steps:
- name: 📦 Checkout Code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 1
- name: 🔐 Prepare Environment
run: |
# Compose `--env-file` is the single source of truth. We do NOT also
# `set -a; source` it elsewhere — that path was double-loading and
# leaking vars to subshells unintentionally.
cp "${ENV_SOURCE}" "${ENV_FILE}"
echo "DEPLOY_TAG=${DEPLOY_TAG}" >> "${ENV_FILE}"
# Stash the previous successful tag (if any) for the rollback step.
mkdir -p "${DEPLOY_STATE_DIR}"
if [ -f "${LAST_GOOD_TAG_FILE}" ]; then
PREVIOUS_TAG=$(cat "${LAST_GOOD_TAG_FILE}")
echo "PREVIOUS_TAG=${PREVIOUS_TAG}" >> "$GITHUB_ENV"
echo "ℹ️ Previous good tag: ${PREVIOUS_TAG}"
else
echo "PREVIOUS_TAG=" >> "$GITHUB_ENV"
echo "ℹ️ No previous good tag stashed (first deploy or fresh state dir)"
fi
echo "✅ Environment ready"
- name: 💾 Pre-deploy disk guard
run: |
echo "📊 Disk before prune:"
df -h / | tail -1
# Free space proactively. Without this, --no-cache builds can fill
# the root volume between deploys and silently OOM/ENOSPC the build
# step (job ends in <2min with no error). Runs before build, not after.
docker image prune -af --filter "until=24h" 2>/dev/null || true
docker builder prune -af --filter "until=24h" 2>/dev/null || true
# Hard guard: refuse to build when <10 GB free. Fail loud here
# rather than fail silently mid-build.
AVAIL_KB=$(df --output=avail / | tail -1)
AVAIL_GB=$((AVAIL_KB / 1024 / 1024))
echo "📊 Disk after prune: ${AVAIL_GB} GB free"
if [ "${AVAIL_GB}" -lt 10 ]; then
echo "::error::Less than 10 GB free on /. Aborting deploy. SSH to host and run 'docker system prune -af --volumes'."
df -h /
docker system df
exit 1
fi
- name: 📂 Verify build context (monorepo root)
run: |
if [ ! -d packages/email-templates ]; then
echo "::error::packages/email-templates missing. Build context must be repo root (context: .). Check checkout includes the workspace."
exit 1
fi
if ! grep -q 'email-templates' apps/hocuspocus.server/docker/Dockerfile.bun; then
echo "::error::apps/hocuspocus.server/docker/Dockerfile.bun must COPY packages/email-templates."
exit 1
fi
if ! grep -q 'email-templates' apps/webapp/docker/Dockerfile.bun; then
echo "::error::apps/webapp/docker/Dockerfile.bun must COPY packages/email-templates."
exit 1
fi
echo "✅ Build context OK (repo root, email-templates present)"
- name: 🏗️ Build Docker Images
env:
DOCKER_BUILDKIT: '1'
COMPOSE_DOCKER_CLI_BUILD: '1'
run: |
echo "🔨 Building images with tag: ${DEPLOY_TAG}"
# hocuspocus-server and hocuspocus-worker share `docsplus-hocuspocus`;
# building both via compose with --no-cache duplicates context transfer
# and ties up the bake plan. Build via hocuspocus-server only; the
# worker reuses the resulting tag at `up` time.
#
# --no-cache: required as long as the prod entrypoint script changes
# are layered late in the Dockerfile and we don't yet have stable
# layer ordering. If/when entrypoint COPY moves to the last layer,
# we can drop --no-cache and gain ~5 min per deploy.
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
build --no-cache rest-api hocuspocus-server
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
build --parallel webapp admin-dashboard
echo "✅ Images built"
- name: 🔧 Ensure Infrastructure
run: |
echo "🔧 Ensuring infrastructure..."
docker network create docsplus-network 2>/dev/null || true
# Start Traefik and Redis (--no-recreate keeps existing if running)
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
up -d --no-recreate traefik redis
# Force-start Traefik if somehow not running
if ! docker ps --filter "name=traefik" --filter "status=running" --format '{{.Names}}' | grep -q traefik; then
echo "⚠️ Traefik not running, starting..."
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" up -d traefik
sleep 15
fi
# Wait for healthy
echo "⏳ Waiting for Traefik..."
for i in {1..30}; do
if docker ps --filter "name=traefik" --filter "health=healthy" --format '{{.Names}}' | grep -q traefik; then
echo "✅ Traefik healthy"
break
fi
[ "${i}" -eq 30 ] && echo "⚠️ Traefik health timeout, continuing..."
sleep 2
done
- name: 🚀 Deploy Services (Blue-Green)
run: |
echo "🚀 Starting zero-downtime deployment..."
deploy_service() {
local SERVICE="$1"
local TARGET="$2"
local CURRENT
CURRENT=$(docker ps --filter "label=com.docker.compose.service=${SERVICE}" -q | wc -l | tr -d ' ')
local SCALE_UP=$((CURRENT + TARGET))
echo ""
echo "📦 Deploying ${SERVICE} (current: ${CURRENT}, target: ${TARGET})..."
# Scale UP first (keeps old containers serving traffic)
if [ "${SCALE_UP}" -gt "${CURRENT}" ]; then
echo "⬆️ Scaling up to ${SCALE_UP}..."
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
up -d --no-deps --scale "${SERVICE}=${SCALE_UP}" "${SERVICE}"
# Wait for healthy. 60×2s = 120s — Next.js cold start can hit
# 60-90s right after a --no-cache build. Was 30×2s = 60s before
# which produced false-fail rollbacks.
echo "⏳ Waiting for healthy containers..."
for i in {1..60}; do
local HEALTHY
HEALTHY=$(docker ps --filter "label=com.docker.compose.service=${SERVICE}" --filter "health=healthy" -q | wc -l)
if [ "${HEALTHY}" -ge "${TARGET}" ]; then
echo "✅ ${HEALTHY} healthy containers"
break
fi
[ $((i % 10)) -eq 0 ] && echo " ... ${HEALTHY}/${TARGET} healthy (attempt ${i}/60)"
sleep 2
done
fi
# Scale to target (compose removes old containers)
echo "📏 Scaling to target: ${TARGET}..."
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
up -d --no-deps --scale "${SERVICE}=${TARGET}" "${SERVICE}"
sleep 2
}
deploy_service "webapp" 2
deploy_service "rest-api" 2
deploy_service "hocuspocus-server" 2
deploy_service "hocuspocus-worker" 1
deploy_service "admin-dashboard" 1
echo ""
echo "✅ All services deployed"
- name: 🩺 Verify Deployment
run: |
echo "🩺 Verifying deployment..."
sleep 10
# Infrastructure check
echo "📊 Infrastructure:"
for svc in traefik docsplus-redis; do
if docker ps --filter "name=${svc}" --filter "status=running" --format '{{.Names}}' | grep -q "${svc}"; then
echo " ✅ ${svc}: running"
else
echo " ❌ ${svc}: NOT running"
docker logs "${svc}" --tail 30 2>/dev/null || true
exit 1
fi
done
# Service running + healthy check
echo "📊 Services:"
for svc in webapp rest-api hocuspocus-server hocuspocus-worker admin-dashboard; do
RUNNING=$(docker ps --filter "label=com.docker.compose.service=${svc}" --filter "status=running" --format "{{.Names}}" | wc -l)
HEALTHY=$(docker ps --filter "label=com.docker.compose.service=${svc}" --filter "health=healthy" --format "{{.Names}}" | wc -l)
if [ "${RUNNING}" -gt 0 ]; then
echo " ✅ ${svc}: ${RUNNING} running, ${HEALTHY} healthy"
else
echo " ❌ ${svc}: NOT running"
exit 1
fi
done
# Internal smoke test — hit container health endpoints via the
# docker network, NOT via the public DNS+TLS stack. A transient
# ACME / Let's Encrypt hiccup must not trigger a false-fail rollback.
echo ""
echo "🔍 Internal smoke tests..."
smoke() {
local SVC="$1" PORT="$2" PATH_="$3"
if docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" exec -T "${SVC}" \
bun -e "fetch('http://localhost:${PORT}${PATH_}').then(r => r.ok ? process.exit(0) : process.exit(1)).catch(() => process.exit(1))"; then
echo " ✅ ${SVC} internal health"
else
echo " ❌ ${SVC} internal health"
return 1
fi
}
smoke webapp 3000 /api/health
smoke rest-api 4000 /health
smoke hocuspocus-server 4001 /health
smoke hocuspocus-worker 4002 /health
smoke admin-dashboard 3100 /api/health
# Public-URL probe is now informational only — does NOT fail the deploy.
# Real public-availability monitoring belongs in uptime-kuma, not here.
echo ""
echo "🌐 Public URL probe (informational):"
PUBLIC_CODE=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 10 https://docs.plus/ 2>/dev/null || echo "000")
echo " https://docs.plus/ → ${PUBLIC_CODE}"
API_CODE=$(curl -sf -o /dev/null -w "%{http_code}" --max-time 10 https://prodback.docs.plus/api/health 2>/dev/null || echo "000")
echo " https://prodback.docs.plus/api/health → ${API_CODE}"
echo ""
echo "✅ Deployment verified"
- name: 📁 Sync compose files for break-glass
if: success()
run: |
# Replaces the previous "Sync Production Directory" step which re-`up`'d
# services from a different cwd — that re-up broke the blue-green
# guarantee. Now we only COPY the active compose + env files to a
# stable path so a human SSH'd in can run, e.g.:
# cd /opt/projects/prod.docs.plus/.deploy/current
# docker compose -f docker-compose.prod.yml --env-file .env.production ps
# without having to know the runner's _work directory.
mkdir -p "${DEPLOY_STATE_DIR}/current"
cp "${COMPOSE_FILE}" "${DEPLOY_STATE_DIR}/current/${COMPOSE_FILE}"
cp "${ENV_FILE}" "${DEPLOY_STATE_DIR}/current/${ENV_FILE}"
echo "✅ Synced compose+env to ${DEPLOY_STATE_DIR}/current/"
- name: 💾 Stash this tag as last-good
# Only on success — failure path is handled by the rollback step.
if: success()
run: |
# Persist current tag for the next deploy's rollback target.
mkdir -p "${DEPLOY_STATE_DIR}"
# Keep the previous one as last-good-tag.previous for one-step-back debugging.
if [ -f "${LAST_GOOD_TAG_FILE}" ]; then
cp "${LAST_GOOD_TAG_FILE}" "${LAST_GOOD_TAG_FILE}.previous"
fi
echo "${DEPLOY_TAG}" > "${LAST_GOOD_TAG_FILE}"
echo "✅ Stashed last-good-tag = ${DEPLOY_TAG}"
- name: 🧹 Cleanup
if: success()
continue-on-error: true # cleanup failure shouldn't fail an otherwise green deploy
run: |
docker image prune -f
docker image prune -f --filter "until=24h" 2>/dev/null || true
echo "✅ Cleanup complete"
- name: 📊 Summary
if: success()
run: |
echo "======================================"
echo "✅ DEPLOYMENT SUCCESSFUL"
echo "======================================"
echo "Tag: ${DEPLOY_TAG}"
echo "Previous tag: ${PREVIOUS_TAG:-<none>}"
echo ""
echo "Services:"
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | grep -E "(traefik|docsplus|webapp|rest-api|hocuspocus)" | head -15
echo ""
echo "URLs:"
echo " - https://docs.plus"
echo " - https://prodback.docs.plus"
echo "======================================"
- name: 🚨 Rollback on Failure
if: failure()
run: |
echo "⚠️ Deployment failed — attempting rollback..."
if [ -z "${PREVIOUS_TAG:-}" ]; then
echo "::warning::No PREVIOUS_TAG stashed — cannot auto-rollback."
echo "📊 Current state:"
docker ps --format "table {{.Names}}\t{{.Status}}" | head -15
exit 0
fi
echo "↩️ Rolling back to: ${PREVIOUS_TAG}"
# Multi-image precondition (A2): ALL service images for the previous
# tag must still exist locally. The cleanup step honors --filter
# until=24h, so within a 24h window this works reliably; outside
# that window we fail loudly rather than partially-rollback into a
# mixed-version cluster.
MISSING=()
for img in docsplus-webapp docsplus-rest-api docsplus-hocuspocus docsplus-admin; do
if ! docker image inspect "${img}:${PREVIOUS_TAG}" >/dev/null 2>&1; then
MISSING+=("${img}:${PREVIOUS_TAG}")
fi
done
if [ "${#MISSING[@]}" -gt 0 ]; then
echo "::error::Cannot auto-rollback. Missing images for previous tag:"
for img in "${MISSING[@]}"; do
echo " - ${img}"
done
echo "Manual recovery: bring traffic back via the existing healthy containers."
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" up -d --no-recreate 2>/dev/null || true
exit 1
fi
# Override DEPLOY_TAG in the env file and re-deploy with previous images.
sed -i.bak "s|^DEPLOY_TAG=.*|DEPLOY_TAG=${PREVIOUS_TAG}|" "${ENV_FILE}"
rm -f "${ENV_FILE}.bak"
docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \
up -d --force-recreate \
webapp rest-api hocuspocus-server hocuspocus-worker admin-dashboard
# Post-rollback verification (A3). Give containers a moment to bind
# ports + pass first healthcheck, then run the same internal smoke
# set the forward path runs. If rollback itself can't come healthy,
# we want the workflow to fail RED so the on-call sees it instead
# of a misleading "rollback complete" green check.
echo ""
echo "⏳ Waiting 30s for rolled-back containers to settle..."
sleep 30
smoke() {
local SVC="$1" PORT="$2" PATH_="$3"
if docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" exec -T "${SVC}" \
bun -e "fetch('http://localhost:${PORT}${PATH_}').then(r => r.ok ? process.exit(0) : process.exit(1)).catch(() => process.exit(1))"; then
echo " ✅ ${SVC} internal health (post-rollback)"
return 0
else
echo " ❌ ${SVC} internal health (post-rollback)"
return 1
fi
}
ROLLBACK_OK=1
smoke webapp 3000 /api/health || ROLLBACK_OK=0
smoke rest-api 4000 /health || ROLLBACK_OK=0
smoke hocuspocus-server 4001 /health || ROLLBACK_OK=0
smoke hocuspocus-worker 4002 /health || ROLLBACK_OK=0
smoke admin-dashboard 3100 /api/health || ROLLBACK_OK=0
echo ""
echo "📊 Post-rollback state:"
docker ps --format "table {{.Names}}\t{{.Status}}" | head -15
if [ "${ROLLBACK_OK}" -ne 1 ]; then
echo "::error::Rollback to ${PREVIOUS_TAG} did not pass smoke tests. Manual intervention required."
exit 1
fi
echo "✅ Rollback to ${PREVIOUS_TAG} verified healthy"
# ===========================================================================
# STAGE 4 — OBSERVABILITY DEPLOY (ordered after app deploy)
# ===========================================================================
observability-deploy:
name: 🔭 Deploy Observability
needs: [triage]
# Independent monitoring infra — deploys whenever `observability` is in the set.
# Deliberately does NOT `needs: deploy`: GitHub does not reliably run a
# reusable-workflow call job under `always()` when a needed job (the app deploy)
# is skipped, so depending on `deploy` made `(build): observability` silently
# skip. The single prod runner serializes app vs observability deploys anyway,
# and monitoring should stay up regardless of an app-deploy outcome.
if: |
github.event_name == 'push' &&
needs.triage.outputs.has_observability == 'true'
uses: ./.github/workflows/observability.docs.plus.yml
with:
action: setup
# ===========================================================================
# STAGE 5 — UPTIME KUMA DEPLOY (trailing, ordered last)
# ===========================================================================
uptime-kuma-deploy:
name: 🔔 Deploy Uptime Kuma
runs-on: prod.docs.plus
timeout-minutes: 10
# Job-level group serializes a SECOND workflow run's uptime-kuma deploy behind the
# first (cross-run). It does NOT shield this job from the workflow-level `…-quality`
# group cancelling the parent run — that residual is the pre-existing, accepted
# blue-green exposure (Global Constraints "preserve, do not regress"); this plan does
# not widen it. Matches the app deploy's own `…-deploy` group semantics.
concurrency:
group: uptime-kuma-deploy
cancel-in-progress: false
needs: [triage, deploy, observability-deploy]
# Always LAST. Fail-fast: any app domain in the set requires deploy success, and
# any observability in the set requires observability success (a skip from an
# upstream failure is not success). Absent domains skip and don't block.
if: |
always() &&
github.event_name == 'push' &&
needs.triage.outputs.has_uptime_kuma == 'true' &&
(needs.triage.outputs.app_deploy != 'true' || needs.deploy.result == 'success') &&
(needs.triage.outputs.has_observability != 'true' || needs.observability-deploy.result == 'success')
permissions: { contents: read }
steps:
- name: 🚀 Deploy
run: |
UPTIME_KUMA_IMAGE='louislam/uptime-kuma:1@sha256:bb1bcecbc3e3ffb1cb0f8fc5f9c3cdaa78c1dfb56d98d64e06da13ebfc6dba0d'
docker network create docsplus-network 2>/dev/null || true
docker stop uptime-kuma 2>/dev/null || true
docker rm uptime-kuma 2>/dev/null || true
docker run -d --name uptime-kuma --network docsplus-network --restart unless-stopped \
-v uptime-kuma-data:/app/data \
--label "traefik.enable=true" \
--label "traefik.http.routers.uptime.rule=Host(\`status.docs.plus\`)" \
--label "traefik.http.routers.uptime.entrypoints=websecure" \
--label "traefik.http.routers.uptime.tls.certresolver=letsencrypt" \
--label "traefik.http.services.uptime.loadbalancer.server.port=3001" \
"${UPTIME_KUMA_IMAGE}"
# Health poll replaces blind sleep 15 (D6 cheap win).
for i in $(seq 1 20); do
if docker exec uptime-kuma wget -qO- http://localhost:3001 >/dev/null 2>&1; then echo "✅ uptime-kuma healthy"; break; fi
[ "$i" -eq 20 ] && echo "⚠️ uptime-kuma health timeout (continuing)"; sleep 2
done