Compare commits
171 Commits
devin/1791
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| a667a92cd8 | |||
| d2a6f6e8ad | |||
| 1d0a39dbc6 | |||
|
|
19c7eb13dc | ||
|
|
8bbfc4e8f5 | ||
|
|
96f3d62081 | ||
|
|
0156177c74 | ||
|
|
219f428208 | ||
|
|
2f42afa78f | ||
|
|
b6196f8666 | ||
|
|
03170c5482 | ||
|
|
537a622409 | ||
|
|
08d4c80bd7 | ||
|
|
666ab88106 | ||
|
|
e1081e31b5 | ||
|
|
95b4ca8c5e | ||
|
|
3518708a99 | ||
|
|
60f25dd7ee | ||
|
|
5863b206a2 | ||
|
|
fdea749005 | ||
|
|
f2f33931ba | ||
|
|
8e3e827b02 | ||
|
|
f31334bd7a | ||
|
|
8cd6e1e0fa | ||
|
|
05499dba2b | ||
|
|
ee8581136e | ||
|
|
bfad481ade | ||
|
|
90413bc6fe | ||
|
|
99357a6823 | ||
|
|
480cd6b456 | ||
|
|
417c002213 | ||
|
|
2dc3ea22ae | ||
|
|
0d8a1c5dd9 | ||
|
|
09e9a299a7 | ||
|
|
dfebe58580 | ||
|
|
9c42b7b7ee | ||
|
|
e62d80fa58 | ||
|
|
494ed2c06f | ||
|
|
7496316875 | ||
|
|
fdc9b2f430 | ||
|
|
2fcc56518e | ||
|
|
43112e5c4d | ||
|
|
dbed87ce55 | ||
|
|
8fc5f86cac | ||
|
|
ddecd1e65a | ||
|
|
f8fbca7cbb | ||
|
|
005d8d12b4 | ||
|
|
348dd803d3 | ||
|
|
7e19faa5e9 | ||
|
|
d7c794e919 | ||
|
|
52a4df2f1a | ||
|
|
214428e2a5 | ||
|
|
68dcaa7893 | ||
|
|
acbb701a82 | ||
|
|
bf6a450aa9 | ||
|
|
7d31f5b625 | ||
|
|
449a953316 | ||
|
|
3660182787 | ||
|
|
0e82f1827c | ||
|
|
b2a89f8ce4 | ||
|
|
061c804062 | ||
|
|
4f926b3efc | ||
|
|
e31d2ec479 | ||
|
|
ae19fbedb1 | ||
|
|
f2304c0d0a | ||
|
|
c2257ae993 | ||
|
|
5f70e3821e | ||
|
|
5a1619da9d | ||
|
|
e4ec7eb1a3 | ||
|
|
85cfbc8c58 | ||
|
|
5ee2049ab4 | ||
|
|
a2e81c2ef5 | ||
|
|
fe328eca7e | ||
|
|
5f1ff79387 | ||
|
|
76074c5973 | ||
|
|
1ff9da55a5 | ||
|
|
3386052ce8 | ||
|
|
4c0df6762d | ||
|
|
c53ee85e14 | ||
|
|
aafcb4f759 | ||
|
|
2050fcf23e | ||
|
|
e41855d01f | ||
|
|
49f325d225 | ||
|
|
169376ef9e | ||
|
|
edd1060e3e | ||
|
|
3e0f5302dd | ||
|
|
2fa359a3b5 | ||
|
|
0ba1acf583 | ||
|
|
fdbca291d3 | ||
|
|
eea71832ec | ||
|
|
04cd135503 | ||
|
|
ae9eca5c7e | ||
|
|
6b8c66c249 | ||
|
|
0911c5390b | ||
|
|
9af270a533 | ||
|
|
10cdecefd7 | ||
|
|
4890945ea1 | ||
|
|
2f18278b9e | ||
|
|
3b1c50b9ec | ||
|
|
bb1b93255d | ||
|
|
3d7372142a | ||
|
|
bdc9d38e1a | ||
|
|
776ec6dcc7 | ||
|
|
67a2077ce0 | ||
|
|
d3612255bb | ||
|
|
a54b50a971 | ||
|
|
e1167f9a61 | ||
|
|
ed76eb5ac0 | ||
|
|
a4d9ca5f20 | ||
|
|
617aa26037 | ||
|
|
7943d9c390 | ||
|
|
0004215fbe | ||
|
|
f90d87724d | ||
|
|
b7a564eac9 | ||
|
|
fa6c935b12 | ||
|
|
46bd956229 | ||
|
|
de4328b4cf | ||
|
|
8b9c9b3b70 | ||
|
|
e240d8a0f0 | ||
|
|
4d70aada93 | ||
|
|
4ee78aebb2 | ||
|
|
7d4c57cab3 | ||
|
|
b5891f3365 | ||
|
|
17a981ca3b | ||
|
|
94f5ca195a | ||
|
|
4b7b626230 | ||
|
|
eaa7956558 | ||
|
|
28b7603fdd | ||
|
|
6be59ae374 | ||
|
|
f334207ccd | ||
|
|
0ca6e7e642 | ||
|
|
f63f459c73 | ||
|
|
e978cc65e1 | ||
|
|
2040492cd3 | ||
|
|
702d46adc3 | ||
|
|
df6bca3608 | ||
|
|
70038e6809 | ||
|
|
5bef598a69 | ||
|
|
a0bbddac51 | ||
|
|
6b3ca85e79 | ||
|
|
89c2c3949c | ||
|
|
43f344ad9b | ||
|
|
1090284452 | ||
|
|
665bd5cf22 | ||
|
|
e8597e8af1 | ||
|
|
51f76294a0 | ||
|
|
2acacc87a7 | ||
|
|
3d238f403d | ||
|
|
dbfc5b06f8 | ||
|
|
df99a87994 | ||
|
|
99b86daacf | ||
|
|
f6da948b36 | ||
|
|
4414ded2cf | ||
|
|
816a92904a | ||
|
|
1ad9214b51 | ||
|
|
cc15dab16a | ||
|
|
6f0700c82a | ||
|
|
414e9f600e | ||
|
|
6a6cc3d94a | ||
|
|
ab193988c1 | ||
|
|
4fbc4c1ff4 | ||
|
|
cc977ec737 | ||
|
|
ca0bc79f3c | ||
|
|
0559ddee09 | ||
|
|
28ebed87a5 | ||
|
|
effdf1a6f5 | ||
|
|
a8b095375b | ||
|
|
0652590962 | ||
|
|
4488078535 | ||
|
|
90bfcce34b | ||
|
|
4ad2c1d02a |
380
.github/aws-deploy/package-lock.json
generated
vendored
Normal file
380
.github/aws-deploy/package-lock.json
generated
vendored
Normal file
@ -0,0 +1,380 @@
|
||||
{
|
||||
"name": "warmbly-aws-deploy",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "warmbly-aws-deploy",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-ecs": "3.1155.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/client-ecs": {
|
||||
"version": "3.1155.0",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/client-ecs/-/client-ecs-3.1155.0.tgz",
|
||||
"integrity": "sha512-SbADWBcBwkEKM1h8N2EtpqyLpax7t6M0Awxuy1GEvDiI8v38S70BFSvSd+hmMQoOWKncmH8dWoMqDFh7PHT9KA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.5",
|
||||
"@aws-sdk/credential-provider-node": "^3.972.88",
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/fetch-http-handler": "^5.8.0",
|
||||
"@smithy/node-http-handler": "^4.12.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/core": {
|
||||
"version": "3.978.5",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.5.tgz",
|
||||
"integrity": "sha512-ruI0AWqcrfL20XDS6SwZIJFPOBZ6N0ADa9nLzWlBhxk1aoUjbUX3JkTR9vEmk7o35XpAHi68ahZ4T6RE2U9XyA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@aws-sdk/xml-builder": "^3.972.41",
|
||||
"@aws/lambda-invoke-store": "^0.3.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/signature-v4": "^5.7.4",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"bowser": "^2.11.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-env": {
|
||||
"version": "3.972.76",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.76.tgz",
|
||||
"integrity": "sha512-YxmL1Ar7bV9ZkJvtv+lj+MF6le76NnZnbmgySyZuxpZaCv0/1lZ3KxKy6RIj1h9ui2hWnVjZxbmTMeO0y/NrQA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.5",
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-http": {
|
||||
"version": "3.972.78",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.78.tgz",
|
||||
"integrity": "sha512-3VHknDRngoebsaqyT1iR75b5cBT2zfEtODK6yYmYltljkK1HEAQLs2FEd1IpWqAeOYdMGaG9wikngWLTufY7RA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.5",
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/fetch-http-handler": "^5.8.0",
|
||||
"@smithy/node-http-handler": "^4.12.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-ini": {
|
||||
"version": "3.973.21",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.21.tgz",
|
||||
"integrity": "sha512-aLx9hbCX6mGAyv4ZZd8UagJ4RASHVJw6BFDSIyuKmLkqNL6H47EjBPK6qaJ4B5TDV6mc+vMa0SJFYyo4mkvVQg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.5",
|
||||
"@aws-sdk/credential-provider-env": "^3.972.76",
|
||||
"@aws-sdk/credential-provider-http": "^3.972.78",
|
||||
"@aws-sdk/credential-provider-login": "^3.972.83",
|
||||
"@aws-sdk/credential-provider-process": "^3.972.76",
|
||||
"@aws-sdk/credential-provider-sso": "^3.973.20",
|
||||
"@aws-sdk/credential-provider-web-identity": "^3.972.82",
|
||||
"@aws-sdk/nested-clients": "^3.997.50",
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/credential-provider-imds": "^4.5.2",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-login": {
|
||||
"version": "3.972.83",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.83.tgz",
|
||||
"integrity": "sha512-9bJU6FjRacF4FK/QHf8Dyxhur7B1grRj8C1Yh4IkpoFW2exQflPdCv0VrbemLW9ZctLNle+AE7TETs9KNvuLYg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.5",
|
||||
"@aws-sdk/nested-clients": "^3.997.50",
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-node": {
|
||||
"version": "3.972.88",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.88.tgz",
|
||||
"integrity": "sha512-AG0KID5qhQPzrFkh5b0Cws1Amytvzo0L0Xj6hMgJXEwD6ZXAqninDhSNAWJiN5zU4YVVZnuidnqwPwYJ04IamA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/credential-provider-env": "^3.972.76",
|
||||
"@aws-sdk/credential-provider-http": "^3.972.78",
|
||||
"@aws-sdk/credential-provider-ini": "^3.973.21",
|
||||
"@aws-sdk/credential-provider-process": "^3.972.76",
|
||||
"@aws-sdk/credential-provider-sso": "^3.973.20",
|
||||
"@aws-sdk/credential-provider-web-identity": "^3.972.82",
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/credential-provider-imds": "^4.5.2",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-process": {
|
||||
"version": "3.972.76",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.76.tgz",
|
||||
"integrity": "sha512-pdhKFTXv1zPYhh92MTlp0jVbLWvUEV9v+rziuEkoskGGQ2iJskxHTO38nA8UAbT/N4IM3n8OZJ9rIY5IBO15/A==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.5",
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-sso": {
|
||||
"version": "3.973.20",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.20.tgz",
|
||||
"integrity": "sha512-DlxrAB3MnjNLx/da7b89W+3DyJQYso/q7Ns15MsST2IV2nkAYQOYTUCaffNSjgrpMP99Dm2mV2jxzcbyF94ZfA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.5",
|
||||
"@aws-sdk/nested-clients": "^3.997.50",
|
||||
"@aws-sdk/token-providers": "3.1155.0",
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/credential-provider-web-identity": {
|
||||
"version": "3.972.82",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.82.tgz",
|
||||
"integrity": "sha512-LUo46IqPTJ4xKBsrLuoaxiv/D2GTBCcFDMD1y3jKIqPR/p7Rw14RJFZ0iRWftqZtyLPQX9JMGnSi3mBu4y/pNA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.5",
|
||||
"@aws-sdk/nested-clients": "^3.997.50",
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/nested-clients": {
|
||||
"version": "3.997.50",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.50.tgz",
|
||||
"integrity": "sha512-nKb8qu4QsnpZTq/6JIXc72mzuzfh7BK4sUQkpQznPec2z4W7Ob2xYkcM4i5e1yGTBnbJVeFv/0sf3jcz85hclA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.5",
|
||||
"@aws-sdk/signature-v4-multi-region": "^3.996.49",
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/fetch-http-handler": "^5.8.0",
|
||||
"@smithy/node-http-handler": "^4.12.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/signature-v4-multi-region": {
|
||||
"version": "3.996.49",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.49.tgz",
|
||||
"integrity": "sha512-nvuAZNRrRRVCiARRKsYlQ0VnXiEQiq7vXPH+UA+LSo+kN3Djl9eo/V2r0udkw85PnsNshmv2Em1clld5M3j67w==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/signature-v4": "^5.7.4",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/token-providers": {
|
||||
"version": "3.1155.0",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1155.0.tgz",
|
||||
"integrity": "sha512-iIcQnYLhWuq9qOLQbIWFrxF3PF0svVeZvHnUS4DUnO4zUFmFHIyGyjqrntXG+e/5PUgfavV0pYfDIKHxTzN2LA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/core": "^3.978.5",
|
||||
"@aws-sdk/nested-clients": "^3.997.50",
|
||||
"@aws-sdk/types": "^3.975.0",
|
||||
"@smithy/core": "^3.36.1",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/types": {
|
||||
"version": "3.975.0",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.975.0.tgz",
|
||||
"integrity": "sha512-WQlhs6juByzWOzj5fwxOx0mPJaEqMcG/sDpl3Id61B3XhKE63xY+0QWcnG+sWR/sCfprmreyUo1lM7IKZmHMzw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-sdk/xml-builder": {
|
||||
"version": "3.972.41",
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.41.tgz",
|
||||
"integrity": "sha512-ctjVSyCMegrWfXlx6VqzSBFI6UqmQ5ZlnfMhdLIiWmhoH8UAQxSCP5N3OpG7X3k4LnS7ou74C4mt20+bfTW2aQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@aws/lambda-invoke-store": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz",
|
||||
"integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@smithy/core": {
|
||||
"version": "3.36.2",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.36.2.tgz",
|
||||
"integrity": "sha512-oRw0rB9QSUiCZfEs23G5F4Af+b2rQt1lItIICWu5IejFNd+uPM8SWqiKLMpQ6bX1Q7slsLbjuoE2niErFtkVPQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@smithy/credential-provider-imds": {
|
||||
"version": "4.5.2",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz",
|
||||
"integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/core": "^3.33.2",
|
||||
"@smithy/types": "^4.17.2",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@smithy/fetch-http-handler": {
|
||||
"version": "5.8.0",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.8.0.tgz",
|
||||
"integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/core": "^3.33.3",
|
||||
"@smithy/types": "^4.18.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@smithy/node-http-handler": {
|
||||
"version": "4.12.1",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz",
|
||||
"integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/core": "^3.33.3",
|
||||
"@smithy/types": "^4.18.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@smithy/signature-v4": {
|
||||
"version": "5.7.4",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.4.tgz",
|
||||
"integrity": "sha512-tHy0K0VtqNd5Y7Y41h0a0Lhh0L1GzC08dTWg0F7vRJWFtTENg7IZikf3wQkanYIRdb7ngoIPMTmqgUi401fEeQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@smithy/core": "^3.35.0",
|
||||
"@smithy/types": "^4.19.0",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@smithy/types": {
|
||||
"version": "4.19.0",
|
||||
"resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.19.0.tgz",
|
||||
"integrity": "sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bowser": {
|
||||
"version": "2.14.1",
|
||||
"resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz",
|
||||
"integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||
"license": "0BSD"
|
||||
}
|
||||
}
|
||||
}
|
||||
8
.github/aws-deploy/package.json
vendored
Normal file
8
.github/aws-deploy/package.json
vendored
Normal file
@ -0,0 +1,8 @@
|
||||
{
|
||||
"name": "warmbly-aws-deploy",
|
||||
"private": true,
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-ecs": "3.1155.0"
|
||||
}
|
||||
}
|
||||
124
.github/scripts/aws-deploy.cjs
vendored
Normal file
124
.github/scripts/aws-deploy.cjs
vendored
Normal file
@ -0,0 +1,124 @@
|
||||
const {
|
||||
ECSClient, DescribeTaskDefinitionCommand, DescribeServicesCommand,
|
||||
RunTaskCommand, DescribeTasksCommand, UpdateServiceCommand,
|
||||
} = require("@aws-sdk/client-ecs");
|
||||
const { createHash } = require("node:crypto");
|
||||
|
||||
function configuration(env) {
|
||||
const cluster = env.AWS_ECS_CLUSTER;
|
||||
const match = /^arn:aws:ecs:us-east-1:(\d{12}):cluster\/([\w-]+)$/.exec(cluster || "");
|
||||
if (!match) throw new Error("AWS_ECS_CLUSTER must be a us-east-1 cluster ARN");
|
||||
const prefix = env.AWS_ECS_SERVICE_PREFIX || match[2];
|
||||
if (!/^[\w-]+$/.test(prefix)) throw new Error("Invalid service prefix");
|
||||
const definitions = {};
|
||||
const counts = {};
|
||||
for (const name of ["migration", "backend", "consumer"]) {
|
||||
const value = env[`AWS_${name.toUpperCase()}_TASK_DEFINITION`];
|
||||
const expected = `arn:aws:ecs:us-east-1:${match[1]}:task-definition/${prefix}-${name}:`;
|
||||
if (!value?.startsWith(expected) || !/^\d+$/.test(value.slice(expected.length))) {
|
||||
throw new Error(`Provide an explicit ${name} task definition revision`);
|
||||
}
|
||||
definitions[name] = value;
|
||||
if (name !== "migration") {
|
||||
const raw = env[`AWS_${name.toUpperCase()}_DESIRED_COUNT`] || "preserve";
|
||||
if (raw !== "preserve" && !/^[0-5]$/.test(raw)) throw new Error("Desired counts must be preserve or 0 through 5");
|
||||
counts[name] = raw === "preserve" ? undefined : Number(raw);
|
||||
}
|
||||
}
|
||||
return { cluster, prefix, definitions, counts, run: `${env.GITHUB_RUN_ID || "operator"}/${env.GITHUB_RUN_ATTEMPT || "1"}` };
|
||||
}
|
||||
|
||||
function assertResponse(out) {
|
||||
if (out.failures?.length) throw new Error("ECS reported a resource or task-placement failure");
|
||||
return out;
|
||||
}
|
||||
|
||||
async function waitForService(client, config, name, definition, sleep) {
|
||||
for (let attempt = 0; attempt < 180; attempt++) {
|
||||
const out = assertResponse(await client.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-${name}`] })));
|
||||
const service = out.services?.[0];
|
||||
if (!service || service.deployments?.some((d) => d.rolloutState === "FAILED")) throw new Error(`${name} deployment failed`);
|
||||
if (service.deployments?.length === 1 && service.deployments[0].rolloutState === "COMPLETED") {
|
||||
if (service.taskDefinition !== definition) throw new Error(`${name} deployment rolled back`);
|
||||
if (service.runningCount === service.desiredCount && service.pendingCount === 0) return;
|
||||
}
|
||||
await sleep(5000);
|
||||
}
|
||||
throw new Error(`${name} deployment did not become healthy within 15 minutes`);
|
||||
}
|
||||
|
||||
async function deploy(client, config, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), report = console.log) {
|
||||
const definitions = {};
|
||||
for (const name of ["migration", "backend", "consumer"]) {
|
||||
const out = await client.send(new DescribeTaskDefinitionCommand({ taskDefinition: config.definitions[name] }));
|
||||
const definition = out.taskDefinition;
|
||||
const container = definition?.containerDefinitions?.find((c) => c.name === name);
|
||||
if (!container || definition.status !== "ACTIVE" || !/@sha256:[a-f0-9]{64}$/.test(container.image || "")) throw new Error(`Invalid or unpinned ${name} task definition`);
|
||||
definitions[name] = { definition, container };
|
||||
}
|
||||
const migration = definitions.migration.container;
|
||||
if (migration.image !== definitions.backend.container.image || migration.entryPoint?.join(" ") !== "/app/migrate") throw new Error("Migration must use /app/migrate from the exact backend image");
|
||||
const migrationSecret = migration.secrets?.find((s) => s.name === "PRIMARY_DB");
|
||||
const directSecret = definitions.backend.container.secrets?.find((s) => s.name === "PRIMARY_DB_DIRECT");
|
||||
if (!directSecret || directSecret.valueFrom !== migrationSecret?.valueFrom) throw new Error("Migration and backend must reference the same direct database secret version");
|
||||
if (!definitions.backend.container.environment?.some((e) => e.name === "DB_MIGRATIONS_ON_START" && e.value === "false")) throw new Error("Backend startup migrations must be disabled for this deployment");
|
||||
const current = assertResponse(await client.send(new DescribeServicesCommand({ cluster: config.cluster, services: [`${config.prefix}-backend`, `${config.prefix}-consumer`] })));
|
||||
if (current.services?.length !== 2 || current.services.some((s) => s.status !== "ACTIVE")) throw new Error("Both ECS services must already exist");
|
||||
const backend = current.services.find((s) => s.serviceName === `${config.prefix}-backend`);
|
||||
if (!backend?.networkConfiguration?.awsvpcConfiguration) throw new Error("Backend task networking is missing");
|
||||
const token = createHash("sha256").update(`${config.run}/${config.definitions.migration}`).digest("hex");
|
||||
const started = assertResponse(await client.send(new RunTaskCommand({
|
||||
cluster: config.cluster, taskDefinition: config.definitions.migration,
|
||||
launchType: "FARGATE", platformVersion: "1.4.0", count: 1,
|
||||
networkConfiguration: backend.networkConfiguration, clientToken: token,
|
||||
startedBy: `migration-${token.slice(0, 24)}`,
|
||||
})));
|
||||
const task = started.tasks?.[0]?.taskArn;
|
||||
if (!task) throw new Error("Migration task was not created");
|
||||
report(`Migration task: ${task}`);
|
||||
let migrated = false;
|
||||
for (let attempt = 0; attempt < 240; attempt++) {
|
||||
const out = assertResponse(await client.send(new DescribeTasksCommand({ cluster: config.cluster, tasks: [task] })));
|
||||
const status = out.tasks?.[0];
|
||||
if (!status) throw new Error("Migration task is missing");
|
||||
if (status.lastStatus === "STOPPED") {
|
||||
const container = status.containers?.find((c) => c.name === "migration");
|
||||
if (status.stopCode !== "EssentialContainerExited" || container?.exitCode !== 0) throw new Error("Migration failed; inspect its CloudWatch logs before retrying");
|
||||
migrated = true;
|
||||
break;
|
||||
}
|
||||
await sleep(5000);
|
||||
}
|
||||
if (!migrated) throw new Error("Migration status is unresolved; no services were updated. Inspect the running task before retrying");
|
||||
report("Migration completed successfully");
|
||||
const updated = [];
|
||||
try {
|
||||
for (const name of ["backend", "consumer"]) {
|
||||
const previous = current.services.find((s) => s.serviceName === `${config.prefix}-${name}`);
|
||||
updated.push(previous);
|
||||
await client.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: config.definitions[name], ...(config.counts[name] === undefined ? {} : { desiredCount: config.counts[name] }) }));
|
||||
await waitForService(client, config, name, config.definitions[name], sleep);
|
||||
report(`${name} service is stable on the requested revision`);
|
||||
}
|
||||
} catch (error) {
|
||||
for (const previous of updated.reverse()) {
|
||||
try {
|
||||
await client.send(new UpdateServiceCommand({ cluster: config.cluster, service: previous.serviceName, taskDefinition: previous.taskDefinition, desiredCount: previous.desiredCount }));
|
||||
report(`Restored previous deployment configuration for ${previous.serviceName}; verify its health`);
|
||||
} catch {
|
||||
report(`Rollback needs operator attention for ${previous.serviceName}`);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
const client = new ECSClient({ region: "us-east-1", maxAttempts: 2 });
|
||||
Promise.resolve().then(() => deploy(client, configuration(process.env))).catch((error) => {
|
||||
console.error(error?.$metadata ? `AWS request failed: ${error.name}` : error.message);
|
||||
process.exitCode = 1;
|
||||
}).finally(() => client.destroy());
|
||||
}
|
||||
|
||||
module.exports = { configuration, deploy };
|
||||
89
.github/scripts/aws-deploy.test.cjs
vendored
Normal file
89
.github/scripts/aws-deploy.test.cjs
vendored
Normal file
@ -0,0 +1,89 @@
|
||||
const { test } = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const { configuration, deploy } = require("./aws-deploy.cjs");
|
||||
|
||||
const root = "arn:aws:ecs:us-east-1:000000000000:";
|
||||
const digest = "sha256:" + "a".repeat(64);
|
||||
const secret = "arn:aws:secretsmanager:us-east-1:000000000000:secret:fixture:PRIMARY_DB_DIRECT::version";
|
||||
function config() {
|
||||
return configuration({
|
||||
AWS_ECS_CLUSTER: root + "cluster/fixture",
|
||||
AWS_MIGRATION_TASK_DEFINITION: root + "task-definition/fixture-migration:2",
|
||||
AWS_BACKEND_TASK_DEFINITION: root + "task-definition/fixture-backend:2",
|
||||
AWS_CONSUMER_TASK_DEFINITION: root + "task-definition/fixture-consumer:2",
|
||||
});
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const calls = [];
|
||||
const services = ["backend", "consumer"].map((name) => ({
|
||||
serviceName: `fixture-${name}`, status: "ACTIVE", desiredCount: 1, runningCount: 1, pendingCount: 0,
|
||||
taskDefinition: root + `task-definition/fixture-${name}:1`, deployments: [{ rolloutState: "COMPLETED" }],
|
||||
networkConfiguration: { awsvpcConfiguration: { subnets: ["subnet-fixture"], securityGroups: ["sg-fixture"], assignPublicIp: "ENABLED" } },
|
||||
}));
|
||||
const client = { async send(command) {
|
||||
const name = command.constructor.name;
|
||||
calls.push({ name, input: command.input });
|
||||
if (name === "DescribeTaskDefinitionCommand") {
|
||||
const service = /fixture-(\w+):/.exec(command.input.taskDefinition)[1];
|
||||
return { taskDefinition: { status: "ACTIVE", containerDefinitions: [{
|
||||
name: service, image: `registry/${service === "migration" ? "backend" : service}@${service === "migration" && options.imageMismatch ? "sha256:" + "b".repeat(64) : digest}`,
|
||||
entryPoint: service === "migration" ? ["/app/migrate"] : undefined,
|
||||
environment: [{ name: "DB_MIGRATIONS_ON_START", value: "false" }],
|
||||
secrets: [{ name: service === "migration" ? "PRIMARY_DB" : "PRIMARY_DB_DIRECT", valueFrom: secret }],
|
||||
}] } };
|
||||
}
|
||||
if (name === "DescribeServicesCommand") return { services: services.filter((s) => command.input.services.includes(s.serviceName)).map((s) => structuredClone(s)) };
|
||||
if (name === "RunTaskCommand") return options.placementFailure ? { failures: [{ reason: "fixture" }] } : { tasks: [{ taskArn: root + "task/fixture/migration" }] };
|
||||
if (name === "DescribeTasksCommand") return { tasks: [{ lastStatus: "STOPPED", stopCode: "EssentialContainerExited", containers: [{ name: "migration", exitCode: options.migrationFailure ? 1 : 0 }] }] };
|
||||
if (name === "UpdateServiceCommand") {
|
||||
if (command.input.service === "fixture-consumer" && command.input.taskDefinition.endsWith(":2") && options.consumerFailure) throw new Error("consumer update failed");
|
||||
const service = services.find((s) => s.serviceName === command.input.service);
|
||||
service.taskDefinition = command.input.taskDefinition;
|
||||
if (command.input.desiredCount !== undefined) service.runningCount = service.desiredCount = command.input.desiredCount;
|
||||
if (options.circuitBreakerRollback && command.input.taskDefinition.endsWith(":2")) service.taskDefinition = root + "task-definition/fixture-backend:1";
|
||||
return {};
|
||||
}
|
||||
throw new Error(`Unexpected command ${name}`);
|
||||
} };
|
||||
return { client, calls };
|
||||
}
|
||||
|
||||
test("migration success gates both service updates and preserves desired counts", async () => {
|
||||
const { client, calls } = fixture();
|
||||
await deploy(client, config(), async () => {}, () => {});
|
||||
const migration = calls.findIndex((c) => c.name === "DescribeTasksCommand");
|
||||
const updates = calls.filter((c) => c.name === "UpdateServiceCommand");
|
||||
assert.equal(updates.length, 2);
|
||||
assert.ok(calls.indexOf(updates[0]) > migration);
|
||||
assert.ok(updates.every((c) => !Object.hasOwn(c.input, "desiredCount")));
|
||||
assert.equal(calls.filter((c) => c.name === "RunTaskCommand").length, 1);
|
||||
});
|
||||
|
||||
for (const failure of ["migrationFailure", "placementFailure", "imageMismatch"]) {
|
||||
test(`${failure} leaves existing services untouched`, async () => {
|
||||
const { client, calls } = fixture({ [failure]: true });
|
||||
await assert.rejects(deploy(client, config(), async () => {}, () => {}));
|
||||
assert.ok(calls.every((c) => c.name !== "UpdateServiceCommand"));
|
||||
});
|
||||
}
|
||||
|
||||
test("consumer rollout failure restores previous service revisions without database rollback", async () => {
|
||||
const { client, calls } = fixture({ consumerFailure: true });
|
||||
await assert.rejects(deploy(client, config(), async () => {}, () => {}), /consumer update failed/);
|
||||
const updates = calls.filter((c) => c.name === "UpdateServiceCommand");
|
||||
assert.equal(updates.length, 4);
|
||||
assert.ok(updates.slice(2).every((c) => c.input.taskDefinition.endsWith(":1") && c.input.desiredCount === 1));
|
||||
assert.equal(calls.filter((c) => c.name === "RunTaskCommand").length, 1);
|
||||
});
|
||||
|
||||
test("a circuit breaker rollback cannot be reported as a successful deployment", async () => {
|
||||
const { client } = fixture({ circuitBreakerRollback: true });
|
||||
await assert.rejects(deploy(client, config(), async () => {}, () => {}), /rolled back/);
|
||||
});
|
||||
|
||||
test("deployment requires explicit revisions in the selected account and region", () => {
|
||||
assert.throws(() => configuration({ AWS_ECS_CLUSTER: root.replace("us-east-1", "eu-west-1") + "cluster/fixture" }));
|
||||
const c = config();
|
||||
assert.equal(c.counts.backend, undefined);
|
||||
});
|
||||
116
.github/workflows/ci.yml
vendored
116
.github/workflows/ci.yml
vendored
@ -28,6 +28,7 @@ jobs:
|
||||
outputs:
|
||||
go-services: ${{ steps.filter.outputs.go-services }}
|
||||
migrations: ${{ steps.filter.outputs.migrations }}
|
||||
aws-deployment: ${{ steps.filter.outputs.aws-deployment }}
|
||||
tracking: ${{ steps.filter.outputs.tracking }}
|
||||
realtime: ${{ steps.filter.outputs.realtime }}
|
||||
web: ${{ steps.filter.outputs.web }}
|
||||
@ -65,6 +66,11 @@ jobs:
|
||||
migrations:
|
||||
- 'internal/infrastructure/db/migrations/**'
|
||||
- 'scripts/check-migrations.sh'
|
||||
aws-deployment:
|
||||
- '.github/workflows/ci.yml'
|
||||
- '.github/workflows/deploy-aws.yml'
|
||||
- '.github/scripts/aws-deploy*.cjs'
|
||||
- '.github/aws-deploy/**'
|
||||
tracking:
|
||||
- 'tracking/**'
|
||||
realtime:
|
||||
@ -175,11 +181,20 @@ jobs:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
# A pinned release verified by digest; resolving "latest" lists every protobuf release and fails when that listing is cut short.
|
||||
- name: Set up protoc
|
||||
uses: arduino/setup-protoc@v3
|
||||
with:
|
||||
version: "33.x"
|
||||
repo-token: ${{ github.token }}
|
||||
env:
|
||||
PROTOC_VERSION: "33.6"
|
||||
PROTOC_SHA256: 9c49962391ff8b5754342509efc3885db91dfe11a358e1e98c9a71518f0ad199
|
||||
run: |
|
||||
set -eu
|
||||
archive="$RUNNER_TEMP/protoc.zip"
|
||||
curl -fsSL --retry 5 --retry-all-errors -o "$archive" \
|
||||
"https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip"
|
||||
echo "${PROTOC_SHA256} ${archive}" | sha256sum -c -
|
||||
unzip -q -o "$archive" -d "$RUNNER_TEMP/protoc"
|
||||
echo "$RUNNER_TEMP/protoc/bin" >> "$GITHUB_PATH"
|
||||
"$RUNNER_TEMP/protoc/bin/protoc" --version
|
||||
|
||||
- name: Install required Go tools
|
||||
run: make setup-tools
|
||||
@ -207,12 +222,20 @@ jobs:
|
||||
run: |
|
||||
set -o pipefail
|
||||
go run ./cmd/migrate
|
||||
go test -race -v -p 1 ./internal/repository ./internal/app/analytics -run '^TestLive(WarmupPartnerCountsMatchIndividualCandidates|WarmupSummariesMatchIndividualReads|LinkedMailboxStatusBatchPreservesDetails)$' -count=1 | tee linked-mailbox-batch.log
|
||||
for test in TestLiveWarmupPartnerCountsMatchIndividualCandidates TestLiveWarmupSummariesMatchIndividualReads TestLiveLinkedMailboxStatusBatchPreservesDetails; do
|
||||
go test -race -v -p 1 ./internal/repository ./internal/app/analytics -run '^TestLive(WarmupPartnerCountsMatchIndividualCandidates|WarmupSummariesMatchIndividualReads|LinkedMailboxStatusBatchPreservesDetails|WarmupPartnerWeekCacheMatchesExactReadAndKeepsTheCapLive|WarmupPartnerWeekCacheRefreshesDetachedAndServesStale|WarmupPartnerCandidatesCarryReciprocityAndHonourTheCap|AccountStatusPageMatchesSingleReadsInBoundedQueries)$' -count=1 | tee linked-mailbox-batch.log
|
||||
for test in TestLiveWarmupPartnerCountsMatchIndividualCandidates TestLiveWarmupSummariesMatchIndividualReads TestLiveLinkedMailboxStatusBatchPreservesDetails TestLiveWarmupPartnerWeekCacheMatchesExactReadAndKeepsTheCapLive TestLiveWarmupPartnerWeekCacheRefreshesDetachedAndServesStale TestLiveWarmupPartnerCandidatesCarryReciprocityAndHonourTheCap TestLiveAccountStatusPageMatchesSingleReadsInBoundedQueries; do
|
||||
grep -q -- "--- PASS: $test " linked-mailbox-batch.log
|
||||
done
|
||||
if grep -q -- '--- SKIP:' linked-mailbox-batch.log; then exit 1; fi
|
||||
|
||||
- name: Contact campaign empty-step regression (PostgreSQL)
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -race -v ./internal/repository -run '^TestLiveContactCampaignStateEmptySteps$' -count=1 | tee contact-campaign-steps.log
|
||||
grep -q -- '--- PASS: TestLiveContactCampaignStateEmptySteps ' contact-campaign-steps.log
|
||||
|
||||
- name: Campaign send admission regressions (PostgreSQL)
|
||||
env:
|
||||
PRIMARY_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
@ -237,6 +260,39 @@ jobs:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
run: go test -race -v -p 1 ./internal/repository -run '^TestLive(MessageMapBatch|FolderGenerationLookup|MailboxLookupIndexes)' -count=1
|
||||
|
||||
- name: CPU query pressure and accounting parity regressions (PostgreSQL)
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -race -v -p 1 ./internal/repository -run '^TestLive(TaskLedgerRangesAndClocksMatchLegacy|TaskLedgerDayRangePreservesDST|TaskQueryPressureUsesBoundedHistory|WarmupHealthCountsShareScansWithoutChangingSignals|DashboardOverallBoundsHistoryAndPreservesScopedCounts|VerificationBacklogFollowsShelfLives|VerificationReopenOnProviderConnect|SegmentCampaignLinks|PoolLinkTouchIsThrottledButKeepsTheVersion)$' -count=1 | tee cpu-query-pressure.log
|
||||
if grep -q -- '--- SKIP:' cpu-query-pressure.log; then exit 1; fi
|
||||
for test in TestLiveTaskLedgerRangesAndClocksMatchLegacy TestLiveTaskLedgerDayRangePreservesDST TestLiveTaskQueryPressureUsesBoundedHistory TestLiveWarmupHealthCountsShareScansWithoutChangingSignals TestLiveDashboardOverallBoundsHistoryAndPreservesScopedCounts TestLiveVerificationBacklogFollowsShelfLives TestLiveVerificationReopenOnProviderConnect TestLiveSegmentCampaignLinks TestLivePoolLinkTouchIsThrottledButKeepsTheVersion; do
|
||||
grep -q "^--- PASS: $test " cpu-query-pressure.log
|
||||
done
|
||||
|
||||
- name: Campaign pool query batching regressions (PostgreSQL)
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -race -v -p 1 ./internal/repository ./internal/scheduler -run '^TestLive(FindRoutedPairs|CampaignPool|LeadHold|PlanCampaignDay|WarmupHealthCountsShareScansWithoutChangingSignals|TaskLedgerRangesAndClocksMatchLegacy)' -count=1 | tee campaign-pool-batch.log
|
||||
if grep -q -- '--- SKIP:' campaign-pool-batch.log; then exit 1; fi
|
||||
for test in TestLiveFindRoutedPairsPriorityMatchesLegacyOrdering TestLiveCampaignPoolReadsStayBatched TestLiveCampaignPoolBatchFailuresKeepSafety TestLiveCampaignPoolStandalonePlacementIsBatched; do
|
||||
grep -q -- "--- PASS: $test " campaign-pool-batch.log
|
||||
done
|
||||
|
||||
- name: Cloud standing and missing-evidence holds (PostgreSQL)
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -race -v -p 1 ./internal/repository -run '^TestLiveCloud' -count=1 | tee cloud-standing.log
|
||||
if grep -q -- '--- SKIP:' cloud-standing.log; then exit 1; fi
|
||||
for test in TestLiveCloudStandingFreshnessIsBoundedWithoutInventingAProviderBlock TestLiveCloudUnknownEvidenceDoesNotHideARealHold TestLiveCloudStaleLiveHoldStaysKnown TestLiveCloudPendingEnrollmentAndDisconnectAreUnknown; do
|
||||
grep -q -- "--- PASS: $test " cloud-standing.log
|
||||
done
|
||||
|
||||
- name: Scheduled-job retirement regressions (PostgreSQL)
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
@ -248,8 +304,15 @@ jobs:
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -race ./internal/repository -run '^TestLive(Monitoring|LastWarmupSendFailure|PersistentMailboxLoadingFailure|DispatchRetry|LineageQueueRevisions|AdminPlatformOverview|WarmupRecoversOldPreparation|WarmupPreparationRecoveryRetains|WarmupFilingRequiresAcknowledgement|WarmupFilingAuthority|WarmupTimeWindow|WarmupInvalidWindow)' -count=1
|
||||
go test -race ./internal/tasks -run '^Test(LiveWarmup(SafetyHoldReschedules|Recovery|TaskDoesNotApplyResults|DispatchDefersInvalidHours)|WarmupSenderDistinguishes)' -count=1
|
||||
go test -race -v ./internal/tasks -run '^Test(LiveWarmup(SafetyHoldReschedules|Recovery|UnknownRetirement|TaskDoesNotApplyResults|DispatchDefersInvalidHours)|WarmupSenderDistinguishes)' -count=1 | tee warmup-recovery.log
|
||||
if grep -q -- '--- SKIP: TestLiveWarmupUnknownRetirement' warmup-recovery.log; then exit 1; fi
|
||||
for test in TestLiveWarmupUnknownRetirementPreservesEvidenceAndFencesReplayOnce TestLiveWarmupUnknownRetirementAllowsDistinctNewSendAndKeepsCooldown TestLiveWarmupUnknownRetirementAndDefinitiveResultSerialize TestLiveWarmupUnknownRetirementSkipsProtectedRowsWithoutStarvingLaterMailboxes TestLiveWarmupUnknownRetirementReconcilerSeedsOnceAndClearsActiveWarnings; do
|
||||
grep -q -- "--- PASS: $test " warmup-recovery.log || exit 1
|
||||
done
|
||||
go test -race -v ./internal/infrastructure/db -run '^TestLiveWarmupUnknownRetirementUpgrade' -count=1 | tee warmup-retirement-upgrade.log
|
||||
grep -q -- '--- PASS: TestLiveWarmupUnknownRetirementUpgrade ' warmup-retirement-upgrade.log
|
||||
go test -race ./internal/app/consumer -run '^TestLiveStoredSendResult' -count=1
|
||||
|
||||
- name: Warmup database query-pressure regressions (PostgreSQL)
|
||||
@ -312,6 +375,14 @@ jobs:
|
||||
go test -race -v -p 1 ./internal/repository ./internal/app/consumer ./internal/tasks ./internal/app/orgtransfer -run '^TestLive(Warmup(ReservationPreservesOriginalChargeDay|RefundCASProtectsConcurrentAndRestampedTasks|RefundAndResultRollbackTogether|ReplyRefundUsesChargedShapeAfterParentDeletion|ChargeAndRefundUseUTCInNonUTCSession|ChargeProofRollsBackWhenCounterWriteFails|CancellationPathsRefundOnlyProvenCharge|GenericCompletionDoesNotProveChargedAccounting|SendFailureGivesTheDayBack|SendGiveBackIsIdempotent|SafetyHoldReschedulesInsteadOfBlockingTheDispatcher|Recovery(RetiresExistingUnstartedDispatchOnce|HandlesPartialReservations|PreservesExecutionAndReceiptEvidence|KeepsOtherUnknownAndRestrictiveHolds|AndBeginCannotBothWin))|AppliedWarmupFailureRepairsOnlyRefundedChargeDeadLetters|GiveBackDailySend(ReversesADispatchedWarmupSend|LeavesOtherDaysAlone)|AccountDailyUsage(CountsWarmupTasksNotTheStatisticsRow|ScopesWarmupToTypeAndDay)|ImportWarmupStatisticsCannotGrantChargeRefundAuthority|Legacy(FailedStatusCannotProveNonExecution|JSONAndAvroFailurePersistConservativeAdmission)|AuthoritativeDeadLetterResultsReconcileWithoutReplay|DeadLetterResultEffectsAndSettlementRollBackTogether|Campaign(DeadLetterWorkerResultReleasesReservationOnce|ForeignKeyCleanupDoesNotProveWakeupOnly)|UserEmailDeadLetterUsesUnderlyingSendTaskType|UnknownSendOperatorResolutionReconcilesAndAudits|RecordedWorkerOutcomeSurvivesLostEvent|UnreservedLegacyFailedTaskRetainsUnknownHold|UnknownResolutionRejectsMissingEvidenceAndLiveExecution|SendRecoveryResolutionRequiresEvidenceAndRetainsUnknown|UnstartedDispatchRetainsDeferredOutcomeConflict)$' -count=1
|
||||
go test -race -v -p 1 ./internal/infrastructure/db -run '^TestLiveWarmupChargeUpgradeKeepsLegacyUnverifiedAndPreservesEvidence$' -count=1
|
||||
|
||||
- name: Mailbox placement health recovery (PostgreSQL)
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
run: |
|
||||
set -o pipefail
|
||||
go test -race -v ./internal/app/warmup -run '^TestLiveHealthPlacementRecoveryUsesTheDisplayedUTCWindow$' -count=1 | tee placement-health-recovery.log
|
||||
grep -q -- '^--- PASS: TestLiveHealthPlacementRecoveryUsesTheDisplayedUTCWindow ' placement-health-recovery.log
|
||||
|
||||
- name: Linked warmup scheduling regressions (PostgreSQL)
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
@ -422,7 +493,12 @@ jobs:
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
run: |
|
||||
go test -race -p 1 ./internal/repository ./internal/app/consumer ./internal/app/orgtransfer ./internal/infrastructure/db -run '^TestLive(SyncArrival|SyncTruth|EmailErrorDedupe)' -count=1
|
||||
go test -race -p 1 ./internal/repository ./internal/app/consumer ./internal/app/orgtransfer ./internal/infrastructure/db -run '^TestLive(SyncArrival|SyncTruth|EmailErrorDedupe|EmailErrorResolveMethod)' -count=1
|
||||
|
||||
- name: Workspace archive spec against the live schema (PostgreSQL)
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
run: go test -race -v -p 1 ./internal/app/orgtransfer -run '^TestLive(EveryOrgTableIsRegistered|TablesAreInDependencyOrder|ResetColumnsHaveSomethingToFallBackOn)$' -count=1
|
||||
|
||||
- name: Payload-free arrival delivery diagnostics regressions (PostgreSQL)
|
||||
env:
|
||||
@ -435,6 +511,11 @@ jobs:
|
||||
run: |
|
||||
go test -race -p 1 ./internal/repository -run '^TestLive(OrganizationScopedCounts|UniboxVisibility|EmailCursorVisibility|NotificationAudience)' -count=1
|
||||
|
||||
- name: Unibox indexed search, paging and interrupted reads (PostgreSQL)
|
||||
env:
|
||||
WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable
|
||||
run: go test -race -v -p 1 ./internal/repository ./internal/infrastructure/db -run '^TestLiveUnibox(Search|ReadStream|SearchUpgrade)' -count=1
|
||||
|
||||
- name: Notification digest authorization retries (Redis)
|
||||
env:
|
||||
WARMBLY_TEST_REDIS: redis://127.0.0.1:6379/15
|
||||
@ -810,10 +891,27 @@ jobs:
|
||||
# The one job to mark required in branch protection. Skipped jobs pass
|
||||
# (their tree did not change); anything failed or cancelled fails it.
|
||||
# Dependency scanning lives in security.yml, off the PR path on purpose.
|
||||
aws-deployment-ci:
|
||||
name: AWS deployment gates
|
||||
needs: changes
|
||||
if: needs.changes.outputs.aws-deployment == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Install deployment client
|
||||
run: npm ci --prefix .github/aws-deploy --omit=dev --ignore-scripts --no-audit --no-fund
|
||||
- name: Check migration and rollback gates
|
||||
env:
|
||||
NODE_PATH: ${{ github.workspace }}/.github/aws-deploy/node_modules
|
||||
run: node --test .github/scripts/aws-deploy.test.cjs
|
||||
|
||||
ci-status:
|
||||
name: CI Status
|
||||
runs-on: ubuntu-latest
|
||||
needs: [changes, migrations-ci, go-ci, web-ci, qa-ci, admin-ci, site-ci, forms-ci, installer-ci, cli-installer-ci, make-ci, rust-ci, elixir-ci, ios-ci, frontend-images, pages-ci]
|
||||
needs: [changes, migrations-ci, go-ci, web-ci, qa-ci, admin-ci, site-ci, forms-ci, installer-ci, cli-installer-ci, make-ci, rust-ci, elixir-ci, ios-ci, frontend-images, pages-ci, aws-deployment-ci]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Check CI status
|
||||
|
||||
27
.github/workflows/coolify.yml
vendored
Normal file
27
.github/workflows/coolify.yml
vendored
Normal file
@ -0,0 +1,27 @@
|
||||
name: Coolify Compose
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- deploy/docker-compose.coolify.yml
|
||||
- deploy/check-coolify.sh
|
||||
- .github/workflows/coolify.yml
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- deploy/docker-compose.coolify.yml
|
||||
- deploy/check-coolify.sh
|
||||
- .github/workflows/coolify.yml
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Validate image-only Coolify stack
|
||||
run: bash deploy/check-coolify.sh
|
||||
65
.github/workflows/deploy-aws.yml
vendored
Normal file
65
.github/workflows/deploy-aws.yml
vendored
Normal file
@ -0,0 +1,65 @@
|
||||
name: Deploy AWS control plane
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
migration_task_definition:
|
||||
description: Terraform migration task definition ARN with an explicit revision
|
||||
required: true
|
||||
type: string
|
||||
backend_task_definition:
|
||||
description: Terraform backend task definition ARN with an explicit revision
|
||||
required: true
|
||||
type: string
|
||||
consumer_task_definition:
|
||||
description: Terraform consumer task definition ARN with an explicit revision
|
||||
required: true
|
||||
type: string
|
||||
backend_desired_count:
|
||||
description: Backend desired count (preserve or 0 through 5)
|
||||
default: preserve
|
||||
type: string
|
||||
consumer_desired_count:
|
||||
description: Consumer desired count (preserve or 0 through 5)
|
||||
default: preserve
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: aws-production-control-plane
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
environment: aws-production
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: Install deployment client
|
||||
run: npm ci --prefix .github/aws-deploy --omit=dev --ignore-scripts --no-audit --no-fund
|
||||
- uses: aws-actions/configure-aws-credentials@v4
|
||||
with:
|
||||
role-to-assume: ${{ vars.AWS_DEPLOY_ROLE_ARN }}
|
||||
aws-region: us-east-1
|
||||
- name: Migrate, verify and deploy
|
||||
env:
|
||||
NODE_PATH: ${{ github.workspace }}/.github/aws-deploy/node_modules
|
||||
AWS_ECS_CLUSTER: ${{ vars.AWS_ECS_CLUSTER }}
|
||||
AWS_ECS_SERVICE_PREFIX: ${{ vars.AWS_ECS_SERVICE_PREFIX }}
|
||||
AWS_MIGRATION_TASK_DEFINITION: ${{ inputs.migration_task_definition }}
|
||||
AWS_BACKEND_TASK_DEFINITION: ${{ inputs.backend_task_definition }}
|
||||
AWS_CONSUMER_TASK_DEFINITION: ${{ inputs.consumer_task_definition }}
|
||||
AWS_BACKEND_DESIRED_COUNT: ${{ inputs.backend_desired_count }}
|
||||
AWS_CONSUMER_DESIRED_COUNT: ${{ inputs.consumer_desired_count }}
|
||||
run: node .github/scripts/aws-deploy.cjs
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@ -58,6 +58,9 @@ go.work.sum
|
||||
# Local Go build cache
|
||||
.gocache/
|
||||
|
||||
# Local cache deployment metadata
|
||||
.elasticache/
|
||||
|
||||
# Node dependencies (per-tree .gitignores cover admin/site/docs/web; this
|
||||
# also catches a stray root-level install / pnpm store)
|
||||
node_modules/
|
||||
|
||||
63
AGENTS.md
63
AGENTS.md
@ -43,6 +43,60 @@ Migrations are numbered against `main`, not against your branch:
|
||||
- run `make check-migrations` (also a prerequisite of `make lint`, and its own CI job) before pushing anything that adds a migration
|
||||
- if a duplicate does reach `main`, renumber the migration that has NOT been released yet. The other one is already recorded in deployments' `schema_migrations`, and renumbering it makes them re-apply it
|
||||
|
||||
### Deployment and rollback compatibility is required
|
||||
|
||||
Every change must support the mixed versions present during a rolling deployment
|
||||
and the supported rollback window. This includes database schemas and stored
|
||||
data, public and internal APIs, worker protocols, queued events, cache formats
|
||||
and credentials. State the supported version window when it changes.
|
||||
|
||||
- The previous application release must keep working after the new migration.
|
||||
The new release must accept data and queued messages written by that previous
|
||||
release. A rollback must preserve writes accepted by the new release.
|
||||
- Use expand-and-contract changes: add compatible fields or tables first, deploy
|
||||
tolerant readers and any coordinated dual writes, backfill with a resumable
|
||||
job, then switch callers. Remove or rename old representations only in a later
|
||||
release, after every old service and worker has drained and the rollback
|
||||
window has closed. Never combine the expansion and removal in one deployment.
|
||||
- A new required field needs a compatible default or a staged nullable period.
|
||||
New event variants and enum values need readers that can handle them before
|
||||
producers emit them. Retained Kafka messages, retries and delayed jobs count
|
||||
as old producers too; preserve codec and schema compatibility.
|
||||
- Keep credential and encryption-key overlap until all readers and writers use
|
||||
replacements. Rollback instances must use credentials and keys compatible with
|
||||
the current data. Reverting configuration is not a data migration.
|
||||
- Hosted deployments run one migration job before rolling application services,
|
||||
using the release's backend image and a direct database connection. Services
|
||||
verify schema readiness. Migration failure blocks deployment; never force a
|
||||
migration version, clear a dirty flag or run a down migration automatically.
|
||||
Self-host startup migrations remain supported by default.
|
||||
- Review schema-lock duration and backfill cost. Keep large backfills out of the
|
||||
deployment transaction. Validate old-code/new-schema behavior, new-code/old-data
|
||||
behavior and rollback with representative fixtures when a change touches a
|
||||
shared contract. Document any intentional breaking change and its transition
|
||||
plan before deploying it.
|
||||
|
||||
### Self-hosting remains a first-class deployment
|
||||
|
||||
Hosted infrastructure changes must be opt-in and preserve existing self-hosts.
|
||||
The hosted service and self-hosts use the same application code and release images.
|
||||
|
||||
- An existing self-host's configuration must keep working after an upgrade.
|
||||
New cloud, pooler and CI settings are optional and default to the existing
|
||||
behavior. In particular, direct `PRIMARY_DB` connections and backend startup
|
||||
migrations remain the self-host defaults.
|
||||
- A self-host requires no AWS account, hosted control plane, PlanetScale,
|
||||
Confluent, GitHub Actions, paid license or cloud credential. Preserve the
|
||||
supported local database, cache, event bus, blob storage and key providers.
|
||||
- Keep the one-command installer, interactive wizard, unattended flags,
|
||||
documented Compose deployments, backup/restore and workspace portability
|
||||
functional. An upgrade must preserve configuration, encryption keys and data.
|
||||
- Configure hosted-only deployment policies in infrastructure or explicit
|
||||
settings. Do not infer a cloud dependency from production mode, and do not
|
||||
require a hosted service to install, start, upgrade or recover a self-host.
|
||||
- Verify the default self-host path as well as any new hosted path. Update the
|
||||
self-host configuration and upgrade docs when adding an optional setting.
|
||||
|
||||
Docs stay in sync:
|
||||
|
||||
- the customer docs site lives in `docs/` (Fumadocs, served at docs.warmbly.com); content is MDX under `docs/content/docs/` in three sections: `guides/` (product behavior), `learn/` (fundamentals), `api/` (API reference)
|
||||
@ -83,7 +137,7 @@ When you add one, work through:
|
||||
- **Scope.** The `WHERE` fragment selecting that table's rows for one organization, with `$1` as the org id. Use a subquery against a parent when the table has no `organization_id` of its own.
|
||||
- **Order.** `Tables` is applied top to bottom on import, so a table must sit below everything it references.
|
||||
- **Group.** Which `models.OrgDataGroup` it belongs to. If a NOT NULL foreign key crosses a group boundary, add the dependency to `Requires` in `models.OrgDataGroupCatalog` — otherwise a user who deselects the target group gets an import that aborts on a constraint. Nullable crossings need nothing; the importer blanks them.
|
||||
- **Secrets.** Any column holding ciphertext needs a `SecretColumn` with the right `KeyDomain`. Warmbly has two and they are not interchangeable: `KeyDomainInstance` is `CREDENTIALS_ENCRYPTION_KEY` (mailbox credentials, which the worker reads without an org context), `KeyDomainOrgDEK` is the per-organization DEK (everything else). Getting this wrong produces mailboxes that authenticate against nothing.
|
||||
- **Secrets.** Any column holding ciphertext needs a `SecretColumn` with the right `KeyDomain`. Warmbly has three and they are not interchangeable: `KeyDomainInstance` is `CREDENTIALS_ENCRYPTION_KEY`, `KeyDomainOrgDEK` is the per-organization DEK, and `KeyDomainMailbox` is mailbox credentials, which hold either (a `dek1:` prefix marks the org DEK, chosen by `MAILBOX_CREDENTIALS_SEALING`; see `internal/app/mailcred`). Getting this wrong produces mailboxes that authenticate against nothing.
|
||||
- **Instance-local columns.** Anything naming a worker, a queue handle, a Stripe object, or a sync checkpoint belongs in `ResetOnImport`, or the whole table in `ImportSkip` when it only means something on the instance that wrote it.
|
||||
- **Blobs.** A column holding an object-storage key needs a `BlobColumn` so the bytes travel with the rows.
|
||||
|
||||
@ -260,7 +314,7 @@ Anything one person types that Warmbly later shows to someone else is content in
|
||||
|
||||
- a server-class (`Internal`) error answers the caller with one fixed sentence and a request id. The real message is logged against that id. `errx.NewPublic` is the narrow exception, for a message an operator can act on, and never for one built from an underlying error
|
||||
- no secret, credential, token or full DSN may reach a log line, an error message or an analytics event. Errors sent to PostHog go through `internal/observability/errs`, and the database wrapper strips parameter values
|
||||
- ciphertext columns carry the right key domain. `KeyDomainInstance` is `CREDENTIALS_ENCRYPTION_KEY`, `KeyDomainOrgDEK` is the per-organization DEK. They are not interchangeable
|
||||
- ciphertext columns carry the right key domain. `KeyDomainInstance` is `CREDENTIALS_ENCRYPTION_KEY`, `KeyDomainOrgDEK` is the per-organization DEK, `KeyDomainMailbox` opens either per value. They are not interchangeable, and mailbox credentials go through `mailcred.Sealer`, never an encrypter directly
|
||||
|
||||
### Dependencies and configuration
|
||||
|
||||
@ -492,10 +546,12 @@ This is modeled in:
|
||||
|
||||
Migration 000156 guarantees exactly one pool per type on every instance, under `models.WarmupPoolFreeID` and `models.WarmupPoolPremiumID` (`warmup_pools_pool_type_key` makes it structural, and the migration moves any pre-existing pool onto those ids). Nothing else may insert into `warmup_pools`: not the sandbox, not the dev scripts, not a test fixture.
|
||||
|
||||
Crossing tiers is an exchange and lives in one place: `WarmupPartnerCandidates` in `pg_warmup.go` returns a sender's own tier plus, when the sender has fewer recipients outside its own workspace than `max(WarmupPoolTierFallbackFloor, warmup_max)`, that many proven free mailboxes from other workspaces (healthy, never blocked, members for `WarmupPoolFallbackMinAgeDays`, workspace not restricted or suspended), best first (Google or Microsoft, then members for `WarmupPoolBorrowSeasonedDays`, then actively sending; random within a rank), and for a free sender that meets the same bar, the premium mailboxes that verifiably wrote to it within `WarmupPoolReturnVisitDays`. Siblings never count toward the floor: a customer with many mailboxes and few premium peers otherwise never borrows and warms against itself. `models.WarmupPoolBorrowsFrom` and `models.WarmupPoolReturnsTo` are the two directions and mirror each other, so nothing unsolicited from the free tier reaches a paying inbox: the paying side always opens the exchange. Before this half existed a thin premium tier sent into the free tier and received almost nothing (#633). Every candidate carries the pool it was drawn from (`PoolType`) and how it got there (`Origin`); the selector and the scheduler both read that one method, so they cannot disagree about who is reachable. A routing rule of weight 0 is an exclusion, not a weight: the candidate is dropped before the draw (and refused on the reply-back path), so a pool of one cannot smuggle it back, and a tick with nothing left ends in `errAllPartnersExcluded` rather than mailing an excluded address (#501). The selector draws its own tier's fresh partners before a borrowed one, but a return visit ranks with the own tier, or a free mailbox with a hundred fresh siblings would never pay a paying inbox back. Every candidate is gated with `CanParticipate` pinned to the pool it was drawn from; gating a borrowed recipient against the sender's pool is what made borrowing dead for months (#495). The reply-back (`directedWarmupPartner`) crosses tiers on the same terms and is refused when the free workspace is restricted.
|
||||
Crossing tiers is an exchange and lives in one place: `WarmupPartnerCandidates` in `pg_warmup.go` returns a sender's own tier plus, when the sender has fewer recipients outside its own workspace than `max(WarmupPoolTierFallbackFloor, warmup_max)`, that many proven free mailboxes from other workspaces (healthy, never blocked, members for `WarmupPoolFallbackMinAgeDays`, workspace not restricted or suspended), best first (Google or Microsoft, then members for `WarmupPoolBorrowSeasonedDays`, then actively sending; random within a rank), and for a free sender that meets the same bar, the premium mailboxes that verifiably wrote to it within `WarmupPoolReturnVisitDays`. Siblings never count toward the floor: a customer with many mailboxes and few premium peers otherwise never borrows and warms against itself. `models.WarmupPoolBorrowsFrom` and `models.WarmupPoolReturnsTo` are the two directions and mirror each other, so nothing unsolicited from the free tier reaches a paying inbox: the paying side always opens the exchange. Before this half existed a thin premium tier sent into the free tier and received almost nothing (#633). Every candidate carries the pool it was drawn from (`PoolType`) and how it got there (`Origin`); the selector and the scheduler both read that one method, so they cannot disagree about who is reachable. A routing rule of weight 0 is an exclusion, not a weight: the candidate is dropped before the draw (and refused on the reply-back path), so a pool of one cannot smuggle it back, and a tick with nothing left ends in `errAllPartnersExcluded` rather than mailing an excluded address (#501). The selector draws its own tier's fresh partners before a borrowed one, but a return visit ranks with the own tier, or a free mailbox with a hundred fresh siblings would never pay a paying inbox back. Every candidate is gated with `CanParticipate` pinned to the pool it was drawn from; gating a borrowed recipient against the sender's pool is what made borrowing dead for months (#495). The reply-back (`directedWarmupPartner`) crosses tiers on the same terms (a free mailbox answers a paid one only while healthy, the bar the return cohort holds it to) and is refused when the free workspace is restricted. It crosses on `CanParticipate`'s `not_in_pool`, so that reason must stay distinct from an ineligible member, or every cross-tier reply is closed unsent.
|
||||
|
||||
Reciprocity is a weight and a cap, both read off the candidate. Each carries its verified sends and arrivals over seven days; `Starvation` (how far behind an inbox is on what it sent) multiplies its draw weight by up to `1 + reciprocityBoostK`, fading to nothing at parity, so the pool's traffic flows to whoever is owed the most and settles there instead of overshooting. `InboundDailyCap` (`WarmupInboundDailyMultiple` times daily sends, between `WarmupInboundDailyFloor` and `WarmupInboundDailyCeiling`) is applied inside `WarmupPartnerCandidates`, and a free sender sees every inbox as full at `WarmupFreeInboundSharePercent` of it, so the rest of each inbox's day is kept for premium senders. A recipient at its cap is offered to nobody for the rest of the day and the scheduler's per-day volume cap (`min(target, len(candidates))`) sees the same set. A sender with nobody left to write to sends nothing that tick and rechecks later; do not add a fallback that mails somebody anyway.
|
||||
|
||||
Reachability is read off the same candidate: `Delivered7d` is the pool's handed-off warmup mail to it (past `config.WarmupRecipientArrivalGraceHours`) and `Arrived7d` what its own sync verified (`warmup_tokens.consumed_at`). The draw multiplies by `ArrivalRate`, and `models.WarmupRecipientUnreachable` (none arrived, at least `WarmupRecipientUnreachableMinDelivered` from at least `WarmupRecipientUnreachableMinSenderOrgs` workspaces) drops the candidate in `capPartnerCandidates`, so the selector, the scheduler and the counts agree, and the Advisor's `warmup_mail_not_arriving` reads the same rule. A send the sender's provider accepted and later bounced looks like success to the worker, so arrival, not the send result, is the evidence. Keep the two-workspace floor: without it one broken sender takes its recipients out of the pool.
|
||||
|
||||
Keep this separation intact. Free-tier accounts should not silently mix into premium warmup traffic, and dedicated-worker accounts should still follow the intended warmup pool policy explicitly rather than by accident.
|
||||
|
||||
## Worker Networking Rules
|
||||
@ -530,6 +586,7 @@ Rules:
|
||||
- **read capabilities while the session is healthy.** A session the server's reply just closed reports none, so anything the ladder needs (`eSearch`, `condStore`) is recorded at connect
|
||||
- **a fix for one provider must not move the others.** #858 fixed Zoho by moving every server onto ESEARCH, which stalled every OVH mailbox for days. Add the provider's real reply to the dialect table in `search_modseq_test.go` (or its equivalent for the command you touch) and keep the whole table green, rather than writing a test that only knows the one server
|
||||
- **check a change to what the sync sends against real servers before it ships.** The cheap version: one login per distinct host, from the worker each mailbox already sits on (a new client IP can earn a provider security challenge), running the changed command and reporting only per-host results. Never print a credential or message content while doing it
|
||||
- **a message the server describes unreadably costs that message, never the folder.** go-imap reads `message/global` in a BODYSTRUCTURE as `message/rfc822`, so one RFC 6533 bounce dropped the session on every pass. `envelopeRung` in `envelope_ladder.go` steps the batch's UIDs down on an unreadable envelope answer (structure, then bare with the body parsed locally by go-message, then one at a time, then left unsynced), remembered on the `Client` per folder generation
|
||||
- **keep the server's words.** An `[ALERT]` reaches the owner verbatim through `imapErrDetail` (Zoho's `The server details you are using seems incorrect` is how they learn the account lives in another Zoho region). Map one in `classifyAlert` only when it is a known throttle or credentials case
|
||||
|
||||
## Encryption Model
|
||||
|
||||
@ -7,7 +7,7 @@ import * as DialogPrimitive from "@radix-ui/react-dialog";
|
||||
import { ShieldCheck } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { reauth } from "@/lib/api/client/auth";
|
||||
import { reauth, reauthMethods } from "@/lib/api/client/auth";
|
||||
import { APIError } from "@/lib/api/client";
|
||||
|
||||
interface ReauthDetail {
|
||||
@ -22,6 +22,8 @@ export function ReauthDialog() {
|
||||
const [code, setCode] = useState("");
|
||||
const [error, setError] = useState("");
|
||||
const [busy, setBusy] = useState(false);
|
||||
// Null while loading or when the lookup failed, which offers both fields.
|
||||
const [methods, setMethods] = useState<{ password: boolean; totp: boolean } | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const handler = (e: Event) => {
|
||||
@ -31,7 +33,9 @@ export function ReauthDialog() {
|
||||
setBusy(false);
|
||||
const detail = (e as CustomEvent<ReauthDetail>).detail;
|
||||
detail.ack?.();
|
||||
setMethods(null);
|
||||
setPending(detail);
|
||||
reauthMethods().then(setMethods, () => setMethods(null));
|
||||
};
|
||||
window.addEventListener("reauth-required", handler);
|
||||
return () => window.removeEventListener("reauth-required", handler);
|
||||
@ -62,6 +66,17 @@ export function ReauthDialog() {
|
||||
}
|
||||
}
|
||||
|
||||
const showPassword = methods === null || methods.password;
|
||||
const showCode = methods === null || methods.totp;
|
||||
const description =
|
||||
showPassword && showCode
|
||||
? "This change needs a fresh check. Enter your password, or a code from your authenticator."
|
||||
: showPassword
|
||||
? "This change needs a fresh check. Enter your password."
|
||||
: showCode
|
||||
? "This change needs a fresh check. Enter a code from your authenticator app."
|
||||
: "This account has no password or authenticator code to confirm with here. Sign in to the admin panel again and repeat this within five minutes.";
|
||||
|
||||
return (
|
||||
<DialogPrimitive.Root open={pending !== null} onOpenChange={(open) => !open && cancel()}>
|
||||
<DialogPrimitive.Portal>
|
||||
@ -82,37 +97,47 @@ export function ReauthDialog() {
|
||||
Confirm it is you
|
||||
</DialogPrimitive.Title>
|
||||
<DialogPrimitive.Description className="text-[13px] leading-relaxed text-muted-foreground">
|
||||
This change needs a fresh check. Enter your password, or a code from your
|
||||
authenticator.
|
||||
{description}
|
||||
</DialogPrimitive.Description>
|
||||
</div>
|
||||
</div>
|
||||
<div className="grid gap-2.5">
|
||||
<Input
|
||||
type="password"
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
placeholder="Password"
|
||||
autoComplete="current-password"
|
||||
autoFocus
|
||||
/>
|
||||
<Input
|
||||
type="text"
|
||||
value={code}
|
||||
onChange={(e) => setCode(e.target.value)}
|
||||
placeholder="Two-factor or recovery code"
|
||||
autoComplete="one-time-code"
|
||||
data-ph-mask=""
|
||||
/>
|
||||
</div>
|
||||
{showPassword || showCode ? (
|
||||
<div className="grid gap-2.5">
|
||||
{showPassword && (
|
||||
<Input
|
||||
type="password"
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
placeholder="Password"
|
||||
aria-label="Password"
|
||||
autoComplete="current-password"
|
||||
autoFocus
|
||||
/>
|
||||
)}
|
||||
{showCode && (
|
||||
<Input
|
||||
type="text"
|
||||
value={code}
|
||||
onChange={(e) => setCode(e.target.value)}
|
||||
placeholder="Two-factor or recovery code"
|
||||
aria-label="Two-factor or recovery code"
|
||||
autoComplete="one-time-code"
|
||||
autoFocus={!showPassword}
|
||||
data-ph-mask=""
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
) : null}
|
||||
{error && <p className="text-sm text-[var(--admin-danger)]">{error}</p>}
|
||||
<div className="flex flex-col-reverse gap-2 sm:flex-row sm:justify-end">
|
||||
<Button type="button" variant="outline" onClick={cancel}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button type="submit" disabled={busy || (!password && !code)}>
|
||||
{busy ? "Checking…" : "Confirm"}
|
||||
</Button>
|
||||
{(showPassword || showCode) && (
|
||||
<Button type="submit" disabled={busy || (!password && !code)}>
|
||||
{busy ? "Checking…" : "Confirm"}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</form>
|
||||
</DialogPrimitive.Content>
|
||||
|
||||
@ -73,6 +73,16 @@ export function logout(): Promise<void> {
|
||||
});
|
||||
}
|
||||
|
||||
// Which of password and authenticator code this account can confirm with.
|
||||
export function reauthMethods(): Promise<{ password: boolean; totp: boolean }> {
|
||||
return Request<{ password: boolean; totp: boolean }>({
|
||||
method: "GET",
|
||||
url: "/v1/auth/reauth/methods",
|
||||
authorization: true,
|
||||
skipReauthPrompt: true,
|
||||
});
|
||||
}
|
||||
|
||||
// Re-proves the operator behind the session; the changes gated on a recent
|
||||
// confirmation (join tokens, admin grants, workspace archives) wait for it.
|
||||
export function reauth(input: { password?: string; code?: string }): Promise<{ valid_for_seconds: number }> {
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
# DATABASES / SERVICES
|
||||
PRIMARY_DB="postgresql://username:password@localhost:5433/database"
|
||||
# For a transaction pooler, set PRIMARY_DB_DIRECT and DB_POOL_MODE=transaction.
|
||||
REDIS="redis://arbitrary_usrname:password@ipaddress:6379/0"
|
||||
KAFKA="amqp://username:password@ip:port/vhost"
|
||||
GEO_DB_PATH="./database.mmdb"
|
||||
|
||||
@ -44,6 +44,7 @@ import (
|
||||
behaviorapp "github.com/warmbly/warmbly/internal/app/behavior"
|
||||
"github.com/warmbly/warmbly/internal/app/bootstrap"
|
||||
"github.com/warmbly/warmbly/internal/app/campaign"
|
||||
"github.com/warmbly/warmbly/internal/app/campaignimport"
|
||||
"github.com/warmbly/warmbly/internal/app/cipher"
|
||||
"github.com/warmbly/warmbly/internal/app/clay"
|
||||
"github.com/warmbly/warmbly/internal/app/cliauth"
|
||||
@ -80,6 +81,7 @@ import (
|
||||
"github.com/warmbly/warmbly/internal/app/leadsync"
|
||||
"github.com/warmbly/warmbly/internal/app/mailboxavatar"
|
||||
"github.com/warmbly/warmbly/internal/app/mailboximport"
|
||||
"github.com/warmbly/warmbly/internal/app/mailcred"
|
||||
"github.com/warmbly/warmbly/internal/app/mcp"
|
||||
"github.com/warmbly/warmbly/internal/app/monitoring"
|
||||
"github.com/warmbly/warmbly/internal/app/nativeactions"
|
||||
@ -204,6 +206,7 @@ func main() {
|
||||
var cloudLinkService cloudlink.Service
|
||||
var cliAuthService cliauth.Service
|
||||
var campaignService campaign.CampaignService
|
||||
var campaignImportService *campaignimport.Service
|
||||
var analyticsService analytics.AnalyticsService
|
||||
var rateLimitService ratelimit.RateLimitService
|
||||
var sequenceService sequence.SequenceService
|
||||
@ -479,19 +482,32 @@ func main() {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
primaryDB, err := db.New(ctx, primaryDBEndpoint)
|
||||
directDBEndpoint, err := db.DirectEndpoint(primaryDBEndpoint, cfg.LoadPrimaryDBDirectEndpoint(ctx))
|
||||
if err != nil {
|
||||
errs.CaptureFatal(err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Run database migrations
|
||||
log.Println("Running database migrations...")
|
||||
if err := db.RunMigrations(primaryDBEndpoint); err != nil {
|
||||
errs.CaptureFatal(err)
|
||||
log.Fatal("Failed to run migrations: ", err)
|
||||
migrationsOnStart := cfg.LoadMigrationsOnStart(ctx)
|
||||
if migrationsOnStart {
|
||||
log.Println("Running database migrations...")
|
||||
if err := db.RunMigrations(directDBEndpoint); err != nil {
|
||||
errs.CaptureFatal(err)
|
||||
log.Fatal("Failed to run migrations: ", err)
|
||||
}
|
||||
log.Println("Database migrations completed")
|
||||
}
|
||||
primaryDB, err := db.NewWithDirect(ctx, primaryDBEndpoint, directDBEndpoint)
|
||||
if err != nil {
|
||||
errs.CaptureFatal(err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
if !migrationsOnStart {
|
||||
if err := db.WaitForSchema(ctx, primaryDB); err != nil {
|
||||
errs.CaptureFatal(err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
log.Println("Database migrations completed")
|
||||
// Once, not per warmup tick: the pools are fixed rows, and their absence
|
||||
// (a data-only restore, a manual delete) otherwise fails every tick quietly.
|
||||
if n, perr := instancecheck.CountSeededWarmupPools(ctx, primaryDB.Pool); perr == nil && n != 2 {
|
||||
@ -669,7 +685,6 @@ func main() {
|
||||
errs.CaptureFatal(cerr)
|
||||
log.Fatal("Invalid CREDENTIALS_ENCRYPTION_KEY: ", cerr)
|
||||
}
|
||||
emailRepostory := repository.NewEmailRepostory(primaryDB, credEncrypter)
|
||||
campaignRepostory := repository.NewCampaignRepostory(primaryDB)
|
||||
sequenceRepostory := repository.NewSequenceRepostory(primaryDB)
|
||||
contactRepostory := repository.NewContactRepostory(primaryDB)
|
||||
@ -696,6 +711,13 @@ func main() {
|
||||
errs.CaptureFatal(err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
cipherService = cipher.NewService(kms, cache, encryptedKeys)
|
||||
mailboxCredentials, merr := mailcred.FromEnv(credEncrypter, cipherService)
|
||||
if merr != nil {
|
||||
errs.CaptureFatal(merr)
|
||||
log.Fatal(merr)
|
||||
}
|
||||
emailRepostory := repository.NewEmailRepositoryWithSealer(primaryDB, mailboxCredentials)
|
||||
|
||||
folderRepostory := repository.NewGroupRepostory(primaryDB, models.Folders)
|
||||
tagRepostory := repository.NewGroupRepostory(primaryDB, models.Tags)
|
||||
@ -958,7 +980,11 @@ func main() {
|
||||
// TOTP 2FA: the secret is sealed with a server-wide key (the per-user DEK
|
||||
// is unreachable at login time). Wire the challenger into auth so the login
|
||||
// gate can issue a pending challenge.
|
||||
twofaService = twofa.NewService(repository.NewTOTPRepository(primaryDB.Pool), userRepostory, tokenService, cache, twofa.DeriveKey(authCfg.TwoFASecret))
|
||||
var twoFAReadKeys [][32]byte
|
||||
for _, secret := range authCfg.TwoFAReadSecrets {
|
||||
twoFAReadKeys = append(twoFAReadKeys, twofa.DeriveKey(secret))
|
||||
}
|
||||
twofaService = twofa.NewService(repository.NewTOTPRepository(primaryDB.Pool), userRepostory, tokenService, cache, twofa.DeriveKey(authCfg.TwoFASecret), twoFAReadKeys...)
|
||||
authService.WireTwoFA(twofaService)
|
||||
// The native app proves itself with App Attest where a browser solves Turnstile.
|
||||
appAttestService = appattest.New(repository.NewAppAttestRepository(primaryDB.Pool), cache, appattest.Config{
|
||||
@ -1155,7 +1181,6 @@ func main() {
|
||||
log.Printf("Warning: bootstrap failed: %v", berr)
|
||||
}
|
||||
|
||||
cipherService = cipher.NewService(kms, cache, encryptedKeys)
|
||||
emailMessageMapForHandler = repository.NewDurableEmailMessageMapRepository(primaryDB, cipherService)
|
||||
|
||||
// Third-party integrations: OAuth connect flows + encrypted token
|
||||
@ -1243,9 +1268,8 @@ func main() {
|
||||
// the mailbox-connect redirect_uri from it sends the provider
|
||||
// "0.0.0.0:8080/addresses/google/callback".
|
||||
oauth2Cfg := config.LoadOauth2(oauthPublicBaseURL(apiCfg.Hostname))
|
||||
// Recipient unsubscribe links live on the API origin, signed under the
|
||||
// auth secret; the same base the OAuth callbacks are built on.
|
||||
unsubSigner = unsublink.New(authCfg.AuthSecret, oauthPublicBaseURL(apiCfg.Hostname))
|
||||
// Recipient links use a separate purpose key when one is configured.
|
||||
unsubSigner = unsublink.NewWithReadKeys(authCfg.UnsubscribeSigningSecret, oauthPublicBaseURL(apiCfg.Hostname), authCfg.UnsubscribeReadSecrets, authCfg.UnsubscribeReadUntil)
|
||||
emailService = email.NewServiceWithWorker(
|
||||
emailRepostory,
|
||||
cipherService,
|
||||
@ -1262,6 +1286,7 @@ func main() {
|
||||
// Every connect path checks the workspace's mailbox allowance
|
||||
// (fair use for paid plans, the free cap otherwise).
|
||||
emailService.WireMailboxAllowance(organizationService)
|
||||
emailService.WireMailboxCredentials(mailboxCredentials)
|
||||
// Seed Graph delta cursors when the reconciler reloads mailboxes.
|
||||
emailService.WireGraphDelta(repository.NewEmailGraphDeltaRepository(primaryDB))
|
||||
// The Gmail equivalent: without it a reloaded mailbox re-bootstraps its
|
||||
@ -1303,7 +1328,7 @@ func main() {
|
||||
// Self-hosted warmup pool link, both roles. The cloud side hands out
|
||||
// codes and runs enrolled mailboxes as warmup-only accounts; the
|
||||
// self-hosted side keeps the link and mirrors the cloud's state.
|
||||
poolLinkRepository := repository.NewPoolLinkRepository(primaryDB.Pool)
|
||||
poolLinkRepository := repository.NewPoolLinkRepository(primaryDB.Pool, primaryDB.ConnectSession)
|
||||
poolLinkService = poollink.NewService(poolLinkRepository, emailRepostory, emailService, analyticsService, warmupRepository, featureGateService, organizationService, planRepository)
|
||||
emailService.WirePoolLink(poolLinkRepository)
|
||||
if g, ok := featureGateService.(interface{ WirePoolLink(feature.PoolLinkReader) }); ok {
|
||||
@ -1312,12 +1337,16 @@ func main() {
|
||||
if w, ok := poolLinkService.(poollink.CacheWirer); ok {
|
||||
w.WireCache(cache)
|
||||
}
|
||||
cloudLinkRepository := repository.NewCloudLinkRepository(primaryDB.Pool, credEncrypter)
|
||||
if w, ok := poolLinkService.(interface{ WireCampaigns(poollink.CampaignUsage) }); ok {
|
||||
w.WireCampaigns(campaignRepostory)
|
||||
}
|
||||
cloudLinkRepository := repository.NewCloudLinkRepository(primaryDB.Pool, credEncrypter, primaryDB.ConnectSession)
|
||||
emailService.WireCloudLink(cloudLinkRepository)
|
||||
cloudLinkService = cloudlink.NewService(cloudLinkRepository, emailRepostory, emailService)
|
||||
cloudLinkService.OnStandingChange(func(ctx context.Context, change models.CloudLinkStandingChange) {
|
||||
state, _, err := warmupRepository.GetHealthState(ctx, change.EmailAccountID)
|
||||
if err == nil {
|
||||
state, cloudUnknown, err := warmupRepository.GetKnownHealthState(ctx, change.EmailAccountID)
|
||||
// A missing Cloud report is no verdict; only a real hold beside it moves the band.
|
||||
if err == nil && (!cloudUnknown || state.Holds()) {
|
||||
err = workerRepository.SetEmailAccountRiskBand(ctx, change.EmailAccountID, models.RiskBandFromHealth(state))
|
||||
}
|
||||
if err != nil {
|
||||
@ -1331,6 +1360,21 @@ func main() {
|
||||
// Deleting a mailbox must also revoke its cloud-held credential.
|
||||
emailService.WireCloudUnenroll(cloudLinkService)
|
||||
emailService.WireCloudCredentials(cloudLinkService)
|
||||
// Warmup edits for Cloud-warmed mailboxes go to Warmbly Cloud, which owns them.
|
||||
emailService.WireCloudWarmup(cloudLinkService)
|
||||
cloudLinkService.OnSettingsChange(func(ctx context.Context, ids []uuid.UUID) {
|
||||
for _, id := range ids {
|
||||
acc, xerr := emailRepostory.GetByID(ctx, id)
|
||||
if xerr != nil || acc == nil {
|
||||
continue
|
||||
}
|
||||
orgID := ""
|
||||
if acc.OrganizationID != nil {
|
||||
orgID = acc.OrganizationID.String()
|
||||
}
|
||||
streamingPublisher.PublishAccountSynced(ctx, orgID, acc.UserID, acc.ID.String(), acc.Email, acc.Provider, acc.Status)
|
||||
}
|
||||
})
|
||||
|
||||
rateLimitRepository := repository.NewRateLimitRepository(primaryDB)
|
||||
rateLimitService = ratelimit.NewService(cache, rateLimitRepository)
|
||||
@ -1454,6 +1498,10 @@ func main() {
|
||||
aware.WireWorkerLiveness(tasks.NewWorkerLiveness(workerRepository, cache))
|
||||
}
|
||||
campaignService = campaign.NewService(campaignRepostory, taskRepository, emailRepostory, campaignLogRepository, featureGateService, dailyThrottleService, schedulerService, tasksClient, streamingPublisher)
|
||||
// Reuses campaignService/sequenceService/contactService as-is: no new
|
||||
// write path, just one call that drives the same three a wizard import
|
||||
// and a manual sequence build would make separately.
|
||||
campaignImportService = campaignimport.NewService(campaignService, sequenceService, contactService)
|
||||
// The launch gate refuses a list that is known to be largely
|
||||
// undeliverable, using the same projection preflight reports.
|
||||
if aware, ok := campaignService.(campaign.AudienceAware); ok {
|
||||
@ -1513,6 +1561,9 @@ func main() {
|
||||
if aware, ok := emailSendService.(emailsend.TrackedLinksAware); ok {
|
||||
aware.WireTrackedLinks(trackedLinkRepository)
|
||||
}
|
||||
if aware, ok := emailSendService.(emailsend.SuppressionAware); ok {
|
||||
aware.WireSuppression(advancedRepository)
|
||||
}
|
||||
composeService = compose.NewService(emailRepostory, repository.NewComposeRepository(primaryDB))
|
||||
// uniboxService is constructed here (rather than alongside the
|
||||
// other service constructors above) because cancel-scheduled
|
||||
@ -1768,8 +1819,9 @@ func main() {
|
||||
}
|
||||
// Never hand a send to a worker that stopped heartbeating: nothing
|
||||
// would execute it and nothing would report it, so the step would
|
||||
// look sent forever. The worker reconciler re-places the mailbox and
|
||||
// the dead-letter retry replays the task once it has.
|
||||
// look sent forever. The worker reconciler re-places the mailbox; a
|
||||
// campaign retries on its next pass, and a unibox send is refused and
|
||||
// reported to the mailbox owner.
|
||||
if liveness, ok := emailSender.(interface{ WireWorkerLiveness(tasks.WorkerLiveness) }); ok {
|
||||
liveness.WireWorkerLiveness(tasks.NewWorkerLiveness(workerRepository, cache))
|
||||
}
|
||||
@ -1818,7 +1870,7 @@ func main() {
|
||||
// Mailbox imports: files and pasted lists connected in the background,
|
||||
// with each domain's mail host detected from its DNS.
|
||||
// One prover for every DNS ownership proof, keyed by the instance secret.
|
||||
domainProver := domainproof.New(authCfg.AuthSecret)
|
||||
domainProver := domainproof.New(authCfg.DomainProofSecret)
|
||||
|
||||
// Whole-domain connects: Google Workspace delegation to the instance's
|
||||
// service account, Microsoft 365 consent to its Outlook app.
|
||||
@ -1997,7 +2049,8 @@ func main() {
|
||||
// snapshot instead of running the planner (lead supply, per-mailbox
|
||||
// history) on the request. A no-op without the snapshot store or planner.
|
||||
if campaignService != nil {
|
||||
go campaignService.StartSendPlanSnapshotter(ctx, time.Minute)
|
||||
interval, maxAge := sendPlanSnapshotTiming()
|
||||
go campaignService.StartSendPlanSnapshotter(ctx, interval, maxAge)
|
||||
}
|
||||
|
||||
// Segment-linked campaigns: enrol contacts that drifted into a linked
|
||||
@ -2045,7 +2098,7 @@ func main() {
|
||||
repository.NewMailboxErasureRepository(primaryDB),
|
||||
s3,
|
||||
credEncrypter,
|
||||
).Start(ctx, 1*time.Minute)
|
||||
).WithOrganizationKeys(cipherService).Start(ctx, 1*time.Minute)
|
||||
|
||||
// Workspace archives: export a whole organization to a portable file
|
||||
// and import one back, so a workspace can move between instances.
|
||||
@ -2363,24 +2416,25 @@ func main() {
|
||||
NodeLogs: nodelogs.New(authCache),
|
||||
BrokerDiagnostics: monitoringBus,
|
||||
|
||||
TokenService: tokenService,
|
||||
PasskeyService: passkeyService,
|
||||
AppAttestService: appAttestService,
|
||||
UserService: userService,
|
||||
EmailService: emailService,
|
||||
MailboxImportService: mailboxImportService,
|
||||
ContactImportService: contactImportService,
|
||||
DelegationService: delegationService,
|
||||
VendorConnService: vendorConnService,
|
||||
SendingDomainService: sendingDomainService,
|
||||
CampaignService: campaignService,
|
||||
AnalyticsService: analyticsService,
|
||||
RateLimitService: rateLimitService,
|
||||
ContactService: contactService,
|
||||
SegmentService: segmentService,
|
||||
FormService: formService,
|
||||
SequenceService: sequenceService,
|
||||
UniboxService: uniboxService,
|
||||
TokenService: tokenService,
|
||||
PasskeyService: passkeyService,
|
||||
AppAttestService: appAttestService,
|
||||
UserService: userService,
|
||||
EmailService: emailService,
|
||||
MailboxImportService: mailboxImportService,
|
||||
ContactImportService: contactImportService,
|
||||
DelegationService: delegationService,
|
||||
VendorConnService: vendorConnService,
|
||||
SendingDomainService: sendingDomainService,
|
||||
CampaignService: campaignService,
|
||||
CampaignImportService: campaignImportService,
|
||||
AnalyticsService: analyticsService,
|
||||
RateLimitService: rateLimitService,
|
||||
ContactService: contactService,
|
||||
SegmentService: segmentService,
|
||||
FormService: formService,
|
||||
SequenceService: sequenceService,
|
||||
UniboxService: uniboxService,
|
||||
|
||||
FolderService: folderService,
|
||||
TagService: tagService,
|
||||
@ -2670,6 +2724,20 @@ func localTasksPollInterval() time.Duration {
|
||||
return d
|
||||
}
|
||||
|
||||
// sendPlanSnapshotTiming reads SEND_PLAN_SNAPSHOT_INTERVAL (default 1m) and
|
||||
// SEND_PLAN_SNAPSHOT_MAX_AGE (default three intervals), both Go durations.
|
||||
func sendPlanSnapshotTiming() (time.Duration, time.Duration) {
|
||||
interval, err := time.ParseDuration(getenvDefault("SEND_PLAN_SNAPSHOT_INTERVAL", "1m"))
|
||||
if err != nil || interval <= 0 {
|
||||
interval = time.Minute
|
||||
}
|
||||
maxAge, err := time.ParseDuration(os.Getenv("SEND_PLAN_SNAPSHOT_MAX_AGE"))
|
||||
if err != nil || maxAge <= 0 {
|
||||
maxAge = 3 * interval
|
||||
}
|
||||
return interval, maxAge
|
||||
}
|
||||
|
||||
// wsHealthURL derives the realtime service's HTTP /health URL from the
|
||||
// public websocket URI (ws[s]://host[:port]/... -> http[s]://host[:port]/health).
|
||||
func wsHealthURL(wsURI string) string {
|
||||
|
||||
@ -32,6 +32,7 @@ import (
|
||||
"github.com/warmbly/warmbly/internal/app/inboxtag"
|
||||
"github.com/warmbly/warmbly/internal/app/instancesettings"
|
||||
"github.com/warmbly/warmbly/internal/app/integration"
|
||||
"github.com/warmbly/warmbly/internal/app/mailcred"
|
||||
"github.com/warmbly/warmbly/internal/app/nativeactions"
|
||||
"github.com/warmbly/warmbly/internal/app/notification"
|
||||
"github.com/warmbly/warmbly/internal/app/opsnotify"
|
||||
@ -97,7 +98,7 @@ func main() {
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
primaryDB, err := db.New(ctx, primaryDBEndpoint)
|
||||
primaryDB, err := db.NewWithDirect(ctx, primaryDBEndpoint, cfg.LoadPrimaryDBDirectEndpoint(ctx))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@ -207,7 +208,12 @@ func main() {
|
||||
errs.CaptureFatal(err)
|
||||
log.Fatal("Invalid CREDENTIALS_ENCRYPTION_KEY: ", err)
|
||||
}
|
||||
emailRepo := repository.NewEmailRepostory(primaryDB, credEncrypter)
|
||||
mailboxCredentials, err := mailcred.FromEnv(credEncrypter, cipherService)
|
||||
if err != nil {
|
||||
errs.CaptureFatal(err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
emailRepo := repository.NewEmailRepositoryWithSealer(primaryDB, mailboxCredentials)
|
||||
uniboxRepo := repository.NewUniboxRepository(primaryDB)
|
||||
mailboxRepo := repository.NewMailboxRepository(primaryDB)
|
||||
emailHistoryIDRepo := repository.NewEmailHistoryIDRepository(primaryDB)
|
||||
@ -529,8 +535,8 @@ func main() {
|
||||
ArrivalOutbox: repository.NewDurableEmailMessageMapRepository(primaryDB, cipherService),
|
||||
EmailAccountErrorRepository: emailAccountErrorRepo,
|
||||
WarmupRepo: warmupRepo,
|
||||
PoolLinkRepo: repository.NewPoolLinkRepository(primaryDB.Pool),
|
||||
CloudLink: cloudlink.NewService(repository.NewCloudLinkRepository(primaryDB.Pool, credEncrypter), emailRepo, nil),
|
||||
PoolLinkRepo: repository.NewPoolLinkRepository(primaryDB.Pool, primaryDB.ConnectSession),
|
||||
CloudLink: cloudlink.NewService(repository.NewCloudLinkRepository(primaryDB.Pool, credEncrypter, primaryDB.ConnectSession), emailRepo, nil),
|
||||
WarmupContentRepo: repository.NewWarmupContentRepository(primaryDB.Pool),
|
||||
WarmupEngagementRepo: repository.NewWarmupEngagementRepository(primaryDB.Pool),
|
||||
WarmupRecoveryRepo: repository.NewWarmupRecoveryRepository(primaryDB.Pool),
|
||||
@ -606,10 +612,13 @@ func main() {
|
||||
// reply refused by a since-fixed check is still attributed to its lead.
|
||||
go jobsService.StartIncomingReplyRepair(ctx)
|
||||
go jobsService.StartReplyOptOutRecheck(ctx)
|
||||
go jobsService.StartHistoricalReplyCleanup(ctx)
|
||||
|
||||
// Mirrors Warmbly Cloud's warmup verdicts onto the mailboxes it warms, so
|
||||
// a cloud quarantine pauses their campaigns here too. No-op when unlinked.
|
||||
go jobsService.StartCloudStandingSync(ctx, 5*time.Minute)
|
||||
// Mirrors the warmup settings Warmbly Cloud owns, so edits made there show here.
|
||||
go jobsService.StartCloudSettingsSync(ctx, 15*time.Minute)
|
||||
|
||||
// Start dead worker detection (every 5 minutes)
|
||||
go jobsService.StartDeadWorkerDetection(ctx, 5*time.Minute)
|
||||
@ -618,6 +627,11 @@ func main() {
|
||||
// result never came back, so a lead is never held in flight forever.
|
||||
go jobsService.StartStuckSendReclaimer(ctx, 5*time.Minute)
|
||||
|
||||
if mailboxCredentials.SealsForOrganizations() {
|
||||
// Moves credentials stored under the instance key onto workspace keys.
|
||||
go jobsService.StartMailboxCredentialReseal(ctx)
|
||||
}
|
||||
|
||||
// Mirror Redis heartbeats into workers.last_seen_at every 60s
|
||||
// so the admin dashboard can render liveness without touching Redis.
|
||||
go jobsService.StartWorkerHeartbeatSync(ctx, 60*time.Second)
|
||||
|
||||
@ -15,6 +15,10 @@ import (
|
||||
|
||||
func main() {
|
||||
url := os.Getenv("PRIMARY_DB")
|
||||
url, err := db.DirectEndpoint(url, os.Getenv("PRIMARY_DB_DIRECT"))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if url == "" {
|
||||
log.Fatal("PRIMARY_DB is not set (e.g. postgres://warmbly:warmbly@localhost:15432/warmbly_dev?sslmode=disable)")
|
||||
}
|
||||
|
||||
@ -104,7 +104,11 @@ func main() {
|
||||
// assume migrations are already applied, but running them here makes it
|
||||
// robust standalone: without it, new tables (e.g. crm_task_types) are
|
||||
// missing and the seed aborts partway, which looks like "nothing seeded".
|
||||
if err := db.RunMigrations(dsn); err != nil {
|
||||
directDSN, err := db.DirectEndpoint(dsn, os.Getenv("PRIMARY_DB_DIRECT"))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := db.RunMigrations(directDSN); err != nil {
|
||||
log.Fatalf("migrations: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@ -58,9 +58,21 @@ const (
|
||||
// every worker and websocket connection, which reads as a broken restore.
|
||||
var backupKeys = []string{
|
||||
"CREDENTIALS_ENCRYPTION_KEY",
|
||||
"CREDENTIALS_ENCRYPTION_READ_KEYS",
|
||||
"KMS_LOCAL_MASTER_KEY",
|
||||
"AUTH_SECRET",
|
||||
"TWOFA_SECRET",
|
||||
"TWOFA_READ_SECRETS",
|
||||
"UNSUBSCRIBE_SIGNING_SECRET",
|
||||
"UNSUBSCRIBE_SIGNING_READ_SECRETS",
|
||||
"UNSUBSCRIBE_SIGNING_READ_UNTIL",
|
||||
"DOMAIN_PROOF_SECRET",
|
||||
"INTERNAL_API_TOKEN",
|
||||
"INTERNAL_API_PREVIOUS_TOKENS",
|
||||
"INTERNAL_API_PREVIOUS_UNTIL",
|
||||
"NODE_BROKER_TOKEN",
|
||||
"NODE_BROKER_PREVIOUS_TOKENS",
|
||||
"NODE_BROKER_PREVIOUS_UNTIL",
|
||||
"SECRET_KEY_BASE",
|
||||
}
|
||||
|
||||
@ -116,7 +128,7 @@ func runBackup(ctx context.Context, args []string) error {
|
||||
return fmt.Errorf("%s already exists. Pass --force to overwrite it, or --out with another path.", target)
|
||||
}
|
||||
|
||||
dsn, err := dbEndpoint(ctx)
|
||||
dsn, err := directDBEndpoint(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -254,7 +266,7 @@ func runRestore(ctx context.Context, args []string) error {
|
||||
return fmt.Errorf("this bundle is layout version %d and this warmblyctl reads version %d. Restore it with the Warmbly it was written by (%s).", m.Version, bundleVersion, m.AppVersion)
|
||||
}
|
||||
|
||||
dsn, err := dbEndpoint(ctx)
|
||||
dsn, err := directDBEndpoint(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -305,6 +317,14 @@ func runRestore(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
fmt.Println(" database restored")
|
||||
if err := analyzeDatabase(ctx, dsn); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
warn("the database restored, but ANALYZE did not finish, so queries plan without statistics until autovacuum gathers them. Run ANALYZE by hand:\n%s", err)
|
||||
} else {
|
||||
fmt.Println(" statistics analyzed")
|
||||
}
|
||||
|
||||
if !*noBlobs && m.BlobFiles > 0 {
|
||||
root := blobRoot()
|
||||
@ -491,6 +511,20 @@ func restoreDatabase(ctx context.Context, dsn, bundle string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// analyzeDatabase gathers planner statistics, which a dump does not carry.
|
||||
func analyzeDatabase(ctx context.Context, dsn string) error {
|
||||
cmd := pgCommand(ctx, "psql", dsn, "-v", "ON_ERROR_STOP=1", "-q", "-c", "ANALYZE;")
|
||||
var stderr strings.Builder
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
if msg := strings.TrimSpace(stderr.String()); msg != "" {
|
||||
return errors.New(msg)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pgCommand builds a libpq invocation with the password moved out of argv.
|
||||
//
|
||||
// A connection string on the command line is readable by anyone on the host who
|
||||
|
||||
29
cmd/warmblyctl/backup_key_rotation_test.go
Normal file
29
cmd/warmblyctl/backup_key_rotation_test.go
Normal file
@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBackupKeyListsRoundTrip(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dump := filepath.Join(root, "database.sql")
|
||||
if err := os.WriteFile(dump, []byte("SELECT 1;\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys := map[string]string{"CREDENTIALS_ENCRYPTION_KEY": "fixture-primary", "CREDENTIALS_ENCRYPTION_READ_KEYS": `["fixture-reader"]`, "TWOFA_SECRET": "fixture-mfa", "TWOFA_READ_SECRETS": `["reader,with,commas"]`, "UNSUBSCRIBE_SIGNING_READ_UNTIL": "2027-10-22T00:00:00Z", "NODE_BROKER_PREVIOUS_TOKENS": `["fixture-broker"]`}
|
||||
bundle := filepath.Join(root, "bundle.tar.gz")
|
||||
if err := writeBundle(bundle, manifest{Version: 1}, dump, nil, keys); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
restored, err := readKeys(bundle)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, want := range keys {
|
||||
if restored[key] != want {
|
||||
t.Fatalf("backup must preserve %s exactly", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/warmbly/warmbly/internal/app/cipher"
|
||||
"github.com/warmbly/warmbly/internal/app/credits"
|
||||
"github.com/warmbly/warmbly/internal/app/mailcred"
|
||||
"github.com/warmbly/warmbly/internal/app/organization"
|
||||
"github.com/warmbly/warmbly/internal/app/orgtransfer"
|
||||
"github.com/warmbly/warmbly/internal/app/token"
|
||||
@ -46,7 +47,11 @@ func connect(ctx context.Context) (*conn, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
handle, err := db.New(ctx, dsn)
|
||||
directDSN, err := directDBEndpoint(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
handle, err := db.NewWithDirect(ctx, dsn, directDSN)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("PRIMARY_DB is not a usable connection string (%s): %w", redact(dsn), err)
|
||||
}
|
||||
@ -175,6 +180,9 @@ func (c *conn) orgTransferService(ctx context.Context) (orgtransfer.Service, err
|
||||
if credEncrypter == nil {
|
||||
warn("CREDENTIALS_ENCRYPTION_KEY is not set, so mailbox credentials cannot be read or written by this command.")
|
||||
}
|
||||
if _, err := mailcred.ModeFromEnv(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return orgtransfer.NewService(
|
||||
repository.NewOrgTransferRepository(c.db),
|
||||
@ -233,6 +241,20 @@ func dbEndpoint(ctx context.Context) (string, error) {
|
||||
return "", errors.New("PRIMARY_DB is not set, so there is no database to talk to.\nRun this inside the backend container, where the environment is already correct:\n docker compose -p warmbly exec backend warmblyctl status\nOr export PRIMARY_DB first.")
|
||||
}
|
||||
|
||||
func directDBEndpoint(ctx context.Context) (string, error) {
|
||||
primary, err := dbEndpoint(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
direct := os.Getenv("PRIMARY_DB_DIRECT")
|
||||
if direct == "" {
|
||||
if cfg, err := config.NewConfig(ctx); err == nil {
|
||||
direct = cfg.LoadPrimaryDBDirectEndpoint(ctx)
|
||||
}
|
||||
}
|
||||
return db.DirectEndpoint(primary, direct)
|
||||
}
|
||||
|
||||
func redisEndpoint(ctx context.Context) (string, error) {
|
||||
if v := strings.TrimSpace(os.Getenv("REDIS")); v != "" {
|
||||
return v, nil
|
||||
|
||||
464
cmd/warmblyctl/keys.go
Normal file
464
cmd/warmblyctl/keys.go
Normal file
@ -0,0 +1,464 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/app/mailcred"
|
||||
"github.com/warmbly/warmbly/internal/app/orgtransfer"
|
||||
"github.com/warmbly/warmbly/internal/app/twofa"
|
||||
"github.com/warmbly/warmbly/internal/config"
|
||||
"github.com/warmbly/warmbly/internal/pkg/encrypt"
|
||||
"github.com/warmbly/warmbly/internal/pkg/secretset"
|
||||
)
|
||||
|
||||
// A rotation ships the new primary with the old key as a read key, reseals, and only then retires the old key.
|
||||
|
||||
// keyDomain names which configured key seals a column.
|
||||
type keyDomain string
|
||||
|
||||
const (
|
||||
domainInstance keyDomain = "instance"
|
||||
domainMFA keyDomain = "mfa"
|
||||
)
|
||||
|
||||
// sealedColumn is one column holding values sealed under an instance-wide key.
|
||||
type sealedColumn struct {
|
||||
Table string `json:"table"`
|
||||
Column string `json:"column"`
|
||||
Domain keyDomain `json:"domain"`
|
||||
guard string
|
||||
}
|
||||
|
||||
// instanceOnlyColumns are sealed under an instance-wide key on tables that never travel in a workspace archive.
|
||||
var instanceOnlyColumns = []sealedColumn{
|
||||
{Table: "cloud_link", Column: "token", Domain: domainInstance},
|
||||
{Table: "mailbox_erasures", Column: "refresh_token", Domain: domainInstance},
|
||||
{Table: "user_totp_settings", Column: "totp_secret_sealed", Domain: domainMFA},
|
||||
}
|
||||
|
||||
// sealedColumns is every column the instance keys seal: the archive registry's instance-key and mailbox secrets plus the instance-only ones.
|
||||
func sealedColumns() []sealedColumn {
|
||||
var out []sealedColumn
|
||||
for _, t := range orgtransfer.Tables {
|
||||
for _, s := range t.Secrets {
|
||||
if s.Domain == orgtransfer.KeyDomainInstance || s.Domain == orgtransfer.KeyDomainMailbox {
|
||||
out = append(out, sealedColumn{Table: t.Name, Column: s.Column, Domain: domainInstance, guard: s.Guard})
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(out, instanceOnlyColumns...)
|
||||
}
|
||||
|
||||
// columnCount is where one column's values stand against the configured keys.
|
||||
type columnCount struct {
|
||||
sealedColumn
|
||||
Primary int `json:"primary"`
|
||||
ReadKey int `json:"read_key"`
|
||||
Workspace int `json:"workspace"`
|
||||
Unsealed int `json:"unsealed"`
|
||||
Unreadable int `json:"unreadable"`
|
||||
Resealed int `json:"resealed,omitempty"`
|
||||
Raced int `json:"changed_meanwhile,omitempty"`
|
||||
Missing bool `json:"missing,omitempty"`
|
||||
}
|
||||
|
||||
// keyReport is the whole answer, shaped for --json as much as for a terminal.
|
||||
type keyReport struct {
|
||||
Applied bool `json:"applied"`
|
||||
InstanceReadKeys int `json:"instance_read_keys"`
|
||||
MFAReadKeys int `json:"mfa_read_keys"`
|
||||
MFAConfigured bool `json:"mfa_configured"`
|
||||
Columns []columnCount `json:"columns"`
|
||||
Windows []graceWindow `json:"grace_windows"`
|
||||
}
|
||||
|
||||
// graceWindow is one previous-credential acceptance window and whether it is still open.
|
||||
type graceWindow struct {
|
||||
Name string `json:"name"`
|
||||
Until time.Time `json:"until"`
|
||||
Open bool `json:"open"`
|
||||
Entries int `json:"entries"`
|
||||
}
|
||||
|
||||
// keyring is the configured keys the backend opens and seals with.
|
||||
type keyring struct {
|
||||
instance *encrypt.Encrypter
|
||||
instanceRead int
|
||||
mfa *[32]byte
|
||||
mfaRead [][32]byte
|
||||
}
|
||||
|
||||
var errUnreadableValues = errors.New("some stored values do not open under any configured key")
|
||||
|
||||
func runKeys(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
keysUsage(os.Stderr)
|
||||
return errors.New("`keys` needs a subcommand. Pick one from the list above.")
|
||||
}
|
||||
switch args[0] {
|
||||
case "help", "-h", "--help":
|
||||
keysUsage(os.Stdout)
|
||||
return nil
|
||||
case "status":
|
||||
return runKeysScan(ctx, "keys status", args[1:], false)
|
||||
case "reseal":
|
||||
return runKeysScan(ctx, "keys reseal", args[1:], true)
|
||||
case "generate":
|
||||
return runKeysGenerate(args[1:])
|
||||
}
|
||||
keysUsage(os.Stderr)
|
||||
return fmt.Errorf("unknown subcommand `keys %s`. Pick one from the list above.", args[0])
|
||||
}
|
||||
|
||||
func keysUsage(w *os.File) {
|
||||
fmt.Fprint(w, "Rotate the instance-wide encryption keys without losing what they seal.\n\nUsage:\n warmblyctl keys <subcommand> [flags]\n\nSubcommands:\n")
|
||||
for _, c := range commands {
|
||||
if !strings.HasPrefix(c.name, "keys ") {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, " %-28s %s\n %s\n", strings.TrimPrefix(c.name, "keys "), c.summary, c.example)
|
||||
}
|
||||
}
|
||||
|
||||
func runKeysScan(ctx context.Context, name string, args []string, reseal bool) error {
|
||||
fs := newFlagSet(name)
|
||||
asJSON := fs.Bool("json", false, "print the report as JSON")
|
||||
var apply *bool
|
||||
var batch *int
|
||||
if reseal {
|
||||
apply = fs.Bool("apply", false, "write the resealed values; without it nothing is written")
|
||||
batch = fs.Int("batch", 500, "rows read per query")
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := noExtraArgs(fs); err != nil {
|
||||
return err
|
||||
}
|
||||
size := 500
|
||||
if batch != nil {
|
||||
if *batch < 1 || *batch > 10000 {
|
||||
return errors.New("--batch must be between 1 and 10000")
|
||||
}
|
||||
size = *batch
|
||||
}
|
||||
write := apply != nil && *apply
|
||||
|
||||
keys, err := loadKeyring(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, err := connect(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer c.close()
|
||||
|
||||
report := keyReport{Applied: write, InstanceReadKeys: keys.instanceRead, MFAReadKeys: len(keys.mfaRead), MFAConfigured: keys.mfa != nil, Windows: graceWindows(time.Now())}
|
||||
for _, col := range sealedColumns() {
|
||||
if col.Domain == domainMFA && keys.mfa == nil {
|
||||
continue
|
||||
}
|
||||
count, err := scanColumn(ctx, c, keys, col, size, write)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.Columns = append(report.Columns, count)
|
||||
}
|
||||
|
||||
if *asJSON {
|
||||
if err := json.NewEncoder(os.Stdout).Encode(report); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
printKeyReport(report, reseal)
|
||||
}
|
||||
for _, col := range report.Columns {
|
||||
if col.Unreadable > 0 {
|
||||
return errUnreadableValues
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadKeyring resolves the keys exactly as the backend does at boot.
|
||||
func loadKeyring(ctx context.Context) (*keyring, error) {
|
||||
instance, err := encrypt.FromEnv()
|
||||
if err != nil {
|
||||
return nil, errors.New("CREDENTIALS_ENCRYPTION_KEY or CREDENTIALS_ENCRYPTION_READ_KEYS is invalid: each must be 64 hex characters")
|
||||
}
|
||||
if instance == nil {
|
||||
return nil, errors.New("CREDENTIALS_ENCRYPTION_KEY is not set. Run this inside the backend container, where the keys are configured.")
|
||||
}
|
||||
reads, _ := secretset.Parse(os.Getenv("CREDENTIALS_ENCRYPTION_READ_KEYS"))
|
||||
k := &keyring{instance: instance}
|
||||
for _, r := range reads {
|
||||
if r != os.Getenv("CREDENTIALS_ENCRYPTION_KEY") {
|
||||
k.instanceRead++
|
||||
}
|
||||
}
|
||||
|
||||
var primary string
|
||||
var mfaReads []string
|
||||
if cfg, err := config.NewConfig(ctx); err == nil {
|
||||
if auth, aerr := cfg.LoadAuthConfig(ctx); aerr == nil {
|
||||
primary, mfaReads = auth.TwoFASecret, auth.TwoFAReadSecrets
|
||||
}
|
||||
}
|
||||
if primary == "" {
|
||||
primary = strings.TrimSpace(os.Getenv("TWOFA_SECRET"))
|
||||
if primary == "" {
|
||||
primary = strings.TrimSpace(os.Getenv("AUTH_SECRET"))
|
||||
}
|
||||
if mfaReads, err = secretset.Parse(os.Getenv("TWOFA_READ_SECRETS")); err != nil {
|
||||
return nil, fmt.Errorf("TWOFA_READ_SECRETS: %w", err)
|
||||
}
|
||||
}
|
||||
if primary != "" {
|
||||
key := twofa.DeriveKey(primary)
|
||||
k.mfa = &key
|
||||
for _, s := range mfaReads {
|
||||
if s != primary {
|
||||
k.mfaRead = append(k.mfaRead, twofa.DeriveKey(s))
|
||||
}
|
||||
}
|
||||
}
|
||||
return k, nil
|
||||
}
|
||||
|
||||
// scanColumn classifies every value in one column and, with write, reseals the read-key ones by compare-and-swap.
|
||||
func scanColumn(ctx context.Context, c *conn, keys *keyring, col sealedColumn, size int, write bool) (columnCount, error) {
|
||||
out := columnCount{sealedColumn: col}
|
||||
var exists bool
|
||||
if err := c.db.Pool.QueryRow(ctx, `SELECT to_regclass($1) IS NOT NULL`, col.Table).Scan(&exists); err != nil {
|
||||
return out, fmt.Errorf("reading %s: %w", col.Table, err)
|
||||
}
|
||||
if !exists {
|
||||
out.Missing = true
|
||||
return out, nil
|
||||
}
|
||||
table, column := pgx.Identifier{col.Table}.Sanitize(), pgx.Identifier{col.Column}.Sanitize()
|
||||
filter := ""
|
||||
if col.guard != "" {
|
||||
filter = " AND " + pgx.Identifier{col.guard}.Sanitize()
|
||||
}
|
||||
query := fmt.Sprintf(`SELECT ctid::text AS row_tid, %[2]s::text FROM %[1]s WHERE %[2]s IS NOT NULL AND %[2]s::text <> ''%[3]s AND ctid > $1::tid ORDER BY ctid LIMIT $2`, table, column, filter)
|
||||
// The ctid and the old value together are the compare: a row rewritten meanwhile matches neither.
|
||||
update := fmt.Sprintf(`UPDATE %[1]s SET %[2]s = $1 WHERE ctid = $2::tid AND %[2]s::text = $3`, table, column)
|
||||
|
||||
// One snapshot for the reads, so a tuple this run rewrites is never read back.
|
||||
snap, err := c.db.Pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly})
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err)
|
||||
}
|
||||
defer func() { _ = snap.Rollback(ctx) }()
|
||||
|
||||
after := "(0,0)"
|
||||
for {
|
||||
rows, err := snap.Query(ctx, query, after, size)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err)
|
||||
}
|
||||
type row struct{ ctid, value string }
|
||||
var page []row
|
||||
for rows.Next() {
|
||||
var r row
|
||||
if err := rows.Scan(&r.ctid, &r.value); err != nil {
|
||||
rows.Close()
|
||||
return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err)
|
||||
}
|
||||
page = append(page, r)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return out, fmt.Errorf("reading %s.%s: %w", col.Table, col.Column, err)
|
||||
}
|
||||
for _, r := range page {
|
||||
resealed, changed, err := resealValue(keys, col, r.value)
|
||||
switch {
|
||||
case errors.Is(err, errUnsealed):
|
||||
out.Unsealed++
|
||||
continue
|
||||
case errors.Is(err, errWorkspaceSealed):
|
||||
out.Workspace++
|
||||
continue
|
||||
case err != nil:
|
||||
out.Unreadable++
|
||||
continue
|
||||
case !changed:
|
||||
out.Primary++
|
||||
continue
|
||||
}
|
||||
out.ReadKey++
|
||||
if !write {
|
||||
continue
|
||||
}
|
||||
tag, err := c.db.Pool.Exec(ctx, update, resealed, r.ctid, r.value)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("resealing %s.%s: %w", col.Table, col.Column, err)
|
||||
}
|
||||
if tag.RowsAffected() == 1 {
|
||||
out.Resealed++
|
||||
} else {
|
||||
out.Raced++
|
||||
}
|
||||
}
|
||||
if len(page) < size {
|
||||
return out, nil
|
||||
}
|
||||
after = page[len(page)-1].ctid
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
errUnsealed = errors.New("value is stored outside the sealed format")
|
||||
errWorkspaceSealed = errors.New("value is sealed under its workspace key")
|
||||
)
|
||||
|
||||
func resealValue(keys *keyring, col sealedColumn, value string) (string, bool, error) {
|
||||
if col.Domain == domainMFA {
|
||||
return twofa.Reseal(*keys.mfa, keys.mfaRead, value)
|
||||
}
|
||||
// A workspace-sealed value needs no instance key, so a rotation leaves it alone.
|
||||
if mailcred.IsOrgSealed(value) {
|
||||
return "", false, errWorkspaceSealed
|
||||
}
|
||||
if !encrypt.LooksSealed(value) {
|
||||
return "", false, errUnsealed
|
||||
}
|
||||
return keys.instance.Reseal(value)
|
||||
}
|
||||
|
||||
// graceWindows reports every previous-credential window the instance is configured with.
|
||||
func graceWindows(now time.Time) []graceWindow {
|
||||
var out []graceWindow
|
||||
for _, w := range []struct{ name, list string }{
|
||||
{"INTERNAL_API_PREVIOUS_UNTIL", "INTERNAL_API_PREVIOUS_TOKENS"},
|
||||
{"NODE_BROKER_PREVIOUS_UNTIL", "NODE_BROKER_PREVIOUS_TOKENS"},
|
||||
{"UNSUBSCRIBE_SIGNING_READ_UNTIL", "UNSUBSCRIBE_SIGNING_READ_SECRETS"},
|
||||
} {
|
||||
raw := strings.TrimSpace(os.Getenv(w.name))
|
||||
entries, _ := secretset.Parse(os.Getenv(w.list))
|
||||
if raw == "" && len(entries) == 0 {
|
||||
continue
|
||||
}
|
||||
until, _ := time.Parse(time.RFC3339, raw)
|
||||
out = append(out, graceWindow{Name: w.name, Until: until, Open: now.Before(until), Entries: len(entries)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func printKeyReport(r keyReport, reseal bool) {
|
||||
if reseal && !r.Applied {
|
||||
fmt.Print("Dry run: nothing will be written. Pass --apply to reseal.\n\n")
|
||||
}
|
||||
fmt.Printf("Instance credential key: primary plus %s.\n", counted(r.InstanceReadKeys, "read key", "read keys"))
|
||||
if r.MFAConfigured {
|
||||
fmt.Printf("MFA key: primary plus %s.\n", counted(r.MFAReadKeys, "read key", "read keys"))
|
||||
} else {
|
||||
fmt.Println("MFA key: not configured here, so TOTP enrollments are not checked.")
|
||||
}
|
||||
fmt.Printf("\n %-44s %8s %9s %10s %9s %11s\n", "column", "primary", "read key", "workspace", "unsealed", "unreadable")
|
||||
var onRead, unreadable, resealed, raced int
|
||||
cols := append([]columnCount(nil), r.Columns...)
|
||||
sort.SliceStable(cols, func(i, j int) bool { return cols[i].Domain < cols[j].Domain })
|
||||
for _, c := range cols {
|
||||
label := c.Table + "." + c.Column
|
||||
if c.Missing {
|
||||
fmt.Printf(" %-44s %s\n", label, "table not present")
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" %-44s %8d %9d %10d %9d %11d\n", label, c.Primary, c.ReadKey, c.Workspace, c.Unsealed, c.Unreadable)
|
||||
onRead += c.ReadKey
|
||||
unreadable += c.Unreadable
|
||||
resealed += c.Resealed
|
||||
raced += c.Raced
|
||||
}
|
||||
for _, w := range r.Windows {
|
||||
state := "closed"
|
||||
if w.Open {
|
||||
state = "open until " + w.Until.UTC().Format(time.RFC3339)
|
||||
}
|
||||
fmt.Printf("\n%s: %s, %s", w.Name, state, counted(w.Entries, "previous value", "previous values"))
|
||||
}
|
||||
if len(r.Windows) > 0 {
|
||||
fmt.Println()
|
||||
}
|
||||
fmt.Println()
|
||||
switch {
|
||||
case r.Applied:
|
||||
fmt.Printf("Resealed %s under the primary key.", counted(resealed, "value", "values"))
|
||||
if raced > 0 {
|
||||
fmt.Printf(" %s changed while this ran; run it again to finish them.", counted(raced, "value", "values"))
|
||||
}
|
||||
fmt.Println()
|
||||
case onRead > 0:
|
||||
fmt.Printf("%s still open only under a read key. `warmblyctl keys reseal --apply` moves them to the primary.\n", counted(onRead, "value is", "values are"))
|
||||
default:
|
||||
fmt.Println("Every sealed value opens under the primary key. Retire a read key only once no rollback target, pending job or retained backup still needs it.")
|
||||
}
|
||||
if unreadable > 0 {
|
||||
fmt.Printf("%s under no configured key. They were left untouched: restore the key that sealed them as a read key before retiring anything.\n", counted(unreadable, "value opens", "values open"))
|
||||
}
|
||||
}
|
||||
|
||||
func counted(n int, one, many string) string {
|
||||
return fmt.Sprintf("%d %s", n, plural(n, one, many))
|
||||
}
|
||||
|
||||
// generatedKeys are the instance-wide values `keys generate` can mint, with their byte length.
|
||||
var generatedKeys = []struct {
|
||||
name string
|
||||
bytes int
|
||||
}{
|
||||
{"CREDENTIALS_ENCRYPTION_KEY", 32},
|
||||
{"AUTH_SECRET", 32},
|
||||
{"TWOFA_SECRET", 32},
|
||||
{"UNSUBSCRIBE_SIGNING_SECRET", 32},
|
||||
{"DOMAIN_PROOF_SECRET", 32},
|
||||
{"INTERNAL_API_TOKEN", 32},
|
||||
{"NODE_BROKER_TOKEN", 32},
|
||||
{"SECRET_KEY_BASE", 64},
|
||||
}
|
||||
|
||||
func runKeysGenerate(args []string) error {
|
||||
fs := newFlagSet("keys generate")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, name := range fs.Args() {
|
||||
want[strings.ToUpper(name)] = true
|
||||
}
|
||||
known := map[string]bool{}
|
||||
for _, k := range generatedKeys {
|
||||
known[k.name] = true
|
||||
}
|
||||
for name := range want {
|
||||
if !known[name] {
|
||||
return fmt.Errorf("%s is not a key this command generates", name)
|
||||
}
|
||||
}
|
||||
for _, k := range generatedKeys {
|
||||
if len(want) > 0 && !want[k.name] {
|
||||
continue
|
||||
}
|
||||
buf := make([]byte, k.bytes)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s=%s\n", k.name, hex.EncodeToString(buf))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
100
cmd/warmblyctl/keys_test.go
Normal file
100
cmd/warmblyctl/keys_test.go
Normal file
@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/app/mailcred"
|
||||
"github.com/warmbly/warmbly/internal/app/orgtransfer"
|
||||
"github.com/warmbly/warmbly/internal/app/twofa"
|
||||
"github.com/warmbly/warmbly/internal/pkg/encrypt"
|
||||
)
|
||||
|
||||
func TestSealedColumnsCoverEveryInstanceKeySecret(t *testing.T) {
|
||||
got := map[string]keyDomain{}
|
||||
for _, c := range sealedColumns() {
|
||||
got[c.Table+"."+c.Column] = c.Domain
|
||||
}
|
||||
for _, want := range []string{
|
||||
"email_accounts_smtp_imap.smtp_password",
|
||||
"email_accounts_smtp_imap.imap_password",
|
||||
"email_accounts_oauth.access_token",
|
||||
"email_accounts_oauth.refresh_token",
|
||||
"webhook_endpoints.secret",
|
||||
"oauth_applications.webhook_secret",
|
||||
"cloud_link.token",
|
||||
"mailbox_erasures.refresh_token",
|
||||
} {
|
||||
if got[want] != domainInstance {
|
||||
t.Fatalf("%s must be scanned under the instance credential key", want)
|
||||
}
|
||||
}
|
||||
if got["user_totp_settings.totp_secret_sealed"] != domainMFA {
|
||||
t.Fatal("TOTP enrollments must be scanned under the MFA key")
|
||||
}
|
||||
|
||||
for _, c := range instanceOnlyColumns {
|
||||
if _, ok := orgtransfer.ExcludedTables[c.Table]; c.Domain == domainInstance && !ok {
|
||||
t.Fatalf("%s is listed as instance-only but is not in orgtransfer.ExcludedTables", c.Table)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResealValueClassifiesEveryShape(t *testing.T) {
|
||||
oldKey, newKey := strings.Repeat("11", 32), strings.Repeat("22", 32)
|
||||
t.Setenv("CREDENTIALS_ENCRYPTION_KEY", oldKey)
|
||||
t.Setenv("CREDENTIALS_ENCRYPTION_READ_KEYS", "")
|
||||
old, err := encrypt.FromEnv()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := old.Encrypt("smtp-password")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("CREDENTIALS_ENCRYPTION_KEY", newKey)
|
||||
t.Setenv("CREDENTIALS_ENCRYPTION_READ_KEYS", `["`+oldKey+`"]`)
|
||||
current, err := encrypt.FromEnv()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys := &keyring{instance: current, instanceRead: 1}
|
||||
col := sealedColumn{Table: "email_accounts_smtp_imap", Column: "smtp_password", Domain: domainInstance}
|
||||
|
||||
resealed, changed, err := resealValue(keys, col, legacy)
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("a read-key value must be resealed, got changed=%v err=%v", changed, err)
|
||||
}
|
||||
if _, changed, err := resealValue(keys, col, resealed); err != nil || changed {
|
||||
t.Fatal("a resealed value must open under the primary")
|
||||
}
|
||||
if _, _, err := resealValue(keys, col, mailcred.OrgPrefix+"c2VhbGVk"); !errors.Is(err, errWorkspaceSealed) {
|
||||
t.Fatal("a workspace-sealed value must be counted apart and never touched by an instance rotation")
|
||||
}
|
||||
if _, _, err := resealValue(keys, col, "ya29.a0Af-legacy-token"); !errors.Is(err, errUnsealed) {
|
||||
t.Fatal("a value outside the sealed format must be reported unsealed, never rewritten")
|
||||
}
|
||||
foreign := hex.EncodeToString(make([]byte, 64))
|
||||
if _, _, err := resealValue(keys, col, foreign); err == nil || errors.Is(err, errUnsealed) {
|
||||
t.Fatal("a sealed value no key opens must be reported unreadable")
|
||||
}
|
||||
|
||||
mfaOld, mfaNew := twofa.DeriveKey("old-mfa-secret"), twofa.DeriveKey("new-mfa-secret")
|
||||
sealed, err := twofa.Seal(mfaOld, "JBSWY3DPEHPK3PXP")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys.mfa, keys.mfaRead = &mfaNew, [][32]byte{mfaOld}
|
||||
mfa := sealedColumn{Table: "user_totp_settings", Column: "totp_secret_sealed", Domain: domainMFA}
|
||||
if _, changed, err := resealValue(keys, mfa, sealed); err != nil || !changed {
|
||||
t.Fatal("an enrollment sealed under the MFA read key must be resealed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateRefusesUnknownNames(t *testing.T) {
|
||||
if err := runKeysGenerate([]string{"NOT_A_KEY"}); err == nil {
|
||||
t.Fatal("an unknown key name must be refused")
|
||||
}
|
||||
}
|
||||
@ -88,6 +88,8 @@ func dispatch(ctx context.Context, args []string) error {
|
||||
return runFleet(ctx, args[1:])
|
||||
case "inbox-tag":
|
||||
return runInboxTag(ctx, args[1:])
|
||||
case "keys":
|
||||
return runKeys(ctx, args[1:])
|
||||
}
|
||||
if _, ok := apiFamilies[args[0]]; ok {
|
||||
return runAPIResource(ctx, args[0], args[1:])
|
||||
@ -119,6 +121,9 @@ var commands = []command{
|
||||
{"hash-password", "Print an argon2 hash for WARMBLY_BOOTSTRAP_PASSWORD_HASH", "printf '%s' 'your-password' | docker compose -p warmbly exec -T backend warmblyctl hash-password"},
|
||||
{"backup", "Write the whole instance (database, blobs, keys) to one restorable bundle", composeExec + "backup --out /data/blobs/warmbly-backup.tar.gz"},
|
||||
{"restore", "Restore a bundle onto this instance, replacing everything on it", composeExec + "restore --file /data/blobs/warmbly-backup.tar.gz"},
|
||||
{"keys status", "Count stored secrets by the key that opens them, and show open grace windows", composeExec + "keys status"},
|
||||
{"keys reseal", "Rewrite secrets a read key opens under the primary key; a dry run without --apply", composeExec + "keys reseal --apply"},
|
||||
{"keys generate", "Print fresh values for the instance-wide keys and tokens", composeExec + "keys generate CREDENTIALS_ENCRYPTION_KEY"},
|
||||
{"inbox-tag backfill", "Classify mail that arrived before automatic tagging was switched on", composeExec + "inbox-tag backfill --org you@example.com --days 30 --dry-run"},
|
||||
{"inbox-tag follow-ups", "Recompute who owes whom a reply from stored mail and classifications", composeExec + "inbox-tag follow-ups --org you@example.com"},
|
||||
{"org list", "List the workspaces on this instance with their id, owner, and size", composeExec + "org list"},
|
||||
@ -197,11 +202,17 @@ Environment:
|
||||
settings) and CREDENTIALS_ENCRYPTION_KEY. Run them inside the backend
|
||||
container, where those are already set.
|
||||
|
||||
keys status and keys reseal read CREDENTIALS_ENCRYPTION_KEY with
|
||||
CREDENTIALS_ENCRYPTION_READ_KEYS, and TWOFA_SECRET with TWOFA_READ_SECRETS,
|
||||
exactly as the backend does. keys generate needs nothing.
|
||||
|
||||
Exit status:
|
||||
status exits non-zero when a check is at error severity, so it works as a
|
||||
post-deploy gate. Use status --quiet from cron: it prints the checks alone,
|
||||
and nothing at all when none fire. status --json always exits 0, because it
|
||||
doubles as a liveness probe; read .summary.error there instead.
|
||||
keys status and keys reseal exit non-zero when a stored value opens under no
|
||||
configured key, which is the signal not to retire one.
|
||||
|
||||
Run `+"`warmblyctl <command> --help`"+` for one command's flags.
|
||||
`)
|
||||
|
||||
@ -223,7 +223,9 @@ func main() {
|
||||
lanes = 1
|
||||
}
|
||||
log.Printf("Worker command concurrency: %d ordered mailbox lanes", lanes)
|
||||
if err := eventbus.SubscribeKeyed(ctx, bus, []string{workerTopic}, "worker-"+workerID.String(), lanes, workerService.ResolveCommandKey, workerService.Receive); err != nil {
|
||||
// A send runs past the bus's default handler budget, so the command subscription carries its own.
|
||||
commandCtx := eventbus.WithHandlerTimeout(ctx, worker.CommandBudget)
|
||||
if err := eventbus.SubscribeKeyed(commandCtx, bus, []string{workerTopic}, "worker-"+workerID.String(), lanes, workerService.ResolveCommandKey, workerService.Receive); err != nil {
|
||||
log.Println("event bus subscribe ended:", err)
|
||||
}
|
||||
|
||||
|
||||
@ -30,6 +30,12 @@ deploy/
|
||||
|
||||
The tracking Dockerfile lives at `tracking/Dockerfile`, and the frontends build from `web/Dockerfile` and `admin/Dockerfile` (nginx static builds with runtime config injection). The self-host compose is `docker-compose.yml` at the repo root.
|
||||
|
||||
## Coolify
|
||||
|
||||
Use [`docker-compose.coolify.yml`](docker-compose.coolify.yml) for an image-only deployment managed by Coolify. Paste it into Docker Compose Empty, or select `/deploy/docker-compose.coolify.yml` with the Docker Compose build pack and `/` as the base directory. It has no source builds, published host ports, development secrets, or Docker socket mounts. Configure the required environment values and public service domains before deploying. Full steps: [Deploy on Coolify](https://docs.warmbly.com/development/coolify/).
|
||||
|
||||
Validate the definition with `bash deploy/check-coolify.sh` (Docker Compose v2 and `jq` required). This checks interpolation, mandatory secrets, internal wiring, persistence, and the absence of source builds or host ports. It does not deploy containers or verify Coolify's proxy.
|
||||
|
||||
## Building images
|
||||
|
||||
```bash
|
||||
|
||||
60
deploy/check-coolify.sh
Normal file
60
deploy/check-coolify.sh
Normal file
@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||
compose=(docker compose --env-file /dev/null -f deploy/docker-compose.coolify.yml)
|
||||
|
||||
# Disposable interpolation fixtures, never deployment credentials.
|
||||
export AUTH_SECRET=coolify-validation-auth-secret-at-least-32-characters
|
||||
export INTERNAL_API_TOKEN=coolify-validation-edge-token
|
||||
export NODE_BROKER_TOKEN=coolify-validation-worker-token
|
||||
export SECRET_KEY_BASE=coolify-validation-phoenix-secret-at-least-sixty-four-characters-long
|
||||
export KMS_LOCAL_MASTER_KEY=Y29vbGlmeS12YWxpZGF0aW9uLW9ubHktMzItYnl0ZXM=
|
||||
export CREDENTIALS_ENCRYPTION_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
export POSTGRES_PASSWORD=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
|
||||
export APP_URL=https://app.example.com
|
||||
export ADMIN_URL=https://admin.example.com
|
||||
export API_PUBLIC_URL=https://api.example.com
|
||||
export REALTIME_HOST=realtime.example.com
|
||||
export TRACKING_DOMAIN=track.example.com
|
||||
export FORMS_DOMAIN=forms.example.com
|
||||
export WARMBLY_TAG=coolify-validation
|
||||
|
||||
config=$("${compose[@]}" config --format json)
|
||||
check() {
|
||||
if ! jq -e "$1" >/dev/null <<<"$config"; then
|
||||
printf 'Coolify check failed: %s\n' "$2" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
check '(.services | keys) == ["admin", "backend", "consumer", "forms", "nats", "postgres", "realtime", "redis", "tracking", "web", "worker"]' 'complete stack'
|
||||
check 'all(.services[]; (has("build") or has("ports") or has("env_file") or has("container_name") or has("privileged")) | not)' 'no builds, published ports, external env files, fixed container names, or privileges'
|
||||
check 'all(.services[].volumes[]?; .type == "volume")' 'named volumes only, no host or Docker socket mounts'
|
||||
check '(.volumes | keys) == ["blobs", "nats_data", "postgres_data", "redis_data", "worker_state"]' 'all persistent stores'
|
||||
check '. as $config | all(["backend", "consumer", "worker", "forms", "tracking", "realtime", "web", "admin"][]; $config.services[.].image == "ghcr.io/warmbly/warmbly/" + . + ":coolify-validation")' 'matching service images with one release tag'
|
||||
check 'all([.services.backend, .services.consumer, .services.worker, .services.tracking, .services.forms, .services.realtime][]; .environment.APP_ENV == "prod")' 'production environment'
|
||||
check '.services.backend.environment.AUTH_SECRET == .services.realtime.environment.JWT_SECRET' 'shared JWT secret'
|
||||
check '.services.backend.environment.NODE_BROKER_TOKEN == .services.worker.environment.ENCRYPTED_KEYS_WORKER_TOKEN and .services.tracking.environment.INTERNAL_API_TOKEN == .services.backend.environment.INTERNAL_API_TOKEN and .services.forms.environment.INTERNAL_API_TOKEN == .services.backend.environment.INTERNAL_API_TOKEN' 'separate worker and edge tokens'
|
||||
check '.services.worker.environment.MAIL_TLS_INSECURE == "false" and .services.worker.environment.ENCRYPTED_KEYS_PROVIDER == "http" and (.services.worker.environment | has("PRIMARY_DB") | not)' 'worker TLS and database boundary'
|
||||
check '.services.postgres.environment.POSTGRES_PASSWORD as $password | .services.backend.environment.PRIMARY_DB == .services.consumer.environment.PRIMARY_DB and .services.backend.environment.PRIMARY_DB == .services.realtime.environment.DATABASE_URL and (.services.backend.environment.PRIMARY_DB | contains($password))' 'database clients share credentials'
|
||||
check '.services.backend.environment.UPDATER_URL == "none" and (.services | has("updater") | not)' 'Coolify owns updates'
|
||||
check '.services.backend.environment.WEBSOCKET_URL == "wss://realtime.example.com/socket/websocket" and .services.web.environment.WEBSOCKET_URL == .services.backend.environment.WEBSOCKET_URL and .services.realtime.environment.CHECK_ORIGIN_HOSTS == "https://app.example.com,https://admin.example.com"' 'secure realtime URL and browser origins'
|
||||
check '.services.web.environment.WARMBLY_API_URL == "https://api.example.com" and .services.admin.environment.WARMBLY_API_URL == "https://api.example.com"' 'runtime frontend API URLs'
|
||||
check '.services.backend.environment.MAIL_TRANSPORT == .services.consumer.environment.MAIL_TRANSPORT' 'platform mail reaches notifications'
|
||||
check 'all([.services.backend, .services.consumer][]; .environment.GEODB_PATH == "/data/blobs/geo/GeoLite2-City.mmdb")' 'GeoIP path configured without requiring an external database'
|
||||
check 'all(["backend", "tracking", "realtime", "forms", "web", "admin"][] as $name | .services[$name].healthcheck.test; any(.[]; contains("http://127.0.0.1:")))' 'HTTP probes use the container IPv4 loopback'
|
||||
check 'all(["consumer", "worker", "tracking", "realtime", "forms", "web", "admin"][] as $name | .services[$name].depends_on.backend.condition; . == "service_healthy")' 'wait for backend migrations'
|
||||
|
||||
for required in AUTH_SECRET INTERNAL_API_TOKEN NODE_BROKER_TOKEN SECRET_KEY_BASE KMS_LOCAL_MASTER_KEY CREDENTIALS_ENCRYPTION_KEY POSTGRES_PASSWORD APP_URL ADMIN_URL API_PUBLIC_URL REALTIME_HOST TRACKING_DOMAIN FORMS_DOMAIN; do
|
||||
if env -u "$required" "${compose[@]}" config --quiet >/dev/null 2>&1; then
|
||||
printf 'Coolify accepted missing required variable: %s\n' "$required" >&2
|
||||
exit 1
|
||||
fi
|
||||
if env "$required=" "${compose[@]}" config --quiet >/dev/null 2>&1; then
|
||||
printf 'Coolify accepted empty required variable: %s\n' "$required" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
printf 'Coolify Compose configuration checks passed.\n'
|
||||
257
deploy/docker-compose.coolify.yml
Normal file
257
deploy/docker-compose.coolify.yml
Normal file
@ -0,0 +1,257 @@
|
||||
# Paste into Coolify's Docker Compose Empty resource, or select this file
|
||||
# with the Docker Compose build pack. No installer, build, or host ports.
|
||||
# Setup: https://docs.warmbly.com/development/coolify/
|
||||
x-runtime-env: &runtime-env
|
||||
APP_ENV: prod
|
||||
AWS_CONFIG_ENABLED: "false"
|
||||
DEPLOYMENT_MODE: self_hosted
|
||||
EVENTBUS_PROVIDER: nats
|
||||
NATS_URL: nats://nats:4222
|
||||
CODEC_PROVIDER: json
|
||||
REDIS: redis://redis:6379
|
||||
KMS_PROVIDER: local
|
||||
KMS_LOCAL_MASTER_KEY: ${KMS_LOCAL_MASTER_KEY:?}
|
||||
CREDENTIALS_ENCRYPTION_KEY: ${CREDENTIALS_ENCRYPTION_KEY:?}
|
||||
CREDENTIALS_ENCRYPTION_READ_KEYS: ${CREDENTIALS_ENCRYPTION_READ_KEYS:-}
|
||||
BLOB_PROVIDER: filesystem
|
||||
BLOB_FS_ROOT: /data/blobs
|
||||
TASKS_PROVIDER: local
|
||||
PUBSUB_ENABLED: "false"
|
||||
BILLING_PROVIDER: none
|
||||
CAPTCHA_PROVIDER: none
|
||||
AUTH_SECRET: ${AUTH_SECRET:?}
|
||||
TWOFA_SECRET: ${TWOFA_SECRET:-}
|
||||
TWOFA_READ_SECRETS: ${TWOFA_READ_SECRETS:-}
|
||||
UNSUBSCRIBE_SIGNING_SECRET: ${UNSUBSCRIBE_SIGNING_SECRET:-}
|
||||
UNSUBSCRIBE_SIGNING_READ_SECRETS: ${UNSUBSCRIBE_SIGNING_READ_SECRETS:-}
|
||||
UNSUBSCRIBE_SIGNING_READ_UNTIL: ${UNSUBSCRIBE_SIGNING_READ_UNTIL:-}
|
||||
DOMAIN_PROOF_SECRET: ${DOMAIN_PROOF_SECRET:-}
|
||||
INTERNAL_API_TOKEN: ${INTERNAL_API_TOKEN:?}
|
||||
INTERNAL_API_PREVIOUS_TOKENS: ${INTERNAL_API_PREVIOUS_TOKENS:-}
|
||||
INTERNAL_API_PREVIOUS_UNTIL: ${INTERNAL_API_PREVIOUS_UNTIL:-}
|
||||
NODE_BROKER_TOKEN: ${NODE_BROKER_TOKEN:?}
|
||||
NODE_BROKER_PREVIOUS_TOKENS: ${NODE_BROKER_PREVIOUS_TOKENS:-}
|
||||
NODE_BROKER_PREVIOUS_UNTIL: ${NODE_BROKER_PREVIOUS_UNTIL:-}
|
||||
|
||||
x-server-env: &server-env
|
||||
<<: *runtime-env
|
||||
PRIMARY_DB: postgres://warmbly:${POSTGRES_PASSWORD:?}@postgres:5432/warmbly?sslmode=disable
|
||||
APP_URL: ${APP_URL:?}
|
||||
API_PUBLIC_URL: ${API_PUBLIC_URL:?}
|
||||
BLOB_PUBLIC_BASE_URL: ${API_PUBLIC_URL:?}/public
|
||||
CORS_ALLOW_ORIGINS: ${APP_URL:?},${ADMIN_URL:?}
|
||||
TRUSTED_PROXIES: ${TRUSTED_PROXIES:-}
|
||||
WEBSOCKET_URL: wss://${REALTIME_HOST:?}/socket/websocket
|
||||
TRACKING_DOMAIN: ${TRACKING_DOMAIN:?}
|
||||
FORMS_DOMAIN: ${FORMS_DOMAIN:?}
|
||||
GEODB_PATH: /data/blobs/geo/GeoLite2-City.mmdb
|
||||
MAIL_TRANSPORT: ${MAIL_TRANSPORT:-log}
|
||||
EMAIL_NAME: ${EMAIL_NAME:-Warmbly}
|
||||
EMAIL_ADDRESS: ${EMAIL_ADDRESS:-noreply@example.com}
|
||||
SMTP_HOST: ${SMTP_HOST:-}
|
||||
SMTP_PORT: ${SMTP_PORT:-587}
|
||||
SMTP_USERNAME: ${SMTP_USERNAME:-}
|
||||
SMTP_PASSWORD: ${SMTP_PASSWORD:-}
|
||||
SMTP_SECURITY: ${SMTP_SECURITY:-starttls}
|
||||
SMTP_AUTH: ${SMTP_AUTH:-auto}
|
||||
AUTH_LOGIN_CODE: ${AUTH_LOGIN_CODE:-off}
|
||||
REQUIRE_EMAIL_VERIFICATION: ${REQUIRE_EMAIL_VERIFICATION:-false}
|
||||
DISABLE_REGISTRATION: ${DISABLE_REGISTRATION:-invite_only}
|
||||
|
||||
x-http-healthcheck: &http-healthcheck
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 60s
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: warmbly
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?}
|
||||
POSTGRES_DB: warmbly
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U warmbly -d warmbly"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
restart: unless-stopped
|
||||
command: ["redis-server", "--appendonly", "yes"]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
|
||||
nats:
|
||||
image: nats:2.10-alpine
|
||||
restart: unless-stopped
|
||||
command: ["-js", "-sd", "/data", "-m", "8222"]
|
||||
volumes:
|
||||
- nats_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8222/healthz"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
|
||||
backend:
|
||||
image: ghcr.io/warmbly/warmbly/backend:${WARMBLY_TAG:-prod}
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 60s
|
||||
environment:
|
||||
<<: *server-env
|
||||
API_HOST: "0.0.0.0:8080"
|
||||
GIN_MODE: release
|
||||
ENCRYPTED_KEYS_PROVIDER: postgres
|
||||
SECRET_KEY_BASE: ${SECRET_KEY_BASE:?}
|
||||
UPDATER_URL: none
|
||||
BOX_GOOGLE_CLIENT_ID: ${BOX_GOOGLE_CLIENT_ID:-}
|
||||
BOX_GOOGLE_CLIENT_SECRET: ${BOX_GOOGLE_CLIENT_SECRET:-}
|
||||
BOX_GOOGLE_OAUTH_CONNECT: ${BOX_GOOGLE_OAUTH_CONNECT:-false}
|
||||
BOX_OUTLOOK_CLIENT_ID: ${BOX_OUTLOOK_CLIENT_ID:-}
|
||||
BOX_OUTLOOK_CLIENT_SECRET: ${BOX_OUTLOOK_CLIENT_SECRET:-}
|
||||
volumes:
|
||||
- blobs:/data/blobs
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
nats: { condition: service_healthy }
|
||||
healthcheck:
|
||||
<<: *http-healthcheck
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8080/health"]
|
||||
|
||||
consumer:
|
||||
image: ghcr.io/warmbly/warmbly/consumer:${WARMBLY_TAG:-prod}
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 60s
|
||||
environment:
|
||||
<<: *server-env
|
||||
ENCRYPTED_KEYS_PROVIDER: postgres
|
||||
volumes:
|
||||
- blobs:/data/blobs
|
||||
depends_on:
|
||||
backend: { condition: service_healthy }
|
||||
nats: { condition: service_healthy }
|
||||
|
||||
worker:
|
||||
image: ghcr.io/warmbly/warmbly/worker:${WARMBLY_TAG:-prod}
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 60s
|
||||
environment:
|
||||
<<: *runtime-env
|
||||
MAIL_TLS_INSECURE: "false"
|
||||
ENCRYPTED_KEYS_PROVIDER: http
|
||||
ENCRYPTED_KEYS_BACKEND_URL: http://backend:8080
|
||||
ENCRYPTED_KEYS_WORKER_TOKEN: ${NODE_BROKER_TOKEN:?}
|
||||
WORKER_STATE_DIR: /data/state
|
||||
BOX_GOOGLE_CLIENT_ID: ${BOX_GOOGLE_CLIENT_ID:-}
|
||||
BOX_GOOGLE_CLIENT_SECRET: ${BOX_GOOGLE_CLIENT_SECRET:-}
|
||||
BOX_OUTLOOK_CLIENT_ID: ${BOX_OUTLOOK_CLIENT_ID:-}
|
||||
BOX_OUTLOOK_CLIENT_SECRET: ${BOX_OUTLOOK_CLIENT_SECRET:-}
|
||||
volumes:
|
||||
- blobs:/data/blobs
|
||||
- worker_state:/data/state
|
||||
depends_on:
|
||||
backend: { condition: service_healthy }
|
||||
nats: { condition: service_healthy }
|
||||
|
||||
tracking:
|
||||
image: ghcr.io/warmbly/warmbly/tracking:${WARMBLY_TAG:-prod}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
APP_ENV: prod
|
||||
AWS_CONFIG_ENABLED: "false"
|
||||
TRACKING_HOST: "0.0.0.0"
|
||||
TRACKING_PORT: "3000"
|
||||
EVENTBUS_PROVIDER: nats
|
||||
NATS_URL: nats://nats:4222
|
||||
BACKEND_INTERNAL_URL: http://backend:8080
|
||||
INTERNAL_API_TOKEN: ${INTERNAL_API_TOKEN:?}
|
||||
TRACKING_TRUSTED_PROXIES: ${TRACKING_TRUSTED_PROXIES:-}
|
||||
depends_on:
|
||||
backend: { condition: service_healthy }
|
||||
nats: { condition: service_healthy }
|
||||
healthcheck:
|
||||
<<: *http-healthcheck
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:3000/health"]
|
||||
|
||||
realtime:
|
||||
image: ghcr.io/warmbly/warmbly/realtime:${WARMBLY_TAG:-prod}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
APP_ENV: prod
|
||||
PHX_HOST: ${REALTIME_HOST:?}
|
||||
PORT: "4000"
|
||||
DATABASE_URL: postgres://warmbly:${POSTGRES_PASSWORD:?}@postgres:5432/warmbly?sslmode=disable
|
||||
DATABASE_SSL: "false"
|
||||
REDIS_URL: redis://redis:6379
|
||||
JWT_SECRET: ${AUTH_SECRET:?}
|
||||
SECRET_KEY_BASE: ${SECRET_KEY_BASE:?}
|
||||
PUBSUB_ENABLED: "false"
|
||||
CHECK_ORIGIN_HOSTS: ${APP_URL:?},${ADMIN_URL:?}
|
||||
depends_on:
|
||||
backend: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
healthcheck:
|
||||
<<: *http-healthcheck
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:4000/health"]
|
||||
|
||||
forms:
|
||||
image: ghcr.io/warmbly/warmbly/forms:${WARMBLY_TAG:-prod}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
APP_ENV: prod
|
||||
GIN_MODE: release
|
||||
FORMS_PORT: "8090"
|
||||
BACKEND_INTERNAL_URL: http://backend:8080
|
||||
INTERNAL_API_TOKEN: ${INTERNAL_API_TOKEN:?}
|
||||
TRUSTED_PROXIES: ${FORMS_TRUSTED_PROXIES:-}
|
||||
depends_on:
|
||||
backend: { condition: service_healthy }
|
||||
healthcheck:
|
||||
<<: *http-healthcheck
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8090/health"]
|
||||
|
||||
web:
|
||||
image: ghcr.io/warmbly/warmbly/web:${WARMBLY_TAG:-prod}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
WARMBLY_API_URL: ${API_PUBLIC_URL:?}
|
||||
WARMBLY_APP_URL: ${APP_URL:?}
|
||||
WARMBLY_GMAIL_OAUTH_CONNECT: ${BOX_GOOGLE_OAUTH_CONNECT:-false}
|
||||
WEBSOCKET_URL: wss://${REALTIME_HOST:?}/socket/websocket
|
||||
depends_on:
|
||||
backend: { condition: service_healthy }
|
||||
healthcheck:
|
||||
<<: *http-healthcheck
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:80/config.js"]
|
||||
|
||||
admin:
|
||||
image: ghcr.io/warmbly/warmbly/admin:${WARMBLY_TAG:-prod}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
WARMBLY_API_URL: ${API_PUBLIC_URL:?}
|
||||
WARMBLY_DASHBOARD_URL: ${APP_URL:?}
|
||||
WARMBLY_ENV_LABEL: production
|
||||
depends_on:
|
||||
backend: { condition: service_healthy }
|
||||
healthcheck:
|
||||
<<: *http-healthcheck
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:80/config.js"]
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
nats_data:
|
||||
blobs:
|
||||
worker_state:
|
||||
12
deploy/terraform/.gitignore
vendored
Normal file
12
deploy/terraform/.gitignore
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
.terraform/
|
||||
*.tfstate
|
||||
*.tfstate.*
|
||||
*.tfplan
|
||||
*.tfvars
|
||||
!*.tfvars.example
|
||||
crash.log
|
||||
override.tf
|
||||
override.tf.json
|
||||
*_override.tf
|
||||
*_override.tf.json
|
||||
backend.hcl
|
||||
110
deploy/terraform/README.md
Normal file
110
deploy/terraform/README.md
Normal file
@ -0,0 +1,110 @@
|
||||
# Control plane infrastructure
|
||||
|
||||
Use separate Terraform roots and state keys for each provider. AWS provisioning
|
||||
can proceed independently of PlanetScale and Confluent operator authentication.
|
||||
|
||||
| Root | Scope | State |
|
||||
| --- | --- | --- |
|
||||
| `aws/bootstrap` | Protected state bucket | Private bootstrap state outside the checkout |
|
||||
| `aws/foundation` | Network, ECS cluster, repositories, logs, certificates, PrivateLink, Valkey, configuration stores | `warmbly/production/foundation.tfstate` |
|
||||
| `aws/runtime` | Application task roles, ECS services, HTTPS ingress and dependency checks | `warmbly/production/runtime.tfstate` |
|
||||
| `aws/workers` | ARM EC2 worker hosts, static IPs, encrypted disks and configuration containers | `warmbly/production/workers.tfstate` |
|
||||
| `planetscale` | Existing Postgres branch, with a database metadata lookup | `warmbly/production/planetscale.tfstate` |
|
||||
| `confluent` | Existing environment, Basic Kafka cluster and service account, with a Schema Registry lookup | `warmbly/production/confluent.tfstate` |
|
||||
|
||||
The remote states use the bootstrap bucket with encryption, versioning and
|
||||
S3 locking. Provider lock files belong in source control. State, saved plans,
|
||||
credentials and installation-specific tfvars belong outside the checkout.
|
||||
|
||||
## Adopt existing data services
|
||||
|
||||
Initialize each root with `-backend-config="bucket=$STATE_BUCKET"`. The
|
||||
PlanetScale and Confluent roots contain import blocks and require the actual
|
||||
identifiers and configuration of the existing resources. Supply those values in
|
||||
private tfvars files.
|
||||
|
||||
Authenticate PlanetScale with `PLANETSCALE_SERVICE_TOKEN_ID` and
|
||||
`PLANETSCALE_SERVICE_TOKEN`. Authenticate Confluent with
|
||||
`CONFLUENT_CLOUD_API_KEY` and `CONFLUENT_CLOUD_API_SECRET`. Supply operator
|
||||
credentials through the process environment from a protected source. Keep them
|
||||
separate from the application credentials and out of Terraform input variables.
|
||||
Short-lived, read-scoped credentials are sufficient for the initial adoption.
|
||||
Set `kafka_region` to the existing cluster's actual region before planning its
|
||||
adoption. Compute placement in AWS does not relocate a Kafka cluster.
|
||||
|
||||
Review the adoption plan before applying it. Its managed resources must show
|
||||
imports only, with zero creates, updates, replacements or deletes. Correct any
|
||||
configuration mismatch before applying. Apply the saved plan, then verify a
|
||||
subsequent plan reports no changes. Imported resources have `prevent_destroy`.
|
||||
|
||||
Terraform manages configuration metadata. SQL rows and migrations, Kafka
|
||||
messages and consumer offsets, runtime-created worker topics, schema versions,
|
||||
access rules and existing application credentials retain their current owners.
|
||||
Bring additional resource types under Terraform only after checking their
|
||||
current settings and ownership. Resource import does not copy or restore data.
|
||||
|
||||
The PlanetScale provider currently exposes Postgres databases as data sources
|
||||
and branches as managed resources. This root therefore adopts the production
|
||||
branch and reads database metadata. PostgreSQL parameter overrides retain their
|
||||
existing owner through `ignore_changes = [parameters]`.
|
||||
|
||||
## Configuration and credentials
|
||||
|
||||
Use Standard String parameters under `/<name>/config/<service>/<VARIABLE>` for
|
||||
approved non-secret environment values. The AWS root validates parameter names
|
||||
against an allowlist and rejects credential-bearing URLs. Connection strings,
|
||||
passwords, tokens and encryption material belong in Secrets Manager.
|
||||
|
||||
Service secret containers hold JSON configuration supplied outside Terraform.
|
||||
Preserved source values can carry a `MIGRATION_SOURCE` version label. The
|
||||
operator-only rotation secret holds replacement material under `AWSPENDING`.
|
||||
These labels describe preparation; workload configuration and the deployed task
|
||||
revision determine which values an application actually uses.
|
||||
|
||||
The Valkey password lives in its own secret. Terraform retrieves it through an
|
||||
ephemeral secret-version resource and passes it to `auth_token_wo`, so the
|
||||
password is omitted from plans and state. The `auth_token_wo_version` field is
|
||||
the deliberate update trigger for later password changes.
|
||||
|
||||
ECS task definitions should inject parameters and service secrets through
|
||||
execution-role access scoped to their ARNs. Injected values are read on task
|
||||
startup; changing a stored value does not refresh a running task. S3, KMS and SES
|
||||
calls should use task roles.
|
||||
|
||||
## Coordinate credential changes
|
||||
|
||||
Prepare a dependency map for every value before changing it. Record the writer,
|
||||
all readers, encrypted data dependencies, replacement procedure and rollback.
|
||||
Preserving current values in a new store is separate from rotating them.
|
||||
|
||||
| Material | Coordination |
|
||||
| --- | --- |
|
||||
| Auth signing key | Backend, consumer and realtime share the signing value. Replacing it invalidates existing tokens. |
|
||||
| MFA sealing key | Preserve enrollment readability. When `TWOFA_SECRET` is unset, the sealing key derives from `AUTH_SECRET`. |
|
||||
| Mailbox encryption key | Re-encrypt affected stored credentials and coordinate every worker before replacing it. |
|
||||
| Organization KMS key | Preserve access to existing wrapped DEKs. IAM access changes and KMS key-material rotation are separate operations. |
|
||||
| Internal and broker tokens | Update both ends of internal APIs, including forms, tracking and external workers. |
|
||||
| Kafka and Schema Registry keys | Create replacements, validate scopes, update every producer and consumer, then retire old access. |
|
||||
| OAuth and integration secrets | Coordinate provider-side credentials, backend and worker token refresh paths, and any callbacks. |
|
||||
| Valkey password and endpoint | Update every cache client together and verify shared-state behavior before draining the old cache. |
|
||||
|
||||
Re-encrypting stored values under a new key does not revoke the upstream
|
||||
credentials represented by those values. Treat those as separate operations
|
||||
when both are required. A rollback must use valid credentials and the keys that
|
||||
match its data, rather than restoring an incompatible configuration bundle.
|
||||
|
||||
## Networking
|
||||
|
||||
PlanetScale uses a same-region interface endpoint and its Private Host with TLS
|
||||
verification. The cache uses private subnets and permits access from the app
|
||||
security group. Local tools and external workers need a suitable private network
|
||||
path before they can use those endpoints.
|
||||
|
||||
Keep Confluent Basic on TLS/SASL during adoption. PrivateLink for Kafka requires
|
||||
a compatible Confluent cluster offering and connectivity for every worker, so it
|
||||
is a separate topology and cost decision. The S3 gateway endpoint provides a
|
||||
private route to S3. Other dependencies still require a working egress path.
|
||||
|
||||
See [AWS provisioning](aws/README.md) for the foundation and cache setup.
|
||||
See [application runtime](aws/runtime/README.md) and
|
||||
[worker hosts](aws/workers/README.md) for the remaining deployment roots.
|
||||
12
deploy/terraform/aws/.gitignore
vendored
Normal file
12
deploy/terraform/aws/.gitignore
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
.terraform/
|
||||
*.tfstate
|
||||
*.tfstate.*
|
||||
*.tfplan
|
||||
*.tfvars
|
||||
!*.tfvars.example
|
||||
crash.log
|
||||
override.tf
|
||||
override.tf.json
|
||||
*_override.tf
|
||||
*_override.tf.json
|
||||
backend.hcl
|
||||
109
deploy/terraform/aws/README.md
Normal file
109
deploy/terraform/aws/README.md
Normal file
@ -0,0 +1,109 @@
|
||||
# AWS foundation
|
||||
|
||||
Terraform provisions the control plane's foundation in `us-east-1`. The existing
|
||||
Postgres database, Kafka cluster, object bucket and KMS key remain external to
|
||||
this state. The sibling PlanetScale and Confluent roots adopt their existing
|
||||
configuration. This root does not deploy application services or change
|
||||
production traffic. A standalone cache-check task definition supports verification.
|
||||
|
||||
## Bootstrap state storage
|
||||
|
||||
Use an AWS operator identity with temporary credentials when available. Keep
|
||||
bootstrap state and saved plans outside the checkout in a private directory.
|
||||
|
||||
```sh
|
||||
mkdir -p "$HOME/.local/state/warmbly-aws/bootstrap"
|
||||
chmod 700 "$HOME/.local/state/warmbly-aws"
|
||||
terraform -chdir=deploy/terraform/aws/bootstrap init
|
||||
terraform -chdir=deploy/terraform/aws/bootstrap plan \
|
||||
-state="$HOME/.local/state/warmbly-aws/bootstrap/terraform.tfstate" \
|
||||
-out="$HOME/.local/state/warmbly-aws/bootstrap/create.tfplan"
|
||||
terraform -chdir=deploy/terraform/aws/bootstrap apply \
|
||||
"$HOME/.local/state/warmbly-aws/bootstrap/create.tfplan"
|
||||
```
|
||||
|
||||
The bucket has encryption, versioning, public access blocking, TLS enforcement
|
||||
and a destruction guard. Keep a protected backup of bootstrap state. Subsequent
|
||||
bootstrap plans must use that same state path. The foundation uses S3 state with
|
||||
native locking; it needs no DynamoDB table.
|
||||
|
||||
## Provision the foundation
|
||||
|
||||
Use the bootstrap output as `STATE_BUCKET`. Set `planetscale_service_name` to the
|
||||
existing branch's Private Service Name. An empty value defers the endpoint.
|
||||
Keep installation-specific values in an untracked tfvars file outside the repo.
|
||||
|
||||
```sh
|
||||
terraform -chdir=deploy/terraform/aws/foundation init \
|
||||
-backend-config="bucket=$STATE_BUCKET"
|
||||
terraform -chdir=deploy/terraform/aws/foundation plan \
|
||||
-var-file=/private/path/foundation.tfvars \
|
||||
-out=/private/path/foundation.tfplan
|
||||
terraform -chdir=deploy/terraform/aws/foundation apply \
|
||||
/private/path/foundation.tfplan
|
||||
```
|
||||
|
||||
Resources:
|
||||
|
||||
- A dedicated VPC with public and isolated private subnets in two zones.
|
||||
- A free S3 gateway endpoint. No NAT gateway or load balancer is started.
|
||||
- An optional two-zone PlanetScale interface endpoint. Endpoint hours and data
|
||||
processing are billed by AWS; PlanetScale also bills private traffic.
|
||||
- An ECS cluster with Fargate capacity providers, defaulting to on-demand.
|
||||
- Immutable ECR repositories and log groups with 14-day retention for backend,
|
||||
consumer, forms, realtime and tracking.
|
||||
- Five Secrets Manager containers. Secret **values** are written separately,
|
||||
never supplied as Terraform variables or stored in Terraform state.
|
||||
- A separate rotation secret holds pending replacements for the coordinated
|
||||
application and worker rollout. Keep it accessible only to operators.
|
||||
- A private single-node Valkey 9.0 `cache.t4g.micro`, with TLS, encryption at
|
||||
rest, password authentication, one-day snapshots and no replica failover.
|
||||
- A separate Valkey credential container. An ephemeral secret read feeds the
|
||||
write-only authentication field, keeping the password out of state.
|
||||
- Standard String Parameter Store entries for allowlisted non-secret values
|
||||
supplied through `service_configuration`.
|
||||
- Optional DNS-validated ACM certificates. Their validation records are output
|
||||
for the existing DNS provider; Terraform does not change DNS.
|
||||
|
||||
Public subnets provide internet access only to future tasks explicitly assigned
|
||||
a public IP. Private subnets have no internet route. Applications still need
|
||||
outbound connectivity for Kafka, Schema Registry, identity providers and other
|
||||
APIs even when Postgres uses PrivateLink. Runtime networking, application task roles,
|
||||
ingress, service deployment and coordinated credential activation are separate
|
||||
work required before this foundation can serve the application.
|
||||
|
||||
## Bootstrap the cache password
|
||||
|
||||
The cache credential container must have a JSON secret version with a
|
||||
`REDIS_PASSWORD` field before Terraform provisions the cache. On a fresh
|
||||
installation, apply only `aws_secretsmanager_secret.cache` to create the
|
||||
container, then populate it securely outside Terraform. Use a strong random
|
||||
alphanumeric password of 16 to 128 characters. Follow with a full saved plan and
|
||||
apply, including the installation's configuration tfvars.
|
||||
|
||||
The smallest node supports the application's standalone Redis clients and
|
||||
multi-key scripts. Serverless requires cluster-aware clients and different
|
||||
multi-key handling. `noeviction` preserves shared coordination keys under memory
|
||||
pressure; monitor memory and write failures as usage grows.
|
||||
|
||||
`cache_check_task_definition` is a one-shot Fargate definition that checks
|
||||
certificate-verified TLS and password authentication with `AUTH` and `PING`.
|
||||
Run it in a public subnet with a public IP and the application task security
|
||||
group. It retrieves only the cache password and writes a success or failure
|
||||
message to its own one-day log group. It exits after checking and creates no
|
||||
application service.
|
||||
|
||||
The cache has no public endpoint. Fargate tasks can reach it inside the VPC;
|
||||
local tools and external workers need a private route or tunnel. Creating it
|
||||
does not switch any existing Redis client.
|
||||
|
||||
See [the Fargate preparation guide](https://docs.warmbly.com/development/aws-fargate/)
|
||||
for sizing and the deployment sequence.
|
||||
|
||||
## Verify
|
||||
|
||||
```sh
|
||||
terraform fmt -check -recursive deploy/terraform/aws
|
||||
terraform -chdir=deploy/terraform/aws/bootstrap validate
|
||||
terraform -chdir=deploy/terraform/aws/foundation validate
|
||||
```
|
||||
26
deploy/terraform/aws/bootstrap/.terraform.lock.hcl
generated
Normal file
26
deploy/terraform/aws/bootstrap/.terraform.lock.hcl
generated
Normal file
@ -0,0 +1,26 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/hashicorp/aws" {
|
||||
version = "6.70.0"
|
||||
constraints = "~> 6.0"
|
||||
hashes = [
|
||||
"h1:L+05gOXA+kdYTtayIw0WxdaTp3JryZCMZpfrJVhoJA0=",
|
||||
"zh:068ca7ed4fc77e3d63d7fac54515a426a32f3dc143b55a6ad00963fe68289955",
|
||||
"zh:2d8c016b3809ab32f536b01bf2c98fcbdeb066b7b7e50f756ff6ff1e0b0539c7",
|
||||
"zh:450d960b60ea81268bd355600340787254f750d76e0075208a6fb903d7336b09",
|
||||
"zh:50447833cce21288ffc331342f9d83854e80515e2d86e8f5776a07ce18866b20",
|
||||
"zh:58acc041b080778ded75320e241ebcf6cf2ed145bede644b86a4784ce46086c6",
|
||||
"zh:7e899a44a1da95fefb1b04b231cf7cdb5871f482f60a710ede78bf25ac265724",
|
||||
"zh:8b141d13f4b84d7a47bc5a4855d0fc985191317b3249603532d4de03db5741c5",
|
||||
"zh:9035357f7d065ce8d35a9b05337977910683219c0f13fa09155d28f9b32e23a7",
|
||||
"zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
|
||||
"zh:9ba0990a876d2641a185ca88263719afc4d5ac43533bdb147437f0073d2cd74d",
|
||||
"zh:a8773c7a02e1b3290df89351ee493fd8b74f807f92a7f69534e05901d4515761",
|
||||
"zh:ab0ebb298578eeeffe0fbf0a8b27ce99acd4419b7049272f1ab87e8c47229407",
|
||||
"zh:b88a3f22127bfb17604adc53e9bdb8ed0eddf4b9b77bf19871cba2e7fcd5b4a3",
|
||||
"zh:c7b1fb41e7f086b670f78932ede2b539903a8d71bac06f55f14a1a33d09a1ccf",
|
||||
"zh:df482bd93f51125f78feab7742aae1d7f22c7108ee2dc47787c4a1d611e6ccfd",
|
||||
"zh:fca401ffaa42d5f350d667123569de51a23de8c30b87ce0bd9d86fc80835fd4d",
|
||||
]
|
||||
}
|
||||
80
deploy/terraform/aws/bootstrap/main.tf
Normal file
80
deploy/terraform/aws/bootstrap/main.tf
Normal file
@ -0,0 +1,80 @@
|
||||
terraform {
|
||||
required_version = ">= 1.10, < 2.0"
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 6.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = "us-east-1"
|
||||
default_tags {
|
||||
tags = {
|
||||
Application = "warmbly"
|
||||
ManagedBy = "terraform"
|
||||
Purpose = "terraform-state"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_caller_identity" "current" {}
|
||||
|
||||
resource "aws_s3_bucket" "state" {
|
||||
bucket = "warmbly-terraform-use1-${data.aws_caller_identity.current.account_id}"
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_public_access_block" "state" {
|
||||
bucket = aws_s3_bucket.state.id
|
||||
block_public_acls = true
|
||||
block_public_policy = true
|
||||
ignore_public_acls = true
|
||||
restrict_public_buckets = true
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_versioning" "state" {
|
||||
bucket = aws_s3_bucket.state.id
|
||||
versioning_configuration {
|
||||
status = "Enabled"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_server_side_encryption_configuration" "state" {
|
||||
bucket = aws_s3_bucket.state.id
|
||||
rule {
|
||||
apply_server_side_encryption_by_default {
|
||||
sse_algorithm = "AES256"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "state" {
|
||||
statement {
|
||||
sid = "RequireTLS"
|
||||
effect = "Deny"
|
||||
actions = ["s3:*"]
|
||||
resources = [aws_s3_bucket.state.arn, "${aws_s3_bucket.state.arn}/*"]
|
||||
principals {
|
||||
type = "*"
|
||||
identifiers = ["*"]
|
||||
}
|
||||
condition {
|
||||
test = "Bool"
|
||||
variable = "aws:SecureTransport"
|
||||
values = ["false"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_policy" "state" {
|
||||
bucket = aws_s3_bucket.state.id
|
||||
policy = data.aws_iam_policy_document.state.json
|
||||
}
|
||||
|
||||
output "state_bucket" {
|
||||
value = aws_s3_bucket.state.id
|
||||
}
|
||||
26
deploy/terraform/aws/foundation/.terraform.lock.hcl
generated
Normal file
26
deploy/terraform/aws/foundation/.terraform.lock.hcl
generated
Normal file
@ -0,0 +1,26 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/hashicorp/aws" {
|
||||
version = "6.70.0"
|
||||
constraints = "~> 6.0"
|
||||
hashes = [
|
||||
"h1:L+05gOXA+kdYTtayIw0WxdaTp3JryZCMZpfrJVhoJA0=",
|
||||
"zh:068ca7ed4fc77e3d63d7fac54515a426a32f3dc143b55a6ad00963fe68289955",
|
||||
"zh:2d8c016b3809ab32f536b01bf2c98fcbdeb066b7b7e50f756ff6ff1e0b0539c7",
|
||||
"zh:450d960b60ea81268bd355600340787254f750d76e0075208a6fb903d7336b09",
|
||||
"zh:50447833cce21288ffc331342f9d83854e80515e2d86e8f5776a07ce18866b20",
|
||||
"zh:58acc041b080778ded75320e241ebcf6cf2ed145bede644b86a4784ce46086c6",
|
||||
"zh:7e899a44a1da95fefb1b04b231cf7cdb5871f482f60a710ede78bf25ac265724",
|
||||
"zh:8b141d13f4b84d7a47bc5a4855d0fc985191317b3249603532d4de03db5741c5",
|
||||
"zh:9035357f7d065ce8d35a9b05337977910683219c0f13fa09155d28f9b32e23a7",
|
||||
"zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
|
||||
"zh:9ba0990a876d2641a185ca88263719afc4d5ac43533bdb147437f0073d2cd74d",
|
||||
"zh:a8773c7a02e1b3290df89351ee493fd8b74f807f92a7f69534e05901d4515761",
|
||||
"zh:ab0ebb298578eeeffe0fbf0a8b27ce99acd4419b7049272f1ab87e8c47229407",
|
||||
"zh:b88a3f22127bfb17604adc53e9bdb8ed0eddf4b9b77bf19871cba2e7fcd5b4a3",
|
||||
"zh:c7b1fb41e7f086b670f78932ede2b539903a8d71bac06f55f14a1a33d09a1ccf",
|
||||
"zh:df482bd93f51125f78feab7742aae1d7f22c7108ee2dc47787c4a1d611e6ccfd",
|
||||
"zh:fca401ffaa42d5f350d667123569de51a23de8c30b87ce0bd9d86fc80835fd4d",
|
||||
]
|
||||
}
|
||||
106
deploy/terraform/aws/foundation/cache-check.tf
Normal file
106
deploy/terraform/aws/foundation/cache-check.tf
Normal file
@ -0,0 +1,106 @@
|
||||
data "aws_caller_identity" "current" {}
|
||||
|
||||
data "aws_iam_policy_document" "cache_check_trust" {
|
||||
statement {
|
||||
actions = ["sts:AssumeRole"]
|
||||
principals {
|
||||
type = "Service"
|
||||
identifiers = ["ecs-tasks.amazonaws.com"]
|
||||
}
|
||||
condition {
|
||||
test = "StringEquals"
|
||||
variable = "aws:SourceAccount"
|
||||
values = [data.aws_caller_identity.current.account_id]
|
||||
}
|
||||
condition {
|
||||
test = "ArnLike"
|
||||
variable = "aws:SourceArn"
|
||||
values = ["arn:aws:ecs:us-east-1:${data.aws_caller_identity.current.account_id}:*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_role" "cache_check" {
|
||||
name = "${var.name}-cache-check"
|
||||
assume_role_policy = data.aws_iam_policy_document.cache_check_trust.json
|
||||
}
|
||||
|
||||
resource "aws_cloudwatch_log_group" "cache_check" {
|
||||
name = "/ecs/${var.name}/cache-check"
|
||||
retention_in_days = 1
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "cache_check" {
|
||||
statement {
|
||||
actions = ["secretsmanager:GetSecretValue"]
|
||||
resources = [aws_secretsmanager_secret.cache.arn]
|
||||
}
|
||||
statement {
|
||||
actions = ["logs:CreateLogStream", "logs:PutLogEvents"]
|
||||
resources = ["${aws_cloudwatch_log_group.cache_check.arn}:*"]
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy" "cache_check" {
|
||||
name = "cache-connectivity-check"
|
||||
role = aws_iam_role.cache_check.id
|
||||
policy = data.aws_iam_policy_document.cache_check.json
|
||||
}
|
||||
|
||||
resource "aws_ecs_task_definition" "cache_check" {
|
||||
family = "${var.name}-cache-check"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
network_mode = "awsvpc"
|
||||
cpu = "256"
|
||||
memory = "512"
|
||||
execution_role_arn = aws_iam_role.cache_check.arn
|
||||
runtime_platform {
|
||||
cpu_architecture = "ARM64"
|
||||
operating_system_family = "LINUX"
|
||||
}
|
||||
container_definitions = jsonencode([{
|
||||
name = "cache-check"
|
||||
image = "public.ecr.aws/docker/library/node:22-alpine"
|
||||
essential = true
|
||||
command = ["node", "-e", <<-JS
|
||||
const tls = require('node:tls');
|
||||
const host = process.env.CACHE_HOST;
|
||||
const password = process.env.REDIS_PASSWORD;
|
||||
const fail = () => { console.error('Valkey connection check failed'); process.exit(1); };
|
||||
const socket = tls.connect({ host, port: 6379, servername: host, rejectUnauthorized: true }, () => {
|
||||
socket.write('*2\r\n$4\r\nAUTH\r\n$' + Buffer.byteLength(password) + '\r\n' + password + '\r\n*1\r\n$4\r\nPING\r\n');
|
||||
});
|
||||
let reply = '';
|
||||
let verified = false;
|
||||
socket.setTimeout(15000, fail);
|
||||
socket.on('error', fail);
|
||||
socket.on('close', () => { if (!verified) fail(); });
|
||||
socket.on('data', chunk => {
|
||||
reply += chunk.toString();
|
||||
if (reply.includes('-')) fail();
|
||||
if (reply === '+OK\r\n+PONG\r\n') {
|
||||
verified = true;
|
||||
console.log('Valkey TLS and password authentication verified');
|
||||
socket.setTimeout(0);
|
||||
socket.end();
|
||||
}
|
||||
});
|
||||
JS
|
||||
]
|
||||
environment = [{ name = "CACHE_HOST", value = aws_elasticache_replication_group.cache.primary_endpoint_address }]
|
||||
secrets = [{ name = "REDIS_PASSWORD", valueFrom = "${aws_secretsmanager_secret.cache.arn}:REDIS_PASSWORD::" }]
|
||||
logConfiguration = {
|
||||
logDriver = "awslogs"
|
||||
options = {
|
||||
awslogs-group = aws_cloudwatch_log_group.cache_check.name
|
||||
awslogs-region = "us-east-1"
|
||||
awslogs-stream-prefix = "check"
|
||||
}
|
||||
}
|
||||
}])
|
||||
depends_on = [aws_iam_role_policy.cache_check]
|
||||
}
|
||||
|
||||
output "cache_check_task_definition" {
|
||||
value = aws_ecs_task_definition.cache_check.arn
|
||||
}
|
||||
82
deploy/terraform/aws/foundation/cache.tf
Normal file
82
deploy/terraform/aws/foundation/cache.tf
Normal file
@ -0,0 +1,82 @@
|
||||
resource "aws_secretsmanager_secret" "cache" {
|
||||
name = "${var.name}/cache"
|
||||
description = "Valkey password; values are provisioned outside Terraform"
|
||||
recovery_window_in_days = 30
|
||||
}
|
||||
|
||||
ephemeral "aws_secretsmanager_secret_version" "cache" {
|
||||
secret_id = aws_secretsmanager_secret.cache.id
|
||||
}
|
||||
|
||||
resource "aws_security_group" "cache" {
|
||||
name_prefix = "${var.name}-cache-"
|
||||
description = "Valkey access from application tasks only"
|
||||
vpc_id = aws_vpc.main.id
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "cache" {
|
||||
security_group_id = aws_security_group.cache.id
|
||||
referenced_security_group_id = aws_security_group.tasks.id
|
||||
ip_protocol = "tcp"
|
||||
from_port = 6379
|
||||
to_port = 6379
|
||||
}
|
||||
|
||||
resource "aws_elasticache_subnet_group" "cache" {
|
||||
name = "${var.name}-cache"
|
||||
subnet_ids = aws_subnet.private[*].id
|
||||
}
|
||||
|
||||
resource "aws_elasticache_parameter_group" "cache" {
|
||||
name = "${var.name}-cache"
|
||||
family = "valkey9"
|
||||
parameter {
|
||||
name = "maxmemory-policy"
|
||||
value = "noeviction"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_elasticache_replication_group" "cache" {
|
||||
replication_group_id = "${var.name}-cache"
|
||||
description = "Shared application cache with standalone Redis protocol compatibility"
|
||||
engine = "valkey"
|
||||
engine_version = "9.0"
|
||||
node_type = "cache.t4g.micro"
|
||||
num_cache_clusters = 1
|
||||
automatic_failover_enabled = false
|
||||
multi_az_enabled = false
|
||||
port = 6379
|
||||
subnet_group_name = aws_elasticache_subnet_group.cache.name
|
||||
security_group_ids = [aws_security_group.cache.id]
|
||||
parameter_group_name = aws_elasticache_parameter_group.cache.name
|
||||
at_rest_encryption_enabled = true
|
||||
transit_encryption_enabled = true
|
||||
transit_encryption_mode = "required"
|
||||
auth_token_wo = jsondecode(ephemeral.aws_secretsmanager_secret_version.cache.secret_string).REDIS_PASSWORD
|
||||
auth_token_wo_version = 1
|
||||
snapshot_retention_limit = 1
|
||||
snapshot_window = "05:00-06:00"
|
||||
maintenance_window = "sun:06:00-sun:07:00"
|
||||
auto_minor_version_upgrade = true
|
||||
apply_immediately = false
|
||||
|
||||
tags = {
|
||||
managed_by = "aws-skills"
|
||||
skill = "elasticache"
|
||||
skill_version = "1.0.0"
|
||||
created_by = "elasticache-skill"
|
||||
generation_model = "gpt-6"
|
||||
}
|
||||
}
|
||||
|
||||
output "cache" {
|
||||
value = {
|
||||
replication_group_id = aws_elasticache_replication_group.cache.id
|
||||
endpoint = aws_elasticache_replication_group.cache.primary_endpoint_address
|
||||
port = aws_elasticache_replication_group.cache.port
|
||||
credential_secret = aws_secretsmanager_secret.cache.arn
|
||||
tls_required = true
|
||||
node_type = aws_elasticache_replication_group.cache.node_type
|
||||
replica_count = 0
|
||||
}
|
||||
}
|
||||
49
deploy/terraform/aws/foundation/configuration.tf
Normal file
49
deploy/terraform/aws/foundation/configuration.tf
Normal file
@ -0,0 +1,49 @@
|
||||
variable "service_configuration" {
|
||||
description = "Non-secret environment values stored as individual Standard String parameters"
|
||||
type = map(map(string))
|
||||
default = {}
|
||||
|
||||
validation {
|
||||
condition = alltrue(flatten([
|
||||
for service, configuration in var.service_configuration : [
|
||||
contains(["backend", "consumer", "forms", "realtime", "tracking"], service),
|
||||
alltrue([
|
||||
for key, value in configuration : contains([
|
||||
"APP_ENV", "APP_URL", "API_PUBLIC_URL", "API_HOST", "FORMS_DOMAIN",
|
||||
"TRACKING_DOMAIN", "WEBSOCKET_URL", "CORS_ALLOW_ORIGINS",
|
||||
"CHECK_ORIGIN_HOSTS", "PHX_HOST", "PORT", "FORMS_PORT", "TRACKING_HOST",
|
||||
"EVENTBUS_PROVIDER", "CODEC_PROVIDER", "KAFKA_BOOTSTRAP_SERVERS",
|
||||
"SCHEMA_REGISTRY_URL", "KMS_PROVIDER", "KMS_AWS_KEY_ID",
|
||||
"ENCRYPTED_KEYS_PROVIDER", "BLOB_PROVIDER", "BLOB_BUCKET",
|
||||
"BLOB_PUBLIC_BASE_URL", "AWS_REGION", "MAIL_TRANSPORT", "EMAIL_NAME",
|
||||
"EMAIL_ADDRESS", "BILLING_PROVIDER", "DEPLOYMENT_MODE", "WEBAUTHN_RP_ID",
|
||||
"WEBAUTHN_RP_ORIGINS", "INBOX_TAGGING_ENABLED", "RELEASES_ENABLED",
|
||||
"GEODB_PATH", "DATABASE_POOL_SIZE", "DATABASE_SSL_CA_FILE",
|
||||
"SEARCH_PROVIDER", "AI_PROVIDER", "AI_MODEL"
|
||||
], key) && !can(regex("://[^/@]+:[^/@]+@", value)) && !can(regex("(?i)(password|secret|token|api_key)=", value))
|
||||
])
|
||||
]
|
||||
]))
|
||||
error_message = "Only approved non-secret configuration names and values may enter Parameter Store. Store credentials and connection strings in Secrets Manager."
|
||||
}
|
||||
}
|
||||
|
||||
locals {
|
||||
configuration_parameters = merge({}, [
|
||||
for service, configuration in var.service_configuration : {
|
||||
for key, value in configuration : "${service}/${key}" => value
|
||||
}
|
||||
]...)
|
||||
}
|
||||
|
||||
resource "aws_ssm_parameter" "configuration" {
|
||||
for_each = local.configuration_parameters
|
||||
name = "/${var.name}/config/${each.key}"
|
||||
type = "String"
|
||||
tier = "Standard"
|
||||
value = each.value
|
||||
}
|
||||
|
||||
output "configuration_parameter_arns" {
|
||||
value = { for key, parameter in aws_ssm_parameter.configuration : key => parameter.arn }
|
||||
}
|
||||
226
deploy/terraform/aws/foundation/main.tf
Normal file
226
deploy/terraform/aws/foundation/main.tf
Normal file
@ -0,0 +1,226 @@
|
||||
terraform {
|
||||
required_version = ">= 1.11, < 2.0"
|
||||
backend "s3" {
|
||||
key = "warmbly/production/foundation.tfstate"
|
||||
region = "us-east-1"
|
||||
encrypt = true
|
||||
use_lockfile = true
|
||||
}
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 6.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = "us-east-1"
|
||||
default_tags {
|
||||
tags = {
|
||||
Application = "warmbly"
|
||||
Environment = "production"
|
||||
ManagedBy = "terraform"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
variable "name" {
|
||||
type = string
|
||||
default = "warmbly-production"
|
||||
}
|
||||
|
||||
variable "vpc_cidr" {
|
||||
type = string
|
||||
default = "10.42.0.0/16"
|
||||
}
|
||||
|
||||
variable "planetscale_service_name" {
|
||||
description = "Private Service Name from the existing PlanetScale Postgres branch. Empty defers endpoint creation."
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "certificate_domains" {
|
||||
description = "Certificate primary names mapped to their additional names. Publish the output validation records with your DNS provider."
|
||||
type = map(list(string))
|
||||
default = {}
|
||||
}
|
||||
|
||||
data "aws_availability_zones" "available" {
|
||||
state = "available"
|
||||
filter {
|
||||
name = "opt-in-status"
|
||||
values = ["opt-in-not-required"]
|
||||
}
|
||||
}
|
||||
|
||||
locals {
|
||||
zones = slice(data.aws_availability_zones.available.names, 0, 2)
|
||||
services = toset(["backend", "consumer", "forms", "realtime", "tracking"])
|
||||
}
|
||||
|
||||
resource "aws_vpc" "main" {
|
||||
cidr_block = var.vpc_cidr
|
||||
enable_dns_support = true
|
||||
enable_dns_hostnames = true
|
||||
tags = { Name = var.name }
|
||||
}
|
||||
|
||||
resource "aws_internet_gateway" "main" {
|
||||
vpc_id = aws_vpc.main.id
|
||||
}
|
||||
|
||||
resource "aws_subnet" "public" {
|
||||
count = 2
|
||||
vpc_id = aws_vpc.main.id
|
||||
availability_zone = local.zones[count.index]
|
||||
cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index)
|
||||
map_public_ip_on_launch = false
|
||||
tags = { Name = "${var.name}-public-${count.index + 1}" }
|
||||
}
|
||||
|
||||
resource "aws_subnet" "private" {
|
||||
count = 2
|
||||
vpc_id = aws_vpc.main.id
|
||||
availability_zone = local.zones[count.index]
|
||||
cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index + 10)
|
||||
tags = { Name = "${var.name}-private-${count.index + 1}" }
|
||||
}
|
||||
|
||||
resource "aws_route_table" "public" {
|
||||
vpc_id = aws_vpc.main.id
|
||||
}
|
||||
|
||||
resource "aws_route" "internet" {
|
||||
route_table_id = aws_route_table.public.id
|
||||
destination_cidr_block = "0.0.0.0/0"
|
||||
gateway_id = aws_internet_gateway.main.id
|
||||
}
|
||||
|
||||
resource "aws_route_table_association" "public" {
|
||||
count = 2
|
||||
subnet_id = aws_subnet.public[count.index].id
|
||||
route_table_id = aws_route_table.public.id
|
||||
}
|
||||
|
||||
resource "aws_route_table" "private" {
|
||||
vpc_id = aws_vpc.main.id
|
||||
}
|
||||
|
||||
resource "aws_route_table_association" "private" {
|
||||
count = 2
|
||||
subnet_id = aws_subnet.private[count.index].id
|
||||
route_table_id = aws_route_table.private.id
|
||||
}
|
||||
|
||||
resource "aws_security_group" "tasks" {
|
||||
name_prefix = "${var.name}-tasks-"
|
||||
description = "Application tasks; inbound rules are added with the runtime deployment"
|
||||
vpc_id = aws_vpc.main.id
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_egress_rule" "tasks" {
|
||||
security_group_id = aws_security_group.tasks.id
|
||||
ip_protocol = "-1"
|
||||
cidr_ipv4 = "0.0.0.0/0"
|
||||
}
|
||||
|
||||
resource "aws_security_group" "database_endpoint" {
|
||||
name_prefix = "${var.name}-database-"
|
||||
description = "PlanetScale endpoint, reachable only by application tasks"
|
||||
vpc_id = aws_vpc.main.id
|
||||
}
|
||||
|
||||
resource "aws_vpc_security_group_ingress_rule" "database" {
|
||||
for_each = toset(["5432", "6432"])
|
||||
security_group_id = aws_security_group.database_endpoint.id
|
||||
referenced_security_group_id = aws_security_group.tasks.id
|
||||
ip_protocol = "tcp"
|
||||
from_port = tonumber(each.value)
|
||||
to_port = tonumber(each.value)
|
||||
}
|
||||
|
||||
resource "aws_vpc_endpoint" "s3" {
|
||||
vpc_id = aws_vpc.main.id
|
||||
service_name = "com.amazonaws.us-east-1.s3"
|
||||
vpc_endpoint_type = "Gateway"
|
||||
route_table_ids = [aws_route_table.public.id, aws_route_table.private.id]
|
||||
}
|
||||
|
||||
resource "aws_vpc_endpoint" "database" {
|
||||
count = var.planetscale_service_name == "" ? 0 : 1
|
||||
vpc_id = aws_vpc.main.id
|
||||
service_name = var.planetscale_service_name
|
||||
vpc_endpoint_type = "Interface"
|
||||
subnet_ids = aws_subnet.private[*].id
|
||||
security_group_ids = [aws_security_group.database_endpoint.id]
|
||||
private_dns_enabled = true
|
||||
}
|
||||
|
||||
resource "aws_ecs_cluster" "main" {
|
||||
name = var.name
|
||||
}
|
||||
|
||||
resource "aws_ecs_cluster_capacity_providers" "main" {
|
||||
cluster_name = aws_ecs_cluster.main.name
|
||||
capacity_providers = ["FARGATE", "FARGATE_SPOT"]
|
||||
default_capacity_provider_strategy {
|
||||
capacity_provider = "FARGATE"
|
||||
weight = 1
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_ecr_repository" "service" {
|
||||
for_each = local.services
|
||||
name = "warmbly/${each.key}"
|
||||
image_tag_mutability = "IMMUTABLE"
|
||||
image_scanning_configuration {
|
||||
scan_on_push = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_cloudwatch_log_group" "service" {
|
||||
for_each = local.services
|
||||
name = "/ecs/${var.name}/${each.key}"
|
||||
retention_in_days = 14
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret" "service" {
|
||||
for_each = local.services
|
||||
name = "${var.name}/${each.key}"
|
||||
description = "Runtime configuration for ${each.key}; values are populated outside Terraform"
|
||||
recovery_window_in_days = 30
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret" "rotation" {
|
||||
name = "${var.name}/rotation"
|
||||
description = "Pending key replacements for coordinated application and worker rollout"
|
||||
recovery_window_in_days = 30
|
||||
}
|
||||
|
||||
resource "aws_acm_certificate" "service" {
|
||||
for_each = var.certificate_domains
|
||||
domain_name = each.key
|
||||
subject_alternative_names = each.value
|
||||
validation_method = "DNS"
|
||||
lifecycle {
|
||||
create_before_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
output "runtime" {
|
||||
value = {
|
||||
cluster_arn = aws_ecs_cluster.main.arn
|
||||
vpc_id = aws_vpc.main.id
|
||||
public_subnet_ids = aws_subnet.public[*].id
|
||||
private_subnet_ids = aws_subnet.private[*].id
|
||||
tasks_sg_id = aws_security_group.tasks.id
|
||||
secret_arns = { for name, secret in aws_secretsmanager_secret.service : name => secret.arn }
|
||||
rotation_secret_arn = aws_secretsmanager_secret.rotation.arn
|
||||
repository_urls = { for name, repo in aws_ecr_repository.service : name => repo.repository_url }
|
||||
database_endpoint = aws_vpc_endpoint.database[*].id
|
||||
certificate_arns = { for name, cert in aws_acm_certificate.service : name => cert.arn }
|
||||
certificate_records = { for name, cert in aws_acm_certificate.service : name => cert.domain_validation_options }
|
||||
}
|
||||
}
|
||||
26
deploy/terraform/aws/runtime/.terraform.lock.hcl
generated
Normal file
26
deploy/terraform/aws/runtime/.terraform.lock.hcl
generated
Normal file
@ -0,0 +1,26 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/hashicorp/aws" {
|
||||
version = "6.70.0"
|
||||
constraints = "~> 6.0"
|
||||
hashes = [
|
||||
"h1:L+05gOXA+kdYTtayIw0WxdaTp3JryZCMZpfrJVhoJA0=",
|
||||
"zh:068ca7ed4fc77e3d63d7fac54515a426a32f3dc143b55a6ad00963fe68289955",
|
||||
"zh:2d8c016b3809ab32f536b01bf2c98fcbdeb066b7b7e50f756ff6ff1e0b0539c7",
|
||||
"zh:450d960b60ea81268bd355600340787254f750d76e0075208a6fb903d7336b09",
|
||||
"zh:50447833cce21288ffc331342f9d83854e80515e2d86e8f5776a07ce18866b20",
|
||||
"zh:58acc041b080778ded75320e241ebcf6cf2ed145bede644b86a4784ce46086c6",
|
||||
"zh:7e899a44a1da95fefb1b04b231cf7cdb5871f482f60a710ede78bf25ac265724",
|
||||
"zh:8b141d13f4b84d7a47bc5a4855d0fc985191317b3249603532d4de03db5741c5",
|
||||
"zh:9035357f7d065ce8d35a9b05337977910683219c0f13fa09155d28f9b32e23a7",
|
||||
"zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
|
||||
"zh:9ba0990a876d2641a185ca88263719afc4d5ac43533bdb147437f0073d2cd74d",
|
||||
"zh:a8773c7a02e1b3290df89351ee493fd8b74f807f92a7f69534e05901d4515761",
|
||||
"zh:ab0ebb298578eeeffe0fbf0a8b27ce99acd4419b7049272f1ab87e8c47229407",
|
||||
"zh:b88a3f22127bfb17604adc53e9bdb8ed0eddf4b9b77bf19871cba2e7fcd5b4a3",
|
||||
"zh:c7b1fb41e7f086b670f78932ede2b539903a8d71bac06f55f14a1a33d09a1ccf",
|
||||
"zh:df482bd93f51125f78feab7742aae1d7f22c7108ee2dc47787c4a1d611e6ccfd",
|
||||
"zh:fca401ffaa42d5f350d667123569de51a23de8c30b87ce0bd9d86fc80835fd4d",
|
||||
]
|
||||
}
|
||||
89
deploy/terraform/aws/runtime/README.md
Normal file
89
deploy/terraform/aws/runtime/README.md
Normal file
@ -0,0 +1,89 @@
|
||||
# Application runtime
|
||||
|
||||
This root reads the foundation state and creates application services in
|
||||
`us-east-1`. Keep installation-specific inputs outside the checkout. Supply
|
||||
digest-pinned ARM64 images, valid Fargate sizes, desired task counts, secret key
|
||||
names and immutable secret version IDs. Secret values never enter Terraform.
|
||||
|
||||
```sh
|
||||
terraform -chdir=deploy/terraform/aws/runtime init \
|
||||
-backend-config="bucket=$STATE_BUCKET"
|
||||
terraform -chdir=deploy/terraform/aws/runtime plan \
|
||||
-var-file=/private/path/runtime.tfvars.json \
|
||||
-out=/private/path/runtime.tfplan
|
||||
terraform -chdir=deploy/terraform/aws/runtime apply /private/path/runtime.tfplan
|
||||
```
|
||||
|
||||
Start at desired count zero. Separate execution roles retrieve each service's
|
||||
configuration and write its logs. Application roles receive reviewed existing
|
||||
permissions or IAM Policy Autopilot output through `application_policies`.
|
||||
|
||||
## Migrate before deployment
|
||||
|
||||
Terraform registers the application and migration task definitions. Service
|
||||
revision and desired-count changes are deployment operations; Terraform ignores
|
||||
those two fields after service creation. The workflow handles backend and consumer
|
||||
rollouts. Forms, realtime and tracking cutovers use separate explicit ECS service
|
||||
updates after their own checks. Start new services at zero and use explicit
|
||||
counts when activating them. Terraform output task definition ARNs describe
|
||||
prepared revisions, not necessarily running revisions.
|
||||
|
||||
Set `github_repository` to the repository running `.github/workflows/deploy-aws.yml`.
|
||||
The root creates an OIDC provider and a deployment role scoped to the migration
|
||||
task and backend/consumer services. Import an existing account-wide GitHub OIDC
|
||||
provider instead of creating a second one. Restrict the `aws-production` GitHub
|
||||
environment to trusted deployment branches. Set its variables:
|
||||
|
||||
| Variable | Value |
|
||||
| --- | --- |
|
||||
| `AWS_DEPLOY_ROLE_ARN` | Terraform `github_deploy_role_arn` output |
|
||||
| `AWS_ECS_CLUSTER` | Runtime `cluster_arn` output |
|
||||
| `AWS_ECS_SERVICE_PREFIX` | Runtime `name` input |
|
||||
|
||||
Prepare a backend secret version with `PRIMARY_DB_DIRECT`, pointing to port 5432.
|
||||
The migration task injects only that key, with no application task role. The
|
||||
workflow never retrieves database credentials. Its backend and migration tasks
|
||||
must use the same immutable image digest and direct secret reference. Keep the
|
||||
default `migration_secret_key=PRIMARY_DB_DIRECT` for workflow deployments.
|
||||
|
||||
Dispatch the workflow with explicit Terraform task definition revisions for
|
||||
migration, backend and consumer. It runs `/app/migrate` inside the VPC over the
|
||||
existing PrivateLink route. An unsuccessful or unresolved migration blocks all
|
||||
service updates. A successful migration permits sequential backend and consumer
|
||||
rollouts, verifying that the requested revision becomes stable. A rollout failure
|
||||
restores the previous service revisions and counts, retaining the current schema.
|
||||
Inspect rollback health separately. The workflow never runs down migrations.
|
||||
Task definitions use `skip_destroy` so earlier revisions remain active for
|
||||
rollback. Retain their referenced images and secret versions throughout the
|
||||
rollback window; retire them separately after it closes.
|
||||
|
||||
The runtime sets `DB_MIGRATIONS_ON_START=false` for the backend. New application
|
||||
images must support that setting and schema readiness before activation. Keep
|
||||
migrations additive and compatible with the previous release throughout rollout
|
||||
and rollback. Do not interrupt an active migration; if a workflow is cancelled or
|
||||
times out, inspect the ECS task before rerunning it.
|
||||
|
||||
Run `database_check_task_definition` to exercise the backend image's actual Go
|
||||
client against its configured pooler without starting the API or background jobs.
|
||||
It collects operator status into temporary task storage and logs only completion.
|
||||
The separate dependency validation task checks provider connectivity and stored
|
||||
credential decryption. Neither check activates the backend service.
|
||||
|
||||
Tasks use public IPs for outbound access, avoiding NAT gateway charges. Inbound
|
||||
application traffic is restricted to the HTTPS load balancer. The backend also
|
||||
accepts private task traffic through service discovery. Restrict the load
|
||||
balancer to approved caller CIDRs, such as Cloudflare's published ranges. Public
|
||||
DNS changes are separate from deploying the services.
|
||||
|
||||
Run `validation_task_definition` once in a public subnet with a public IP and the
|
||||
task security group. It checks private database DNS and logins, stored mailbox
|
||||
and MFA decryption, KMS, both caches, Kafka metadata, Schema Registry and S3 using
|
||||
the actual backend task role. It writes no production data, logs only counts and
|
||||
results, and exits. Observe its stopped exit code and logs before activation.
|
||||
The pinned validation packages are installed in temporary task storage.
|
||||
|
||||
Keep one cache authoritative throughout deployment overlap. A backend starts
|
||||
background jobs immediately; its desired count is the
|
||||
activation control. Preserve compatible images, encryption material and shared
|
||||
coordination while the previous deployment drains. Verify target health and
|
||||
HTTPS routes before switching DNS. New injected values require replacing tasks.
|
||||
165
deploy/terraform/aws/runtime/checks/dependencies.cjs
Normal file
165
deploy/terraform/aws/runtime/checks/dependencies.cjs
Normal file
@ -0,0 +1,165 @@
|
||||
const crypto = require("node:crypto");
|
||||
const dns = require("node:dns/promises");
|
||||
const https = require("node:https");
|
||||
const { Client } = require("pg");
|
||||
const { Kafka, logLevel } = require("kafkajs");
|
||||
const { createClient } = require("redis");
|
||||
const { KMSClient, DecryptCommand } = require("@aws-sdk/client-kms");
|
||||
const { S3Client, ListObjectsV2Command } = require("@aws-sdk/client-s3");
|
||||
|
||||
const results = [];
|
||||
const config = { region: process.env.AWS_REGION, maxAttempts: 2 };
|
||||
const kms = new KMSClient(config);
|
||||
const s3 = new S3Client(config);
|
||||
const timeout = setTimeout(() => {
|
||||
console.error("Dependency validation exceeded its time limit");
|
||||
process.exit(1);
|
||||
}, 240000);
|
||||
|
||||
async function check(name, fn) {
|
||||
try {
|
||||
const detail = await fn();
|
||||
results.push({ name, ok: true, ...detail });
|
||||
console.log(JSON.stringify(results.at(-1)));
|
||||
} catch {
|
||||
results.push({ name, ok: false });
|
||||
console.error(JSON.stringify(results.at(-1)));
|
||||
}
|
||||
}
|
||||
|
||||
function readSecrets(variable) {
|
||||
const values = JSON.parse(process.env[variable] || "[]");
|
||||
if (!Array.isArray(values) || values.length > 4 || values.some((value) => typeof value !== "string" || !value)) throw new Error("Invalid read keys");
|
||||
return values;
|
||||
}
|
||||
|
||||
function verifyCipher(keys, encoded, encoding) {
|
||||
const raw = Buffer.from(encoded, encoding);
|
||||
if (raw.length < 28) throw new Error("Invalid ciphertext");
|
||||
for (const key of keys) {
|
||||
const parts = [];
|
||||
try {
|
||||
const decipher = crypto.createDecipheriv("aes-256-gcm", key, raw.subarray(0, 12));
|
||||
decipher.setAuthTag(raw.subarray(raw.length - 16));
|
||||
parts.push(decipher.update(raw.subarray(12, raw.length - 16)));
|
||||
parts.push(decipher.final());
|
||||
return;
|
||||
} catch {
|
||||
// Try the next explicitly configured reader.
|
||||
} finally {
|
||||
parts.forEach((part) => part.fill(0));
|
||||
}
|
||||
}
|
||||
throw new Error("Ciphertext verification failed");
|
||||
}
|
||||
|
||||
async function withDatabase(connectionString, fn) {
|
||||
const client = new Client({ connectionString, connectionTimeoutMillis: 10000, query_timeout: 20000 });
|
||||
client.on("error", () => {});
|
||||
try {
|
||||
await client.connect();
|
||||
await client.query("BEGIN READ ONLY");
|
||||
await client.query("SET LOCAL statement_timeout = '15s'");
|
||||
return await fn(client);
|
||||
} finally {
|
||||
await client.end().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
function requestJSON(url, headers = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.get(url, { headers, timeout: 15000 }, (res) => {
|
||||
let body = "";
|
||||
res.on("data", (chunk) => { body += chunk; });
|
||||
res.on("end", () => {
|
||||
if (res.statusCode !== 200) return reject(new Error("Unexpected response"));
|
||||
try { resolve(JSON.parse(body)); } catch { reject(new Error("Unexpected JSON")); }
|
||||
});
|
||||
});
|
||||
req.on("timeout", () => req.destroy());
|
||||
req.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await check("planetscale_private_dns", async () => {
|
||||
const host = new URL(process.env.PRIMARY_DB).hostname;
|
||||
const addresses = await dns.resolve4(host);
|
||||
if (!addresses.length || addresses.some((ip) => !ip.startsWith("10."))) throw new Error("Private DNS required");
|
||||
return { private_addresses: addresses.length };
|
||||
});
|
||||
|
||||
for (const [name, variable] of [["backend_database", "PRIMARY_DB"], ["consumer_database", "CONSUMER_DB"], ["realtime_database", "REALTIME_DB"]]) {
|
||||
await check(name, () => withDatabase(process.env[variable], async (db) => {
|
||||
const out = await db.query("SELECT count(*)::int AS readable_tables FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind='r' AND has_table_privilege(c.oid,'SELECT')");
|
||||
const version = await db.query("SELECT version,dirty FROM schema_migrations");
|
||||
if (!out.rows[0].readable_tables || version.rows.some((r) => r.dirty)) throw new Error("Database validation failed");
|
||||
return { readable_tables: out.rows[0].readable_tables, migration_clean: true };
|
||||
}));
|
||||
}
|
||||
|
||||
await check("stored_credentials_and_kms", () => withDatabase(process.env.PRIMARY_DB, async (db) => {
|
||||
const keys = [process.env.CREDENTIALS_ENCRYPTION_KEY, ...readSecrets("CREDENTIALS_ENCRYPTION_READ_KEYS")].map((value) => Buffer.from(value, "hex"));
|
||||
const mfaKeys = [process.env.TWOFA_SECRET || process.env.AUTH_SECRET, ...readSecrets("TWOFA_READ_SECRETS")].map((value) => crypto.createHash("sha256").update(value).digest());
|
||||
try {
|
||||
const fields = await db.query("SELECT v AS value,false AS signing_secret FROM email_accounts_smtp_imap CROSS JOIN LATERAL (VALUES (smtp_host),(smtp_user),(smtp_password),(imap_host),(imap_user),(imap_password)) fields(v) WHERE v IS NOT NULL AND v<>'' UNION ALL SELECT v,false FROM email_accounts_oauth CROSS JOIN LATERAL (VALUES(access_token),(refresh_token)) fields(v) WHERE v IS NOT NULL AND v<>'' UNION ALL SELECT secret,true FROM webhook_endpoints WHERE secret IS NOT NULL AND secret<>'' UNION ALL SELECT webhook_secret,true FROM oauth_applications WHERE webhook_secret IS NOT NULL AND webhook_secret<>'' UNION ALL SELECT token,false FROM cloud_link WHERE token IS NOT NULL AND token<>'' UNION ALL SELECT refresh_token,false FROM mailbox_erasures WHERE refresh_token IS NOT NULL AND refresh_token<>''");
|
||||
let legacySigningFields = 0;
|
||||
for (const row of fields.rows) {
|
||||
if (row.signing_secret && row.value.startsWith("whsec_")) legacySigningFields++;
|
||||
else verifyCipher(keys, row.value, "hex");
|
||||
}
|
||||
const mfa = await db.query("SELECT totp_secret_sealed FROM user_totp_settings WHERE totp_secret_sealed IS NOT NULL AND totp_secret_sealed<>''");
|
||||
for (const row of mfa.rows) verifyCipher(mfaKeys, row.totp_secret_sealed, "base64");
|
||||
const wrapped = await db.query("SELECT encrypted_data_key FROM organization_encrypted_keys");
|
||||
if (!wrapped.rowCount) throw new Error("No data key available to verify");
|
||||
for (let offset = 0; offset < wrapped.rowCount; offset += 5) {
|
||||
await Promise.all(wrapped.rows.slice(offset, offset + 5).map(async (row) => {
|
||||
const out = await kms.send(new DecryptCommand({ CiphertextBlob: Buffer.from(row.encrypted_data_key, "base64") }));
|
||||
try { if (out.Plaintext?.length !== 32) throw new Error("Invalid data key"); }
|
||||
finally { out.Plaintext?.fill(0); }
|
||||
}));
|
||||
}
|
||||
return { credential_fields: fields.rowCount - legacySigningFields, legacy_signing_fields: legacySigningFields, mfa_enrollments: mfa.rowCount, organization_keys: wrapped.rowCount, kms_decrypt_as_task_role: true };
|
||||
} finally {
|
||||
[...keys, ...mfaKeys].forEach((key) => key.fill(0));
|
||||
}
|
||||
}));
|
||||
|
||||
await check("existing_shared_cache", async () => {
|
||||
const client = createClient({ url: process.env.REDIS, socket: { connectTimeout: 10000, reconnectStrategy: false } });
|
||||
client.on("error", () => {});
|
||||
try { await client.connect(); if (await client.ping() !== "PONG") throw new Error("Cache check failed"); }
|
||||
finally { client.destroy(); }
|
||||
return {};
|
||||
});
|
||||
await check("private_valkey", async () => {
|
||||
const client = createClient({ password: process.env.NEW_REDIS_PASSWORD, socket: { host: process.env.CACHE_HOST, port: 6379, tls: true, servername: process.env.CACHE_HOST, rejectUnauthorized: true, connectTimeout: 10000, reconnectStrategy: false } });
|
||||
client.on("error", () => {});
|
||||
try { await client.connect(); if (await client.ping() !== "PONG") throw new Error("Cache check failed"); }
|
||||
finally { client.destroy(); }
|
||||
return { tls_verified: true };
|
||||
});
|
||||
await check("kafka_replacement_key", async () => {
|
||||
const kafka = new Kafka({ clientId: "aws-migration-validation", brokers: process.env.KAFKA_BOOTSTRAP_SERVERS.split(","), ssl: true, sasl: { mechanism: "plain", username: process.env.KAFKA_SASL_USERNAME, password: process.env.KAFKA_SASL_PASSWORD }, logLevel: logLevel.NOTHING, connectionTimeout: 10000, requestTimeout: 15000, retry: { retries: 1 } });
|
||||
const admin = kafka.admin({ allowAutoTopicCreation: false });
|
||||
try { await admin.connect(); const out = await admin.fetchTopicMetadata({ topics: ["tracking-events"] }); return { existing_topics: out.topics.length }; }
|
||||
finally { await admin.disconnect(); }
|
||||
});
|
||||
await check("schema_registry_replacement_key", async () => {
|
||||
const authorization = "Basic " + Buffer.from(process.env.SCHEMA_REGISTRY_KEY + ":" + process.env.SCHEMA_REGISTRY_SECRET).toString("base64");
|
||||
const subjects = await requestJSON(process.env.SCHEMA_REGISTRY_URL.replace(/\/$/, "") + "/subjects", { authorization });
|
||||
if (!Array.isArray(subjects)) throw new Error("Invalid Schema Registry response");
|
||||
return { subjects: subjects.length };
|
||||
});
|
||||
await check("blob_bucket_as_task_role", async () => {
|
||||
await s3.send(new ListObjectsV2Command({ Bucket: process.env.BLOB_BUCKET, MaxKeys: 1 }));
|
||||
return {};
|
||||
});
|
||||
kms.destroy();
|
||||
s3.destroy();
|
||||
clearTimeout(timeout);
|
||||
console.log(JSON.stringify({ checks: results.length, passed: results.filter((r) => r.ok).length, production_writes: false }));
|
||||
process.exit(results.every((r) => r.ok) ? 0 : 1);
|
||||
}
|
||||
|
||||
main().catch(() => { console.error("Dependency validation failed; credential values omitted"); process.exit(1); });
|
||||
21
deploy/terraform/aws/runtime/checks/realtime.exs
Normal file
21
deploy/terraform/aws/runtime/checks/realtime.exs
Normal file
@ -0,0 +1,21 @@
|
||||
try do
|
||||
{:ok, _} = Application.ensure_all_started(:ecto_sql)
|
||||
{:ok, _} = Application.ensure_all_started(:postgrex)
|
||||
|
||||
{:ok, _} =
|
||||
Realtime.Repo.start_link(
|
||||
pool_size: 2,
|
||||
show_sensitive_data_on_connection_error: false
|
||||
)
|
||||
|
||||
for value <- 1..4 do
|
||||
{:ok, %{rows: [[^value]]}} =
|
||||
Ecto.Adapters.SQL.query(Realtime.Repo, "SELECT $1::integer", [value], log: false)
|
||||
end
|
||||
|
||||
IO.puts("Ecto/Postgrex pooled prepared queries succeeded")
|
||||
rescue
|
||||
_ ->
|
||||
IO.puts("Realtime database client check failed; connection details omitted")
|
||||
System.halt(1)
|
||||
end
|
||||
91
deploy/terraform/aws/runtime/ci.tf
Normal file
91
deploy/terraform/aws/runtime/ci.tf
Normal file
@ -0,0 +1,91 @@
|
||||
variable "github_repository" {
|
||||
description = "Repository allowed to deploy through the protected GitHub environment"
|
||||
type = string
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "github_environment" {
|
||||
type = string
|
||||
default = "aws-production"
|
||||
}
|
||||
|
||||
resource "aws_iam_openid_connect_provider" "github" {
|
||||
count = var.github_repository == null ? 0 : 1
|
||||
url = "https://token.actions.githubusercontent.com"
|
||||
client_id_list = ["sts.amazonaws.com"]
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "github_trust" {
|
||||
count = var.github_repository == null ? 0 : 1
|
||||
statement {
|
||||
actions = ["sts:AssumeRoleWithWebIdentity"]
|
||||
principals {
|
||||
type = "Federated"
|
||||
identifiers = [aws_iam_openid_connect_provider.github[0].arn]
|
||||
}
|
||||
condition {
|
||||
test = "StringEquals"
|
||||
variable = "token.actions.githubusercontent.com:aud"
|
||||
values = ["sts.amazonaws.com"]
|
||||
}
|
||||
condition {
|
||||
test = "StringEquals"
|
||||
variable = "token.actions.githubusercontent.com:sub"
|
||||
values = ["repo:${var.github_repository}:environment:${var.github_environment}"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_role" "github_deploy" {
|
||||
count = var.github_repository == null ? 0 : 1
|
||||
name = "${var.name}-github-deploy"
|
||||
assume_role_policy = data.aws_iam_policy_document.github_trust[0].json
|
||||
max_session_duration = 3600
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "github_deploy" {
|
||||
count = var.github_repository == null ? 0 : 1
|
||||
statement {
|
||||
actions = ["ecs:DescribeTaskDefinition"]
|
||||
resources = ["*"]
|
||||
}
|
||||
statement {
|
||||
actions = ["ecs:DescribeServices", "ecs:UpdateService"]
|
||||
resources = [for service in ["backend", "consumer"] : "arn:aws:ecs:us-east-1:${data.aws_caller_identity.current.account_id}:service/${var.name}/${var.name}-${service}"]
|
||||
}
|
||||
statement {
|
||||
actions = ["ecs:DescribeTasks"]
|
||||
resources = ["arn:aws:ecs:us-east-1:${data.aws_caller_identity.current.account_id}:task/${var.name}/*"]
|
||||
}
|
||||
statement {
|
||||
actions = ["ecs:RunTask"]
|
||||
resources = ["arn:aws:ecs:us-east-1:${data.aws_caller_identity.current.account_id}:task-definition/${var.name}-migration:*"]
|
||||
condition {
|
||||
test = "ArnEquals"
|
||||
variable = "ecs:cluster"
|
||||
values = [local.foundation.cluster_arn]
|
||||
}
|
||||
}
|
||||
statement {
|
||||
actions = ["iam:PassRole"]
|
||||
resources = concat([aws_iam_role.migration_execution.arn], flatten([
|
||||
for name in ["backend", "consumer"] : [aws_iam_role.execution[name].arn, aws_iam_role.application[name].arn]
|
||||
]))
|
||||
condition {
|
||||
test = "StringEquals"
|
||||
variable = "iam:PassedToService"
|
||||
values = ["ecs-tasks.amazonaws.com"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy" "github_deploy" {
|
||||
count = var.github_repository == null ? 0 : 1
|
||||
name = "migrate-before-deploy"
|
||||
role = aws_iam_role.github_deploy[0].id
|
||||
policy = data.aws_iam_policy_document.github_deploy[0].json
|
||||
}
|
||||
|
||||
output "github_deploy_role_arn" {
|
||||
value = var.github_repository == null ? null : aws_iam_role.github_deploy[0].arn
|
||||
}
|
||||
399
deploy/terraform/aws/runtime/main.tf
Normal file
399
deploy/terraform/aws/runtime/main.tf
Normal file
@ -0,0 +1,399 @@
|
||||
terraform {
|
||||
required_version = ">= 1.11, < 2.0"
|
||||
backend "s3" {
|
||||
key = "warmbly/production/runtime.tfstate"
|
||||
region = "us-east-1"
|
||||
encrypt = true
|
||||
use_lockfile = true
|
||||
}
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 6.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = "us-east-1"
|
||||
default_tags {
|
||||
tags = {
|
||||
Application = "warmbly"
|
||||
Environment = "production"
|
||||
ManagedBy = "terraform"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
variable "state_bucket" { type = string }
|
||||
variable "name" {
|
||||
type = string
|
||||
default = "warmbly-production"
|
||||
}
|
||||
variable "namespace" {
|
||||
type = string
|
||||
default = "warmbly.internal"
|
||||
}
|
||||
variable "certificate_primary_domain" { type = string }
|
||||
variable "origin_ipv4_cidrs" {
|
||||
type = set(string)
|
||||
validation {
|
||||
condition = length(var.origin_ipv4_cidrs) > 0 && alltrue([for cidr in var.origin_ipv4_cidrs : can(cidrnetmask(cidr)) && cidr != "0.0.0.0/0"])
|
||||
error_message = "Provide explicit origin caller CIDRs, such as Cloudflare's published IPv4 ranges."
|
||||
}
|
||||
}
|
||||
variable "application_policies" {
|
||||
description = "Reviewed runtime policy JSON by service, supplied outside the checkout"
|
||||
type = map(string)
|
||||
default = {}
|
||||
}
|
||||
variable "services" {
|
||||
description = "Digest-pinned images and secret metadata; secret values never enter Terraform"
|
||||
type = map(object({
|
||||
image = string
|
||||
cpu = number
|
||||
memory = number
|
||||
desired_count = optional(number, 0)
|
||||
port = optional(number)
|
||||
host = optional(string)
|
||||
health_path = optional(string, "/health")
|
||||
secret_version_id = string
|
||||
secret_keys = set(string)
|
||||
environment = optional(map(string), {})
|
||||
}))
|
||||
validation {
|
||||
condition = (var.github_repository == null || alltrue([for name in ["backend", "consumer"] : contains(keys(var.services), name)])) && alltrue([for name, service in var.services :
|
||||
contains(["backend", "consumer", "forms", "realtime", "tracking"], name) &&
|
||||
can(regex("@sha256:[a-f0-9]{64}$", service.image)) &&
|
||||
((service.cpu == 256 && contains([512, 1024, 2048], service.memory)) ||
|
||||
(service.cpu == 512 && contains([1024, 2048, 3072, 4096], service.memory)) ||
|
||||
(service.cpu == 1024 && service.memory >= 2048 && service.memory <= 8192 && service.memory % 1024 == 0) ||
|
||||
(service.cpu == 2048 && service.memory >= 4096 && service.memory <= 16384 && service.memory % 1024 == 0)) &&
|
||||
service.desired_count >= 0 && service.desired_count == floor(service.desired_count) &&
|
||||
!contains(service.secret_keys, "AWS_ACCESS_KEY_ID") && !contains(service.secret_keys, "AWS_SECRET_ACCESS_KEY") &&
|
||||
alltrue([for key, value in service.environment :
|
||||
!can(regex("(?i)(passwords?|secrets?|tokens?|private_keys?|access_keys?|master_keys?|encryption_keys?|read_keys|primary_db(_direct)?|database_url|redis_url)$", key)) &&
|
||||
!can(regex("://[^/@]+:[^/@]+@", value))
|
||||
])
|
||||
])
|
||||
error_message = "Use supported Fargate CPU/memory pairs, known services, immutable image digests, integer counts and secret references rather than inline credentials. GitHub deployment requires backend and consumer services."
|
||||
}
|
||||
}
|
||||
|
||||
data "terraform_remote_state" "foundation" {
|
||||
backend = "s3"
|
||||
config = {
|
||||
bucket = var.state_bucket
|
||||
key = "warmbly/production/foundation.tfstate"
|
||||
region = "us-east-1"
|
||||
}
|
||||
}
|
||||
data "aws_caller_identity" "current" {}
|
||||
locals {
|
||||
foundation = data.terraform_remote_state.foundation.outputs.runtime
|
||||
parameters = data.terraform_remote_state.foundation.outputs.configuration_parameter_arns
|
||||
public_services = {
|
||||
for name, service in var.services : name => service if service.port != null && service.host != null
|
||||
}
|
||||
service_environment = {
|
||||
for name, service in var.services : name => merge(service.environment, name == "backend" ? { DB_MIGRATIONS_ON_START = "false" } : {})
|
||||
}
|
||||
parameter_secrets = {
|
||||
for name, service in var.services : name => [
|
||||
for key, arn in local.parameters : {
|
||||
name = split("/", key)[1]
|
||||
valueFrom = arn
|
||||
} if startswith(key, "${name}/") && !contains(keys(local.service_environment[name]), split("/", key)[1]) && !contains(service.secret_keys, split("/", key)[1])
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "trust" {
|
||||
statement {
|
||||
actions = ["sts:AssumeRole"]
|
||||
principals {
|
||||
type = "Service"
|
||||
identifiers = ["ecs-tasks.amazonaws.com"]
|
||||
}
|
||||
condition {
|
||||
test = "StringEquals"
|
||||
variable = "aws:SourceAccount"
|
||||
values = [data.aws_caller_identity.current.account_id]
|
||||
}
|
||||
condition {
|
||||
test = "ArnLike"
|
||||
variable = "aws:SourceArn"
|
||||
values = ["arn:aws:ecs:us-east-1:${data.aws_caller_identity.current.account_id}:*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
resource "aws_iam_role" "execution" {
|
||||
for_each = var.services
|
||||
name = "${var.name}-${each.key}-execution"
|
||||
assume_role_policy = data.aws_iam_policy_document.trust.json
|
||||
}
|
||||
resource "aws_iam_role" "application" {
|
||||
for_each = var.services
|
||||
name = "${var.name}-${each.key}-task"
|
||||
assume_role_policy = data.aws_iam_policy_document.trust.json
|
||||
}
|
||||
data "aws_iam_policy_document" "execution" {
|
||||
for_each = var.services
|
||||
statement {
|
||||
actions = ["secretsmanager:GetSecretValue"]
|
||||
resources = [local.foundation.secret_arns[each.key]]
|
||||
}
|
||||
statement {
|
||||
actions = ["ssm:GetParameters"]
|
||||
resources = ["arn:aws:ssm:us-east-1:${data.aws_caller_identity.current.account_id}:parameter/${var.name}/config/${each.key}/*"]
|
||||
}
|
||||
statement {
|
||||
actions = ["logs:CreateLogStream", "logs:PutLogEvents"]
|
||||
resources = ["arn:aws:logs:us-east-1:${data.aws_caller_identity.current.account_id}:log-group:/ecs/${var.name}/${each.key}:*"]
|
||||
}
|
||||
statement {
|
||||
actions = ["ecr:GetAuthorizationToken"]
|
||||
resources = ["*"]
|
||||
}
|
||||
statement {
|
||||
actions = ["ecr:BatchCheckLayerAvailability", "ecr:GetDownloadUrlForLayer", "ecr:BatchGetImage"]
|
||||
resources = ["arn:aws:ecr:us-east-1:${data.aws_caller_identity.current.account_id}:repository/${split(".amazonaws.com/", local.foundation.repository_urls[each.key])[1]}"]
|
||||
}
|
||||
}
|
||||
resource "aws_iam_role_policy" "execution" {
|
||||
for_each = var.services
|
||||
name = "runtime-configuration-and-logs"
|
||||
role = aws_iam_role.execution[each.key].id
|
||||
policy = data.aws_iam_policy_document.execution[each.key].json
|
||||
}
|
||||
resource "aws_iam_role_policy" "application" {
|
||||
for_each = var.application_policies
|
||||
name = "application-services"
|
||||
role = aws_iam_role.application[each.key].id
|
||||
policy = each.value
|
||||
}
|
||||
|
||||
resource "aws_security_group" "ingress" {
|
||||
name_prefix = "${var.name}-ingress-"
|
||||
description = "HTTPS origin access from approved callers"
|
||||
vpc_id = local.foundation.vpc_id
|
||||
}
|
||||
resource "aws_vpc_security_group_ingress_rule" "https" {
|
||||
for_each = var.origin_ipv4_cidrs
|
||||
security_group_id = aws_security_group.ingress.id
|
||||
cidr_ipv4 = each.value
|
||||
ip_protocol = "tcp"
|
||||
from_port = 443
|
||||
to_port = 443
|
||||
}
|
||||
resource "aws_vpc_security_group_ingress_rule" "internal_https" {
|
||||
security_group_id = aws_security_group.ingress.id
|
||||
referenced_security_group_id = local.foundation.tasks_sg_id
|
||||
ip_protocol = "tcp"
|
||||
from_port = 443
|
||||
to_port = 443
|
||||
}
|
||||
resource "aws_vpc_security_group_egress_rule" "ingress" {
|
||||
security_group_id = aws_security_group.ingress.id
|
||||
referenced_security_group_id = local.foundation.tasks_sg_id
|
||||
ip_protocol = "tcp"
|
||||
from_port = 1
|
||||
to_port = 65535
|
||||
}
|
||||
resource "aws_vpc_security_group_ingress_rule" "service" {
|
||||
for_each = local.public_services
|
||||
security_group_id = local.foundation.tasks_sg_id
|
||||
referenced_security_group_id = aws_security_group.ingress.id
|
||||
ip_protocol = "tcp"
|
||||
from_port = each.value.port
|
||||
to_port = each.value.port
|
||||
}
|
||||
resource "aws_vpc_security_group_ingress_rule" "internal_backend" {
|
||||
security_group_id = local.foundation.tasks_sg_id
|
||||
referenced_security_group_id = local.foundation.tasks_sg_id
|
||||
ip_protocol = "tcp"
|
||||
from_port = 8080
|
||||
to_port = 8080
|
||||
}
|
||||
resource "aws_lb" "main" {
|
||||
name = "${var.name}-ingress"
|
||||
load_balancer_type = "application"
|
||||
internal = false
|
||||
security_groups = [aws_security_group.ingress.id]
|
||||
subnets = local.foundation.public_subnet_ids
|
||||
idle_timeout = 120
|
||||
drop_invalid_header_fields = true
|
||||
}
|
||||
resource "aws_lb_target_group" "service" {
|
||||
for_each = local.public_services
|
||||
name = "${var.name}-${each.key}"
|
||||
port = each.value.port
|
||||
protocol = "HTTP"
|
||||
target_type = "ip"
|
||||
vpc_id = local.foundation.vpc_id
|
||||
deregistration_delay = 60
|
||||
health_check {
|
||||
path = each.value.health_path
|
||||
matcher = "200"
|
||||
interval = 30
|
||||
timeout = 5
|
||||
healthy_threshold = 2
|
||||
unhealthy_threshold = 3
|
||||
}
|
||||
}
|
||||
resource "aws_lb_listener" "https" {
|
||||
load_balancer_arn = aws_lb.main.arn
|
||||
port = 443
|
||||
protocol = "HTTPS"
|
||||
ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06"
|
||||
certificate_arn = local.foundation.certificate_arns[var.certificate_primary_domain]
|
||||
default_action {
|
||||
type = "fixed-response"
|
||||
fixed_response {
|
||||
content_type = "text/plain"
|
||||
status_code = "404"
|
||||
message_body = "Not found"
|
||||
}
|
||||
}
|
||||
}
|
||||
resource "aws_lb_listener_certificate" "additional" {
|
||||
for_each = { for domain, arn in local.foundation.certificate_arns : domain => arn if domain != var.certificate_primary_domain }
|
||||
listener_arn = aws_lb_listener.https.arn
|
||||
certificate_arn = each.value
|
||||
}
|
||||
resource "aws_lb_listener_rule" "service" {
|
||||
for_each = local.public_services
|
||||
listener_arn = aws_lb_listener.https.arn
|
||||
priority = 100 + index(sort(keys(local.public_services)), each.key)
|
||||
condition {
|
||||
host_header { values = [each.value.host] }
|
||||
}
|
||||
action {
|
||||
type = "forward"
|
||||
target_group_arn = aws_lb_target_group.service[each.key].arn
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_service_discovery_private_dns_namespace" "main" {
|
||||
name = var.namespace
|
||||
vpc = local.foundation.vpc_id
|
||||
}
|
||||
resource "aws_service_discovery_service" "backend" {
|
||||
name = "backend"
|
||||
dns_config {
|
||||
namespace_id = aws_service_discovery_private_dns_namespace.main.id
|
||||
dns_records {
|
||||
ttl = 10
|
||||
type = "A"
|
||||
}
|
||||
routing_policy = "MULTIVALUE"
|
||||
}
|
||||
health_check_custom_config {}
|
||||
lifecycle {
|
||||
ignore_changes = [health_check_custom_config]
|
||||
}
|
||||
}
|
||||
|
||||
locals {
|
||||
service_containers = {
|
||||
for name, service in var.services : name => merge({
|
||||
name = name
|
||||
image = service.image
|
||||
essential = true
|
||||
stopTimeout = 120
|
||||
linuxParameters = { initProcessEnabled = true }
|
||||
environment = [for key, value in local.service_environment[name] : { name = key, value = value }]
|
||||
secrets = concat(local.parameter_secrets[name], [for key in sort(tolist(service.secret_keys)) : {
|
||||
name = key
|
||||
valueFrom = "${local.foundation.secret_arns[name]}:${key}::${service.secret_version_id}"
|
||||
}])
|
||||
logConfiguration = {
|
||||
logDriver = "awslogs"
|
||||
options = {
|
||||
awslogs-group = "/ecs/${var.name}/${name}"
|
||||
awslogs-region = "us-east-1"
|
||||
awslogs-stream-prefix = "service"
|
||||
}
|
||||
}
|
||||
}, jsondecode(service.port == null ? "{}" : jsonencode({
|
||||
portMappings = [{ containerPort = service.port, protocol = "tcp" }]
|
||||
healthCheck = {
|
||||
command = ["CMD-SHELL", "wget -q --spider http://127.0.0.1:${service.port}${service.health_path} || exit 1"]
|
||||
interval = 30
|
||||
timeout = 5
|
||||
retries = 3
|
||||
startPeriod = 60
|
||||
}
|
||||
})))
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_ecs_task_definition" "service" {
|
||||
for_each = var.services
|
||||
skip_destroy = true
|
||||
family = "${var.name}-${each.key}"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
network_mode = "awsvpc"
|
||||
cpu = each.value.cpu
|
||||
memory = each.value.memory
|
||||
execution_role_arn = aws_iam_role.execution[each.key].arn
|
||||
task_role_arn = aws_iam_role.application[each.key].arn
|
||||
runtime_platform {
|
||||
cpu_architecture = "ARM64"
|
||||
operating_system_family = "LINUX"
|
||||
}
|
||||
container_definitions = jsonencode([local.service_containers[each.key]])
|
||||
depends_on = [aws_iam_role_policy.execution, aws_iam_role_policy.application]
|
||||
}
|
||||
resource "aws_ecs_service" "service" {
|
||||
for_each = var.services
|
||||
name = "${var.name}-${each.key}"
|
||||
cluster = local.foundation.cluster_arn
|
||||
task_definition = aws_ecs_task_definition.service[each.key].arn
|
||||
desired_count = each.value.desired_count
|
||||
launch_type = "FARGATE"
|
||||
platform_version = "1.4.0"
|
||||
deployment_minimum_healthy_percent = 100
|
||||
deployment_maximum_percent = 200
|
||||
health_check_grace_period_seconds = contains(keys(local.public_services), each.key) ? 120 : null
|
||||
enable_ecs_managed_tags = true
|
||||
propagate_tags = "SERVICE"
|
||||
lifecycle {
|
||||
ignore_changes = [task_definition, desired_count]
|
||||
}
|
||||
deployment_circuit_breaker {
|
||||
enable = true
|
||||
rollback = true
|
||||
}
|
||||
network_configuration {
|
||||
subnets = local.foundation.public_subnet_ids
|
||||
security_groups = [local.foundation.tasks_sg_id]
|
||||
assign_public_ip = true
|
||||
}
|
||||
dynamic "load_balancer" {
|
||||
for_each = contains(keys(local.public_services), each.key) ? [each.value.port] : []
|
||||
content {
|
||||
target_group_arn = aws_lb_target_group.service[each.key].arn
|
||||
container_name = each.key
|
||||
container_port = load_balancer.value
|
||||
}
|
||||
}
|
||||
dynamic "service_registries" {
|
||||
for_each = each.key == "backend" ? [1] : []
|
||||
content { registry_arn = aws_service_discovery_service.backend.arn }
|
||||
}
|
||||
depends_on = [aws_lb_listener_rule.service]
|
||||
}
|
||||
|
||||
output "runtime" {
|
||||
value = {
|
||||
cluster_arn = local.foundation.cluster_arn
|
||||
load_balancer_dns = aws_lb.main.dns_name
|
||||
load_balancer_arn = aws_lb.main.arn
|
||||
target_group_arns = { for name, group in aws_lb_target_group.service : name => group.arn }
|
||||
task_definitions = { for name, task in aws_ecs_task_definition.service : name => task.arn }
|
||||
task_roles = { for name, role in aws_iam_role.application : name => role.arn }
|
||||
execution_roles = { for name, role in aws_iam_role.execution : name => role.arn }
|
||||
backend_internal = "http://backend.${var.namespace}:8080"
|
||||
}
|
||||
}
|
||||
136
deploy/terraform/aws/runtime/migrations.tf
Normal file
136
deploy/terraform/aws/runtime/migrations.tf
Normal file
@ -0,0 +1,136 @@
|
||||
variable "migration_secret_key" {
|
||||
description = "Direct database URL key in the backend's pinned secret bundle"
|
||||
type = string
|
||||
default = "PRIMARY_DB_DIRECT"
|
||||
validation {
|
||||
condition = contains(["PRIMARY_DB", "PRIMARY_DB_DIRECT"], var.migration_secret_key)
|
||||
error_message = "Migration credentials must come from the backend database configuration."
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_cloudwatch_log_group" "migration" {
|
||||
name = "/ecs/${var.name}/migration"
|
||||
retention_in_days = 14
|
||||
}
|
||||
|
||||
resource "aws_iam_role" "migration_execution" {
|
||||
name = "${var.name}-migration-execution"
|
||||
assume_role_policy = data.aws_iam_policy_document.trust.json
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "migration_execution" {
|
||||
statement {
|
||||
actions = ["secretsmanager:GetSecretValue"]
|
||||
resources = [local.foundation.secret_arns["backend"]]
|
||||
}
|
||||
statement {
|
||||
actions = ["logs:CreateLogStream", "logs:PutLogEvents"]
|
||||
resources = ["${aws_cloudwatch_log_group.migration.arn}:*"]
|
||||
}
|
||||
statement {
|
||||
actions = ["ecr:GetAuthorizationToken"]
|
||||
resources = ["*"]
|
||||
}
|
||||
statement {
|
||||
actions = ["ecr:BatchCheckLayerAvailability", "ecr:GetDownloadUrlForLayer", "ecr:BatchGetImage"]
|
||||
resources = ["arn:aws:ecr:us-east-1:${data.aws_caller_identity.current.account_id}:repository/${split(".amazonaws.com/", local.foundation.repository_urls["backend"])[1]}"]
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy" "migration_execution" {
|
||||
name = "migration-configuration-and-logs"
|
||||
role = aws_iam_role.migration_execution.id
|
||||
policy = data.aws_iam_policy_document.migration_execution.json
|
||||
}
|
||||
|
||||
resource "aws_ecs_task_definition" "migration" {
|
||||
skip_destroy = true
|
||||
family = "${var.name}-migration"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
network_mode = "awsvpc"
|
||||
cpu = "256"
|
||||
memory = "512"
|
||||
execution_role_arn = aws_iam_role.migration_execution.arn
|
||||
runtime_platform {
|
||||
cpu_architecture = "ARM64"
|
||||
operating_system_family = "LINUX"
|
||||
}
|
||||
container_definitions = jsonencode([{
|
||||
name = "migration"
|
||||
image = var.services["backend"].image
|
||||
essential = true
|
||||
entryPoint = ["/app/migrate"]
|
||||
command = []
|
||||
environment = [
|
||||
{ name = "DB_POOL_MODE", value = "direct" }
|
||||
]
|
||||
secrets = [{
|
||||
name = "PRIMARY_DB"
|
||||
valueFrom = "${local.foundation.secret_arns["backend"]}:${var.migration_secret_key}::${var.services["backend"].secret_version_id}"
|
||||
}]
|
||||
logConfiguration = {
|
||||
logDriver = "awslogs"
|
||||
options = {
|
||||
awslogs-group = aws_cloudwatch_log_group.migration.name
|
||||
awslogs-region = "us-east-1"
|
||||
awslogs-stream-prefix = "migration"
|
||||
}
|
||||
}
|
||||
}])
|
||||
depends_on = [aws_iam_role_policy.migration_execution]
|
||||
}
|
||||
|
||||
output "migration_task_definition" {
|
||||
value = aws_ecs_task_definition.migration.arn
|
||||
}
|
||||
|
||||
resource "aws_ecs_task_definition" "database_check" {
|
||||
skip_destroy = true
|
||||
family = "${var.name}-database-check"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
network_mode = "awsvpc"
|
||||
cpu = "256"
|
||||
memory = "512"
|
||||
execution_role_arn = aws_iam_role.execution["backend"].arn
|
||||
task_role_arn = aws_iam_role.application["backend"].arn
|
||||
runtime_platform {
|
||||
cpu_architecture = "ARM64"
|
||||
operating_system_family = "LINUX"
|
||||
}
|
||||
container_definitions = jsonencode([merge({
|
||||
for key, value in local.service_containers["backend"] : key => value if !contains(["portMappings", "healthCheck"], key)
|
||||
}, {
|
||||
name = "database-check"
|
||||
entryPoint = ["/bin/sh", "-c"]
|
||||
command = ["warmblyctl status --json >/tmp/status.json && echo 'Go pooled database client and schema queries succeeded'"]
|
||||
})])
|
||||
}
|
||||
|
||||
output "database_check_task_definition" {
|
||||
value = aws_ecs_task_definition.database_check.arn
|
||||
}
|
||||
|
||||
resource "aws_ecs_task_definition" "realtime_database_check" {
|
||||
skip_destroy = true
|
||||
family = "${var.name}-realtime-database-check"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
network_mode = "awsvpc"
|
||||
cpu = "256"
|
||||
memory = "512"
|
||||
execution_role_arn = aws_iam_role.execution["realtime"].arn
|
||||
runtime_platform {
|
||||
cpu_architecture = "ARM64"
|
||||
operating_system_family = "LINUX"
|
||||
}
|
||||
container_definitions = jsonencode([merge({
|
||||
for key, value in local.service_containers["realtime"] : key => value if !contains(["portMappings", "healthCheck"], key)
|
||||
}, {
|
||||
name = "realtime-database-check"
|
||||
entryPoint = ["/app/bin/realtime", "eval"]
|
||||
command = [file("${path.module}/checks/realtime.exs")]
|
||||
})])
|
||||
}
|
||||
|
||||
output "realtime_database_check_task_definition" {
|
||||
value = aws_ecs_task_definition.realtime_database_check.arn
|
||||
}
|
||||
26
deploy/terraform/aws/runtime/tracking-hosts.tf
Normal file
26
deploy/terraform/aws/runtime/tracking-hosts.tf
Normal file
@ -0,0 +1,26 @@
|
||||
variable "tracking_additional_hosts" {
|
||||
description = "Tracking hostnames with validated DNS and TLS at the edge"
|
||||
type = set(string)
|
||||
default = []
|
||||
validation {
|
||||
condition = alltrue([for host in var.tracking_additional_hosts :
|
||||
length(host) <= 128 && can(regex("^[a-zA-Z0-9][a-zA-Z0-9.-]*\\.[a-zA-Z]{2,}$", host))
|
||||
])
|
||||
error_message = "Provide DNS hostnames supported by an ALB host-header rule."
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_lb_listener_rule" "tracking_alias" {
|
||||
for_each = {
|
||||
for index, hosts in chunklist(sort(tolist(var.tracking_additional_hosts)), 5) : tostring(index) => hosts
|
||||
}
|
||||
listener_arn = aws_lb_listener.https.arn
|
||||
priority = 200 + tonumber(each.key)
|
||||
condition {
|
||||
host_header { values = each.value }
|
||||
}
|
||||
action {
|
||||
type = "forward"
|
||||
target_group_arn = aws_lb_target_group.service["tracking"].arn
|
||||
}
|
||||
}
|
||||
80
deploy/terraform/aws/runtime/validation.tf
Normal file
80
deploy/terraform/aws/runtime/validation.tf
Normal file
@ -0,0 +1,80 @@
|
||||
resource "aws_iam_role" "validation_execution" {
|
||||
name = "${var.name}-validation-execution"
|
||||
assume_role_policy = data.aws_iam_policy_document.trust.json
|
||||
}
|
||||
resource "aws_cloudwatch_log_group" "validation" {
|
||||
name = "/ecs/${var.name}/validation"
|
||||
retention_in_days = 1
|
||||
}
|
||||
data "aws_iam_policy_document" "validation_execution" {
|
||||
statement {
|
||||
actions = ["secretsmanager:GetSecretValue"]
|
||||
resources = concat([
|
||||
for name in ["backend", "consumer", "realtime"] : local.foundation.secret_arns[name]
|
||||
], [data.terraform_remote_state.foundation.outputs.cache.credential_secret])
|
||||
}
|
||||
statement {
|
||||
actions = ["ssm:GetParameters"]
|
||||
resources = [for key in ["backend/KAFKA_BOOTSTRAP_SERVERS", "backend/SCHEMA_REGISTRY_URL", "backend/BLOB_BUCKET"] : local.parameters[key]]
|
||||
}
|
||||
statement {
|
||||
actions = ["logs:CreateLogStream", "logs:PutLogEvents"]
|
||||
resources = ["${aws_cloudwatch_log_group.validation.arn}:*"]
|
||||
}
|
||||
}
|
||||
resource "aws_iam_role_policy" "validation_execution" {
|
||||
name = "dependency-validation"
|
||||
role = aws_iam_role.validation_execution.id
|
||||
policy = data.aws_iam_policy_document.validation_execution.json
|
||||
}
|
||||
resource "aws_ecs_task_definition" "validation" {
|
||||
skip_destroy = true
|
||||
family = "${var.name}-validation"
|
||||
requires_compatibilities = ["FARGATE"]
|
||||
network_mode = "awsvpc"
|
||||
cpu = "512"
|
||||
memory = "1024"
|
||||
execution_role_arn = aws_iam_role.validation_execution.arn
|
||||
task_role_arn = aws_iam_role.application["backend"].arn
|
||||
runtime_platform {
|
||||
cpu_architecture = "ARM64"
|
||||
operating_system_family = "LINUX"
|
||||
}
|
||||
container_definitions = jsonencode([{
|
||||
name = "validation"
|
||||
image = "public.ecr.aws/docker/library/node:22-alpine"
|
||||
essential = true
|
||||
command = ["sh", "-c", "npm install --prefix /tmp/check --omit=dev --ignore-scripts --no-audit --no-fund pg@8.23.1 kafkajs@2.2.4 redis@6.3.0 @aws-sdk/client-kms@3.1155.0 @aws-sdk/client-s3@3.1155.0 >/dev/null 2>&1 && NODE_PATH=/tmp/check/node_modules exec node -e \"$CHECK_SOURCE\""]
|
||||
environment = [
|
||||
{ name = "AWS_REGION", value = "us-east-1" },
|
||||
{ name = "CACHE_HOST", value = data.terraform_remote_state.foundation.outputs.cache.endpoint },
|
||||
{ name = "CHECK_SOURCE", value = file("${path.module}/checks/dependencies.cjs") }
|
||||
]
|
||||
secrets = concat([
|
||||
for key in ["PRIMARY_DB", "CREDENTIALS_ENCRYPTION_KEY", "AUTH_SECRET", "REDIS", "KAFKA_SASL_USERNAME", "KAFKA_SASL_PASSWORD", "SCHEMA_REGISTRY_KEY", "SCHEMA_REGISTRY_SECRET"] : {
|
||||
name = key
|
||||
valueFrom = "${local.foundation.secret_arns["backend"]}:${key}::${var.services["backend"].secret_version_id}"
|
||||
}
|
||||
], [for key in ["CREDENTIALS_ENCRYPTION_READ_KEYS", "TWOFA_SECRET", "TWOFA_READ_SECRETS"] : {
|
||||
name = key
|
||||
valueFrom = "${local.foundation.secret_arns["backend"]}:${key}::${var.services["backend"].secret_version_id}"
|
||||
} if contains(var.services["backend"].secret_keys, key)], [
|
||||
{ name = "CONSUMER_DB", valueFrom = "${local.foundation.secret_arns["consumer"]}:PRIMARY_DB::${var.services["consumer"].secret_version_id}" },
|
||||
{ name = "REALTIME_DB", valueFrom = "${local.foundation.secret_arns["realtime"]}:DATABASE_URL::${var.services["realtime"].secret_version_id}" },
|
||||
{ name = "NEW_REDIS_PASSWORD", valueFrom = "${data.terraform_remote_state.foundation.outputs.cache.credential_secret}:REDIS_PASSWORD::" }
|
||||
], [for key in ["KAFKA_BOOTSTRAP_SERVERS", "SCHEMA_REGISTRY_URL", "BLOB_BUCKET"] : {
|
||||
name = key
|
||||
valueFrom = local.parameters["backend/${key}"]
|
||||
}])
|
||||
logConfiguration = {
|
||||
logDriver = "awslogs"
|
||||
options = {
|
||||
awslogs-group = aws_cloudwatch_log_group.validation.name
|
||||
awslogs-region = "us-east-1"
|
||||
awslogs-stream-prefix = "check"
|
||||
}
|
||||
}
|
||||
}])
|
||||
depends_on = [aws_iam_role_policy.validation_execution, aws_iam_role_policy.application]
|
||||
}
|
||||
output "validation_task_definition" { value = aws_ecs_task_definition.validation.arn }
|
||||
26
deploy/terraform/aws/workers/.terraform.lock.hcl
generated
Normal file
26
deploy/terraform/aws/workers/.terraform.lock.hcl
generated
Normal file
@ -0,0 +1,26 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/hashicorp/aws" {
|
||||
version = "6.70.0"
|
||||
constraints = "~> 6.0"
|
||||
hashes = [
|
||||
"h1:L+05gOXA+kdYTtayIw0WxdaTp3JryZCMZpfrJVhoJA0=",
|
||||
"zh:068ca7ed4fc77e3d63d7fac54515a426a32f3dc143b55a6ad00963fe68289955",
|
||||
"zh:2d8c016b3809ab32f536b01bf2c98fcbdeb066b7b7e50f756ff6ff1e0b0539c7",
|
||||
"zh:450d960b60ea81268bd355600340787254f750d76e0075208a6fb903d7336b09",
|
||||
"zh:50447833cce21288ffc331342f9d83854e80515e2d86e8f5776a07ce18866b20",
|
||||
"zh:58acc041b080778ded75320e241ebcf6cf2ed145bede644b86a4784ce46086c6",
|
||||
"zh:7e899a44a1da95fefb1b04b231cf7cdb5871f482f60a710ede78bf25ac265724",
|
||||
"zh:8b141d13f4b84d7a47bc5a4855d0fc985191317b3249603532d4de03db5741c5",
|
||||
"zh:9035357f7d065ce8d35a9b05337977910683219c0f13fa09155d28f9b32e23a7",
|
||||
"zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
|
||||
"zh:9ba0990a876d2641a185ca88263719afc4d5ac43533bdb147437f0073d2cd74d",
|
||||
"zh:a8773c7a02e1b3290df89351ee493fd8b74f807f92a7f69534e05901d4515761",
|
||||
"zh:ab0ebb298578eeeffe0fbf0a8b27ce99acd4419b7049272f1ab87e8c47229407",
|
||||
"zh:b88a3f22127bfb17604adc53e9bdb8ed0eddf4b9b77bf19871cba2e7fcd5b4a3",
|
||||
"zh:c7b1fb41e7f086b670f78932ede2b539903a8d71bac06f55f14a1a33d09a1ccf",
|
||||
"zh:df482bd93f51125f78feab7742aae1d7f22c7108ee2dc47787c4a1d611e6ccfd",
|
||||
"zh:fca401ffaa42d5f350d667123569de51a23de8c30b87ce0bd9d86fc80835fd4d",
|
||||
]
|
||||
}
|
||||
49
deploy/terraform/aws/workers/README.md
Normal file
49
deploy/terraform/aws/workers/README.md
Normal file
@ -0,0 +1,49 @@
|
||||
# EC2 worker hosts
|
||||
|
||||
This root prepares one ARM `t4g.small` per worker in `us-east-1`, each with an
|
||||
encrypted 16 GiB gp3 disk, static IPv4 address and pinned worker image. Standard
|
||||
CPU credits cap burst charges; depleted credits throttle to baseline. Monitor
|
||||
credit balance and fleet health before changing size or credit mode.
|
||||
|
||||
```sh
|
||||
terraform -chdir=deploy/terraform/aws/workers init \
|
||||
-backend-config="bucket=$STATE_BUCKET"
|
||||
terraform -chdir=deploy/terraform/aws/workers plan \
|
||||
-var-file=/private/path/workers.tfvars.json \
|
||||
-out=/private/path/workers.tfplan
|
||||
terraform -chdir=deploy/terraform/aws/workers apply /private/path/workers.tfplan
|
||||
```
|
||||
|
||||
Supply the SSH public key, operator CIDRs, cache security group ID and worker
|
||||
image metadata through private inputs. SSH is restricted to those CIDRs, and
|
||||
IMDSv2 is required. Hosts have no instance profile or application AWS
|
||||
credentials. Workers use brokered KMS and blob operations through the backend.
|
||||
|
||||
Bootstrap installs Docker and prepares systemd without starting the worker. An
|
||||
operator delivers configuration from the worker's Secrets Manager container
|
||||
over SSH into `/etc/warmbly/node.env`, owned by root with mode `0600`. Credentials
|
||||
never enter user data, Terraform or command arguments. Preserve worker IDs,
|
||||
mailbox encryption keys, OAuth identities and compatible broker configuration.
|
||||
Package installation and image download retry while the static IP association
|
||||
becomes usable during first boot.
|
||||
|
||||
Never run both hosts with the same worker ID. Drain and stop the previous
|
||||
process, confirm it exited, start the replacement, and verify fresh heartbeats,
|
||||
command consumption and mailbox health before advancing to the next host. The
|
||||
prepared service allows 120 seconds for Docker shutdown. Preserve the previous
|
||||
host for rollback. Static AWS IPs remain stable, but differ from the previous
|
||||
provider's addresses, so verify mailbox authentication during the handover.
|
||||
|
||||
Private Valkey access is allowed from the worker security group. Coordinate its
|
||||
activation with every control-plane client, and check configured mail ports and
|
||||
AWS outbound restrictions before moving work. Do not switch one worker to an
|
||||
empty independent cache.
|
||||
|
||||
The dependency check expects `CACHE_HOST` and `NEW_REDIS_PASSWORD` in addition to
|
||||
the worker environment. Supply the replacement password through protected
|
||||
temporary input. It verifies both TLS certificate validation and an authenticated
|
||||
`PING` against private Valkey, without changing the active cache endpoint.
|
||||
|
||||
Instances and addresses have destruction guards. AMI and bootstrap drift do not
|
||||
silently replace a running worker. Roll out host upgrades deliberately while
|
||||
preserving worker identity and static addresses.
|
||||
39
deploy/terraform/aws/workers/bootstrap.sh.tftpl
Normal file
39
deploy/terraform/aws/workers/bootstrap.sh.tftpl
Normal file
@ -0,0 +1,39 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
hostnamectl set-hostname '${name}'
|
||||
retry() {
|
||||
attempts=0
|
||||
while ! "$@"; do
|
||||
attempts=$((attempts + 1))
|
||||
if [ "$attempts" -ge 24 ]; then return 1; fi
|
||||
sleep 5
|
||||
done
|
||||
return 0
|
||||
}
|
||||
retry dnf -y --setopt=retries=20 install docker
|
||||
systemctl enable --now docker
|
||||
install -d -m 700 /etc/warmbly /var/lib/warmbly /var/lib/warmbly/node
|
||||
printf '%s\n' 'WARMBLY_IMAGE_REF=${image}' > /var/lib/warmbly/image-ref
|
||||
chmod 600 /var/lib/warmbly/image-ref
|
||||
cat > /etc/systemd/system/warmbly-worker.service <<'UNIT'
|
||||
[Unit]
|
||||
Description=Warmbly worker
|
||||
After=docker.service network-online.target
|
||||
Requires=docker.service
|
||||
ConditionPathExists=/etc/warmbly/node.env
|
||||
|
||||
[Service]
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
EnvironmentFile=/var/lib/warmbly/image-ref
|
||||
ExecStartPre=-/usr/bin/docker rm -f warmbly-worker
|
||||
ExecStart=/usr/bin/docker run --rm --name warmbly-worker --env-file /etc/warmbly/node.env --env AWS_EC2_METADATA_DISABLED=true --network host -v /var/lib/warmbly/node:/var/lib/warmbly/node $${WARMBLY_IMAGE_REF}
|
||||
ExecStop=/usr/bin/docker stop -t 120 warmbly-worker
|
||||
TimeoutStopSec=150
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNIT
|
||||
systemctl daemon-reload
|
||||
retry docker pull '${image}'
|
||||
printf '%s\n' 'Worker host ready; service stays stopped until an operator completes the handover'
|
||||
42
deploy/terraform/aws/workers/checks/dependencies.cjs
Normal file
42
deploy/terraform/aws/workers/checks/dependencies.cjs
Normal file
@ -0,0 +1,42 @@
|
||||
const https = require("node:https");
|
||||
const { Kafka, logLevel } = require("kafkajs");
|
||||
const { createClient } = require("redis");
|
||||
|
||||
const results = [];
|
||||
const timer = setTimeout(() => process.exit(1), 120000);
|
||||
async function check(name, fn) {
|
||||
try { await fn(); results.push(true); console.log(JSON.stringify({ name, ok: true })); }
|
||||
catch { results.push(false); console.log(JSON.stringify({ name, ok: false })); }
|
||||
}
|
||||
async function main() {
|
||||
await check("shared_cache", async () => {
|
||||
const client = createClient({ url: process.env.REDIS, socket: { connectTimeout: 10000, reconnectStrategy: false } });
|
||||
client.on("error", () => {});
|
||||
try { await client.connect(); if (await client.ping() !== "PONG") throw new Error("Cache check failed"); }
|
||||
finally { client.destroy(); }
|
||||
});
|
||||
await check("private_cache_tls", async () => {
|
||||
if (!process.env.NEW_REDIS_PASSWORD) throw new Error("Replacement cache credential required");
|
||||
const client = createClient({ password: process.env.NEW_REDIS_PASSWORD, socket: { host: process.env.CACHE_HOST, port: 6379, tls: true, servername: process.env.CACHE_HOST, rejectUnauthorized: true, connectTimeout: 10000, reconnectStrategy: false } });
|
||||
client.on("error", () => {});
|
||||
try { await client.connect(); if (await client.ping() !== "PONG") throw new Error("Cache check failed"); }
|
||||
finally { client.destroy(); }
|
||||
});
|
||||
await check("replacement_kafka_key", async () => {
|
||||
const kafka = new Kafka({ clientId: "worker-migration-validation", brokers: process.env.KAFKA_BOOTSTRAP_SERVERS.split(","), ssl: true, sasl: { mechanism: "plain", username: process.env.KAFKA_SASL_USERNAME, password: process.env.KAFKA_SASL_PASSWORD }, logLevel: logLevel.NOTHING, connectionTimeout: 10000, requestTimeout: 15000, retry: { retries: 1 } });
|
||||
const admin = kafka.admin({ allowAutoTopicCreation: false });
|
||||
try { await admin.connect(); await admin.fetchTopicMetadata({ topics: ["tracking-events"] }); }
|
||||
finally { await admin.disconnect(); }
|
||||
});
|
||||
await check("broker_authorization", () => new Promise((resolve, reject) => {
|
||||
const token = process.env.NODE_BROKER_TOKEN || process.env.ENCRYPTED_KEYS_WORKER_TOKEN || process.env.INTERNAL_API_TOKEN;
|
||||
const url = new URL("/api/v1/internal/dek/00000000-0000-0000-0000-000000000000", process.env.ENCRYPTED_KEYS_BACKEND_URL);
|
||||
const req = https.get(url, { headers: { authorization: "Bearer " + token }, timeout: 15000 }, (res) => { res.resume(); res.on("end", () => [200, 404].includes(res.statusCode) ? resolve() : reject(new Error("Broker refused request"))); });
|
||||
req.on("timeout", () => req.destroy());
|
||||
req.on("error", reject);
|
||||
}));
|
||||
clearTimeout(timer);
|
||||
console.log(JSON.stringify({ checks: results.length, passed: results.filter(Boolean).length, worker_started: false }));
|
||||
process.exit(results.every(Boolean) ? 0 : 1);
|
||||
}
|
||||
main().catch(() => process.exit(1));
|
||||
132
deploy/terraform/aws/workers/main.tf
Normal file
132
deploy/terraform/aws/workers/main.tf
Normal file
@ -0,0 +1,132 @@
|
||||
terraform {
|
||||
required_version = ">= 1.11, < 2.0"
|
||||
backend "s3" {
|
||||
key = "warmbly/production/workers.tfstate"
|
||||
region = "us-east-1"
|
||||
encrypt = true
|
||||
use_lockfile = true
|
||||
}
|
||||
required_providers {
|
||||
aws = { source = "hashicorp/aws", version = "~> 6.0" }
|
||||
}
|
||||
}
|
||||
provider "aws" {
|
||||
region = "us-east-1"
|
||||
default_tags {
|
||||
tags = { Application = "warmbly", Environment = "production", ManagedBy = "terraform" }
|
||||
}
|
||||
}
|
||||
variable "state_bucket" { type = string }
|
||||
variable "name" {
|
||||
type = string
|
||||
default = "warmbly-production"
|
||||
}
|
||||
variable "operator_public_key" { type = string }
|
||||
variable "operator_ssh_cidrs" {
|
||||
type = set(string)
|
||||
validation {
|
||||
condition = length(var.operator_ssh_cidrs) > 0 && alltrue([for cidr in var.operator_ssh_cidrs : can(cidrnetmask(cidr)) && cidr != "0.0.0.0/0"])
|
||||
error_message = "SSH must be restricted to explicit operator IPv4 CIDRs."
|
||||
}
|
||||
}
|
||||
variable "cache_security_group_id" { type = string }
|
||||
variable "workers" {
|
||||
type = map(object({
|
||||
image = string
|
||||
zone_index = number
|
||||
}))
|
||||
validation {
|
||||
condition = alltrue([for name, worker in var.workers : can(regex("^[a-z0-9-]+$", name)) && can(regex("@sha256:[a-f0-9]{64}$", worker.image)) && contains([0, 1], worker.zone_index)])
|
||||
error_message = "Use safe host names, immutable ARM64 image digests and one of the foundation's two subnet indexes."
|
||||
}
|
||||
}
|
||||
data "terraform_remote_state" "foundation" {
|
||||
backend = "s3"
|
||||
config = { bucket = var.state_bucket, key = "warmbly/production/foundation.tfstate", region = "us-east-1" }
|
||||
}
|
||||
data "aws_ssm_parameter" "ami" {
|
||||
name = "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64"
|
||||
}
|
||||
resource "aws_secretsmanager_secret" "worker" {
|
||||
for_each = var.workers
|
||||
name = "${var.name}/worker/${each.key}"
|
||||
description = "Worker configuration delivered by an operator; values stay outside Terraform"
|
||||
recovery_window_in_days = 30
|
||||
}
|
||||
resource "aws_key_pair" "operator" {
|
||||
key_name_prefix = "${var.name}-workers-"
|
||||
public_key = var.operator_public_key
|
||||
}
|
||||
resource "aws_security_group" "worker" {
|
||||
name_prefix = "${var.name}-workers-"
|
||||
description = "Outbound mail workers; SSH only from operators"
|
||||
vpc_id = data.terraform_remote_state.foundation.outputs.runtime.vpc_id
|
||||
}
|
||||
resource "aws_vpc_security_group_ingress_rule" "ssh" {
|
||||
for_each = var.operator_ssh_cidrs
|
||||
security_group_id = aws_security_group.worker.id
|
||||
cidr_ipv4 = each.value
|
||||
ip_protocol = "tcp"
|
||||
from_port = 22
|
||||
to_port = 22
|
||||
}
|
||||
resource "aws_vpc_security_group_egress_rule" "worker" {
|
||||
security_group_id = aws_security_group.worker.id
|
||||
ip_protocol = "-1"
|
||||
cidr_ipv4 = "0.0.0.0/0"
|
||||
}
|
||||
resource "aws_vpc_security_group_ingress_rule" "cache" {
|
||||
security_group_id = var.cache_security_group_id
|
||||
referenced_security_group_id = aws_security_group.worker.id
|
||||
ip_protocol = "tcp"
|
||||
from_port = 6379
|
||||
to_port = 6379
|
||||
}
|
||||
resource "aws_instance" "worker" {
|
||||
for_each = var.workers
|
||||
ami = nonsensitive(data.aws_ssm_parameter.ami.value)
|
||||
instance_type = "t4g.small"
|
||||
subnet_id = data.terraform_remote_state.foundation.outputs.runtime.public_subnet_ids[each.value.zone_index]
|
||||
vpc_security_group_ids = [aws_security_group.worker.id]
|
||||
key_name = aws_key_pair.operator.key_name
|
||||
monitoring = false
|
||||
metadata_options {
|
||||
http_endpoint = "enabled"
|
||||
http_tokens = "required"
|
||||
http_put_response_hop_limit = 1
|
||||
instance_metadata_tags = "disabled"
|
||||
}
|
||||
credit_specification { cpu_credits = "standard" }
|
||||
root_block_device {
|
||||
volume_size = 16
|
||||
volume_type = "gp3"
|
||||
encrypted = true
|
||||
delete_on_termination = true
|
||||
}
|
||||
user_data = templatefile("${path.module}/bootstrap.sh.tftpl", { name = each.key, image = each.value.image })
|
||||
tags = { Name = each.key, Role = "worker" }
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
ignore_changes = [ami, user_data]
|
||||
}
|
||||
}
|
||||
resource "aws_eip" "worker" {
|
||||
for_each = var.workers
|
||||
domain = "vpc"
|
||||
tags = { Name = each.key }
|
||||
lifecycle { prevent_destroy = true }
|
||||
}
|
||||
resource "aws_eip_association" "worker" {
|
||||
for_each = var.workers
|
||||
allocation_id = aws_eip.worker[each.key].id
|
||||
instance_id = aws_instance.worker[each.key].id
|
||||
}
|
||||
output "workers" {
|
||||
value = { for name, instance in aws_instance.worker : name => {
|
||||
instance_id = instance.id
|
||||
public_ip = aws_eip.worker[name].public_ip
|
||||
private_ip = instance.private_ip
|
||||
ssh_user = "ec2-user"
|
||||
secret_arn = aws_secretsmanager_secret.worker[name].arn
|
||||
} }
|
||||
}
|
||||
25
deploy/terraform/confluent/.terraform.lock.hcl
generated
Normal file
25
deploy/terraform/confluent/.terraform.lock.hcl
generated
Normal file
@ -0,0 +1,25 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/confluentinc/confluent" {
|
||||
version = "2.91.0"
|
||||
constraints = "~> 2.91.0"
|
||||
hashes = [
|
||||
"h1:0jFJ6LlwvUWtqdq1f+UsY3KeF3TOvy1rccnaNu22ebA=",
|
||||
"zh:0bd70bd4e05be90c3ef710c8b942e6e2623636209ae86911a12c0ed89b7adfb9",
|
||||
"zh:127a3f48ccd4d26cf460a0550fdf86ffff7e284e39b1c25dddc117d2afdc7edb",
|
||||
"zh:2a922ffe0cab1278b67bc03174163546784ff8aa1304d031d687476028c019cc",
|
||||
"zh:42315f0a12d3e20b15802fb79ba787debc062fe4276f87db42b98ac364a6c0d9",
|
||||
"zh:45996f0d147443de8183cb55c1c3b5c921d82d36b09a7dfd5f19f9ec442abea9",
|
||||
"zh:5965e0396afd059e3aac0b61f1ba2494cd34faabfa0014ac8fe6aafa1b3858fd",
|
||||
"zh:641a9b9b554e15bafc3e134c1eabeeca590da04fd9f40c73e1a0eb726ee6f8b7",
|
||||
"zh:688ca7d857c4f1dd3c45e83bae37ffa8550a48de5b69b4eca96ffb431140f499",
|
||||
"zh:84dfef63de88fddc8caf1fb7c38a24da1d4c4b041d32af7b3eb776fb279c8b6f",
|
||||
"zh:99067d2be596a2fdcda70101b818fe44c5756e0262dd4924427f020ad1940265",
|
||||
"zh:cac64a51f1f923467145680f1deac9df428c7d9f5c8ca540918982d23e3c9e38",
|
||||
"zh:d0756e7099cd785d957b0a2b8b92c3ab39709c8096efa9ed8d7b3870edee90aa",
|
||||
"zh:d1e3017038dcb4a9513a783f937cdb7b21671fc48842f2a71e29b4889bf5f42e",
|
||||
"zh:df4c262fc61c3ad5f41ebe0430d174c3828510f7c8e9568c599fc2bb2ebdbb11",
|
||||
"zh:ea60d96b77ce5040b35e81ea1e940d19dc14890e0d691cb5826ee313852d9363",
|
||||
]
|
||||
}
|
||||
127
deploy/terraform/confluent/main.tf
Normal file
127
deploy/terraform/confluent/main.tf
Normal file
@ -0,0 +1,127 @@
|
||||
terraform {
|
||||
required_version = ">= 1.11, < 2.0"
|
||||
backend "s3" {
|
||||
key = "warmbly/production/confluent.tfstate"
|
||||
region = "us-east-1"
|
||||
encrypt = true
|
||||
use_lockfile = true
|
||||
}
|
||||
required_providers {
|
||||
confluent = {
|
||||
source = "confluentinc/confluent"
|
||||
version = "~> 2.91.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "confluent" {}
|
||||
|
||||
variable "environment_id" {
|
||||
type = string
|
||||
validation {
|
||||
condition = startswith(var.environment_id, "env-")
|
||||
error_message = "Provide the ID of the existing Confluent environment."
|
||||
}
|
||||
}
|
||||
|
||||
variable "environment_name" {
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "kafka_cluster_id" {
|
||||
type = string
|
||||
validation {
|
||||
condition = startswith(var.kafka_cluster_id, "lkc-")
|
||||
error_message = "Provide the ID of the existing Kafka cluster."
|
||||
}
|
||||
}
|
||||
|
||||
variable "kafka_cluster_name" {
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "kafka_region" {
|
||||
description = "Region of the existing cluster; adoption must preserve its placement"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "kafka_availability" {
|
||||
description = "Existing availability configuration; adoption must not change it"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "service_account_id" {
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "service_account_name" {
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "service_account_description" {
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
resource "confluent_environment" "production" {
|
||||
display_name = var.environment_name
|
||||
stream_governance {
|
||||
package = "ESSENTIALS"
|
||||
}
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "confluent_kafka_cluster" "production" {
|
||||
display_name = var.kafka_cluster_name
|
||||
availability = var.kafka_availability
|
||||
cloud = "AWS"
|
||||
region = var.kafka_region
|
||||
basic {}
|
||||
environment {
|
||||
id = confluent_environment.production.id
|
||||
}
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "confluent_service_account" "production" {
|
||||
display_name = var.service_account_name
|
||||
description = var.service_account_description
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
data "confluent_schema_registry_cluster" "existing" {
|
||||
environment {
|
||||
id = var.environment_id
|
||||
}
|
||||
}
|
||||
|
||||
import {
|
||||
to = confluent_environment.production
|
||||
id = var.environment_id
|
||||
}
|
||||
|
||||
import {
|
||||
to = confluent_kafka_cluster.production
|
||||
id = "${var.environment_id}/${var.kafka_cluster_id}"
|
||||
}
|
||||
|
||||
import {
|
||||
to = confluent_service_account.production
|
||||
id = var.service_account_id
|
||||
}
|
||||
|
||||
output "event_bus" {
|
||||
value = {
|
||||
environment_id = confluent_environment.production.id
|
||||
cluster_id = confluent_kafka_cluster.production.id
|
||||
bootstrap_endpoint = confluent_kafka_cluster.production.bootstrap_endpoint
|
||||
schema_registry_endpoint = data.confluent_schema_registry_cluster.existing.rest_endpoint
|
||||
service_account_id = confluent_service_account.production.id
|
||||
}
|
||||
}
|
||||
24
deploy/terraform/planetscale/.terraform.lock.hcl
generated
Normal file
24
deploy/terraform/planetscale/.terraform.lock.hcl
generated
Normal file
@ -0,0 +1,24 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/planetscale/planetscale" {
|
||||
version = "1.14.1"
|
||||
constraints = "~> 1.14.1"
|
||||
hashes = [
|
||||
"h1:7saJHztKmxsmeHH69TvHtfk9KPO6sHe/GkuJ21bjUg8=",
|
||||
"zh:2184ff71844d7145abe2661b66bebb652eb634fcc1f3961d1db161946c87a41c",
|
||||
"zh:3273541636e305cfbe9602522d854f04195f3006c14cdac7be70109c324fbf49",
|
||||
"zh:3ff8eed2e41c290e0d05a3ecb15e55e9953b0036576ec6bf1abb8b768eab7f2e",
|
||||
"zh:4c8f4227ef6736f9638cf6fcc284441f5b8efe9e5b6b8a505d58379f34ba97df",
|
||||
"zh:5ad1b122ef1993adf6e0c99ab15e6fb42babe57300f0f252cc4f2b42d3d226ed",
|
||||
"zh:5c8fe1314c52839f1879d20f3e00d97d2de30c2c5c2c9f3c3d0248070d4c5f90",
|
||||
"zh:62c11a290116e25a3bea89a0a8f586bb3d4d2eeae00fb5f1db944b181959c9e1",
|
||||
"zh:71018bfb27a981b9efc8a0a2c5b4dcafad4c03dcb938f3f3468585949e6c1a65",
|
||||
"zh:86e95cf6c6f610ccd22d6ea6814a0fa9ff536569d504512e4b2b57741cc18293",
|
||||
"zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f",
|
||||
"zh:d0d47a78e8d3272bbc9c3715646c6e037affe66dac691bab7b4deabe68791288",
|
||||
"zh:dd93517ec4213000052559cd0a1ec2d8b26fabf7596f2d5d328d3839f731505b",
|
||||
"zh:e6f8920df8a527a18892120f82b765c9e039ed0f89690d9d8e88b10ac9ebec05",
|
||||
"zh:fea4baa7a5147f751ceb159b2745a85b2203ca7e8d01969b38e36337732027e9",
|
||||
]
|
||||
}
|
||||
77
deploy/terraform/planetscale/main.tf
Normal file
77
deploy/terraform/planetscale/main.tf
Normal file
@ -0,0 +1,77 @@
|
||||
terraform {
|
||||
required_version = ">= 1.11, < 2.0"
|
||||
backend "s3" {
|
||||
key = "warmbly/production/planetscale.tfstate"
|
||||
region = "us-east-1"
|
||||
encrypt = true
|
||||
use_lockfile = true
|
||||
}
|
||||
required_providers {
|
||||
planetscale = {
|
||||
source = "planetscale/planetscale"
|
||||
version = "~> 1.14.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "planetscale" {}
|
||||
|
||||
variable "organization" {
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "database" {
|
||||
description = "Existing Postgres database name"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "branch" {
|
||||
description = "Existing production branch name"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "branch_id" {
|
||||
description = "ID of the existing branch to adopt, never a new branch"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "cluster_size" {
|
||||
description = "Current cluster SKU; adoption must not resize the database"
|
||||
type = string
|
||||
}
|
||||
|
||||
data "planetscale_database_postgres" "existing" {
|
||||
organization = var.organization
|
||||
id = var.database
|
||||
}
|
||||
|
||||
resource "planetscale_postgres_branch" "production" {
|
||||
organization = var.organization
|
||||
database = var.database
|
||||
name = var.branch
|
||||
cluster_size = var.cluster_size
|
||||
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
ignore_changes = [parameters]
|
||||
}
|
||||
}
|
||||
|
||||
import {
|
||||
to = planetscale_postgres_branch.production
|
||||
id = jsonencode({
|
||||
organization = var.organization
|
||||
database = var.database
|
||||
id = var.branch_id
|
||||
})
|
||||
}
|
||||
|
||||
output "database" {
|
||||
value = {
|
||||
name = data.planetscale_database_postgres.existing.name
|
||||
branch = planetscale_postgres_branch.production.name
|
||||
branch_id = planetscale_postgres_branch.production.id
|
||||
replicas = planetscale_postgres_branch.production.replicas
|
||||
cluster_sku = planetscale_postgres_branch.production.cluster_size
|
||||
}
|
||||
}
|
||||
@ -74,6 +74,7 @@ x-selfhost-env: &selfhost-env
|
||||
KMS_LOCAL_MASTER_KEY: ${KMS_LOCAL_MASTER_KEY:-Xr0JA7gqF2POy29a7MRByyqddivTNt8WOyKsOXklazk=}
|
||||
KMS_AWS_KEY_ID: ${KMS_AWS_KEY_ID:-}
|
||||
CREDENTIALS_ENCRYPTION_KEY: ${CREDENTIALS_ENCRYPTION_KEY:-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}
|
||||
CREDENTIALS_ENCRYPTION_READ_KEYS: ${CREDENTIALS_ENCRYPTION_READ_KEYS:-}
|
||||
|
||||
# Blob storage. s3 also needs BLOB_BUCKET + the AWS_* values below.
|
||||
BLOB_PROVIDER: ${BLOB_PROVIDER:-filesystem}
|
||||
@ -192,11 +193,21 @@ x-selfhost-env: &selfhost-env
|
||||
PRIMARY_DB: ${PRIMARY_DB:-postgres://warmbly:warmbly@postgres:5432/warmbly_dev?sslmode=disable}
|
||||
REDIS: ${REDIS:-redis://redis:6379}
|
||||
AUTH_SECRET: ${AUTH_SECRET:-local-dev-auth-secret-minimum-32-characters-long}
|
||||
TWOFA_SECRET: ${TWOFA_SECRET:-}
|
||||
TWOFA_READ_SECRETS: ${TWOFA_READ_SECRETS:-}
|
||||
UNSUBSCRIBE_SIGNING_SECRET: ${UNSUBSCRIBE_SIGNING_SECRET:-}
|
||||
UNSUBSCRIBE_SIGNING_READ_SECRETS: ${UNSUBSCRIBE_SIGNING_READ_SECRETS:-}
|
||||
UNSUBSCRIBE_SIGNING_READ_UNTIL: ${UNSUBSCRIBE_SIGNING_READ_UNTIL:-}
|
||||
DOMAIN_PROOF_SECRET: ${DOMAIN_PROOF_SECRET:-}
|
||||
INTERNAL_API_TOKEN: ${INTERNAL_API_TOKEN:-local-dev-internal-token}
|
||||
INTERNAL_API_PREVIOUS_TOKENS: ${INTERNAL_API_PREVIOUS_TOKENS:-}
|
||||
INTERNAL_API_PREVIOUS_UNTIL: ${INTERNAL_API_PREVIOUS_UNTIL:-}
|
||||
# The broker routes (open a data key, sign a blob operation, mint a managed
|
||||
# mailbox's access token). Optional; empty falls back to the token above, and
|
||||
# every service that needs it reads it from here so the two never disagree.
|
||||
NODE_BROKER_TOKEN: ${NODE_BROKER_TOKEN:-}
|
||||
NODE_BROKER_PREVIOUS_TOKENS: ${NODE_BROKER_PREVIOUS_TOKENS:-}
|
||||
NODE_BROKER_PREVIOUS_UNTIL: ${NODE_BROKER_PREVIOUS_UNTIL:-}
|
||||
|
||||
# AI assistant. Omit AI_PROVIDER to run with AI off (endpoints return 503).
|
||||
AI_PROVIDER: ${AI_PROVIDER:-}
|
||||
|
||||
@ -70,6 +70,7 @@ Campaigns, steps and activity logs are scoped to the selected organization for s
|
||||
| GET | `/campaigns` | `READ_CAMPAIGNS` |
|
||||
| GET | `/campaigns-overview` | `READ_CAMPAIGNS` |
|
||||
| POST | `/campaigns-estimate` | `READ_CAMPAIGNS` |
|
||||
| POST | `/campaign-imports` | `WRITE_CAMPAIGNS` |
|
||||
| POST | `/campaigns` | `WRITE_CAMPAIGNS` |
|
||||
| GET | `/campaigns/:id` | `READ_CAMPAIGNS` |
|
||||
| PATCH | `/campaigns/:id` | `WRITE_CAMPAIGNS` |
|
||||
@ -525,6 +526,10 @@ With [multiple dashboards sharing a backend](/development/configuration/#google-
|
||||
|
||||
- `POST /auth/login`, `/auth/login/confirm`, `/auth/register`, `/auth/register/confirm`, `/auth/refresh`, `/auth/reset-password`, `/auth/reset-password/confirm`
|
||||
- `POST /auth/reauth` (re-prove the account holder behind a live session, for the changes that require a recent confirmation)
|
||||
- `GET /auth/reauth/methods` (which ways this account can confirm: password, two-factor code, passkey, and the linked sign-in providers it can re-verify)
|
||||
- `POST /auth/reauth/passkey/begin`, `/auth/reauth/passkey/finish` (confirm with one of the account's own passkeys)
|
||||
- `POST /auth/reauth/sso/begin`, `/auth/reauth/sso/finish` (confirm through the Google, Apple or single sign-on identity linked to the account, in a browser window)
|
||||
- `POST /auth/reauth/native/begin`, `/auth/reauth/native` (confirm with a native Sign in with Apple or Google ID token carrying the session's nonce)
|
||||
- `GET /auth/config` (public deployment capabilities: which sign-in methods this backend has enabled, whether a login code step follows, whether signups are open, whether the instance still needs claiming)
|
||||
- `POST /auth/app-attest/challenge`, `/auth/app-attest/keys` (the native app's [App Attest](/api/reference/account-org/#app-attest-native-apps) challenge and one-time key registration, on their own per-IP budget `APP_ATTEST_IP_RATE_LIMIT`)
|
||||
- `GET /auth/instance` (JWT only: the running Warmbly version of a self-hosted instance and whether a newer release exists, for the dashboard's version pill; a hosted deployment answers `self_hosted: false` and nothing else)
|
||||
@ -552,11 +557,31 @@ These changes require the session to have confirmed the account holder within th
|
||||
| Record, use or remove an admin grant over a whole domain | `POST /emails/grants/google/finish`, `POST /emails/grants/microsoft/finish`, `POST /emails/grants/:id/connect`, `DELETE /emails/grants/:id` |
|
||||
| Admin panel: issue a fleet join token, grant admin permissions, export or import any workspace | `POST /admin/fleet/join-token`, `POST /admin/admins/:userId/grant`, `POST /admin/organizations/:id/exports`, `POST /admin/organizations/:id/imports` |
|
||||
|
||||
Each either hands out a credential that outlives the session that created it, reveals a secret, changes who can get in, cannot be reversed by the person it was done to, or reaches a whole domain's mail. Without a confirmation they answer `403` with code `reauth_required`; confirm with `POST /auth/reauth` (password or a current two-factor code) and retry. See [error codes](/api/error-codes/#confirmation-required).
|
||||
Each either hands out a credential that outlives the session that created it, reveals a secret, changes who can get in, cannot be reversed by the person it was done to, or reaches a whole domain's mail. Without a confirmation they answer `403` with code `reauth_required`; confirm and retry. See [error codes](/api/error-codes/#confirmation-required).
|
||||
|
||||
`GET /auth/reauth/methods` says which ways the account can confirm, so a client offers only those:
|
||||
|
||||
```json
|
||||
{
|
||||
"password": false,
|
||||
"totp": false,
|
||||
"passkey": true,
|
||||
"providers": [{ "id": "google", "label": "Google" }],
|
||||
"native_providers": ["google"],
|
||||
"valid_for_seconds": 300
|
||||
}
|
||||
```
|
||||
|
||||
- **Password or code:** `POST /auth/reauth` with `password` or a current two-factor or recovery `code`.
|
||||
- **Passkey:** `POST /auth/reauth/passkey/begin` returns WebAuthn request `options` limited to the account's own passkeys; pass `options.publicKey` to the browser and send the assertion to `POST /auth/reauth/passkey/finish` as `credential`. The authenticator must verify the user.
|
||||
- **Linked identity in a browser:** `POST /auth/reauth/sso/begin` with `provider` (one of `providers[].id`) returns `url` and `binding`. Open `url` in a window and keep `binding`. The provider returns that window to the dashboard's `/reauth/complete` with a single-use `code`, which only the session that began the flow can redeem: `POST /auth/reauth/sso/finish` with `code` and `binding`. The identity the provider verifies must already be linked to the account; another account is refused with `reauth_identity_mismatch`, never matched by email address.
|
||||
- **Native app:** `POST /auth/reauth/native/begin` returns a `nonce`. Pass it unchanged as the Sign in with Apple request nonce or the Google authorization `nonce`, then send the ID token to `POST /auth/reauth/native` with `provider` (`apple` or `google`). The nonce is single-use and bound to the session.
|
||||
|
||||
Each succeeds with `{ "valid_for_seconds": 300 }` and confirms only the session that made the call.
|
||||
|
||||
This applies to session callers. An API key or OAuth token has no session to confirm and is not the threat here, so it passes straight through to the route's permission gate.
|
||||
|
||||
A completed sign-in counts as a confirmation for the first five minutes of the session. An account with neither a password nor two-factor authentication has nothing else to confirm with, so after those five minutes `POST /auth/reauth` answers it `400` with code `reauth_no_factor` and it signs in again. Once 2FA is on, it confirms with an authenticator code like any other account.
|
||||
A completed sign-in counts as a confirmation for the first five minutes of the session. `POST /auth/reauth` answers an account with neither a password nor two-factor authentication `400` with code `reauth_no_factor`; it confirms with a passkey or its linked identity instead. An account with none of those signs in again.
|
||||
|
||||
`GET /auth/config` gained two fields: `invites_required` (boolean, true when an invitation token is needed to create an account) and `docs_url` (string, the deployment's link to the accounts and access documentation, for a client to surface next to a refusal).
|
||||
|
||||
@ -668,6 +693,8 @@ Alongside them, `api_url` is this API's own public base (for a copyable example
|
||||
|
||||
`PATCH /cloud-link/mailboxes/:id/participation` is self-hosted JWT-only with `MANAGE_EMAILS` and organization ownership. It accepts typed `mode` (`diagnostic` or `off`), `send`, `receive` and optional positive shared/rolling limits. `receive` also authorizes probabilistic spam rescue, reads, importance and stars or flags on verified warmup messages where supported; it does not authorize replies or guarantee reputation improvement. Cloud mirrors it through instance-token-only `PATCH /pool-link/instance/mailboxes/:remoteId/participation`, scoped to the calling instance. These dedicated routes prevent an old Cloud silently ignoring additive consent fields; old state without participation is not explicit diagnostic consent. The existing native lifecycle routes select diagnostic send/receive on start/resume and complete off on stop while retaining history. See [participation and send safety](/development/send-safety/).
|
||||
|
||||
For a mailbox Warmbly Cloud warms, `PATCH /emails/:id` sends `warmup_base`, `warmup_max`, `warmup_increase`, `warmup_reply_rate`, `warmup_start_time`, `warmup_end_time`, `warmup_days` and `warmup` (as pause or resume) to Warmbly Cloud through instance-token-only `PATCH /pool-link/instance/mailboxes/:remoteId`, then stores the values Cloud reports. The rest of the patch is written locally only once Cloud has applied its part, and the timezone travels with it. `test_mode`, `test_send_enabled`, `test_receive_enabled`, `shared_daily_limit` and `rolling_recipient_limit` return `409 cloud_warmup_managed`; use the participation route above. `POST /emails/:id/warmup/start` and `/resume` resume the mailbox on Cloud, `/pause` pauses it there, and `/stop` returns `409 cloud_warmup_managed`. `GET /cloud-link/mailboxes` rows carry `settings_observed_at`, when the local copy last matched Cloud. See [warmup settings](/guides/warmbly-cloud/#warmup-settings) and [the refusals](/api/error-codes/#warmbly-cloud-warmup-refusals).
|
||||
|
||||
Cloud-managed OAuth additionally exposes `POST /pool-link/instance/oauth/start-correlated`, `POST /pool-link/instance/oauth/finish-correlated` and `POST /pool-link/instance/mailboxes/adopt-correlated`, all instance-token only and requiring `protocol: 1`. The authenticated instance response advertises `managed_consent_protocol: 1`. Legacy routes remain unchanged; absence of the capability cannot be bypassed with fields an old replica might ignore. See [the Cloud route and recovery contract](/guides/warmbly-cloud/).
|
||||
|
||||
`POST /internal/worker/warmup-dispatch` is an execution-plane internal-token route, not a public JWT/API-key scope. It inspects, begins and durably finishes a task/mailbox/worker/nonce-bound command. Start repeats authoritative policy checks; unknown or denied authority cannot permit native execution. Receipt/header/model text supplies no authorization.
|
||||
|
||||
@ -22,7 +22,7 @@ All errors follow this structure:
|
||||
|
||||
## HTTP status codes
|
||||
|
||||
### Campaign send-admission diagnostics
|
||||
### Send-admission diagnostics
|
||||
|
||||
These codes appear in campaign activity and task failure metadata, not as HTTP response codes:
|
||||
|
||||
@ -30,9 +30,11 @@ These codes appear in campaign activity and task failure metadata, not as HTTP r
|
||||
|------|---------|
|
||||
| `CAMPAIGN_DAILY_LIMIT_REACHED` | This campaign has reached its configured UTC-day maximum for this sending mailbox. Other mailboxes have separate campaign allowances. |
|
||||
| `MAILBOX_SENDING_PLAN_DENIED` | The mailbox's enabled local-day sending plan is unavailable, outside working hours or on a break, or its daily/hourly quota is exhausted. The scheduler retries when capacity opens. |
|
||||
| `MANUAL_SEND_LIMIT_REACHED` | A unibox send (reply, forward or compose, including `POST /unibox/*` and `POST /emails/:id/send`) would exceed the mailbox's fair-use allowance: recipients on one email, in an hour, in 24 hours, in new conversations over 24 hours, or, across all the workspace's mailboxes, the 24-hour allowance its plan gives. The failure message names which. Unibox sends never count toward campaign or warmup limits. See [sending controls](/guides/unibox/#sending-controls). |
|
||||
| `RECIPIENT_SUPPRESSED` | A recipient is on the workspace suppression list, directly or through its domain. The failure message names the address. A unibox forward is never refused for this. |
|
||||
| `SEND_ADMISSION_DENIED` | Another current send-authority, recipient or mailbox capacity check denied admission. |
|
||||
|
||||
Admission denial before dispatch releases the send reservation without spending a failed-send attempt. See [campaigns](/guides/campaigns/) and [sending behavior](/guides/sending-behavior/) for the separate configurable ceilings and daily targets.
|
||||
Admission denial before dispatch releases the send reservation without spending a failed-send attempt. A refused unibox send ends with status `failed`, is not retried, and is reported to the mailbox owner. See [campaigns](/guides/campaigns/) and [sending behavior](/guides/sending-behavior/) for the separate configurable ceilings and daily targets.
|
||||
|
||||
### Client errors (4xx)
|
||||
|
||||
@ -133,6 +135,7 @@ Resource IDs on email, campaign and contact routes must be UUIDs. Email `tags` a
|
||||
| `lead_cc_limit` | [Set a lead's CC](/api/reference/campaigns/#set-a-leads-cc) was given more than `2` contacts |
|
||||
| `lead_cc_self` | [Set a lead's CC](/api/reference/campaigns/#set-a-leads-cc) named the lead itself as a copy |
|
||||
| `invalid_recipient` | `POST /emails/:id/send`, `POST /unibox/reply` or `POST /unibox/compose` was given a `to`, `cc` or `bcc` entry that is not exactly one email address (`ana@example.com` or `Ana <ana@example.com>`), or no `to` address at all. Put each address in its own entry |
|
||||
| `recipient_suppressed` | `POST /emails/:id/send`, `POST /unibox/reply` or `POST /unibox/compose` named a `to`, `cc` or `bcc` address on the workspace [suppression list](/guides/unsubscribe/#the-suppression-list), directly or through its domain. The `message` names the address and the entry. Nothing was queued. A forward (`POST /unibox/reply` with `forward_message_id`) is not checked against the list |
|
||||
| `invalid_message_id` | `POST /emails/:id/send` or `POST /unibox/reply` was given an `in_reply_to` entry that is not one Message-ID: printable characters with no spaces or line breaks, with or without the surrounding `<` and `>` |
|
||||
|
||||
#### Password refusals
|
||||
@ -184,6 +187,19 @@ Each registered key may make 30 protected calls per hour. Past that the call ans
|
||||
| `cloud_link_upgrade_required` | 409 | This Cloud deployment does not support workspace-scoped connections. Upgrade Cloud first |
|
||||
| `organization_cloud_connected` | 409 | Disconnect the workspace's Cloud connection before deleting it or scheduling deletion of it or its owner account |
|
||||
|
||||
#### Warmbly Cloud warmup refusals
|
||||
|
||||
A mailbox warmed by Warmbly Cloud has its warmup settings changed through the cloud (see [warmup settings](/guides/warmbly-cloud/#warmup-settings)). `PATCH /emails/:id` and the warmup start, pause and resume routes answer with these when that cannot happen. In every case the rest of the patch is not applied on the instance. After `cloud_warmup_not_applied`, the mailbox's copy of its Warmbly Cloud settings is still refreshed to what Warmbly Cloud reported. A replay with the same `Idempotency-Key` returns the stored answer, so retry with a new key.
|
||||
|
||||
| `code` | Status | Meaning |
|
||||
|--------|--------|---------|
|
||||
| `cloud_warmup_value_unsupported` | 422 | Warmbly Cloud does not take 0 from an instance for the starting volume, maximum, daily increase or reply rate. Use 1 or more, or change it in Warmbly Cloud |
|
||||
| `cloud_warmup_not_applied` | 409 | Warmbly Cloud answered but did not apply the named settings. The mailbox now shows the values Warmbly Cloud reported |
|
||||
| `cloud_warmup_managed` | 409 | Participation, the shared daily and recipient limits, and stopping warmup are changed with the mailbox's Warmbly Cloud controls, not on the mailbox |
|
||||
| `cloud_link_disconnecting` | 409 | The workspace is disconnecting from Warmbly Cloud. Retry once it has finished |
|
||||
| `cloud_link_unreachable` | 503 | Warmbly Cloud could not be reached |
|
||||
| `cloud_link_bad_response` | 503 | Warmbly Cloud answered without the mailbox's warmup settings |
|
||||
|
||||
#### Name refusals
|
||||
|
||||
| `code` | Status | Meaning |
|
||||
@ -267,6 +283,7 @@ The [inbox vendor](/guides/mailbox-import/#import-from-an-inbox-vendor) routes (
|
||||
| `domain_redirect_limit` | 409 | Warmbly Cloud already serves 200 redirects for this linked instance |
|
||||
| `pool_link_redirect_not_found` | 404 | On Warmbly Cloud's linked-instance API: Cloud serves no redirect for this domain for the calling instance |
|
||||
| `pool_link_cleartext_mailbox` | 422 | Enrolling a mailbox in [Warmbly Cloud](/guides/warmbly-cloud/), or updating its credential there: SMTP or IMAP uses security `none`. Cloud reaches the mail server over the internet and needs TLS or STARTTLS on both legs. Switch the mailbox to TLS, or keep warming it on the instance |
|
||||
| `pool_link_mailbox_conflict` | 409 | Enrollment found the address already connected in the linked Cloud workspace and could not attach it. Cloud attaches an existing copy only when it is an active SMTP/IMAP mailbox, the server presents exactly the credentials Cloud holds for it, no active Cloud campaign sends from it and no other server has enrolled it. Nothing is attached by address alone. Contact Warmbly support to check the exact instance and mailbox IDs; do not delete or re-enroll the mailbox |
|
||||
| `pool_link_instance_url` | 422 | Starting a Google or Microsoft sign-in through [Warmbly Cloud](/guides/warmbly-cloud/#google-and-microsoft-mailboxes-sign-in-through-warmbly-cloud) from an instance that registered no address of its own when it was linked, so Cloud has nowhere to return the sign-in to. Set `APP_URL` on the instance, disconnect it and link it again |
|
||||
| `pool_link_return_url` | 400 | The same sign-in named a return address that is not on the instance's registered address (same scheme and host) |
|
||||
| `pool_link_oauth_browser` | 400 | The brokered sign-in was not continued from the Warmbly Cloud page that started it, or returned to a different browser. Nothing was connected. Start again from the instance |
|
||||
@ -546,13 +563,15 @@ Signup and invitation refusals carry their own `code`, so a client can branch on
|
||||
|--------|--------|---------|
|
||||
| `reauth_required` | 403 | The action needs a proof of identity newer than the session. Confirm with `POST /v1/auth/reauth`, then retry |
|
||||
| `reauth_limited` | 400 | Too many failed confirmations for this account within the hour. Wait, or sign out and sign in again: a sign-in counts as a confirmation for five minutes |
|
||||
| `reauth_no_factor` | 400 | The account has neither a password nor two-factor authentication, so there is nothing to confirm with. A sign-in counts as a confirmation for five minutes, so sign out and sign in again, then repeat the action; turning on two-factor authentication gives the account a way to confirm without signing in again |
|
||||
| `reauth_no_factor` | 400 | `POST /auth/reauth` was sent a password or code for an account that has neither. Confirm with a passkey or the linked sign-in provider listed by `GET /auth/reauth/methods`; an account with none of those signs out and in again (a sign-in counts as a confirmation for five minutes) |
|
||||
| `reauth_identity_mismatch` | 400 | The Google, Apple or single sign-on account verified during a confirmation is not the one linked to this Warmbly account. Continue with the account you sign in with |
|
||||
| `reauth_expired` | 400 | The confirmation's code or nonce expired or was already used. Start the confirmation again |
|
||||
| `admin_mfa_required` | 403 | An admin route was reached by a session that did not present a second factor. Turn on 2FA or add a passkey, then sign in again |
|
||||
| `passkey_user_verification_required` | 400 | A passkey sign-in came from an authenticator that did not verify the user with a PIN or biometric. Set a PIN on the security key, or sign in another way |
|
||||
| `two_fa_invalid_code` | 400 | The authenticator or recovery code did not match. Wait for a fresh code and try again; the login challenge allows five attempts per pending session |
|
||||
| `password_changed_sign_in_again` | 409 | `POST /auth/me/password` stored the new password but could not issue the calling device a new session. Every earlier token is invalid; discard them and sign in again with the new password |
|
||||
|
||||
`reauth_required` guards the changes that hand out a durable credential, reveal a signing secret, change who can get in, cannot be undone, or reach a whole domain's mail. The full list is under [actions that need a recent confirmation](/api/endpoints/#actions-that-need-a-recent-confirmation). Confirm with a password or a current two-factor code:
|
||||
`reauth_required` guards the changes that hand out a durable credential, reveal a signing secret, change who can get in, cannot be undone, or reach a whole domain's mail. The full list is under [actions that need a recent confirmation](/api/endpoints/#actions-that-need-a-recent-confirmation). List the ways the account can confirm with `GET /v1/auth/reauth/methods` (a passkey and a linked Google, Apple or single sign-on identity are among them; see [endpoints](/api/endpoints/#actions-that-need-a-recent-confirmation)). With a password or a current two-factor code:
|
||||
|
||||
```bash
|
||||
curl -X POST "https://api.warmbly.com/v1/auth/reauth" \
|
||||
@ -567,7 +586,7 @@ curl -X POST "https://api.warmbly.com/v1/auth/reauth" \
|
||||
|
||||
The confirmation is recorded on the session and lasts for the window returned, so a run of related changes only asks once.
|
||||
|
||||
Turning on two-factor authentication (`POST /auth/2fa/enroll/start` and `/confirm`) is guarded too, since a confirmed authenticator is itself a way to confirm. An account with a password confirms as above. An account with neither a password nor 2FA (created through Google, Apple or single sign-on) has nothing to confirm with, so like every confirmed action it is allowed for five minutes after a sign-in; an older session gets `reauth_no_factor` and the fix is to sign out and sign in again. API keys and OAuth tokens have no session to confirm, so these routes are reachable only with a signed-in session; that is deliberate for key creation, which otherwise lets one leaked key mint more.
|
||||
Turning on two-factor authentication (`POST /auth/2fa/enroll/start` and `/confirm`) is guarded too, since a confirmed authenticator is itself a way to confirm. An account with a password confirms as above. An account with neither a password nor 2FA (created through Google, Apple or single sign-on) confirms with its passkey or linked identity. One with none of those is allowed for five minutes after a sign-in like every confirmed action; an older session gets `reauth_no_factor` and the fix is to sign out and sign in again. API keys and OAuth tokens have no session to confirm, so these routes are reachable only with a signed-in session; that is deliberate for key creation, which otherwise lets one leaked key mint more.
|
||||
|
||||
```json
|
||||
{
|
||||
|
||||
@ -17,7 +17,7 @@ Everything in this reference is generated from the same surface the spec describ
|
||||
- **Explore it interactively.** Import the URL into Postman or Insomnia to get a ready-to-call collection with every endpoint, parameter, and schema.
|
||||
- **Validate against it.** Use it for contract tests so your integration breaks loudly when an assumption changes.
|
||||
|
||||
There are no official SDKs yet, so generating a client from this spec (or calling the API over plain HTTP) is the supported path today.
|
||||
Official JavaScript and TypeScript, Python, and Go libraries exist (see [SDKs](/api/sdks/)). For any other language, generating a client from this spec or calling the API over plain HTTP is the supported path.
|
||||
|
||||
The realtime WebSocket gateway has its own machine-readable description in [AsyncAPI 3.1](https://docs.warmbly.com/asyncapi.json); see the [realtime guide](/api/realtime/).
|
||||
|
||||
|
||||
@ -5,6 +5,8 @@ description: Subscribe to live Warmbly events over a WebSocket connection.
|
||||
|
||||
Warmbly pushes dashboard events (campaign sends, opens, clicks, replies, inbox arrivals, audit entries, and more) over a WebSocket. The same socket that powers the live dashboard is available to developers. Events are ordered and carry a monotonic `seq`, and a reconnecting client can resume and replay what it missed.
|
||||
|
||||
Realtime rate limits and the per-user connection cap apply to the user, not an individual workspace. If you own multiple workspace subscriptions, each limit uses the highest value across those subscriptions, not their sum. Enterprise custom limits override the corresponding plan limits for that subscription. Limits are cached for up to five minutes.
|
||||
|
||||
The gateway has a machine-readable AsyncAPI 3.1 description you can generate clients or docs from:
|
||||
|
||||
```
|
||||
@ -47,7 +49,7 @@ After connecting, join one or more topics with a `phx_join` message:
|
||||
| `account:<account_id>` | One mailbox's sync and warmup events | Requires `manage_emails` |
|
||||
| `bulk:<operation_id>` | Progress of one bulk operation | Import/export progress; only the user who started the operation receives its events |
|
||||
|
||||
Events arrive as channel messages whose event name is the event type, for example `EMAIL_SENT`, `EMAIL_OPENED`, `EMAIL_REPLIED`, `EMAIL_RECEIVED`, `AI_DRAFT_READY`, `CAMPAIGN_COMPLETED`, `CAMPAIGN_IDLE` (a continuous campaign ran out of leads and is waiting), `TASK_PROGRESS`, `ACCOUNT_HEALTH_CHANGED`, `ACCOUNT_SYNC_STATE`, `WARMUP_PLACEMENT` (a partner found one of the workspace's warmup emails; `status` is `inbox`, `tabs` or `spam`), `PLACEMENT_TEST_UPDATED`, `AUDIT_CREATED`, `AUTOMATION_RUN`, `MEETING_BOOKED`, `NOTIFICATION_CREATED`, `PAGE_HIT`. Payloads always include `event_type` and a timestamp, plus the relevant ids (`campaign_id`, `contact_id`, `thread_id`, and so on). `EMAIL_OPENED` and `EMAIL_CLICKED` also carry `occurred_at` (when the tracking service saw it; `timestamp` is the publish time), `machine` (true for a mail client prefetch or a security scanner's fetch), and `client`, `client_type` (`app` or `webmail`), `device_hidden` (a mailbox provider's image proxy fetched it, so the device is unknowable), `device_type`, `os`, `browser`, `country_code` and `city` when the consumer could tell.
|
||||
Events arrive as channel messages whose event name is the event type, for example `EMAIL_SENT`, `EMAIL_OPENED`, `EMAIL_REPLIED`, `EMAIL_RECEIVED`, `AI_DRAFT_READY`, `CAMPAIGN_COMPLETED`, `CAMPAIGN_IDLE` (a continuous campaign ran out of leads and is waiting), `TASK_PROGRESS`, `ACCOUNT_HEALTH_CHANGED`, `ACCOUNT_SYNC_STATE`, `WARMUP_PLACEMENT` (a partner found one of the workspace's warmup emails; `status` is `inbox`, `tabs` or `spam`, or `rescue_moved`, `rescue_already_out`, `rescue_gone` or `rescue_failed` when the recipient's mailbox reports what a spam rescue came to), `PLACEMENT_TEST_UPDATED`, `AUDIT_CREATED`, `AUTOMATION_RUN`, `MEETING_BOOKED`, `NOTIFICATION_CREATED`, `PAGE_HIT`. Payloads always include `event_type` and a timestamp, plus the relevant ids (`campaign_id`, `contact_id`, `thread_id`, and so on). `EMAIL_OPENED` and `EMAIL_CLICKED` also carry `occurred_at` (when the tracking service saw it; `timestamp` is the publish time), `machine` (true for a mail client prefetch or a security scanner's fetch), and `client`, `client_type` (`app` or `webmail`), `device_hidden` (a mailbox provider's image proxy fetched it, so the device is unknowable), `device_type`, `os`, `browser`, `country_code` and `city` when the consumer could tell.
|
||||
|
||||
`AI_DRAFT_READY` fires when the [inbox agent](/guides/inbox-agent/) drafts a suggested reply awaiting review; it carries `thread_id` and `draft_id` and requires `access_unibox`.
|
||||
|
||||
|
||||
@ -777,7 +777,7 @@ Auth: Session only (not available to API keys).
|
||||
|
||||
Generates a TOTP secret and the `otpauth://` provisioning URI (returned once). The client renders the URI as a QR code; the remaining fields spell out the same parameters for manual entry.
|
||||
|
||||
Requires a recent confirmation (`POST /auth/reauth`), like confirming enrollment below; without one it answers `403 reauth_required`. An account with neither a password nor 2FA cannot confirm, so it is accepted instead from a session signed in within the last five minutes, and an older session answers `400 reauth_no_factor`.
|
||||
Requires a recent confirmation (`POST /auth/reauth`), like confirming enrollment below; without one it answers `403 reauth_required`. An account with neither a password nor 2FA confirms with a passkey or its linked sign-in provider (`GET /auth/reauth/methods`). One with none of those is accepted from a session signed in within the last five minutes, and an older session answers `400 reauth_no_factor`.
|
||||
|
||||
Auth: Session only (not available to API keys).
|
||||
|
||||
|
||||
@ -268,7 +268,7 @@ Results are scoped to the selected workspace, so every member with access sees t
|
||||
|
||||
Returns where warmup mail landed in partners' mailboxes over a date range: the primary inbox, a Gmail category tab (Promotions, Updates, Social or Forums), or spam. Counts are per UTC day, per recipient provider, and, on the workspace report, per mailbox. Requires an organization context. Auth: **Scope** `READ_ANALYTICS` · **Org permission** `view_analytics`.
|
||||
|
||||
Every figure is measured, not estimated: a delivery is counted when the recipient's own sync finds the warmup email and records the folder it arrived in. `rescued` counts spam placements the recipient's mailbox was told to move back to the inbox; the move itself is not confirmed back, so it is the rescues requested, not a verified count. `unconfirmed` counts completed sends more than 24 hours old that no recipient has reported seeing, bucketed on the day they were sent; a provider filter cannot split it, so it is only reported unfiltered.
|
||||
Every figure is measured, not estimated: a delivery is counted when the recipient's own sync finds the warmup email and records the folder it arrived in. `rescued` counts spam placements the recipient's mailbox was told to move back to the inbox. What each rescue came to is reported back by that mailbox: `rescue_confirmed` is rescues that left the message out of spam (moved, or already out when the rescue ran), `rescue_missing` is rescues whose message was no longer in the mailbox, `rescue_failed` is rescues the provider refused, and `rescue_unreported` is requested rescues with no outcome yet, which includes every rescue from before outcomes were recorded. The three outcome counters also appear on each `groups` entry. `unconfirmed` counts completed sends more than 24 hours old that no recipient has reported seeing, bucketed on the day they were sent; a provider filter cannot split it, so it is only reported unfiltered.
|
||||
|
||||
`rate` is the headline deliverability figure the dashboard shows next to each mailbox: `inbox_rate` is inbox plus tabs over what was delivered in the trailing 7 UTC days, and stays `null` until `min_sample` (`20`) deliveries are in. `scope` is always `major`: the rate is taken over Google, Microsoft and Yahoo recipients only, the providers that filter on sender reputation. A mailbox whose warmup mail reached only other hosts has no rate (`delivered` `0`, `band` `none`). Other hosts are always in `summary`, `daily` and `providers`, and `other_delivered` and `other_inbox_rate` give their share of the same window (`0` and `null` with none). `band` is `good` at `90` or above, `fair` from `80`, `poor` below `80`, `collecting` below the sample floor and `none` with no deliveries. Each day's `rolling_inbox_rate` is the trailing 7-day figure at the same three providers ending on that day; the day's counts cover every host, so filter by provider to read one. Rates are percentages with two decimals.
|
||||
|
||||
@ -290,7 +290,7 @@ A range longer than 366 days, `from` after `to`, a malformed date or an invalid
|
||||
"date_range": { "from": "2026-06-01T00:00:00Z", "to": "2026-06-30T00:00:00Z" },
|
||||
"summary": {
|
||||
"sent": 820, "delivered": 791, "inbox": 702, "tabs": 41, "spam": 48,
|
||||
"rescued": 44, "unconfirmed": 12, "inbox_rate": 93.93, "spam_rate": 6.07
|
||||
"rescued": 44, "rescue_confirmed": 41, "rescue_missing": 1, "rescue_failed": 0, "rescue_unreported": 2, "unconfirmed": 12, "inbox_rate": 93.93, "spam_rate": 6.07
|
||||
},
|
||||
"rate": {
|
||||
"window_days": 7, "min_sample": 20, "scope": "major", "delivered": 196, "inbox": 180, "tabs": 9, "spam": 7,
|
||||
@ -302,8 +302,8 @@ A range longer than 366 days, `from` after `to`, a malformed date or an invalid
|
||||
"sent": 30, "delivered": 29, "inbox": 27, "tabs": 1, "spam": 1, "rescued": 1, "unconfirmed": 0,
|
||||
"inbox_rate": 96.55, "spam_rate": 3.45, "rolling_inbox_rate": 96.43,
|
||||
"groups": [
|
||||
{ "group": "google", "inbox": 15, "tabs": 1, "spam": 0, "rescued": 0 },
|
||||
{ "group": "microsoft", "inbox": 12, "tabs": 0, "spam": 1, "rescued": 1 }
|
||||
{ "group": "google", "inbox": 15, "tabs": 1, "spam": 0, "rescued": 0, "rescue_confirmed": 0, "rescue_missing": 0, "rescue_failed": 0 },
|
||||
{ "group": "microsoft", "inbox": 12, "tabs": 0, "spam": 1, "rescued": 1, "rescue_confirmed": 1, "rescue_missing": 0, "rescue_failed": 0 }
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -615,7 +615,8 @@ Returns the detailed status for one email account: a combined health score (fold
|
||||
"target_volume": 33,
|
||||
"max_volume": 40,
|
||||
"reply_rate": 35,
|
||||
"days_active": 23
|
||||
"days_active": 23,
|
||||
"next_send_at": "2026-06-12T14:37:12Z"
|
||||
},
|
||||
"warmup_health": {
|
||||
"state": "watch",
|
||||
@ -643,7 +644,7 @@ Returns the detailed status for one email account: a combined health score (fold
|
||||
}
|
||||
```
|
||||
|
||||
`warmup_status` is present only when warmup has ever been enabled; `warmup_health` is present only when the mailbox is in a warmup pool. `in_campaign` reports whether the mailbox currently backs a live campaign.
|
||||
`send_recovery_reason` is present only while a send safety hold pauses the mailbox's sending: `authentication` (cleared by reconnecting the mailbox), `permanent`, `conflict` or `unknown`. `warmup_status` is present only when warmup has ever been enabled, and its `next_send_at` is when the mailbox's next warmup send is queued for (absent while nothing is queued, for example while warmup is paused); `warmup_health` is present only when the mailbox is in a warmup pool. `in_campaign` reports whether the mailbox currently backs a live campaign.
|
||||
|
||||
`warmup_placement` is the mailbox's measured warmup inbox rate over the trailing 7 UTC days, the same figure [Get warmup placement](#get-warmup-placement) reports as `rate`, and is absent when nothing was delivered in that window. Once `inbox_rate` is present, `health.score` is capped at it (rounded down), so a mailbox landing any mail in spam no longer reads `100`, and a rate below `90` also sets `health.status` to at least `warning` with an issue naming the rate.
|
||||
|
||||
|
||||
@ -1089,7 +1089,7 @@ The arithmetic adds up: `configured_ceiling` minus every `limits[].emails` minus
|
||||
}
|
||||
```
|
||||
|
||||
`limits[].kind` is one of `campaign_daily_limit`, `campaign_ramp`, `warmup_graduation`, `workspace_risk`, `domain_auth`, `resting`, `warmup_health_hold`, `other_campaigns`, `warmup_health_pace`, `mailbox_hours`, `sending_behavior`, `spacing`, `sending_window`, `not_running`, `org_daily_limit`, `new_lead_cap` or `leads`. `mailboxes[].state` is one of `sending`, `budget_spent`, `hours_closed`, `no_working_day`, `domain_auth`, `resting`, `health_hold`, `no_worker` (no running sending worker holds the mailbox right now; its day is still counted, since it is placed on one again within minutes) or `window_closed`; `mailboxes[].limited_by` names the clamp that set `cap_today`. See the [campaigns guide](/guides/campaigns/#todays-sending-plan) for what each limit means.
|
||||
`limits[].kind` is one of `campaign_daily_limit`, `campaign_ramp`, `warmup_graduation`, `workspace_risk`, `domain_auth`, `resting`, `warmup_health_hold`, `cloud_status_hold`, `send_recovery`, `send_cooldown`, `other_campaigns`, `warmup_health_pace`, `mailbox_hours`, `sending_behavior`, `spacing`, `sending_window`, `not_running`, `org_daily_limit`, `new_lead_cap` or `leads`. `mailboxes[].state` is one of `sending`, `budget_spent`, `hours_closed`, `no_working_day`, `domain_auth`, `resting`, `health_hold`, `cloud_status_hold` (Warmbly Cloud has not reported a current status, so cold sends from the mailbox wait for one; it is not a Cloud quarantine or block), `send_recovery`, `send_cooldown`, `send_authority`, `no_worker` (no running sending worker holds the mailbox right now; its day is still counted, since it is placed on one again within minutes) or `window_closed`; `mailboxes[].limited_by` names the clamp that set `cap_today`. See the [campaigns guide](/guides/campaigns/#todays-sending-plan) for what each limit means.
|
||||
|
||||
## List campaign senders
|
||||
|
||||
|
||||
@ -72,7 +72,7 @@ A `data` array of mailbox objects plus a `pagination` envelope.
|
||||
"warmup_pool_type": "premium",
|
||||
"warmup_start_time": "09:00",
|
||||
"warmup_end_time": "17:00",
|
||||
"warmup_days": 5,
|
||||
"warmup_days": 31,
|
||||
"timezone": "America/New_York",
|
||||
"tags": ["outbound"],
|
||||
"created_at": "2026-05-19T18:00:00Z",
|
||||
@ -163,7 +163,7 @@ Auth: **Scope** `WRITE_EMAILS` · **Org permission** `manage_emails`
|
||||
| `warmup_tag` | string | no | Tag applied to warmup threads. |
|
||||
| `warmup_start_time` | string | no | Daily warmup window start, `HH:MM`. |
|
||||
| `warmup_end_time` | string | no | Daily warmup window end, `HH:MM`. |
|
||||
| `warmup_days` | integer | no | Number of active warmup days per week. |
|
||||
| `warmup_days` | integer | no | Weekdays warmup mail goes out, as a bitmask: 1 Monday, 2 Tuesday, 4 Wednesday, 8 Thursday, 16 Friday, 32 Saturday, 64 Sunday. 31 is Monday to Friday; 0 means every day. For a mailbox Warmbly Cloud warms, this and the other ramp and window fields are sent to Warmbly Cloud; see [warmup settings](/guides/warmbly-cloud/#warmup-settings). |
|
||||
| `warmup_placement` | string | no | Where warmup mail is filed in the mailbox itself: `folder` (default), `inbox`, or `archive`. See [Warmup](/guides/warmup/#in-your-mail-client). |
|
||||
| `warmup_folder` | string | no | Folder (Gmail label) warmup mail is filed into when `warmup_placement` is `folder`. Empty string means the instance default, `Warmbly`. No folder separators or control characters; 64 characters max. |
|
||||
| `warmup_retention_days` | integer | no | How many days warmup mail stays in this mailbox before Warmbly deletes it, wherever `warmup_placement` keeps it: `3` to `3650`, or `0` to follow the instance setting (30 days unless the operator changed it). |
|
||||
@ -718,6 +718,8 @@ Auth: **Scope** `SEND_CAMPAIGNS` · **Org permission** `send_campaigns`. Require
|
||||
|
||||
`task_id` identifies the queued send task. `scheduled_at` is the resolved dispatch time (immediate for `instant`, the next gap for `smart`, or the requested time for `scheduled`).
|
||||
|
||||
A recipient on the workspace [suppression list](/guides/unsubscribe/#the-suppression-list), directly or through its domain, answers `400` with `code: "recipient_suppressed"` and a `message` naming the address. Nothing is queued.
|
||||
|
||||
## Connect a mailbox (onboarding)
|
||||
|
||||
The three onboarding routes connect a new mailbox, and a fourth moves an existing one to an app password. They are **session only (not available to API keys)** because they write user-encrypted provider refresh tokens through the SPA popup flow.
|
||||
|
||||
@ -348,6 +348,8 @@ Sends or schedules a reply or a forward from any mailbox in the organization. Th
|
||||
|
||||
Instant sends are held for the sender's undo window (5 to 120 seconds, default 30) before they actually leave, so `scheduled_at` is that far in the future. Within the window the send can still be cancelled with `DELETE /unibox/scheduled/:task_id`.
|
||||
|
||||
A reply to an address on the workspace [suppression list](/guides/unsubscribe/#the-suppression-list), directly or through its domain, answers `400` with `code: "recipient_suppressed"` and a `message` naming the address, and nothing is queued. A forward is not checked against the list.
|
||||
|
||||
Only Gmail has a provider thread handle, and it belongs to the mailbox that holds the thread. So the reply files into the Gmail thread `thread_id` names only when `email_account_id` holds a message in it; a mailbox that already replied into the conversation from outside continues the thread that reply started. Otherwise it goes out with the same In-Reply-To and References headers and the conversation's subject, which keeps it in the same conversation for the recipient, and starts a conversation of its own in the sending mailbox. The queued send still belongs to `thread_id`, so it is listed with that conversation's scheduled sends.
|
||||
|
||||
### Forwarding a message
|
||||
@ -362,6 +364,8 @@ The Date line is written in the sending mailbox's time zone (its own, else the w
|
||||
|
||||
A forward goes out as a new conversation for its recipient: no provider thread, and no In-Reply-To or References unless you set `in_reply_to` yourself. A `thread_id` sent with it is not used for either. Inside Unibox the forward belongs to the conversation of the message it forwards (that message's `thread_id`), whichever mailbox sends it: it is listed with that conversation's scheduled sends while queued, and once the sending mailbox's Sent folder syncs, the sent copy is returned by `GET /unibox/thread` for that conversation, stored once. A forward in a conversation is never picked as the parent Message-ID for a later reply into it.
|
||||
|
||||
The [suppression list](/guides/unsubscribe/#the-suppression-list) does not apply to a forward: it passes on mail already received to recipients chosen by hand, so it may go to an address or domain the list keeps out of campaigns. Fair use still counts its recipients.
|
||||
|
||||
Two limits:
|
||||
|
||||
- attachments on the original are not forwarded
|
||||
|
||||
@ -354,6 +354,23 @@ Every callback is an HTTP `POST` with a `Content-Type: application/json` body in
|
||||
|
||||
There is no `version` field in the payload. If an event's payload ever changes in a backward-incompatible way, it is published under a new `event_type` name. Existing event type names are never re-versioned in place.
|
||||
|
||||
### Reply received payload
|
||||
|
||||
`campaign.reply_received` carries `contact_email`, `intent` and `confidence` (the workspace keyword list's reading), `subject`, `snippet`, `action_taken`, `thread_id` and `email_account_id`. `contact_id` and `campaign_id` are optional, included only when the reply is linked to the corresponding contact or campaign. When [inbox tagging](/guides/inbox-tagging/) judged the message it also carries the stored verdict:
|
||||
|
||||
```json
|
||||
{
|
||||
"ai_labels": ["Meeting", "Interested"],
|
||||
"ai_intent": "scheduling",
|
||||
"ai_intent_confidence": 0.91,
|
||||
"ai_kind": "human_reply",
|
||||
"ai_relevance": 82,
|
||||
"ai_needs_review": false
|
||||
}
|
||||
```
|
||||
|
||||
`ai_kind` and `ai_intent` are empty strings when tagging was not confident enough to act on them, and the six keys are absent when tagging never judged the message. See [Automations](/guides/automations/#filtering-replies-on-inbox-tagging) for what each holds.
|
||||
|
||||
### Delivery headers
|
||||
|
||||
Each delivery carries these request headers:
|
||||
|
||||
@ -74,7 +74,7 @@ Application-layer secrets are sealed with AES-256-GCM under the DEK and base64-e
|
||||
|
||||
Instance-level secrets that are not owned by any organization are encrypted under a platform DEK (key ID = `uuid.Nil`): the same envelope as organization secrets, the same trust boundary, a different identity. See `internal/app/cipher/`.
|
||||
|
||||
Mailbox credentials are the one exception to the per-organization DEK. They are sealed with `CREDENTIALS_ENCRYPTION_KEY` (`KeyDomainInstance`) because a worker reads them without an organization context.
|
||||
Mailbox credentials (`KeyDomainMailbox`) are sealed under the organization DEK when `MAILBOX_CREDENTIALS_SEALING=organization`, at rest and inside the `ADD_EMAIL` command, so a worker opens them through the same brokered DEK path it uses for message bodies. Values written before that, and mailboxes with no organization, stay under `CREDENTIALS_ENCRYPTION_KEY`; a stored value carries a `dek1:` prefix when it uses the organization key, and readers accept both. The DEK is cached in process for five minutes in front of Redis, and concurrent first uses share one KMS call.
|
||||
|
||||
## Worker model
|
||||
|
||||
|
||||
193
docs/content/docs/development/aws-fargate.mdx
Normal file
193
docs/content/docs/development/aws-fargate.mdx
Normal file
@ -0,0 +1,193 @@
|
||||
---
|
||||
title: Prepare AWS Fargate
|
||||
description: Provision an AWS foundation with Terraform and plan a coordinated control plane cutover.
|
||||
---
|
||||
|
||||
The configurations in `deploy/terraform/` manage infrastructure through separate
|
||||
AWS, PlanetScale and Confluent state files. The AWS root prepares infrastructure
|
||||
in `us-east-1`: a network, ECS cluster, repositories, logs, configuration stores,
|
||||
Valkey and optional database connectivity and certificates. The separate runtime
|
||||
root creates application services, task roles and HTTPS ingress. The worker root
|
||||
prepares ARM EC2 hosts with static IPs. Public traffic changes remain a separate
|
||||
cutover step.
|
||||
|
||||
## Adopt existing resources
|
||||
|
||||
The PlanetScale root adopts an existing Postgres branch and reads its database
|
||||
metadata. The Confluent root adopts an existing environment, Basic Kafka cluster
|
||||
and service account, and reads Schema Registry metadata. Each uses import blocks
|
||||
and destruction guards. An adoption plan must show imports with zero resource
|
||||
creates, updates, replacements or deletes before it is applied.
|
||||
|
||||
Importing configuration does not copy SQL rows, Kafka messages or consumer
|
||||
offsets. SQL migrations, schema versions, access rules and runtime worker topics
|
||||
retain their current owners. Existing credentials are provisioned separately
|
||||
from Terraform.
|
||||
|
||||
## Keep existing data services
|
||||
|
||||
Warmbly can run its control plane on Fargate while keeping PlanetScale Postgres,
|
||||
Confluent Kafka, S3 and AWS KMS. Keep the existing KMS key available so stored
|
||||
organization data keys remain readable. Workers continue to use the brokered
|
||||
interfaces described in [split deployment](/development/split-deployment/).
|
||||
|
||||
PlanetScale Postgres supports an interface VPC endpoint in the database's AWS
|
||||
region. Set `planetscale_service_name` from the branch's Private Service Name.
|
||||
The foundation creates endpoint interfaces in two availability zones and limits
|
||||
database access to the application security group. Use the branch's Private
|
||||
Host with certificate verification in application connection strings.
|
||||
|
||||
Creating an endpoint does not disable the database's public access. Change public
|
||||
access restrictions only after every database client, including operational
|
||||
tools, has a working private path.
|
||||
|
||||
### Database connection pooling
|
||||
|
||||
PlanetScale includes a local PgBouncer with each Postgres database. It uses the
|
||||
same hostname and credentials on port `6432`, including through PrivateLink.
|
||||
Dedicated PgBouncers are a separate paid option. See
|
||||
[PlanetScale's connection pooling documentation](https://planetscale.com/docs/postgres/connecting/pgbouncer).
|
||||
|
||||
The local pooler uses transaction pooling. Route migrations and operations that
|
||||
hold session advisory locks through a direct connection on port `5432`.
|
||||
Set `PRIMARY_DB` to the pooled connection, `PRIMARY_DB_DIRECT` to the direct
|
||||
connection and `DB_POOL_MODE=transaction` for the Go services. Cloud reconciliation
|
||||
and managed operations open direct sessions for their session locks. Backups and
|
||||
the migration binary also use the direct connection. Before enabling transaction
|
||||
mode, configure the application database roles with `statement_timeout=60s` and
|
||||
`idle_in_transaction_session_timeout=5min`. The Go client verifies those effective
|
||||
settings instead of sending timeout startup parameters to PgBouncer. Verify the
|
||||
actual realtime client's prepared statements separately before switching it.
|
||||
|
||||
Budget Postgres server connections across every database role, direct client and
|
||||
deployment replica. PgBouncer's `default_pool_size` applies per user/database
|
||||
pair, so separate service roles can each create a server pool. Its client
|
||||
connection limit is not additional Postgres server capacity. Preserve headroom
|
||||
for migrations and operations throughout the overlap with the old deployment.
|
||||
|
||||
Confluent Kafka PrivateLink requires an Enterprise or Dedicated cluster. Keep the current
|
||||
Kafka transport, codec, topics and consumer groups during a compute migration.
|
||||
Check connectivity for both Kafka and Schema Registry. External workers need
|
||||
their own network path if the cluster becomes private.
|
||||
|
||||
Keep existing tracking hostnames registered with the edge provider, including
|
||||
their TLS certificates and origin configuration. Put those hostnames in
|
||||
`tracking_additional_hosts` so the ALB forwards their original Host headers to
|
||||
the tracking service. The empty default keeps ordinary single-host deployments
|
||||
unchanged. Validate each hostname before moving its traffic.
|
||||
|
||||
## Size tasks from measurements
|
||||
|
||||
Fargate bills provisioned CPU and memory. Its smallest Linux task has 0.25 vCPU
|
||||
and 512 MiB. Use more CPU when measured peaks exceed that allocation and allow
|
||||
memory headroom for startup and bursts. ARM64 images can reduce compute cost;
|
||||
verify that every pinned image includes that architecture, including Kafka
|
||||
variants.
|
||||
|
||||
Keep the shared Redis-compatible cache separate from replaceable app tasks. It
|
||||
coordinates multiple services and workers, so inspect all callers before
|
||||
changing its endpoint. Size the database from its own utilization and query
|
||||
metrics rather than from application container memory.
|
||||
|
||||
The AWS root provisions one Valkey `cache.t4g.micro` with TLS, encryption at rest,
|
||||
password authentication and one-day snapshots. It uses the standalone protocol
|
||||
supported by Warmbly's current clients and multi-key scripts. ElastiCache
|
||||
Serverless requires cluster-aware clients and different multi-key handling.
|
||||
The single-node configuration has no replica failover.
|
||||
|
||||
The new cache starts empty. Shared counters, locks and transient state need a
|
||||
coordinated transition before clients switch endpoints. Its private endpoint is
|
||||
reachable from application tasks in the VPC; external workers and local tools
|
||||
need a private route or tunnel.
|
||||
|
||||
Budget for load balancing, public IPv4 addresses, cache, interface endpoints,
|
||||
logs, secrets and transfer charges in addition to Fargate compute. The foundation
|
||||
starts no NAT gateway or load balancer. PrivateLink does not replace internet
|
||||
egress for other application dependencies.
|
||||
|
||||
The runtime root uses public task IPs for egress and restricts inbound application
|
||||
traffic to its load balancer. Start with zero desired tasks until configuration
|
||||
and dependency checks pass. A Linux task with 0.5 vCPU requires at least 1 GiB of
|
||||
memory; the 512 MiB minimum applies to 0.25 vCPU tasks.
|
||||
|
||||
Mail workers can remain on existing hosts or use the separate EC2 worker root.
|
||||
Graviton `t4g.small` hosts provide 2 vCPUs and 2 GiB of memory with a static public
|
||||
IP per worker. The root uses standard CPU credits to cap burst charges. Bootstrap
|
||||
prepares each host without starting its worker, so identity and credentials can
|
||||
be checked before a sequential handover. Check mailbox authentication when the
|
||||
outbound IP changes, and preserve each worker ID throughout the move.
|
||||
|
||||
## Prepare configuration
|
||||
|
||||
Use the deployment README to bootstrap encrypted, versioned Terraform state and
|
||||
apply the foundation. Installation-specific values and plans belong outside the
|
||||
checkout. Use Standard String parameters in Systems Manager Parameter Store for
|
||||
approved non-secret values such as public URLs, provider names and feature
|
||||
switches. Put passwords, tokens, encryption material and credential-bearing
|
||||
connection strings in Secrets Manager. Reference both stores from ECS task
|
||||
definitions. Running tasks must be replaced to pick up updated injected values.
|
||||
|
||||
The foundation also creates a separate rotation secret for pending key
|
||||
replacements. Keep that secret restricted to operators. Generating pending keys
|
||||
does not activate them or retire the keys used by the running deployment.
|
||||
|
||||
Preserve current configuration before preparing replacements. Keep a dependency
|
||||
map of each value's readers, encrypted data and rollback requirements. The
|
||||
Valkey password uses an ephemeral Terraform read and a write-only authentication
|
||||
argument, so its value is omitted from Terraform state and saved plans.
|
||||
|
||||
Use ECS task roles for S3, KMS and SES access. Give the execution role permission
|
||||
to retrieve only its service's runtime configuration. Backend and consumer need
|
||||
application permissions; worker nodes use brokered operations and carry no cloud
|
||||
credentials.
|
||||
|
||||
## Coordinate the cutover
|
||||
|
||||
1. Verify backups and a restore procedure, including the keys needed to read
|
||||
encrypted data. Pin compatible release images and prepare TLS certificates.
|
||||
2. Create separate database roles and Kafka credentials for the new deployment.
|
||||
Verify access before retiring credentials used by the running deployment.
|
||||
3. Configure a shared cache and internal service addresses. Verify that workers
|
||||
can reach every dependency they require.
|
||||
4. Run the release's migration task and require a successful exit before starting
|
||||
the backend. Set `DB_MIGRATIONS_ON_START=false` in its runtime configuration.
|
||||
The backend and consumer verify schema readiness before starting work. Coordinate
|
||||
scheduler ownership before allowing old and new backends to run scheduled work
|
||||
together. Confirm compatibility before sharing a database across releases.
|
||||
5. Start the remaining services. Check API access, forms, tracking, websocket
|
||||
authentication, Kafka processing and worker heartbeats. Set a stop timeout of
|
||||
at least 60 seconds for services that drain work.
|
||||
6. Move public traffic to healthy targets, drain the previous deployment and
|
||||
confirm that queued work has completed. DNS changes can leave some clients
|
||||
on the previous address until caches expire.
|
||||
7. Retire old credentials after all callers use replacements. If rollback is
|
||||
needed, use the replacement credentials on the rollback deployment too.
|
||||
|
||||
Authentication key rotation invalidates existing signed tokens and requires
|
||||
users to sign in again. MFA secrets are encrypted with `TWOFA_SECRET`, which
|
||||
falls back to `AUTH_SECRET` when unset. Coordinate re-encryption with key changes
|
||||
so enrolled MFA remains readable. Mailbox encryption key changes likewise need
|
||||
an explicit re-encryption and worker rollout procedure.
|
||||
|
||||
Treat database logins and data encryption keys as separate changes. A replacement
|
||||
database login connects to the same stored data; it does not require new mailbox
|
||||
passwords or OAuth grants. Preserve the existing credential encryption key,
|
||||
organization data keys and their KMS decrypt capability during the hosting
|
||||
cutover. Keep OAuth application identities and authorized callback URLs
|
||||
compatible with existing refresh tokens.
|
||||
|
||||
Before activating replacement encryption keys, verify decryption of the stored
|
||||
mailbox credentials, OAuth tokens, webhook secrets, integration credentials and
|
||||
MFA enrollments without logging their contents. Include instance-level data and
|
||||
pending mailbox-erasure tokens in the inventory. Repeat these checks using the
|
||||
new tasks' runtime permissions before moving traffic; operator access alone does
|
||||
not verify a task role. Re-encryption must preserve the underlying credential
|
||||
values and coordinate all readers, writers, queued work and rollback deployments.
|
||||
|
||||
`AUTH_SECRET` also derives signing keys for unsubscribe links and domain ownership
|
||||
proofs. Plan compatibility for previously issued links and existing DNS proofs
|
||||
before changing it. A successful sign-in check alone does not verify these paths.
|
||||
|
||||
The infrastructure foundation alone does not provide a zero-outage migration.
|
||||
Credential rotation, scheduler ownership, shared state and connection draining
|
||||
must all be coordinated before the traffic switch.
|
||||
@ -57,7 +57,7 @@ Five values protect the whole instance. Compose ships a working default for each
|
||||
|
||||
| Variable | Format | What it protects | Restart needed |
|
||||
|---|---|---|---|
|
||||
| `AUTH_SECRET` | 32 characters or more, enforced at boot | JWT and session signing. The realtime service reads the same value as `JWT_SECRET` and applies the same floor. Both refuse to start below it: a shorter HS256 key can be recovered offline from any token the service has issued. It also keys the `warmbly-verify=` values workspaces publish in `_warmbly` `TXT` records to prove a domain, so changing it changes every one of them: [root redirects](/guides/sending-domains/#root-redirects) stop at their next check until the new value is published | yes |
|
||||
| `AUTH_SECRET` | 32 characters or more, enforced at boot | JWT and session signing. The realtime service reads the same value as `JWT_SECRET` and applies the same floor. Both refuse to start below it: a shorter HS256 key can be recovered offline from any token the service has issued. By default it also supplies the MFA sealing, unsubscribe signing and DNS ownership-proof keys. Configure the separate purpose keys below before rotating sessions when those values must remain stable | yes |
|
||||
| `INTERNAL_API_TOKEN` | any random string | The backend's `/api/v1/internal/` routes the tracking and forms services call (click tickets, root redirects, page views, hosted forms). Fleet nodes authenticate with it too when `NODE_BROKER_TOKEN` is unset | yes |
|
||||
| `NODE_BROKER_TOKEN` | any random string | Every internal route only a fleet node calls: organization data keys (read, store and open), blob signing, provider access tokens for a mailbox Warmbly Cloud manages, the message map and sync lookups, worker config and the node heartbeat. Optional, and falls back to `INTERNAL_API_TOKEN`. The same value has to be set on the control plane and on every node, or those routes answer 401; a node joined after it is set receives it in `node.env` in place of `INTERNAL_API_TOKEN`, and a node joined before has to join again. The installer generates one for a new install; an existing install keeps the shared token until you set one | yes |
|
||||
| `NODE_ACCEPT_INTERNAL_TOKEN` | `true` or unset | While `true`, the node-only routes other than the key and blob brokers also accept `INTERNAL_API_TOKEN`. An upgrade does not need it: a heartbeat sent with `INTERNAL_API_TOKEN` is always answered with the node's version and nothing is recorded, so an older node updates itself. Set it only to keep older nodes syncing until they have updated, then remove it: while it is on, the tracking and forms services' token reaches those routes | no |
|
||||
@ -100,6 +100,44 @@ The secret check runs in the backend. The consumer and the workers start happily
|
||||
`KMS_LOCAL_MASTER_KEY` and `CREDENTIALS_ENCRYPTION_KEY` seal every stored credential and every stored message body. Losing them is unrecoverable, and a database backup without them cannot be decrypted.
|
||||
</Callout>
|
||||
|
||||
### Rotate instance keys
|
||||
|
||||
These settings are optional. An existing self-host keeps its current keys and behavior when they are unset. All list values are JSON arrays with at most four nonempty strings; `[]` means no additional keys. Keep them in your secret store or private `.env`.
|
||||
|
||||
| Variable | Purpose | Default |
|
||||
|---|---|---|
|
||||
| `CREDENTIALS_ENCRYPTION_READ_KEYS` | Additional 64-character hex keys that can decrypt instance credential ciphertext. New writes use only `CREDENTIALS_ENCRYPTION_KEY` | unset |
|
||||
| `TWOFA_READ_SECRETS` | Additional secrets that can open existing TOTP enrollments. New enrollments use `TWOFA_SECRET` | unset |
|
||||
| `UNSUBSCRIBE_SIGNING_SECRET` | Signs recipient unsubscribe links independently of sessions. At least 32 characters | `AUTH_SECRET` |
|
||||
| `UNSUBSCRIBE_SIGNING_READ_SECRETS` | Previous secrets accepted only for unsubscribe signatures, with the cutoff below | unset |
|
||||
| `UNSUBSCRIBE_SIGNING_READ_UNTIL` | RFC3339 cutoff required with unsubscribe read secrets. Both the verification time and the signed link expiry must be within this window | unset |
|
||||
| `DOMAIN_PROOF_SECRET` | Keys `_warmbly` DNS ownership proofs independently of sessions. At least 32 characters | `AUTH_SECRET` |
|
||||
| `INTERNAL_API_PREVIOUS_TOKENS` | Previous bearer tokens for the tracking and forms routes, accepted only before the cutoff below | unset |
|
||||
| `INTERNAL_API_PREVIOUS_UNTIL` | RFC3339 cutoff for previous internal tokens. Missing, malformed or elapsed deadlines reject previous tokens | unset |
|
||||
| `NODE_BROKER_PREVIOUS_TOKENS` | Previous bearer tokens for fleet broker routes, accepted only before the cutoff below. These retain broker privileges during the window | unset |
|
||||
| `NODE_BROKER_PREVIOUS_UNTIL` | RFC3339 cutoff for previous broker tokens. Missing, malformed or elapsed deadlines reject previous tokens | unset |
|
||||
|
||||
Changing these settings requires restarting the affected service. Internal-token deadlines are checked on every request, so old tokens stop working at the cutoff without another restart. Tracking and forms overlap does not grant fleet broker access. JWT verification uses only `AUTH_SECRET`; encryption and unsubscribe read keys never authorize sessions.
|
||||
|
||||
For encryption rotation, first deploy a compatible release everywhere with the existing primary key and the replacement included in its read-key list. Confirm that all readers use that baseline before a later deployment activates the replacement primary and retains the old key for reads. Then reseal stored values with [`warmblyctl keys reseal`](/development/warmblyctl/#keys): run it once without `--apply` to see the counts, then with `--apply`, and repeat until `warmblyctl keys status` shows nothing left under a read key. It is resumable and never overwrites a value the backend changed meanwhile. `warmblyctl keys generate` prints replacement values in the right format. The supported rollback floor during rotation is that compatible baseline release, using keys that read both generations. A binary rollback keeps the current compatible key configuration; reverting a primary key alone does not migrate data.
|
||||
|
||||
Retain old keys until stored ciphertext, retained events, pending jobs and the supported rollback window no longer need them. Keep the keys required by older backups for as long as those backups are retained. Organization DEKs and their KMS wrapping keys have a separate lifecycle; changing the instance credential key does not rotate them.
|
||||
|
||||
To rotate sessions while preserving existing MFA enrollments, unsubscribe links and published DNS proofs, first pin `TWOFA_SECRET`, `UNSUBSCRIBE_SIGNING_SECRET` and `DOMAIN_PROOF_SECRET` to their existing resolved values. Then change `AUTH_SECRET` and realtime's matching `JWT_SECRET` together. Users sign in again. Rotate the independent purpose keys separately; issued unsubscribe links remain valid for one year, so their read cutoff must cover the last previously issued link's expiry.
|
||||
|
||||
### Seal mailbox credentials under workspace keys
|
||||
|
||||
`MAILBOX_CREDENTIALS_SEALING` chooses the key that seals newly stored mailbox credentials: SMTP and IMAP hosts, usernames and passwords, and OAuth access and refresh tokens.
|
||||
|
||||
| Value | Stored credentials | Credentials sent to workers |
|
||||
|---|---|---|
|
||||
| `instance` (default) | Sealed with `CREDENTIALS_ENCRYPTION_KEY` | In the worker command, protected by the event bus |
|
||||
| `organization` | Sealed with the workspace's KMS-protected data key | Sealed with the same workspace key, opened only on the worker that runs the mailbox |
|
||||
|
||||
Every release that reads this setting opens both formats, so the setting only decides what new writes use. With `organization`, the consumer also moves credentials already stored under the instance key onto workspace keys, in small resumable batches, and skips a mailbox whose stored value no key opens. Workspace keys are cached in each process for five minutes and in Redis for fifteen, so a busy workspace costs one KMS decrypt per cache period, not one per mailbox.
|
||||
|
||||
Set it on the backend and the consumer, and only after every backend, consumer and worker runs a release that reads it. That release becomes the rollback floor: an older release cannot open a workspace-sealed credential. Keep `CREDENTIALS_ENCRYPTION_KEY` configured, because personal mailboxes with no workspace, mailboxes not yet moved and older backups still use it.
|
||||
|
||||
## Addresses
|
||||
|
||||
Every emailed link (password reset, invitation, the first-run claim link) is built from `APP_URL`. Leave it unset and a self-hosted instance guesses the origin from `CORS_ALLOW_ORIGINS` or `PUBLIC_HOST` instead, and never falls back to the hosted service: a link nobody can open is a support ticket, while a working link to somebody else's dashboard carries a live reset token out of your deployment. Set it.
|
||||
@ -154,7 +192,7 @@ TRUSTED_PROXIES=10.0.0.0/8,172.16.0.0/12
|
||||
| `WARMBLY_BOOTSTRAP_PASSWORD_HASH` | Argon2id PHC string for that owner, as `warmblyctl hash-password` prints it. Preferred over the plaintext form. A malformed hash, or one whose cost parameters are below `m=65536,t=3,p=2` with a 16-byte salt and 32-byte key, is refused and no owner is created | unset | yes |
|
||||
| `WARMBLY_BOOTSTRAP_PASSWORD` | Plaintext convenience form. Held to the same rules as any password (length, breached list); one that fails them is refused and no owner is created. Warns at boot, and leaves a password in your process environment | unset | yes |
|
||||
| `WARMBLY_BOOTSTRAP_ORG` | Name of the organization created with that owner. A value that breaks the [naming rules](/api/error-codes/#name-refusals) is ignored with a warning | derived from the name | yes |
|
||||
| `TWOFA_SECRET` | Key that encrypts stored TOTP secrets. Falls back to `AUTH_SECRET`, so existing deployments keep working; rotating it invalidates every enrolled TOTP secret | `AUTH_SECRET` | yes |
|
||||
| `TWOFA_SECRET` | Key that encrypts stored TOTP secrets. Falls back to `AUTH_SECRET`. Keep matching `TWOFA_READ_SECRETS` while existing enrollments are resealed under a replacement key | `AUTH_SECRET` | yes |
|
||||
| `WEBAUTHN_RP_ID` | Passkey relying party id. Derived from `APP_URL` when unset. Changing it invalidates every enrolled passkey | derived | yes |
|
||||
| `WEBAUTHN_RP_ORIGINS` | Origins accepted for passkey ceremonies | derived from `APP_URL` | yes |
|
||||
| `WEBAUTHN_RP_DISPLAY_NAME` | The name the passkey prompt shows | `Warmbly` | yes |
|
||||
@ -313,7 +351,7 @@ On `filesystem`, a remote worker writes blobs to its own disk rather than a volu
|
||||
| `KAFKA_SASL_USERNAME`, `KAFKA_SASL_PASSWORD` | Broker credentials | unset | yes |
|
||||
| `SCHEMA_REGISTRY_URL`, `SCHEMA_REGISTRY_KEY`, `SCHEMA_REGISTRY_SECRET` | Registry for the Avro codec | unset | yes |
|
||||
| `CODEC_PROVIDER` | `json` or `avro` | `json` under compose, `avro` for a bare binary | yes |
|
||||
| `EVENTBUS_HANDLER_TIMEOUT` | How long one handler may take before the delivery is abandoned | `30s` | yes |
|
||||
| `EVENTBUS_HANDLER_TIMEOUT` | How long one handler may take before the delivery is abandoned. A subscription that needs longer carries its own budget, never shorter than this one: a worker's command subscription runs under 195s, because a send is given 120s (the SMTP client's 90s conversation plus the checks before it) and then records its result. On NATS the ack wait is the subscription's budget plus 5s | `30s` | yes |
|
||||
| `PUBSUB_ENABLED` | `false` uses the Redis bridge for realtime fanout, `true` uses Google Pub/Sub | `false` | yes |
|
||||
| `GCP_PROJECT_ID` | Project when `PUBSUB_ENABLED=true` | unset | yes |
|
||||
|
||||
@ -334,18 +372,36 @@ Go services limit each decoded Avro array or map to 65,536 entries, counted acro
|
||||
|
||||
## Database
|
||||
|
||||
Existing self-hosts keep their current database configuration and startup
|
||||
migrations. A separate direct connection, transaction pooling and a deployment
|
||||
migration job are optional. They do not require cloud services or change the
|
||||
one-command installer defaults.
|
||||
|
||||
| Variable | What it does | Default | Restart needed |
|
||||
|---|---|---|---|
|
||||
| `PRIMARY_DB` | PostgreSQL connection string. Carries inline credentials, so it is never returned by any API | the compose postgres | yes |
|
||||
| `PRIMARY_DB_DIRECT` | Direct PostgreSQL connection for migrations, backups and session advisory locks. Required when `DB_POOL_MODE=transaction`. It must address the same database as `PRIMARY_DB`; its credentials can be different. With AWS configuration loading, the secret path is `postgres/primary_direct` | falls back to `PRIMARY_DB` in direct mode | yes |
|
||||
| `DB_POOL_MODE` | `direct` or `transaction` for the Go services. Transaction mode uses `PRIMARY_DB` for ordinary queries and `PRIMARY_DB_DIRECT` for session operations. It verifies server-side role defaults of nonzero `statement_timeout` at most 60 seconds and `idle_in_transaction_session_timeout` at most 5 minutes on every new pooled client connection. Configure those role defaults before connecting | `direct` | yes |
|
||||
| `DB_MIGRATIONS_ON_START` | Whether the backend applies embedded migrations at startup. Set `false` when a deployment job runs `/app/migrate` first. The backend still waits for a clean schema at the version its image requires. AWS configuration path: `postgres/migrations_on_start` | `true` for self-host compatibility; `false` in the AWS runtime root | yes |
|
||||
| `PGSSLROOTCERT` | Certificate bundle used to verify the database's TLS certificate, read by the Postgres driver. Amazon RDS chains to a root in no public trust store, so `sslmode=verify-full` against it needs AWS's bundle; the backend and consumer images ship it at `/etc/ssl/rds/global-bundle.pem`. Prefer `sslrootcert=` in the DSN so the setting travels with the connection it applies to | unset (system trust store) | yes |
|
||||
| `DB_MAX_CONNS` | Maximum pooled Postgres connections for one Go process (backend, consumer). The ceiling that matters is the database's `max_connections`, and every replica, consumer and `warmblyctl` run takes its own share of it. At boot a process asks the server for that ceiling and lowers its own pool to a quarter of the connections the server leaves unreserved. The quarter assumes four database clients, which is backend and consumer with a rolling deploy running two of each, so it is a per-process guard and not a fleet-wide allocation: more than four clients, or anything else sharing the server, still needs this set deliberately. The clamp only ever lowers the pool, and only when the probe succeeds; a probe that cannot reach the server or read the setting leaves the value below in force. A `pool_max_conns` in the DSN is honoured when this is unset | `25`, lowered to the probed share | yes |
|
||||
| `DB_MAX_CONNS` | Maximum client connections for one Go process (backend, consumer). In direct mode, a successful server-capacity probe lowers the pool to a quarter of unreserved server connections, allowing for backend and consumer deployment overlap. Transaction mode keeps the configured client ceiling and relies on separately configured pooler server limits. Size the aggregate server budget across all roles and direct clients. A `pool_max_conns` in the DSN is honoured when this is unset | `25`, lowered to the probed share in direct mode | yes |
|
||||
| `DB_MIN_CONNS` | Connections each Go process keeps open while idle. Capped at `DB_MAX_CONNS` | `2` | yes |
|
||||
| `DATABASE_URL` | The realtime service's own name for the same database | the compose postgres | yes |
|
||||
| `DATABASE_POOL_SIZE` | Maximum pooled connections **for the realtime service only**. The Go services use the driver default and do not read it | `10` | yes |
|
||||
| `DATABASE_POOL_SIZE` | Maximum client connections **for the realtime service only**. The Go services use `DB_MAX_CONNS` | `10` | yes |
|
||||
| `DATABASE_SSL` | Whether the realtime service connects to Postgres over TLS | `true`, and `false` under compose | yes |
|
||||
| `DATABASE_SSL_CA_FILE` | CA bundle the realtime service verifies Postgres against. Unset verifies against the system store, which has no Amazon RDS root in it, so an RDS database needs this set to `/etc/ssl/rds/global-bundle.pem` (shipped in the image) | unset | yes |
|
||||
|
||||
Migrations are embedded in the backend binary and applied on boot. There is no separate migration step, and a standalone `/app/migrate` binary ships in the image for the cases where you want one.
|
||||
Migrations are embedded in the backend image. Self-hosts apply them on boot by
|
||||
default. Hosted deployments run the same image's `/app/migrate` binary once
|
||||
before rolling services, with `DB_MIGRATIONS_ON_START=false` on the backend.
|
||||
The migration binary uses `PRIMARY_DB_DIRECT` when configured. A failed migration
|
||||
blocks deployment; operators inspect its logs and schema state before retrying.
|
||||
Services do not automatically reverse a migration during application rollback.
|
||||
|
||||
Use additive schema changes that remain usable by the previous release during
|
||||
deployment and rollback. Stage renames, removals, data transformations and event
|
||||
format changes across releases. Backfill large datasets with a resumable job
|
||||
after compatible readers and writers are deployed.
|
||||
|
||||
## Realtime upgrade compatibility
|
||||
|
||||
@ -446,7 +502,7 @@ Keep the copies private. GeoLite2 may not be redistributed, which is also why no
|
||||
| `WORKER_BIND_IP` | Source address to bind outbound connections to, and the seed for a derived `WORKER_ID` | unset | yes |
|
||||
| `WORKER_PUBLIC_IP` | Public IPv4 reported by the worker, never replaced by the backend-observed address, which may be a reverse proxy. Otherwise the node discovers HTTPS egress IPv4 using `api.ipify.org` every ten minutes. Set this explicitly when mail and HTTPS have different egress routes | detected | yes |
|
||||
| `WARMBLY_WORKER_CAPACITY` | Assigned-mailbox planning target for this worker. This guides placement and is not a mailbox-provider send limit | `100` | yes |
|
||||
| `WORKER_COMMAND_CONCURRENCY` | Kafka worker command lanes, from `1` to `64`. Commands are grouped by the mailbox in the payload, including older task-keyed sends. Each mailbox stays in order; independent lanes run concurrently. NATS remains serial. Invalid values use the default | `8` | yes |
|
||||
| `WORKER_COMMAND_CONCURRENCY` | Worker command lanes, from `1` to `64`, on both Kafka and NATS. Commands are grouped by the mailbox in the payload, including older task-keyed sends. Each mailbox stays in order; independent lanes run concurrently, so one slow mail server holds only its own lane. On NATS a command waiting in a lane is kept from being redelivered while it waits. Invalid values use the default | `8` | yes |
|
||||
| `WARMBLY_NODE_REGION` | Free-form label for where this node egresses from, e.g. `eu-central`. Placement prefers a worker near where a mailbox's provider expects sign-ins; unset scores neutral. Written by the join script from `--region` | unset | yes |
|
||||
| `ENCRYPTED_KEYS_BACKEND_URL` | Backend base the worker fetches organization keys from | unset | yes |
|
||||
| `ENCRYPTED_KEYS_WORKER_TOKEN` | The worker's copy of `INTERNAL_API_TOKEN` | unset | yes |
|
||||
@ -571,7 +627,7 @@ Each customer's authorization is tied to the service account's Client ID, so cre
|
||||
A Google grant is recorded only for the workspace that proves it controls the domain, and only when the directory lists the admin address as a super administrator. The proof is one of:
|
||||
|
||||
- **The administrator's Google sign-in**, through the dashboard's own sign-in client (`GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`) and its existing redirect URI (`GOOGLE_REDIRECT_URI`, or `API_PUBLIC_URL` plus `/v1/auth/google/callback`), with the `openid` and `email` scopes only. The callback recognizes the proof and hands it back to the dashboard window rather than signing anyone in, so no new redirect URI needs registering. When that client is not configured, the `BOX_GOOGLE_*` mailbox client is used instead, the same way. Neither depends on `BOX_GOOGLE_OAUTH_CONNECT`.
|
||||
- **A `TXT` record** at `_warmbly.<domain>` holding a `warmbly-verify=` value unique to the workspace and domain, always available and the only proof when neither client is set. The value is keyed by `AUTH_SECRET`.
|
||||
- **A `TXT` record** at `_warmbly.<domain>` holding a `warmbly-verify=` value unique to the workspace and domain, always available and the only proof when neither client is set. The value is keyed by `DOMAIN_PROOF_SECRET`, which defaults to `AUTH_SECRET`.
|
||||
|
||||
Microsoft 365 organizations reuse the Outlook app, `BOX_OUTLOOK_CLIENT_ID` and `BOX_OUTLOOK_CLIENT_SECRET`, and are on whenever those are set. The app registration needs three additions for them:
|
||||
|
||||
@ -673,6 +729,8 @@ An empty `AI_PROVIDER` with a set `AI_API_KEY` falls back to `api.openai.com`, s
|
||||
|
||||
Model compatibility needs no configuration. Backends disagree about the request shape: newer OpenAI models want `max_completion_tokens` rather than `max_tokens`, refuse any temperature but the default, and some refuse function tools unless reasoning is switched off, while most OpenAI compatible backends only know the older shape. The provider learns this from the first rejection of each parameter and remembers it, so a model is walked to a working shape on its first call and every later call starts there.
|
||||
|
||||
The OpenAI-compatible agent loop also handles catalogs larger than 128 tools automatically. It exposes read-only tool discovery and loads up to 20 matching tools at a time, rather than sending the whole catalog in one request. All permitted built-in and connected MCP tools remain discoverable, and their original permissions and approval rules still apply. Existing conversations can continue after upgrading; no database migration or AI configuration change is needed.
|
||||
|
||||
Set these on the backend and the consumer.
|
||||
|
||||
Automatic inbox tagging is read by the backend, consumer, and `warmblyctl`. Both inbox-tagging variables are required; a key by itself does not start classification. See [Automatic inbox tagging](/guides/inbox-tagging/).
|
||||
@ -687,6 +745,8 @@ Per-mailbox `test_mode`, `test_send_enabled` and `test_receive_enabled` separate
|
||||
|---|---|---|
|
||||
| `TASKS_PROVIDER` | `local` (an in-process Postgres poller) or `gcloud` (Cloud Tasks) | `local` |
|
||||
| `TASKS_LOCAL_POLL_INTERVAL` | How often the local poller looks for due work | `1s` |
|
||||
| `SEND_PLAN_SNAPSHOT_INTERVAL` | How often the backend re-walks the send plan of every active campaign whose stored plan is out of date. Each walk reads all of that campaign's leads, so a longer interval lowers database load on large campaigns | `1m` |
|
||||
| `SEND_PLAN_SNAPSHOT_MAX_AGE` | How old a stored send plan may be before it counts as out of date. An out-of-date plan is still served, with `stale: true`, while a fresh one is computed in the background | three times `SEND_PLAN_SNAPSHOT_INTERVAL` (`3m`) |
|
||||
| `BILLING_PROVIDER` | `none` (every feature unlocked, no trial expiry; the dashboard reports the workspace as self-hosted rather than on a free tier and hides billing) or `stripe` | `none` |
|
||||
| `STRIPE_SECRET_KEY`, `STRIPE_WEBHOOK_SECRET`, `STRIPE_PUBLISHABLE_KEY` | Required together when `BILLING_PROVIDER=stripe`. The backend exits at boot if any is missing | unset |
|
||||
|
||||
|
||||
172
docs/content/docs/development/coolify.mdx
Normal file
172
docs/content/docs/development/coolify.mdx
Normal file
@ -0,0 +1,172 @@
|
||||
---
|
||||
title: Deploy on Coolify
|
||||
description: Run Warmbly on your VPS with Coolify, published Docker images, persistent volumes, and HTTPS. No host installer or source build required.
|
||||
---
|
||||
|
||||
Use [deploy/docker-compose.coolify.yml](https://github.com/warmbly/warmbly/blob/main/deploy/docker-compose.coolify.yml), not the repository-root development Compose file. It runs the API, dashboard, admin panel, consumer, worker, tracking, realtime, forms, PostgreSQL, Redis, and NATS entirely in containers.
|
||||
|
||||
There is no `install.sh`, source compilation, Docker socket mount, or host port mapping. Coolify owns the reverse proxy, certificates, and deployments. Data lives in Docker named volumes, so containers are disposable but the data is not.
|
||||
|
||||
## Before you start
|
||||
|
||||
- A working Coolify server with its proxy enabled. Start with at least **4 GB RAM** and enough disk for images, mailbox data, and backups. Actual requirements grow with usage.
|
||||
- An `amd64` or `arm64` VPS with outbound access to GHCR, Docker Hub, your mail providers, and their SMTP/IMAP ports. Warmbly is not an SMTP server; it sends through the mailboxes you connect.
|
||||
- Six DNS names pointing to your VPS. The examples below use `app.example.com`, `admin.example.com`, `api.example.com`, `realtime.example.com`, `track.example.com`, and `forms.example.com`. Replace every example with your own domain.
|
||||
- Public ports **80 and 443** available to Coolify's proxy. Do not open PostgreSQL, Redis, NATS, or the individual application ports to the internet.
|
||||
|
||||
For real outreach, prefer a separate registered domain for tracking rather than a subdomain of the domain serving your dashboard and platform mail. This separates the reputation of campaign links from your own site. The shared example domain below is only to illustrate routing; see [deliverability](/guides/deliverability/).
|
||||
|
||||
<Callout title="Why a generic GitHub deployment can fail">
|
||||
Warmbly is a multi-service application, not one Node or Go web server. Automatic Nixpacks/Railpack detection or selecting a single Dockerfile does not deploy its dependencies. The root Compose file also includes source-build definitions, development defaults, and host ports. The dedicated Coolify file avoids those paths and requires real secrets before starting. Without the failed deployment's logs, this is not a diagnosis of a particular failure.
|
||||
</Callout>
|
||||
|
||||
## 1. Create the Compose resource
|
||||
|
||||
Choose **one** of these methods. They run the same images.
|
||||
|
||||
### Paste the file, no Git integration needed
|
||||
|
||||
1. In your Coolify project and environment, select **+ New**.
|
||||
2. Select **Docker Compose Empty**.
|
||||
3. Paste the entire [raw Coolify Compose file](https://raw.githubusercontent.com/warmbly/warmbly/main/deploy/docker-compose.coolify.yml) into the Compose editor and save it.
|
||||
|
||||
This is the simplest path if you only want to run Warmbly, not maintain a fork. Keep the complete file, including the `x-` anchors at the top and the `volumes` section at the bottom.
|
||||
|
||||
### Use the GitHub repository
|
||||
|
||||
1. Select **+ New**, choose a public Git repository, enter `https://github.com/warmbly/warmbly`, and check the repository.
|
||||
2. Select the **Docker Compose** build pack (called **Compose** under the build strategy in some versions), not Nixpacks, Railpack, or Dockerfile.
|
||||
3. Select branch `main`, set **Base Directory** to `/`, and **Docker Compose Location** to `/deploy/docker-compose.coolify.yml`.
|
||||
4. Save and load/reload the Compose definition. Confirm that the images start with `ghcr.io/warmbly/warmbly/` and that there are no `build:` entries.
|
||||
|
||||
The path is relative to the repository root. Do not point Coolify at `/docker-compose.yml`, and do not add custom build or start commands. A Git checkout selects the Compose definition; `WARMBLY_TAG` selects the actual application release. Disable automatic Git deployments if you want upgrades only when you request them.
|
||||
|
||||
## 2. Set environment variables
|
||||
|
||||
In this resource's **Environment Variables**, supply the values below before deploying. URL values are browser-facing, not Docker service addresses.
|
||||
|
||||
```dotenv
|
||||
APP_URL=https://app.example.com
|
||||
ADMIN_URL=https://admin.example.com
|
||||
API_PUBLIC_URL=https://api.example.com
|
||||
REALTIME_HOST=realtime.example.com
|
||||
TRACKING_DOMAIN=track.example.com
|
||||
FORMS_DOMAIN=forms.example.com
|
||||
WARMBLY_TAG=prod
|
||||
```
|
||||
|
||||
Use **no trailing slash** in the three URLs. The three hostname values have **no scheme, port, or path**. In particular, do not put `:8080` in `API_PUBLIC_URL` or `:4000` in `REALTIME_HOST`. The file builds the secure WebSocket URL automatically.
|
||||
|
||||
`prod` pulls the production release channel. For repeatable deployments, set `WARMBLY_TAG` to a published version such as `v0.6.55` from [GitHub Releases](https://github.com/warmbly/warmbly/releases). Every Warmbly service uses this same tag. Do not use a tag that has not finished publishing, `latest`, or the `-kafka` variants for this NATS stack.
|
||||
|
||||
Generate **seven independent secrets** on a trusted computer. These commands only generate random strings; they do not install Warmbly or change your VPS:
|
||||
|
||||
| Coolify variable | Command to generate its value |
|
||||
| --- | --- |
|
||||
| `POSTGRES_PASSWORD` | `openssl rand -hex 32` |
|
||||
| `AUTH_SECRET` | `openssl rand -hex 32` |
|
||||
| `INTERNAL_API_TOKEN` | `openssl rand -hex 32` |
|
||||
| `NODE_BROKER_TOKEN` | `openssl rand -hex 32` |
|
||||
| `SECRET_KEY_BASE` | `openssl rand -hex 64` |
|
||||
| `KMS_LOCAL_MASTER_KEY` | `openssl rand -base64 32` |
|
||||
| `CREDENTIALS_ENCRYPTION_KEY` | `openssl rand -hex 32` |
|
||||
|
||||
Run the command separately for each row and paste only its output into that variable. Mark these as secrets in Coolify where supported. Keep them available to Compose at deployment/runtime; there is no build stage. Do not paste the command itself or leave a required value empty. Use the generated hexadecimal PostgreSQL password so it is safe inside the database URL without URL encoding.
|
||||
|
||||
The same saved variables are reused across services. `AUTH_SECRET` also becomes realtime's `JWT_SECRET`; `NODE_BROKER_TOKEN` protects worker-only internal calls separately from public-edge services. You do not need to enter per-container copies.
|
||||
|
||||
<Callout type="warn" title="Keep encryption keys and database credentials stable">
|
||||
Back up these variables securely. Losing or replacing `KMS_LOCAL_MASTER_KEY` or `CREDENTIALS_ENCRYPTION_KEY` makes existing encrypted data unreadable. Do not regenerate them on redeploy. Changing `POSTGRES_PASSWORD` in Coolify does not change the password inside an already initialized PostgreSQL volume; rotate it in PostgreSQL deliberately before updating clients.
|
||||
</Callout>
|
||||
|
||||
### Platform email
|
||||
|
||||
The initial default is `MAIL_TRANSPORT=log`, so the instance can boot without a mail relay. Platform emails are written to service logs instead of being delivered. Before inviting other users or relying on password resets, add:
|
||||
|
||||
```dotenv
|
||||
MAIL_TRANSPORT=smtp
|
||||
EMAIL_ADDRESS=noreply@example.com
|
||||
EMAIL_NAME=Warmbly
|
||||
SMTP_HOST=smtp.your-provider.com
|
||||
SMTP_PORT=587
|
||||
SMTP_SECURITY=starttls
|
||||
SMTP_USERNAME=your-smtp-user
|
||||
SMTP_PASSWORD=your-smtp-password
|
||||
```
|
||||
|
||||
Use your provider's actual details and a verified sender. For implicit TLS use `SMTP_PORT=465` and `SMTP_SECURITY=tls`. These variables reach both the backend and consumer, including notifications and digests. They are separate from campaign mailbox credentials. See [platform mail](/development/configuration/#platform-mail).
|
||||
|
||||
## 3. Assign the domains to the right services
|
||||
|
||||
In Coolify, configure a domain for each public service. The port suffix selects the **internal container port**, not a port the user puts in their browser:
|
||||
|
||||
| Service | Coolify domain value | Public browser address |
|
||||
| --- | --- | --- |
|
||||
| `web` | `https://app.example.com:80` | `https://app.example.com` |
|
||||
| `admin` | `https://admin.example.com:80` | `https://admin.example.com` |
|
||||
| `backend` | `https://api.example.com:8080` | `https://api.example.com` |
|
||||
| `realtime` | `https://realtime.example.com:4000` | `wss://realtime.example.com/socket/websocket` |
|
||||
| `tracking` | `https://track.example.com:3000` | `https://track.example.com` |
|
||||
| `forms` | `https://forms.example.com:8090` | `https://forms.example.com` |
|
||||
|
||||
For a Git-based resource, use **Domains**, select the receiving service, and add its domain. For Docker Compose Empty, configure the domain in each service's settings. Labels and screen names vary between Coolify versions, but the service/port mapping above is the important part.
|
||||
|
||||
Do **not** add domains or public port mappings for `postgres`, `redis`, `nats`, `consumer`, or `worker`. Leave Coolify's normal proxy management enabled; this file does not supply manual proxy labels. You do not need a shared/predefined network with unrelated resources. Coolify handles routing and HTTPS certificates for these domains.
|
||||
|
||||
The frontend containers listen on port **80**, not `5173`/`5174`. All `/health` requests belong to their own service; sending the API domain to `web` produces HTML instead of API responses. Compose defines readiness checks per HTTP service. Consumer and worker are background processes without HTTP endpoints.
|
||||
|
||||
### Preserve real client IP addresses
|
||||
|
||||
Set `TRUSTED_PROXIES`, `TRACKING_TRUSTED_PROXIES`, and `FORMS_TRUSTED_PROXIES` to the actual comma-separated source IPs/CIDRs of your Coolify reverse proxy, then redeploy. Determine these from your server's Docker network configuration; there is no universal Coolify subnet. Never use `0.0.0.0/0` or `::/0`. Only a trusted proxy may supply the original visitor's forwarded address.
|
||||
|
||||
The safe default trusts no forwarded addresses. Leaving it unset behind a proxy means visitors share the proxy's IP for rate limits, audit records, and tracking classification. See [network and proxy](/development/configuration/#network-and-proxy).
|
||||
|
||||
## 4. Deploy and claim the instance
|
||||
|
||||
1. Select **Deploy** and inspect the deployment logs. Coolify pulls the images; it should not compile Go, Rust, Elixir, or Node. PostgreSQL, Redis, and NATS start first, then the backend applies database migrations automatically, then the remaining services start.
|
||||
2. Wait for the HTTP services to become healthy and for `consumer` and `worker` to remain running. Initial migrations can take longer than an ordinary restart.
|
||||
3. Open a terminal **inside the `backend` container** using Coolify and run:
|
||||
|
||||
```bash
|
||||
warmblyctl setup-link
|
||||
```
|
||||
|
||||
4. Open the single-use `https://app.example.com/setup?token=...` link it prints and create the owner account. Treat the link as a password. It expires after 24 hours; run the command again to replace a lost or expired link while the instance is unclaimed.
|
||||
5. In the backend terminal, run `warmblyctl status`. Review any findings rather than interpreting an HTTP health check as a complete mail-delivery check. For details, see [first run](/development/first-run/) and [instance health](/development/instance-health/).
|
||||
|
||||
The claim creates the initial platform admin; do not seed demo users into a real instance. After claiming, registration defaults to invitation-only. To access the admin panel, enroll a passkey or TOTP under **Settings > Security** and sign in again with that second factor. See [admin access](/development/admin-panel/).
|
||||
|
||||
Confirm `https://api.example.com/health` responds successfully, the dashboard opens without a CORS error, and a worker appears in the admin fleet view. Then connect a mailbox and verify a controlled send and sync before using the instance for live campaigns. A healthy deployment alone does not prove your VPS can reach a mail provider.
|
||||
|
||||
### Connecting mailboxes
|
||||
|
||||
SMTP/IMAP mailbox connections require no Google or Microsoft app registration. Their TLS certificates are verified by this stack. Do not disable verification to work around provider settings.
|
||||
|
||||
For OAuth mailbox connections, the file also passes `BOX_GOOGLE_CLIENT_ID`, `BOX_GOOGLE_CLIENT_SECRET`, `BOX_GOOGLE_OAUTH_CONNECT`, `BOX_OUTLOOK_CLIENT_ID`, and `BOX_OUTLOOK_CLIENT_SECRET` to the appropriate services. Gmail's UI switch follows `BOX_GOOGLE_OAUTH_CONNECT`. Set up your own provider application and the callback URLs documented under [configuration](/development/configuration/), then redeploy. These are mailbox-connection clients, not sign-in-with-Google credentials.
|
||||
|
||||
## Updates, backups, and existing data
|
||||
|
||||
- **Update through Coolify**, not Warmbly's in-app updater. This stack sets `UPDATER_URL=none` and never mounts the Docker socket. Back up first, set a new published `WARMBLY_TAG` (or refresh the `prod` images), and deploy with fresh image pulls enabled in your Coolify version. Merely restarting an existing container does not pull a new release. Keep all eight application images on the same release.
|
||||
- **Keep the resource and its volumes.** The persistent volumes are `postgres_data`, `redis_data`, `nats_data`, `blobs`, and `worker_state`. Coolify/Compose may prefix their actual Docker names with a resource identifier. Ordinary redeploys reuse them; deleting volumes, recreating the resource under another identifier, or running `docker compose down -v` does not.
|
||||
- **Back up data and secrets together.** Take logical PostgreSQL backups, preserve the blob volume and encryption keys, and include Redis, NATS, and worker state in your recovery plan. A live copy of the PostgreSQL data directory is not a consistent database backup. For a coordinated whole-instance snapshot, stop application services first. See [data control](/development/data-control/).
|
||||
- **Do not assume automatic backups.** PostgreSQL here is a Compose service, not necessarily a Coolify-managed database with scheduled backups. Configure and test your backup/restore process explicitly.
|
||||
- **Existing installations are a migration, not a fresh deploy.** This resource creates its own volumes and does not discover an installer deployment's data. Keep the existing instance and keys intact, and use the [backup/move procedure](/development/data-control/) or [workspace export/import](/guides/workspace-export-import/). Copying only the database or generating new encryption keys is not sufficient. Database migrations run on startup, so restoring an older image is not a guaranteed database rollback.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | What to check |
|
||||
| --- | --- |
|
||||
| Coolify builds source or fails looking for a package/start command | Select Docker Compose and the dedicated file path, or paste it into Docker Compose Empty. Remove Nixpacks/Railpack/Dockerfile settings. |
|
||||
| Required variable error before startup | Fill every `${VARIABLE:?}` value in this resource's environment, save, and redeploy. Keep all YAML anchors when pasting. |
|
||||
| Image not found or registry denied | Check the exact release tag and image name. These are public GHCR images; no Warmbly Cloud login is required. Check VPS DNS/network access and registry limits. |
|
||||
| Port already allocated | This file publishes no host ports. Remove mappings copied from the root Compose file; only the Coolify proxy should bind public 80/443. |
|
||||
| Proxy 502 or “no available server” | Check the failing service's runtime logs and health status, then its domain's internal port. Use `127.0.0.1` in container health checks. A certificate error is a separate DNS/proxy issue. |
|
||||
| PostgreSQL authentication fails after a redeploy | Restore the original `POSTGRES_PASSWORD` for the existing volume or rotate it inside PostgreSQL. Do not delete the data volume. |
|
||||
| Dashboard loads but requests fail | `API_PUBLIC_URL` must be the external HTTPS API origin, with no internal port or `/v1` suffix. The API must route to `backend:8080`, not nginx. Redeploy after changing environment variables so frontend `config.js` is regenerated. |
|
||||
| Realtime disconnects or the browser reports CSP/origin errors | Check DNS/HTTPS for `REALTIME_HOST`, its route to `realtime:4000`, and exact `APP_URL`/`ADMIN_URL` origins. The file shares the JWT secret and configures both WebSocket origin checks and frontend CSP. |
|
||||
| No emails arrive for resets or invitations | `MAIL_TRANSPORT=log` does not deliver mail. Configure SMTP, check backend and consumer logs, and confirm outbound provider ports are allowed. |
|
||||
| Mailbox connects but nothing sends or syncs | Inspect worker/consumer logs, worker heartbeat, provider credentials, TLS, and outbound SMTP/IMAP access. Do not open inbound mail ports or disable TLS checks. |
|
||||
|
||||
For a failure this page does not cover, share the Coolify version, VPS architecture, selected release tag, failing service name, and relevant deployment/runtime error lines. **Redact secrets, claim links, mailbox credentials, and connection strings.**
|
||||
|
||||
These instructions follow Coolify's official [Docker Compose](https://coolify.io/docs/applications/builds/docker-compose), [domain routing](https://coolify.io/docs/knowledge-base/domains), and [health check](https://coolify.io/docs/applications/configuration/health-checks) documentation.
|
||||
@ -36,8 +36,8 @@ Two keys, and they are not interchangeable.
|
||||
|
||||
| Key | Opens | Read by |
|
||||
|---|---|---|
|
||||
| `CREDENTIALS_ENCRYPTION_KEY` | Mailbox SMTP and IMAP credentials, and OAuth tokens | Backend and workers, without an organization context |
|
||||
| `KMS_LOCAL_MASTER_KEY` | The per-organization data keys, which in turn open everything else | Backend and consumer |
|
||||
| `CREDENTIALS_ENCRYPTION_KEY` | Mailbox SMTP and IMAP credentials and OAuth tokens stored under the instance key: everything by default, and only older values and personal mailboxes with `MAILBOX_CREDENTIALS_SEALING=organization` | Backend, consumer and workers |
|
||||
| `KMS_LOCAL_MASTER_KEY` | The per-organization data keys, which in turn open everything else, including mailbox credentials sealed under workspace keys | Backend and consumer, and workers when `KMS_PROVIDER=local` |
|
||||
|
||||
Both are unrecoverable. A database backup without them restores an instance whose mailboxes authenticate against nothing, and there is no way back from that. They are in `.env`, and `install.sh` also writes them to `keys-backup.txt` next to the install, which is on the same disk as the database and therefore not a backup either. Copy them somewhere else.
|
||||
|
||||
@ -177,7 +177,7 @@ Disconnecting a mailbox, or deleting the workspace holding it, removes its rows
|
||||
| The OAuth grant | A Gmail mailbox's refresh token is sent to Google's revocation endpoint, which invalidates it and removes the instance from the customer's third-party access list. Microsoft publishes no per-application revocation endpoint, so an Outlook mailbox's tokens are destroyed locally and the customer removes the app themselves |
|
||||
| The message bodies | Everything the mailbox wrote under `users/<user>/emails/<mailbox>/` in the blob store is deleted, whether that is a filesystem path or an S3 bucket |
|
||||
|
||||
While an erasure is outstanding its row holds a copy of the mailbox's OAuth refresh token, still sealed under `CREDENTIALS_ENCRYPTION_KEY`, because the row it was read from goes with the mailbox and the token is what the revocation sends. It is destroyed with the rest of the row when the erasure finishes.
|
||||
While an erasure is outstanding its row holds a copy of the mailbox's OAuth refresh token, still sealed under the key that sealed the mailbox's credentials, because the row it was read from goes with the mailbox and the token is what the revocation sends. It is destroyed with the rest of the row when the erasure finishes.
|
||||
|
||||
The queue is the `mailbox_erasures` table, and it holds only work still owed: a finished erasure is removed rather than kept, because the row names the address the customer asked to have forgotten. A failure is retried with a widening backoff, capped at six hours, and never abandoned, so an unreachable provider or store delays the erasure instead of losing it. Anything still outstanding after a day is reported by the `mailbox_erasure` job, visible under scheduled jobs in the admin panel.
|
||||
|
||||
@ -209,7 +209,7 @@ docker compose -p warmbly cp backend:/data/blobs/warmbly.tar.gz ./warmbly.tar.gz
|
||||
|
||||
`/data/blobs` is a hand-off, not a destination: it is the one path the container and the host both see. The `&&` matters twice over. `backup` leaves its own output out of the archive, but a bundle left there is swept into the next run, so it has to be deleted; and a `cp` that failed must not be followed by deleting the only copy that exists.
|
||||
|
||||
The bundle is written 0600 and holds every mailbox credential on the instance plus the keys that open them. Treat the file as you would the instance itself.
|
||||
The bundle is written 0600 and holds every mailbox credential on the instance plus the keys that open them. It includes configured credential and MFA read keys, purpose-specific signing keys and bounded internal-token overlap settings. Preserve those values when restoring a bundle that contains mixed generations of ciphertext. Treat the file as you would the instance itself.
|
||||
|
||||
An install from `install.sh --wizard` can schedule this for you: `backup.sh` next to the install, a systemd timer, a retention count, and an optional `aws s3 cp` to somewhere off the host. A backup that only exists on the machine it backs up is not one.
|
||||
|
||||
@ -222,7 +222,7 @@ docker compose -p warmbly exec backend warmblyctl restore --file /data/blobs/war
|
||||
docker compose -p warmbly restart
|
||||
```
|
||||
|
||||
The restore empties the schema and replays the dump, so it replaces everything currently on that instance and asks you to type `restore` first.
|
||||
The restore empties the schema and replays the dump, so it replaces everything currently on that instance and asks you to type `restore` first. It then runs `ANALYZE`, since a dump carries no planner statistics; a failure there only warns.
|
||||
|
||||
Before it does anything it compares the bundle's `CREDENTIALS_ENCRYPTION_KEY` and `KMS_LOCAL_MASTER_KEY` against the destination's and refuses to continue when they differ, printing the two lines to put in `.env`. That check is the point of the command: without it a restore looks like it worked and every mailbox fails to authenticate days later, with no error that names the cause.
|
||||
|
||||
@ -250,7 +250,7 @@ curl -fsSL https://warmbly.com/install.sh | sh -s -- --host <new-hostname>
|
||||
|
||||
### Put the old keys in the new `.env`
|
||||
|
||||
Copy `CREDENTIALS_ENCRYPTION_KEY` and `KMS_LOCAL_MASTER_KEY` from the old install, then recreate the containers so they take:
|
||||
Copy the saved keys from the old install, including `CREDENTIALS_ENCRYPTION_KEY`, `KMS_LOCAL_MASTER_KEY` and any configured read keys or separate MFA and signing keys. See [key rotation](/development/configuration/#rotate-instance-keys). Then recreate the containers so they take:
|
||||
|
||||
```bash
|
||||
docker compose -p warmbly up -d
|
||||
|
||||
@ -5,6 +5,8 @@ description: A step-by-step guide to running the whole Warmbly platform yourself
|
||||
|
||||
Warmbly self-hosts with **no cloud account of any kind**: no AWS, no GCP, no Stripe, no Kafka. One `docker-compose.yml` runs everything on local, open-source pieces, and every external dependency is an environment variable you can flip later.
|
||||
|
||||
Using **Coolify**? Follow [Deploy on Coolify](/development/coolify/) for an image-only Compose stack, exact domain/port mappings, and setup without running an installer on the host.
|
||||
|
||||
<Callout title="Just want it running?">
|
||||
```bash
|
||||
curl -fsSL https://warmbly.com/install.sh | sh
|
||||
|
||||
@ -91,6 +91,8 @@ The consumer registers one handler per type (`internal/app/consumer/events.go`);
|
||||
|
||||
Every `SEND_EMAIL` is answered with exactly one per-task result: `EMAIL_SENT` (the provider accepted the message; the consumer records the wire Message-ID on the task) or `EMAIL_FAILED` (a `SendEmailResult` carrying the error code and message). The control plane stamps a campaign step sent when it hands the send to the worker, so `EMAIL_FAILED` is what walks that back: the consumer marks the task failed, clears the step's `sent_at` and counts the attempt on `campaign_contact_progress`, gives the send back to the campaign's daily counters, writes the failure to the campaign activity log, and reopens a campaign that completed while the send was in flight. After `CampaignSendMaxAttempts` (5) the lead is marked failed and routing drops it; a `RECIPIENT_REJECTED` result (refused at RCPT) skips the retries and is ingested as a bounce instead. Account-level conditions (`EMAIL_AUTH_ERROR`, `EMAIL_DISABLED`, `EMAIL_RATE_LIMITED`, `EMAIL_SERVER_ERROR`) are raised in addition to, never instead of, the per-task result; they carry an `EmailErrorEvent` and act on the mailbox. A worker that does not hold the mailbox yet leaves the send for a few redeliveries before reporting it failed, and the backend never publishes a send to a worker that is not heartbeating.
|
||||
|
||||
A worker gives a `SEND_EMAIL` 120 seconds, enough for the SMTP client's own 90-second conversation with a slow server plus the dispatch checks and body fetch before it; every other command keeps 30 seconds. Its command subscription runs under 195 seconds, which also covers recording the result, so the bus does not hand a send out again while the first delivery is still finishing. Commands run on ordered per-mailbox lanes (`WORKER_COMMAND_CONCURRENCY`) on both buses, so a stalled mail server holds only its own mailbox's commands.
|
||||
|
||||
`UPDATE_FOLDER` reports the canonical folder a provider now has a message in, without the rest of the message. Gmail sends it when a history record adds or removes `INBOX`, `TRASH`, `SPAM`, `SENT` or `DRAFT`, with the folder recomputed from the message's labels, and from a reconciliation that runs every six hours and once per mailbox load: the worker lists the Gmail inbox and asks the backend (`GET /api/v1/internal/sync/provider-folder-messages`) which rows it believes are in inbox, archive, spam or trash, then reports the ones Gmail has elsewhere. The consumer resolves it like `UPDATE_EMAIL`: against the folder the provider last reported, so a message filed in Warmbly stays filed unless the provider itself moved it. The worker also sends it, with `relayed` set, for every message a `MESSAGE_FOLDER` left in its destination, plus the handles the move changed (the new Graph id in `provider_id`, or the IMAP `folder_path`, `mailbox` and `uid`). A relayed one writes `provider_folder` and those handles and never the folder, so the sync's own report of the same placement afterwards reads as no move, and a filing undone meanwhile stays undone.
|
||||
|
||||
`SYNC_STATE` is the worker's relay of a mailbox's sync state (backfill progress and cursor, fair-use throttle, last-synced time). It carries the full state rather than a delta, so a lost event is repaired by the next one; the consumer writes it to `email_sync_state` and the backend hands it back inside `ADD_EMAIL` on the next load, which is what lets a replaced worker resume an import instead of restarting it.
|
||||
@ -145,7 +147,7 @@ EVENTBUS_PROVIDER=nats # nats | kafka (kafka needs the kafka build ta
|
||||
NATS_URL=nats://nats:4222
|
||||
NATS_STREAM_NAME=warmbly # optional, default warmbly
|
||||
NATS_SUBJECT_PREFIX=warmbly # optional, default warmbly
|
||||
EVENTBUS_HANDLER_TIMEOUT=30s # optional, per-message handler timeout
|
||||
EVENTBUS_HANDLER_TIMEOUT=30s # optional, per-message handler timeout (worker commands run under 195s)
|
||||
|
||||
# Codec
|
||||
CODEC_PROVIDER=json # json | avro (avro needs the kafka build tag)
|
||||
|
||||
@ -294,6 +294,10 @@ Put your own compose changes in a `docker-compose.override.yml`, which compose m
|
||||
|
||||
Upgrade the backend first and confirm its database migrations complete before upgrading consumers. Consumers wait up to two minutes for the clean migration version required by their embedded code before starting database-dependent work. If the backend is still old or a migration failed, the consumer exits with an actionable schema-readiness error and can retry through the container restart policy. It does not run migrations or ignore missing tables.
|
||||
|
||||
### Unibox forwarding upgrade order
|
||||
|
||||
Do not use forwarding to suppressed recipients until every backend handling submissions, queued send callbacks and worker/control-plane authorization has been upgraded, old requests have drained and old replicas have retired. During a mixed-backend rollout, pause Unibox submissions and queued send dispatch: an old backend still refuses these forwards. Existing clients can already submit forwards, so frontend ordering alone is insufficient. Verify old-replica retirement explicitly rather than relying on release completion or one healthy backend. No worker schema change or mailbox reconnect is required. See [send safety](/development/send-safety/#shared-send-time-admission).
|
||||
|
||||
### Mailbox sync upgrade order
|
||||
|
||||
The mailbox lookup optimization adds concurrent indexes for normalized send Message-IDs and exact IMAP folder generations, without rewriting existing records. Index construction uses temporary disk space and I/O, so confirm database headroom before upgrading a busy instance and wait for migrations to finish. The new batch message-map endpoint needs no configuration: old workers still use single-message reads, and new workers use that route when an old backend does not support batches. Upgrade the backend first to get the performance benefit, then workers. Sync cadence, draft-deletion reconciliation and durable arrival requirements stay unchanged.
|
||||
|
||||
@ -6,6 +6,8 @@
|
||||
"---Self-hosting---",
|
||||
"install",
|
||||
"deployment-guide",
|
||||
"aws-fargate",
|
||||
"coolify",
|
||||
"bare-metal",
|
||||
"split-deployment",
|
||||
"first-run",
|
||||
|
||||
@ -14,8 +14,8 @@ This control system restricts sending using current authority and observed evide
|
||||
| `test_mode` | New actions accept `diagnostic` or `off`. Historical `legacy` and NULL remain readable without relabeling; newly connected mailboxes default to `off` |
|
||||
| `test_send_enabled` | Enables diagnostic sending only in `diagnostic` mode |
|
||||
| `test_receive_enabled` | Separately enables receiving and provider-supported automated warmup mailbox actions in `diagnostic` mode |
|
||||
| `shared_daily_limit` | Positive mailbox-wide calendar-day ceiling across outbound lanes; PATCH zero clears the override, restoring the default shared ceiling |
|
||||
| `rolling_recipient_limit` | Positive rolling 24-hour ceiling counting each To/CC/BCC occurrence; PATCH zero restores the default ceiling |
|
||||
| `shared_daily_limit` | Positive mailbox-wide calendar-day ceiling across outbound lanes; PATCH zero clears the override, restoring the default shared ceiling. Manual Unibox sends do not count toward it; they have their own fair-use limits |
|
||||
| `rolling_recipient_limit` | Positive rolling 24-hour ceiling counting each To/CC/BCC occurrence; PATCH zero restores the default ceiling. Manual Unibox sends do not count toward it; they have their own fair-use limits |
|
||||
|
||||
Choose sender-only, recipient-only or both explicitly. Receiving participation authorizes probabilistic spam rescue, reads, importance and stars or flags on verified warmup messages where the provider supports them. This applies to existing receive-enabled diagnostic mailboxes too; no database backfill or new setting is required. Recipient-only does not authorize a reply. Sender-only and `off` deny recipient engagement. `off` also excludes diagnostic starts, active-campaign health checks, reply-back and new recipient selection. It does not stop legitimate campaign/manual traffic by itself: pause the campaign or mailbox for that. These are disclosed automated actions, not human engagement or proof of improved reputation. Authorized filing and deletion remain housekeeping, not positive deliverability evidence. Cleanup can remove an already received diagnostic after participation stops.
|
||||
|
||||
@ -31,7 +31,9 @@ Local participation restricts local use of an alias. Individually removing a man
|
||||
|
||||
## Shared send-time admission
|
||||
|
||||
Every new outbound lane goes through a mailbox/task advisory lock and one durable SQL reservation: campaigns, manual/Unibox sends, normal warmup, campaign-backed health checks/reply-back, and placement/seed probes. Campaign progress already reserved for the same task is not counted twice. Cold/manual and diagnostic allocations remain distinct constraints, with optional shared and rolling ceilings layered above them. Calendar days use the mailbox/organization timezone, including daylight-saving changes; rolling recipient totals do not reset at midnight.
|
||||
Every new outbound lane goes through a mailbox/task advisory lock and one durable SQL reservation: campaigns, manual/Unibox sends, normal warmup, campaign-backed health checks/reply-back, and placement/seed probes. Campaign progress already reserved for the same task is not counted twice. Cold and diagnostic allocations remain distinct constraints, with optional shared and rolling ceilings layered above them. Manual Unibox sends (reply, forward, compose) are conversations rather than outreach: they spend none of those allocations and are not refused by them, so a mailbox at its campaign limit can still answer a lead. They are held to fair use instead: a rolling 24-hour recipient allowance sized below the mailbox's provider cap and lowered by connection age, warmup standing and workspace risk, with an hourly, a per-message and a new-conversation bound under it, and a workspace-wide allowance of twice the plan's daily email limit (floor 100) checked under a per-workspace lock. `BILLING_PROVIDER=none` and plans without a daily limit set no workspace allowance (see [unibox sending controls](/guides/unibox/#sending-controls)). The same mailbox standing as every other lane still applies: active, current worker, no recovery hold or cooldown, no suppressed recipient. A Unibox forward is the one send exempt from the suppression list, because it relays received mail to recipients picked by hand; a refusal for a suppressed recipient is `RECIPIENT_SUPPRESSED` and names the address. Calendar days use the mailbox/organization timezone, including daylight-saving changes; rolling recipient totals do not reset at midnight.
|
||||
|
||||
Before using suppressed-recipient forwarding, upgrade every backend handling submissions, queued send callbacks and worker/control-plane authorization, then drain old requests and retire old replicas. An old backend still refuses the forward. Pause Unibox submissions and queued send dispatch during a mixed-backend rollout. Existing clients can already submit forwards, so frontend ordering alone is insufficient. Verify old-replica retirement explicitly; release completion or a healthy backend does not establish it. This exemption requires no worker schema change or mailbox reconnect. See [self-host upgrade order](/development/install/#unibox-forwarding-upgrade-order).
|
||||
|
||||
Reservation requires an active organization-owned mailbox, its current configured provider and worker, current suppression and recovery state, and the exact recipient envelope. Campaign sends additionally require active campaign/lead/progress state and current calendar/pacing policy. Diagnostic partners retain their real pool, role, tier, trust, standing and inbound capacity constraints. Seed mailboxes stay measurements, not conversation partners. Future, missing or stale positive Cloud standing is unavailable evidence; restrictive holds are not erased.
|
||||
|
||||
@ -39,6 +41,12 @@ Provider starts also write immutable per-nonce `outbound_attempts`. A definitive
|
||||
|
||||
The executor rechecks organization, mailbox, worker, provider, durable task/nonce and exact recipients before starting. A command cannot broaden its CC/BCC or switch tenants after reservation. Worker silence, publication uncertainty, provider timeout and executor restart are unknown outcomes, not unsent proof. Their reservations and mailbox hold remain until a correlated definite result resolves them. Terminal replay returns the saved result without resending. A definite native throttle records only the evidence and scope actually observed; legacy codes preserve conservative mailbox/provider cooldowns without invented HTTP or SMTP codes.
|
||||
|
||||
Warmup authorization and execution take shared participation locks before task/mailbox locks. Stop takes the exclusive participation lock and retires provably unstarted requests in both directions in the same transaction as disabling participation. Directional settings retire only that direction; switching the mailbox off retires both. Started sends, stored results, outbound attempts, delivered tokens and ambiguous completed legacy history remain intact. Only recorded warmup charges are refunded, once. Re-enabling an already-off mailbox also retires its older outstanding approvals, covering stops made before this upgrade.
|
||||
|
||||
Warmup commands without current durable execution authority are refused, including old nonce-free commands and compatibility responses from an older backend. Old JSON/Avro payloads remain readable, but readability is not permission to send. Non-warmup compatibility and replay of already-recorded provider results are preserved. Upgrade the backend and execution workers together; an updated worker intentionally will not start legacy warmup against a backend that cannot authorize it. No new environment setting is required.
|
||||
|
||||
Migration 300 adds typed `warmup_stopped_at` evidence and indexed mailbox/recipient lookups. For warmup mailboxes already off at upgrade, it records the upgrade time as a monitoring baseline, not an invented historical stop time. It does not rewrite mailbox history. The operations monitor checks actual execution-start timestamps in the last hour against a recorded stop for both sender and recipient, grouped by executor worker or fleet and deduplicated for six hours. Delayed consumer result application, old execution starts and provider delivery notices do not trigger this alert. Subscribe operator channels to `warmup.dispatch_after_stop` or their normal catch-all policy. Previously stopped mailboxes remain blocked by current eligibility; historical starts before the upgrade are not retrospectively labeled violations.
|
||||
|
||||
Result reconciliation commits task, campaign progress, variant, suppression and warmup-health changes together. Bounce webhooks and notifications enter an idempotent durable outbox in that transaction, and result consumers drain it after commit. Failed delivery retains a leased retry row; a consumer restart does not lose the effect. Stable downstream webhook delivery and notification-feed IDs prevent duplicate stored rows. External integration, Slack/push and realtime delivery remain at least once/best effort, not exactly-once delivery. Core accounting and unknown holds remain durable.
|
||||
|
||||
Warmup authorization records the UTC charge day and whether a reply was counted in the same transaction as the statistics increment. Outbound reservation preserves the initial completion timestamp. A definitive failure or safe unstarted cancellation refunds only that recorded charge, at most once, in the same transaction as the task change. Completion timestamps and dispatch/executor nonces alone never authorize a refund. Pre-upgrade and imported tasks have no positive charge record: known outcomes can still settle, but historical statistics are not decremented by inference. Unknown outcomes remain held and unresolved.
|
||||
@ -47,7 +55,7 @@ If a modern warmup task becomes dead-lettered after its definitive failure and r
|
||||
|
||||
## Evidence-based repair holds
|
||||
|
||||
Authentication, permanent and conflicting-result holds do not clear merely because time passed, a reconnect occurred or an unrelated send succeeded. An organization-authorized operator can use `send_recovery_resolution` in the existing mailbox PATCH after investigating the current held task and reason:
|
||||
Authentication, permanent and conflicting-result holds do not clear merely because time passed or an unrelated send succeeded. An authentication hold clears when the mailbox is reconnected: both reconnect flows (OAuth re-authorization and replacement SMTP/IMAP credentials) only complete after the provider accepted the credential, and SMTP/IMAP credentials pass a worker's SMTP AUTH and IMAP login check first. The release is recorded in `send_recovery_resolutions` with evidence type `credentials_reverified`, scoped to the mailbox's organization, and the hold is then recomputed, so an unresolved unknown send or outcome conflict still holds the mailbox. Permanent and conflicting-result holds are not released by a reconnect. An organization-authorized operator can also use `send_recovery_resolution` in the existing mailbox PATCH after investigating the current held task and reason:
|
||||
|
||||
```json
|
||||
{
|
||||
@ -64,6 +72,18 @@ Authentication, permanent and conflicting-result holds do not clear merely becau
|
||||
|
||||
### Resolving an unknown send
|
||||
|
||||
#### Automatic warmup retirement
|
||||
|
||||
Warmup-only unknown sends have an automatic alternative to confirming the old outcome: permanently retire the individual task without replaying it, then admit new, distinct tasks. After ten minutes, the warmup scheduler checks for an existing execution authorization and attempt, no definitive recorded result or receipt, and an exact `unknown` hold for that task. It rechecks under scheduling, task and mailbox send locks. Unstarted dispatches use their separate safe cancellation and refund path first.
|
||||
|
||||
Retirement records `tasks.warmup_send_retired_at`, cancels execution permanently, preserves `send_result_state='unknown'`, and keeps all original reservation, charge, message ID, nonce, worker and result evidence. It neither refunds nor reconstructs charges, marks the message sent or failed, nor retries it. Late results cannot apply accounting, reauthorize the retired task or change a later task's hold. Other unresolved sends and conflict, authentication or permanent holds remain restrictive. The normal scheduler respects all existing warmup eligibility, cooldown and volume limits.
|
||||
|
||||
This recovery includes eligible historical holds after upgrade. Legacy sends without execution fencing, messages with confirmed receipt/provider evidence, and tasks with a definitive result or operator confirmation awaiting reconciliation are not retired. Campaign and other non-warmup sends keep the provider-confirmation-only policy below. No additional customer action or configuration is required for eligible warmup sends.
|
||||
|
||||
Apply migration 315 before running the updated backend, consumer and scheduler. Upgrade all control-plane replicas before relying on the new policy. An old replica may temporarily reassert a retired task's unknown hold; the updated scheduler repairs that hold without replaying the task or touching a newer restrictive hold. Whole-instance backups and organization send-history exports retain the retirement marker. The down migration refuses to remove recorded retirements; retain the schema and roll forward rather than discarding the replay fence.
|
||||
|
||||
#### Provider-confirmed resolution
|
||||
|
||||
Unknown means the provider outcome is unproven, not that sending failed. Neither an expired timeout nor the absence of a synced Sent copy is proof that nothing was sent. Obtain explicit provider evidence for the exact task before using the mailbox PATCH:
|
||||
|
||||
```json
|
||||
@ -92,9 +112,9 @@ The PATCH atomically queues an immutable audited result; it does **not** immedia
|
||||
|
||||
Campaign routing, placement, day plans and estimates share durable mailbox send admission. Recovery-held or currently cooling mailboxes contribute no immediately available capacity and do not trap a lead when another healthy sender is available. Plans identify send recovery and provider cooldown separately. When the entire pool is unavailable, the campaign defers with a bounded recheck rather than completing or repeatedly dispatching refused sends. An admission read failure also closes the affected mailbox for that pass.
|
||||
|
||||
After provider execution, workers record and publish the result on an independent 45-second context with at most four attempts, each bounded to 10 seconds. Failed reports retain the exact outcome in memory for command redelivery without another provider send. Persisted results survive worker replacement and lost broker events: the consumer replays them through the same idempotent reconciliation. A worker crash before durable recording can still leave an unknown outcome requiring investigation.
|
||||
Provider execution gets 120 seconds, which covers the SMTP client's own 90-second budget for a slow server's verdict after the message, so a send is not cut off waiting for an answer the server was about to give. After provider execution, workers record and publish the result on an independent 45-second context with at most four attempts, each bounded to 10 seconds. Failed reports retain the exact outcome in memory for command redelivery without another provider send. Persisted results survive worker replacement and lost broker events: the consumer replays them through the same idempotent reconciliation. A worker crash before durable recording can still leave an unknown outcome requiring investigation.
|
||||
|
||||
Already-failed tasks with no outbound reservation or execution authorization cannot create unknown provider holds. The recorded-result sweep also reconciles existing invalid holds for such tasks. Real reserved or started unknown sends remain held; upgrades never infer failure from elapsed time. Deploy the backend/consumer (including the additive migration) before updated workers. No new configuration is required; existing genuine unknown holds need definitive worker evidence or the explicit operator confirmation above.
|
||||
Already-failed tasks with no outbound reservation or execution authorization cannot create unknown provider holds. The recorded-result sweep also reconciles existing invalid holds for such tasks. Real reserved or started unknown sends remain held until a definitive result is reconciled or an eligible warmup task is permanently retired; upgrades never infer failure from elapsed time. Deploy the backend/consumer (including the additive migration) before updated workers. No new configuration is required. Non-warmup and ineligible unknown holds need definitive worker evidence or the explicit operator confirmation above.
|
||||
|
||||
## Actual-message authentication
|
||||
|
||||
|
||||
@ -79,6 +79,7 @@ Newer builds return the invite-only refusal with its own machine code, `registra
|
||||
| "No mailbox workers are available" when connecting a mailbox | No worker has a heartbeat inside the last 10 minutes. Check `make status` shows `worker` running and `make logs worker` is clean. An empty `ENCRYPTED_KEYS_BACKEND_URL` or worker token lets a worker start and never register, silently |
|
||||
| Connecting a mailbox fails with `SERVER_UNREACHABLE` on a reachable host | The security setting may not match the server. A server expecting STARTTLS looks unreachable to a client attempting implicit TLS, and vice versa. Any port from 1 to 65535 is accepted, so the port alone no longer decides: set **Security** to SSL / TLS for a server that is encrypted from the first byte (usually SMTP `465`, IMAP `993`) and STARTTLS for one that upgrades in place (usually SMTP `587` or `2525`, IMAP `143`). The error names the step that failed and its cause. A refused or timed out `dial` never reached the server. `tls: first record does not look like a TLS handshake` points to a protocol mismatch. Other `tls:` dial failures describe certificate or handshake problems and do not by themselves mean the security mode is wrong. Anything from `mail from` onwards is the server's own answer |
|
||||
| Sending fails with `SERVER_UNREACHABLE` on port `465` while `587` works | The network between the sending machine and the mail server does not allow implicit TLS submission; some hosting providers block `465` outbound while leaving `587` open. The mailbox's error says `dial <host>:465: connect: connection refused` or `i/o timeout`. Move the mailbox to `587` with STARTTLS, or have the block lifted |
|
||||
| A mailbox shows `SERVER_UNREACHABLE` for a pass or two, and the worker logs `imap: server sent a body structure that could not be read` | One message in the folder is described in a form the IMAP client cannot read, typically an RFC 6533 bounce (`message/global`). Nothing needs doing: the worker asks for those messages without their structure from the next pass and reads them whole, then one at a time, and leaves behind only a message no form can read (logged with its folder and UID). Each step costs at most one failed pass |
|
||||
| A mailbox's error says the sending network blocks outbound port `465`, and the worker's detail page in **Workers** says the same | The worker has seen several servers leave `465` silent while answering on `587`, so the block is its own network, not the mail server. Mailboxes whose server offers `587` keep sending through the automatic fallback; a server that accepts submission only on `465` cannot be reached from that machine. Many cloud providers block outbound `25` and `465` on new servers until asked: request that outbound `465` be opened for the worker's address. The warning clears on its own once any `465` connection succeeds |
|
||||
| A local relay such as Proton Bridge will not connect, and the **None** security option is not there | It appears only for a loopback literal (`localhost`, `127.0.0.0/8`, `::1`) on a self-hosted instance, because the mode is safe only when the credentials never leave the machine. A hostname that resolves to `127.0.0.1` is refused; type the address itself. On hosted Warmbly the worker is not your machine and cannot reach a Bridge at all. See [local mail relays](/guides/mailboxes/#local-mail-relays-proton-bridge) |
|
||||
| A mailbox reports `INSECURE_REMOTE_HOST` | It is stored with the unencrypted security mode but its host is not this machine, so the worker refused to dial it rather than put the password on a wire. Reconnect it with SSL / TLS or STARTTLS, or point it back at the local relay |
|
||||
|
||||
@ -241,6 +241,9 @@ The [shared authentication reference](/api/authentication/) remains the backend
|
||||
| [`fleet remove`](#fleet) | Forgets a node; its mailboxes re-place themselves |
|
||||
| [`backup`](#backup) | Writes the whole instance to one restorable bundle |
|
||||
| [`restore`](#restore) | Restores a bundle onto this instance, replacing everything on it |
|
||||
| [`keys status`](#keys) | Counts stored secrets by the key that opens them, and shows open grace windows |
|
||||
| [`keys reseal`](#keys) | Rewrites secrets a read key opens under the primary key |
|
||||
| [`keys generate`](#keys) | Prints fresh values for the instance-wide keys and tokens |
|
||||
| [`org list`](#org-list) | Lists the workspaces on this instance with their id, owner, and size |
|
||||
| [`org export`](#org-export) | Writes a whole workspace to a portable archive file |
|
||||
| [`org import`](#org-import) | Applies an archive to a workspace on this instance |
|
||||
@ -499,8 +502,45 @@ Before anything is written it compares the bundle's `CREDENTIALS_ENCRYPTION_KEY`
|
||||
|
||||
Blobs are unpacked over the blob root additively: files the bundle knows about are overwritten, anything else is left alone.
|
||||
|
||||
Once the dump has loaded, the restore runs `ANALYZE`, because a dump carries no planner statistics. If that step fails the restore still completes and prints a warning; run `ANALYZE` in `psql` yourself afterwards.
|
||||
|
||||
[Data control](/development/data-control/#moving-an-instance) walks the whole move to a new host.
|
||||
|
||||
## keys
|
||||
|
||||
```bash
|
||||
docker compose -p warmbly exec backend warmblyctl keys status
|
||||
docker compose -p warmbly exec backend warmblyctl keys reseal
|
||||
docker compose -p warmbly exec backend warmblyctl keys reseal --apply
|
||||
docker compose -p warmbly exec backend warmblyctl keys generate CREDENTIALS_ENCRYPTION_KEY
|
||||
```
|
||||
|
||||
The tooling for [rotating instance keys](/development/configuration/#rotate-instance-keys). It reads `CREDENTIALS_ENCRYPTION_KEY` with `CREDENTIALS_ENCRYPTION_READ_KEYS`, and `TWOFA_SECRET` with `TWOFA_READ_SECRETS`, exactly as the backend does, so run it where the backend runs.
|
||||
|
||||
`keys status` reads every column sealed under an instance-wide key (mailbox SMTP and IMAP credentials, OAuth tokens, webhook signing secrets, the Warmbly Cloud link, pending mailbox erasures and TOTP enrolments) and counts each value as one of:
|
||||
|
||||
| State | Means |
|
||||
|---|---|
|
||||
| primary | Opens under the primary key. Nothing to do |
|
||||
| read key | Opens only under a read key. `keys reseal --apply` moves it to the primary |
|
||||
| workspace | Sealed under its workspace key, as mailbox credentials are with `MAILBOX_CREDENTIALS_SEALING=organization`. An instance key rotation does not touch it |
|
||||
| unsealed | Stored outside the sealed format, such as a legacy OAuth token or a `whsec_` webhook secret. The backend seals it on its next write |
|
||||
| unreadable | Sealed, but opens under no configured key. Left untouched by every command here |
|
||||
|
||||
It also lists any `INTERNAL_API_PREVIOUS_UNTIL`, `NODE_BROKER_PREVIOUS_UNTIL` and `UNSUBSCRIBE_SIGNING_READ_UNTIL` window and whether it is still open.
|
||||
|
||||
`keys reseal` is a dry run unless you pass `--apply`. With it, every value a read key opens is rewritten under the primary, one row at a time and only if the row still holds the value that was read, so a credential the backend changes meanwhile is never overwritten. It is safe to stop and run again: a later run picks up where values remain, and a run with nothing left to do writes nothing. Values the primary already opens, workspace-sealed values, unsealed values and unreadable values are never written.
|
||||
|
||||
| Flag | Does |
|
||||
|---|---|
|
||||
| `--apply` | Write the resealed values. Without it, `reseal` reports what it would do |
|
||||
| `--batch` | Rows read per query, 1 to 10000. Defaults to 500 |
|
||||
| `--json` | Print the report as JSON (also on `keys status`) |
|
||||
|
||||
Both exit non-zero when any value is unreadable. Restore the key that sealed it as a read key before retiring anything; an unreadable mailbox credential is a mailbox that can no longer sign in.
|
||||
|
||||
`keys generate` prints `NAME=value` lines for `CREDENTIALS_ENCRYPTION_KEY`, `AUTH_SECRET`, `TWOFA_SECRET`, `UNSUBSCRIBE_SIGNING_SECRET`, `DOMAIN_PROOF_SECRET`, `INTERNAL_API_TOKEN`, `NODE_BROKER_TOKEN` and `SECRET_KEY_BASE`, or only the names you pass. It changes nothing; where each value goes, and in what order, is the rotation procedure.
|
||||
|
||||
## org list
|
||||
|
||||
```bash
|
||||
|
||||
@ -83,7 +83,7 @@ The backend runs as uid `1000`, so the file has to be readable by that user (`ch
|
||||
The Client ID belongs to your instance, not to any one workspace, so a customer's authorization says nothing about which workspace asked for it. A Google grant is therefore recorded only for the workspace that proves it controls the domain, and only when the domain's directory lists the administrator's address as a super administrator. The proof is one of:
|
||||
|
||||
- **The administrator's Google sign-in.** It goes through the dashboard's own Google sign-in client (`GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`) and returns to its existing redirect URI (`GOOGLE_REDIRECT_URI`, or `API_PUBLIC_URL` plus `/v1/auth/google/callback`), asking only for `openid` and `email`. The callback recognizes an administrator's proof and hands it back to the dashboard window that opened it; it signs nobody in. No redirect URI needs registering beyond the one sign-in already uses. When that client is not configured, the `BOX_GOOGLE_*` mailbox client is used the same way, through its own redirect URI (`/addresses/google/callback`) and again with `openid` and `email` only. The sign-in carries a PKCE challenge and an OpenID nonce, and the backend redeems the code with the matching verifier and accepts the ID token only when it echoes that nonce.
|
||||
- **A DNS record.** A `TXT` record at `_warmbly.<domain>` holding a `warmbly-verify=` value unique to the workspace and domain, keyed by `AUTH_SECRET`. Always offered, and the only proof on an instance with neither Google client.
|
||||
- **A DNS record.** A `TXT` record at `_warmbly.<domain>` holding a `warmbly-verify=` value unique to the workspace and domain, keyed by `DOMAIN_PROOF_SECRET` (which defaults to `AUTH_SECRET`). Always offered, and the only proof on an instance with neither Google client.
|
||||
|
||||
Customer administrators sign in to that client from their own organizations, so its OAuth consent screen has to be **External**. An **Internal** consent screen admits only accounts in your own Google organization, and every other administrator is refused and has to use the DNS record.
|
||||
|
||||
|
||||
@ -86,6 +86,7 @@ A check stays quiet until it can be right. Two bounces in thirty sends is not a
|
||||
| Complaint rate | `100` sends in 30 days |
|
||||
| Bounce rate | `50` sends in 30 days |
|
||||
| Spam placement | `20` warmup deliveries in 7 days |
|
||||
| Warmup mail not arriving | `5` warmup emails to the mailbox in 7 days, from at least `2` workspaces |
|
||||
| Reply rate and copy | `200` sends in 30 days |
|
||||
| Per-step comparison | `100` sends on that step |
|
||||
|
||||
@ -110,7 +111,11 @@ Only the first two badge a nav tab.
|
||||
|
||||
**Mailbox configuration**: daily cap above the `50`/day safe band (gently with clean numbers, firmly without); a mailbox under `30` days old sending above `20`/day; a send gap under `5` minutes; unresolved errors on a mailbox campaigns still use; too few mailboxes carrying total volume; an inactive mailbox attached to a running campaign.
|
||||
|
||||
**Warmup**: warmup off while sending cold (critical on a new mailbox); warmup left paused; a ceiling too low to balance cold volume (roughly half the cold cap); reply rate under `20%`, making warmup one-way; quarantined or blocked from the pool. Missing or stale Cloud status is reported separately from a real reputation ban, with steps to refresh it without disconnecting mailboxes.
|
||||
**Warmup**: warmup off while sending cold (critical on a new mailbox); warmup left paused; a ceiling too low to balance cold volume (roughly half the cold cap); reply rate under `20%`, making warmup one-way; temporarily throttled sending, reported separately from quarantine or a pool block; none of the pool's warmup mail to the mailbox arriving, which stops the pool sending to it (see [unreachable mailboxes](/guides/warmup/#mailboxes-the-pool-cannot-reach)). Missing or stale Cloud status is reported separately from a real reputation ban, with steps to refresh it without disconnecting mailboxes.
|
||||
|
||||
A **throttled** mailbox remains in the warmup pool. The advisor explains the automatic reduced sending pace rather than asking you to remove it from campaigns, wait a fixed week or manually re-enter the pool. Keep warmup running and do not increase its cap to bypass the slowdown. For placement throttles, half volume steps down to watch below `15%` spam placement, and watch lifts below `7.5%`; separate ramp limits still apply. Re-entry probation and other cooldowns follow the reason shown on the mailbox. A throttle alone does not mean the connection has failed or the provider has blocked the mailbox.
|
||||
|
||||
The advisor's placement sample covers today and the preceding six UTC dates, matching the Deliverability display and placement-health decisions. After an upgrade, the next advisor evaluation replaces outdated throttle/exclusion guidance automatically. A campaign hold you already applied remains yours to release after checking any other findings.
|
||||
|
||||
**Campaigns**: no follow-ups; reply rate under `1%` at volume, escalating under `0.3%`; one step replying below a quarter of the campaign's best (compared only against siblings, which share an audience); one-click unsubscribe off; a daily limit above what its mailboxes can send; no active sender; follow-up spacing outside `2` to `14` days; under `7` days of leads left; no A/B test at high volume; a sending window too narrow for the volume; contacts enrolled in several running campaigns.
|
||||
|
||||
@ -128,7 +133,7 @@ Two more copy findings run only when the operator has configured TypeSafe (`TYPE
|
||||
|
||||
Opening a recommendation walks three screens: **Why** (with the measured numbers), **What changes** (the exact before and after of every field), and **Done** (with **Undo** where revertible).
|
||||
|
||||
A finding that asks you to stop cold sending from a mailbox, for spam placement or a lost warmup pool standing, applies [Hold from campaigns](/guides/mailboxes/#holding-a-mailbox-yourself) rather than switching the mailbox off, and leaves its warmup as it is. It is offered only while campaigns can still pick the mailbox, so a mailbox that is already held, resting or switched off gets no fix, and **Undo** never releases a hold someone else set.
|
||||
A finding that asks you to stop cold sending from a mailbox, for severe spam placement (`50%` or more on at least `20` major-provider deliveries), quarantine or a pool block, applies [Hold from campaigns](/guides/mailboxes/#holding-a-mailbox-yourself) rather than switching the mailbox off, and leaves its warmup as it is. A recoverable placement slowdown below that severe band does not offer a campaign hold on its own; complaints, bounces and other confirmed problems are still checked separately. The hold is offered only while campaigns can still pick the mailbox, so a mailbox that is already held, resting or switched off gets no fix, and **Undo** never releases a hold someone else set.
|
||||
|
||||
Nothing is applied until you have seen the second screen. The change runs as you, with your permissions, and is written to the audit log. If you can see a finding but lack permission for its fix, applying it is refused rather than silently escalated.
|
||||
|
||||
|
||||
@ -116,6 +116,8 @@ These preferences are yours alone, kept in your browser for each workspace.
|
||||
|
||||
It knows what page you are on, so "summarize this campaign" and "draft a reply here" resolve correctly.
|
||||
|
||||
When the available tools exceed the OpenAI-compatible provider's 128-tool limit, Remie discovers the tools it needs on demand, including tools from connected MCP servers. No permitted capability is removed. Discovery is a read-only step; the action itself keeps its normal permissions and approval requirements. Like other model steps, discovery counts toward the run's step budget and AI credits.
|
||||
|
||||
</Fold>
|
||||
|
||||
<Fold>
|
||||
|
||||
@ -111,7 +111,7 @@ Triggers, conditions, and actions only take effect after **Save**. **Test** save
|
||||
| Inbound webhook | An external system POSTs to this automation's unique URL |
|
||||
| Campaign action | A campaign sequence reaches a "Run automation" step |
|
||||
|
||||
Each trigger carries its own event data (a reply carries contact email, reply intent, classifier confidence). Those are the values your conditions test and your action text inserts. Full per-trigger variable list: [Personalization & expressions](/guides/expressions/).
|
||||
Each trigger carries its own event data (a reply carries contact email, reply intent, classifier confidence, and the inbox tagging verdict when tagging is on). Those are the values your conditions test and your action text inserts. Full per-trigger variable list: [Personalization & expressions](/guides/expressions/).
|
||||
|
||||
**Campaign action** never fires by itself; it runs only from a campaign step, still evaluating your conditions.
|
||||
|
||||
@ -141,6 +141,30 @@ Pick a **field**, an **operator**, and a **value**. Fields depend on the trigger
|
||||
| Text | `is`, `is not`, `contains`, `is present` |
|
||||
| Number | `≥`, `≤`, `is`, `is present` |
|
||||
| Choice (enum) | `is`, `is not`, `is present` |
|
||||
| List | `contains`, `is present` |
|
||||
|
||||
A list `contains` a value only when one item matches it whole, ignoring case, so a reply labelled "Not interested" does not pass `contains Interested`. A list with no items is not present.
|
||||
|
||||
### Filtering replies on inbox tagging
|
||||
|
||||
With [inbox tagging](/guides/inbox-tagging/) on, **Reply received** also carries the verdict tagging stored for that message, so a reply automation filters on the same labels the inbox shows:
|
||||
|
||||
| Field | What it holds |
|
||||
| --- | --- |
|
||||
| `ai_labels` | The labels on the message (Interested, Meeting, Pricing, Question, Wrong person, and your own question labels) |
|
||||
| `ai_intent` | What a human reply wants: `agreed`, `scheduling`, `wants_pricing`, `wants_info`, `wrong_person`, `not_now`, `not_interested`, `opt_out`, and the rest. Empty when tagging could not read it confidently |
|
||||
| `ai_intent_confidence` | How sure tagging was of `ai_intent`, 0 to 1 |
|
||||
| `ai_kind` | What the message is: `human_reply`, `auto_reply_ooo`, `auto_reply_ticket`, `cold_inbound`, `internal`. Empty when tagging was not sure |
|
||||
| `ai_relevance` | The 0 to 100 relevance score |
|
||||
| `ai_needs_review` | Tagging flagged the message for a person to check |
|
||||
|
||||
A reply tagging never judged (tagging off, or a message it skipped) carries none of these keys. Test `ai_needs_review` with `is present` to tell a judged reply from one with no verdict: this key is present even when its value is `false`. Do not use `ai_labels`, `ai_kind` or `ai_intent` for this check, because a judged reply can have an empty list or an empty string. `intent` and `confidence` are still the workspace keyword list's reading, unchanged. To be told about every interested lead, branch on `ai_labels` containing `Interested`, `Meeting`, `Pricing` or `Question`, or use one expression:
|
||||
|
||||
```
|
||||
or (has .ai_labels "Interested") (has .ai_labels "Meeting") (has .ai_labels "Pricing")
|
||||
```
|
||||
|
||||
The built-in Slack and Discord messages name the labels in their title when a reply has them.
|
||||
|
||||
Three special conditions work with any trigger:
|
||||
|
||||
|
||||
@ -226,6 +226,7 @@ The number is the sends already out plus what is still expected, and the list un
|
||||
| Easing out of warmup | A mailbox that warmed starts cold at `5` to `20` a day and adds `5` each clean day (see [easing out of warmup](#easing-out-of-warmup)) |
|
||||
| Workspace sending posture | A restricted workspace sends a fraction of every cap (see [workspace sending posture](/guides/deliverability/#workspace-sending-posture)) |
|
||||
| Domain authentication failing, Resting or in reserve, Held by warmup health | A mailbox that sends nothing cold today, and why |
|
||||
| Waiting for Warmbly Cloud status | [Warmbly Cloud](/guides/warmbly-cloud/#safety-and-enforcement) has not reported a current status for a mailbox it warms. Cold sends from it wait until one arrives; it is not a Cloud quarantine or block |
|
||||
| Used by other campaigns | A mailbox's daily cap is shared by every campaign it is on; these sends went to another campaign today |
|
||||
| Slowed by warmup health | A mailbox on watch or throttled has its sends spaced wider |
|
||||
| Outside the mailbox's hours, Sending behaviour plan | A mailbox in another timezone or with its own [workday](/guides/sending-behavior/) is closed for the rest of today, or its rolled plan gives it fewer sends than its cap |
|
||||
@ -235,7 +236,7 @@ The number is the sends already out plus what is still expected, and the list un
|
||||
| New leads per day | Only this many contacts may receive their first email today; follow-ups keep going |
|
||||
| Not enough leads due | The mailboxes could send more, but no more steps are due today |
|
||||
|
||||
Under the list, the card says how many leads are due now, due later today, waiting on a step's delay, inside an undecided branch window, waiting for their own mailbox (each contact keeps the address they first heard from, so a follow-up whose mailbox is spent waits rather than switching) or held, and when the campaign next works through its queue. Daily budgets reset at midnight UTC whatever the campaign's timezone, so a campaign far from UTC sees its "sent today" reset partway through its own day. Expand the mailbox rows to see each mailbox's cap for this campaign today and which limit set it, what it has sent (for this campaign, and for others), what it is still expected to send, and whether it is sending, closed for the day, out of budget, resting, held, failing authentication or reconnecting (no running sending worker holds it right now; it is placed on one again within minutes and the other mailboxes keep sending meanwhile). Each limit links to the page where it is changed.
|
||||
Under the list, the card says how many leads are due now, due later today, waiting on a step's delay, inside an undecided branch window, waiting for their own mailbox (each contact keeps the address they first heard from, so a follow-up whose mailbox is spent waits rather than switching) or held, and when the campaign next works through its queue. Daily budgets reset at midnight UTC whatever the campaign's timezone, so a campaign far from UTC sees its "sent today" reset partway through its own day. Expand the mailbox rows to see each mailbox's cap for this campaign today and which limit set it, what it has sent (for this campaign, and for others), what it is still expected to send, and whether it is sending, closed for the day, out of budget, resting, held, waiting for a [Warmbly Cloud](/guides/warmbly-cloud/#safety-and-enforcement) status, failing authentication or reconnecting (no running sending worker holds it right now; it is placed on one again within minutes and the other mailboxes keep sending meanwhile). Each limit links to the page where it is changed.
|
||||
|
||||
The sidebar's **sent today** meter reads its denominator the same way: what the workspace's mailboxes can send today under these limits, not their caps added up. The two agree by construction, because both are read through the scheduler.
|
||||
|
||||
@ -582,6 +583,10 @@ When enabled, Warmbly limits a campaign mailbox to at most five diagnostic warmu
|
||||
|
||||
</Fold>
|
||||
|
||||
## Import from Instantly or Smartlead
|
||||
|
||||
`POST /v1/campaign-imports` builds one draft campaign, with its steps in order, from a competitor's export in a single call: send the platform (`instantly` or `smartlead`), its sequence/steps export saved as JSON (from that platform's API), and optionally its lead-list CSV export. Steps land with their subject, body and wait time already set; a leads file is imported and enrolled the same way a manual contact import would be, through the existing column-mapping guesser. A sequence step the import can't place, or a leads file it can't read, leaves the campaign and whatever steps already landed as a draft rather than failing the whole import away. There is no dashboard wizard step for this yet; it's reached through the API.
|
||||
|
||||
## Related guides
|
||||
|
||||
<Cards>
|
||||
|
||||
@ -294,6 +294,8 @@ Filter chips narrow the feed (**Emails**, **Replies**, **Deliv.**, **Notes**, **
|
||||
|
||||
At the top of the tab sits the campaign panel: for each campaign the contact is in, its flow with this contact's progress, the lead status, and what the scheduler will do next. See [Campaigns](/guides/campaigns/) for how the next action is worked out and what its states mean.
|
||||
|
||||
A campaign with no flow steps yet still appears here with **0/0 steps** and **Not started**. Its progress strip stays hidden until steps are added, and the rest of the contact's activity remains available.
|
||||
|
||||
</Fold>
|
||||
|
||||
<Fold>
|
||||
@ -369,6 +371,10 @@ Which replies get one is up to you, in **Settings > Sending > Reply follow-ups**
|
||||
| Negative: not interested | Yes |
|
||||
| Automated: out of office, autoresponders, bounces | No |
|
||||
|
||||
Only mail that arrives after the sender became a contact opens a task. Importing a list or creating contacts through the API does not turn their older messages already in your mailboxes into replies; a reply in one of your campaign threads always counts.
|
||||
|
||||
Before this rule, older mail could open tasks and record reply intents when its sender was imported. Warmbly checks those once per consumer start, but withdraws an effect only when its saved identifiers link it to the exact message and, for a follow-up, the exact task. Older records without those links are left alone, even if their sender, subject or timing matches. A linked task is cancelled only if it is still pending and nobody has edited it. Every cancellation is recorded in the audit log. A task someone already started, edited or completed is left as it is. Temporary cleanup errors are retried before the check advances.
|
||||
|
||||
Automated replies are off deliberately. A vacation notice is not follow-up work, and a week of sending makes enough of them to bury the real replies. Tick **Automated** if you do want one per machine reply, or untick the human ones you do not need. Unticking everything is the same as turning the switch off.
|
||||
|
||||
The classification does more than decide the task: an automated reply never counts as a reply for **stop on reply** or for a **replied** branch in a sequence, and its notification goes to the **Out-of-office detected** preference rather than **New reply**, so muting one does not mute the other.
|
||||
|
||||
@ -101,7 +101,7 @@ Google, Microsoft, and Yahoo family totals are not interchangeable with their co
|
||||
|
||||
## Health bands
|
||||
|
||||
Each mailbox sits in a band, shown as a colored chip. The band controls how Warmbly treats it.
|
||||
Each mailbox sits in a band, shown as a colored chip. The band controls how Warmbly treats it. Watch and throttled are amber warnings on Emails, not red errors, unless a separate account problem is present. A warming mailbox in either band still counts under **Warming** in the summary (shown as slowed), not under **Needs attention**, because it keeps warming and the band lifts on its own; the same goes for a warming mailbox whose only warning is a low inbox rate. They do not apply a flat health-score deduction on top of the inbox-rate cap. Mailbox health is a score out of 100, not an inbox percentage.
|
||||
|
||||
| Band | Spam placement | Complaint rate | Bounce rate | Other triggers |
|
||||
| --- | --- | --- | --- | --- |
|
||||
@ -113,13 +113,15 @@ Each mailbox sits in a band, shown as a colored chip. The band controls how Warm
|
||||
A mailbox enters a band by crossing any single threshold. What each band does:
|
||||
|
||||
- **Healthy**: sending normally, nothing to do.
|
||||
- **Watch**: warmup volume lowered and spacing increased. Cold volume is dampened by the same factor, so it slows across both before anything stricter. Lifts once spam placement is back below `7.5%`.
|
||||
- **Throttled**: warmup keeps running at half volume with double spacing, because warming is how placement recovers, and cold volume is halved. Lifts once spam placement is back below `15%`.
|
||||
- **Watch**: warmup keeps its volume and pace while the pool replies to and rescues more of its mail ([recovery engagement](/guides/warmup/#recovery-engagement)), because warming is how placement recovers. Cold volume is eased to `70%`, since cold mail is what costs reputation. Lifts once spam placement is back below `7.5%`.
|
||||
- **Throttled**: warmup keeps running at half volume with double spacing, because warming is how placement recovers, and cold volume is halved. Steps down to watch once spam placement is back below `15%`, then returns to healthy below `7.5%`. Separate [early-signal ramp limits](/guides/warmup/#holding-the-ramp-on-an-early-signal) still apply to fresh spam landings.
|
||||
- **Quarantine**: out of the shared paid warmup pool for `7 days`, as neither sender nor recipient. Cold sending is paused.
|
||||
- **Blocked**: out of the shared pool for `30 days`.
|
||||
|
||||
Spam placement never goes past throttled. It is a reading of reputation, and warming is how a mailbox recovers, so it slows the mailbox down and lifts on its own; there is nothing to appeal.
|
||||
|
||||
Placement decisions and the headline inbox rate use the same seven UTC dates: today and the preceding six days. This date boundary does not shorten the rolling complaint or tampering windows.
|
||||
|
||||
**Sample floors** stop one bad event flipping a new mailbox: spam placement needs `20+` verified warmup deliveries in `7 days`, complaints need `100+` delivered emails in `30 days`.
|
||||
|
||||
Spam placement is judged at Google, Microsoft and Yahoo only. Other mail hosts run their own filters, so their spam folders are shown and never held against a mailbox; see [how spam placement is judged](/guides/warmup/#how-spam-placement-is-judged).
|
||||
@ -188,13 +190,9 @@ If a campaign has already been paused because every one of its mailboxes was gat
|
||||
|
||||
The list is held per workspace, and the check runs against the sending campaign's workspace before every send. If a campaign somehow has no workspace, that check cannot be applied, so the campaign is paused with the reason in its activity feed instead of sending unchecked.
|
||||
|
||||
Bounces are detected from the delivery-status report the recipient's server sends back. Only permanent failures (`5.x.x`, such as a nonexistent address) record a bounce and suppress; temporary ones (`4.x.x`, such as a full mailbox or greylisting) never do. This works the same for Gmail and Microsoft Graph mailboxes, where the send reports success and the bounce arrives later. Some failure notices carry no status code at all, such as Gmail's notice when a Google Group refuses a post. Those are read from the failed-recipient header the notice carries, and count as permanent unless the notice says delivery is still being retried.
|
||||
Bounces are detected from the delivery-status report the recipient's server sends back. That includes the internationalized form (RFC 6533, `message/global-delivery-status`) that servers such as Postfix write for mail with non-ASCII addresses or headers. Only permanent failures (`5.x.x`, such as a nonexistent address) record a bounce and suppress; temporary ones (`4.x.x`, such as a full mailbox or greylisting) never do. This works the same for Gmail and Microsoft Graph mailboxes, where the send reports success and the bounce arrives later. Some failure notices carry no status code at all, such as Gmail's notice when a Google Group refuses a post. Those are read from the failed-recipient header the notice carries, and count as permanent unless the notice says delivery is still being retried.
|
||||
|
||||
**Spam complaints** arrive the same way. When a recipient presses "mark as spam", their provider mails an abuse report back to the sending mailbox. Warmbly reads those reports, resolves them to the campaign send they name, and records a complaint: the recipient is suppressed and the number reaches your complaint rate. Providers that redact who complained still identify the message, so the right lead is suppressed anyway. Only genuine abuse reports count. A "not spam" report, where someone rescued your email from junk, is not recorded as a complaint. Who gets suppressed is always taken from the send the report names, never from an address written in the report, so a forged report cannot reach a contact that send never went to.
|
||||
|
||||
<Callout type="info" title="Not yet on Microsoft Graph">
|
||||
Complaint reports are read from IMAP and Gmail mailboxes. Microsoft Graph returns a single rendered message body without the report's machine-readable parts, so reports arriving in a Graph-connected mailbox are not detected yet. Bounce handling is unaffected.
|
||||
</Callout>
|
||||
**Spam complaints** arrive the same way. When a recipient presses "mark as spam", their provider mails an abuse report back to the sending mailbox. Warmbly reads those reports, resolves them to the campaign send they name, and records a complaint: the recipient is suppressed and the number reaches your complaint rate. Providers that redact who complained still identify the message, so the right lead is suppressed anyway. Only genuine abuse reports count. A "not spam" report, where someone rescued your email from junk, is not recorded as a complaint. Who gets suppressed is always taken from the send the report names, never from an address written in the report, so a forged report cannot reach a contact that send never went to. This works for IMAP, Gmail and Microsoft Graph mailboxes. Graph syncs only a report's rendered notice, so for a message that looks like a report, the worker reads its raw message from Graph to find the parts that name the send. Temporary Graph failures leave the message pending for a later sync rather than losing the complaint. A missing or oversized raw message does not prevent other mail from syncing. The raw fetch is limited to 1 MiB and 15 seconds; parsing is limited to 16 nesting levels, 64 parts and 1 MiB of decoded report content. Malformed or over-limit reports are not counted as complaints.
|
||||
|
||||
**Domain authentication refusals** are separated from ordinary failures. When a receiving server refuses a message because your sending domain failed its checks (Microsoft's `5.7.515`, Gmail's `5.7.26`), that is not a bad recipient and not an outage: every send from that domain fails the same way until its DNS is fixed. Warmbly says so in the campaign's activity feed rather than blaming the address, does not suppress the recipient, and brings that domain's authentication check forward so the [domain authentication](/guides/deliverability/#domain-authentication) state reflects it.
|
||||
|
||||
|
||||
@ -130,7 +130,7 @@ New reply from {{.contact_email}} on campaign {{.campaign_id}} ({{.intent}})
|
||||
|
||||
| Trigger | Variables |
|
||||
| --- | --- |
|
||||
| Reply received | `contact_email`, `contact_id`, `campaign_id`, `intent`, `confidence`, `subject`, `snippet` |
|
||||
| Reply received | `contact_email`, `contact_id`, `campaign_id`, `intent`, `confidence`, `subject`, `snippet`, and with [inbox tagging](/guides/inbox-tagging/) on: `ai_labels`, `ai_intent`, `ai_intent_confidence`, `ai_kind`, `ai_relevance`, `ai_needs_review` |
|
||||
| Contact created | `contact_email`, `contact_id`, `first_name`, `last_name`, `company`, `phone`, `subscribed`, `source`, `source_detail`, `custom_fields` (a map: `{{.custom_fields.industry}}`), `campaign_ids`, `category_ids` |
|
||||
| Form submitted | `contact_email`, `contact_id`, `first_name`, `last_name`, `company`, `phone`, `form_id`, `form_name`, `submission_id`, `source_url`, `campaign_id`, `data` (every answer by its key: `{{.data.team_size}}`) |
|
||||
| Meeting booked / rescheduled | `invitee_name`, `invitee_email`, `event_name`, `scheduled_for`, `join_url`, `source`, `contact_id` |
|
||||
@ -184,6 +184,8 @@ The same helpers work in campaign emails, automation conditions, and action text
|
||||
| `num a` | Force a value to a number | `trim` | Remove surrounding spaces |
|
||||
| | | `contains` | Contains, case-insensitive |
|
||||
| | | `hasPrefix` `hasSuffix` | Starts or ends with, case-sensitive |
|
||||
| | | `has list item` | The list holds that whole item, case-insensitive |
|
||||
| | | `join list sep` | The list as one line of text |
|
||||
|
||||
Division or modulo by zero returns 0 rather than erroring, so these never break an email.
|
||||
|
||||
|
||||
@ -242,9 +242,10 @@ Tagging languages are part of the workspace settings, so they travel with a [wor
|
||||
|
||||
## What else reads the verdict
|
||||
|
||||
Two other features reuse a stored verdict rather than paying for a second judgment:
|
||||
Three other features reuse a stored verdict rather than paying for a second judgment:
|
||||
|
||||
- **Campaign reply handling.** The reply classifier that drives `stop_on_reply` and the reply branches reads the stored kind and intent when one exists, so an interested reply routes as positive and a declined one as negative with the classifier's real confidence. The intent itself is also stored on the contact's progress, and a sequence can branch on it with the `reply_intent` condition (see [Campaigns](/guides/campaigns/)). Without tagging, the classifier asks TypeSafe one three-way question instead, and without a key it falls back to the platform LLM as before.
|
||||
- **Reply automations and webhooks.** The `campaign.reply_received` event carries the verdict as `ai_labels`, `ai_intent`, `ai_intent_confidence`, `ai_kind`, `ai_relevance` and `ai_needs_review`, held to the same confidence floor as the labels, so an automation can notify you about Interested or Meeting replies (see [Automations](/guides/automations/#filtering-replies-on-inbox-tagging)).
|
||||
- **The inbox agent.** A reply the verdict shows as closed (Not interested, Unsubscribe, Wrong person, Not now), a one-line acknowledgement, a legal threat or a message that needs a person to read it is not drafted, so no credits are spent answering it.
|
||||
|
||||
</Fold>
|
||||
|
||||
@ -247,6 +247,8 @@ The grant is checked again every hour. When Google stops honoring it (the entry
|
||||
|
||||
Connecting mailboxes through a grant and deleting a grant both reach a whole domain's mail, so like recording one they need a recent confirmation of your sign-in.
|
||||
|
||||
To remove a grant, open **Add account > Google > Whole Workspace domain** or **Microsoft > Whole organization**, use the grant's **More** menu, and choose **Remove grant**. Confirm the removal, then complete **Confirm it is you** if prompted. The removal waits for this identity check and continues automatically when it succeeds. A `403` with code `reauth_required` means this check is needed, not that the domain is owned by another instance. Removing the grant stops its mailboxes on this instance; it does not delete mailboxes connected with app passwords on a separate self-hosted instance.
|
||||
|
||||
</Fold>
|
||||
|
||||
<Fold>
|
||||
|
||||
@ -268,6 +268,8 @@ When the provider stops accepting a mailbox's stored credential (a password chan
|
||||
|
||||
A successful reconnect stores the new credential, clears the authentication error, reactivates the mailbox on its existing worker, and it resumes syncing from where it stopped. Nothing else changes: settings, history, warmup progress, and campaign membership all stay.
|
||||
|
||||
A server can also refuse the login only when the mailbox sends, while sync keeps working. Sending (cold and warmup) is then paused, the drawer's **Health** says so, and the same reconnect button sits under it. Entering the current password or app password again is enough when it has not changed. The reconnect checks both the SMTP and IMAP login against your server, and once they pass, the login hold is lifted. Sending resumes unless another send safety hold is still pending, such as a send whose outcome is unconfirmed, which keeps its own hold until it is resolved.
|
||||
|
||||
Reconnecting never counts against the mailbox allowance, so a workspace at its cap can still fix a broken mailbox. A mailbox whose sign-in is held by Warmbly Cloud is reconnected from your cloud workspace instead; the button says so if you try locally.
|
||||
|
||||
</Fold>
|
||||
@ -290,7 +292,7 @@ The IMAP folder cap limits which folders are opened, not which folders exist. Fo
|
||||
|
||||
If Gmail rejects an ordinary import continuation token, the import restarts inside its original fixed date window. Already mapped mail is deduplicated, and the existing message cap and sync budgets still apply. Outlook recovers an expired delta checkpoint by replaying the configured recent-history window, reconciling known messages and following changes from a fresh checkpoint only after recovery completes. Older messages never imported are outside that recovery window. Authentication failures, throttles and transport errors are not treated as expired checkpoints.
|
||||
|
||||
On IMAP, discovering a new folder or a changed UIDVALIDITY generation during or after the initial import schedules a bounded recent-history scan. Generation recovery reconciles known message handles and read state without restarting the global import count. In folders excluded from history imports, such as Junk and Trash, recovery updates only known messages and does not import unseen history. An unreadable or incomplete message body remains retryable rather than being stored as a successful empty message. A folder without usable modification sequences falls back to UIDNEXT checks and periodic flags even if the server advertises CONDSTORE. Changes are read with the standard CONDSTORE fetch. If a server answers that in a form Warmbly cannot read, the worker asks the other supported ways, and when none can be read it uses the same UIDNEXT checks. The mailbox keeps syncing either way. Failed flag publication keeps the previous baseline so it can retry. Background lookups do not mark unread messages as seen.
|
||||
On IMAP, discovering a new folder or a changed UIDVALIDITY generation during or after the initial import schedules a bounded recent-history scan. Generation recovery reconciles known message handles and read state without restarting the global import count. In folders excluded from history imports, such as Junk and Trash, recovery updates only known messages and does not import unseen history. An unreadable or incomplete message body remains retryable rather than being stored as a successful empty message. A folder without usable modification sequences falls back to UIDNEXT checks and periodic flags even if the server advertises CONDSTORE. Changes are read with the standard CONDSTORE fetch. If a server answers that in a form Warmbly cannot read, the worker asks the other supported ways, and when none can be read it uses the same UIDNEXT checks. The mailbox keeps syncing either way. When a server marks a whole folder as changed at once, only mail that arrived since the last pass is imported; older messages it reports again keep their read state in sync but are not imported, and they do not count toward the sync budget. Failed flag publication keeps the previous baseline so it can retry. Background lookups do not mark unread messages as seen.
|
||||
|
||||
On upgraded instances, a newly admitted provider message is recorded together with an encrypted pending arrival before sync advances. The consumer replays that arrival under its original identity after a restart, including related bounce or complaint processing, and provider updates wait for an unresolved arrival. This is an at-least-once recovery path, not a promise that all historical mail can be reconstructed. Old map-only records have no replay payload, and mixed-version instances retain their legacy behavior. See the [self-host upgrade order and limits](/development/install/#mailbox-sync-upgrade-order).
|
||||
|
||||
@ -456,7 +458,9 @@ Tracking uses the same host as everything else, so a [custom tracking domain](#c
|
||||
|
||||
New mailboxes should warm up before carrying campaign volume. Defaults are `10`/day starting volume, `+1`/day ramp, and a `40`/day ceiling.
|
||||
|
||||
Start, pause, resume, or stop from the mailbox's **Warmup** tab, or for several mailboxes at once from the Accounts selection bar. **Pausing keeps your ramp progress; stopping resets it**, so a restart begins at the base volume again.
|
||||
Start, pause, resume, or stop from the mailbox's **Warmup** tab, or for several mailboxes at once from the Accounts selection bar. Pausing keeps your ramp progress and stops new warmup sending. Stopping also disables warmup receiving and engagement without deleting the mailbox or its history.
|
||||
|
||||
Stopping retires warmup requests that have not begun sending, including other mailboxes' requests to this recipient. Restarting does not reactivate those old requests. Directional participation switches retire requests only in the disabled direction. A send already underway or accepted by the provider cannot be recalled. Gmail can therefore send delayed-delivery notices for earlier messages after you stop warmup; a notice arriving later is not necessarily a new Warmbly send.
|
||||
|
||||
The same tab decides where warmup mail lands **in your own mail client**: its own folder (`Warmbly` by default), left in the inbox, or archived. It covers both directions, the warmup this mailbox receives and the copy of what it sends, so neither Inbox nor Sent fills with it. See [keeping warmup out of your inbox](/guides/warmup/#keeping-warmup-out-of-your-inbox).
|
||||
|
||||
@ -556,13 +560,17 @@ A mailbox now has a cold-rotation state, separate from its health:
|
||||
| `resting` | Out of cold rotation to recover. Warmup keeps running, so its reputation stays alive. Leads mid-sequence on it move to another mailbox at their next step |
|
||||
| `reserve` | Held back by you. Never entered or left automatically |
|
||||
|
||||
The row shows **Resting** or **Reserve** even when no campaign is active on that instance, including Cloud accounts used only for warmup. Warmup can continue while cold sending rests. Each instance owns its own cold rotation and manual holds; linking warmup does not copy a manual hold between instances. A connection problem still takes precedence in the row, with the cold-rotation notice available in the drawer.
|
||||
|
||||
A mailbox rests when it is quarantined or blocked from warmup, and returns on its own three days after that lifts. One good hour does not put it back in rotation. Spam placement never rests a mailbox: a `watch` or `throttled` mailbox stays in rotation with its daily cold volume dampened (to 70% and 50%) until placement recovers.
|
||||
|
||||
A missing or stale [Warmbly Cloud](/guides/warmbly-cloud/#safety-and-enforcement) status is not a health signal. It never rests a mailbox, and campaign sends from the mailbox wait until a current status arrives. A rest that is already running stays in place, and its three days count only time with a current status, so a hold that ends while the status is missing does not put the mailbox back on the first fresh report.
|
||||
|
||||
Resting only makes sense while the mailbox is in a warmup pool, because pool health is the signal it recovers on. If a resting mailbox leaves its pool (the workspace loses warmup when a trial ends or a plan changes), there is no health signal to wait for, so the mailbox returns on its own three days after its last rest stamp rather than staying out indefinitely. Pausing warmup on the mailbox does not remove it from the pool, so its health keeps being tracked and the normal three days after it lifts still apply.
|
||||
|
||||
### Putting a resting mailbox back yourself
|
||||
|
||||
You do not have to wait. A resting mailbox's notice in the drawer has a **Put back into campaigns** button. It lands in `active` unless it is still quarantined or blocked from warmup, in which case it stays resting and the drawer says so: a mailbox the pool has taken out should not send cold mail, and there is no override for that short of the standing lifting.
|
||||
You do not have to wait. A resting mailbox's notice in the drawer has a **Put back into campaigns** button. It lands in `active` unless it is still quarantined or blocked from warmup, in which case it stays resting and the drawer says so: a mailbox the pool has taken out should not send cold mail, and there is no override for that short of the standing lifting. A Warmbly Cloud mailbox whose status is unavailable lands in `active` too; its campaign sends wait for a fresh status instead of keeping it resting.
|
||||
|
||||
API keys get the same exit through `POST /emails/:id/release`, which works on a resting mailbox as well as a held one.
|
||||
|
||||
@ -586,6 +594,16 @@ This is not the same as the risk band that decides which sending worker and IP h
|
||||
|
||||
The Accounts list groups mailboxes as **Healthy** (sending normally), **Warming** (ramping through warmup), or **Needs attention** (paused, failing, or not sending). Each row shows the mailbox with how it connects and its tags, a **Status** that says what it is doing (**Sending**, **Warming**, **Active** when it does both, **Resting**, or **Idle**, and **Error**, **Off** or **Reconnect** when something stops it), campaign emails sent today against its cap, warmup sent today against the ramp target, the inbox rate, and the health score. Hover a status for the reason, and click any column header to sort. A drop in health between refreshes notifies you rather than waiting to be noticed.
|
||||
|
||||
Repeated reports of the same error code count once toward the health score, at their highest unresolved severity, rather than making one problem look like several. The drawer groups identical reports while keeping different error details and sending versus sync failures separate. Connection titles name Gmail, Microsoft Outlook or the mail server. **Reported** shows the stored report time: connection and temporary provider-limit observations refresh it, while other error categories may retain their original report time. It is not proof of an ongoing outage.
|
||||
|
||||
A later completed sync automatically clears earlier temporary sync errors. A provider-confirmed send clears earlier temporary sending errors, including provider throttling and quota warnings. Recovered reports no longer appear under **Needs attention** or reduce the health score. Older reports that do not identify which operation failed clear once both sending and sync have succeeded after the report; existing reports with that evidence are cleared on upgrade too. A newer failure remains visible, and simply waiting or completing a queued task does not count as recovery. Recoverable connection failures retry automatically while the mailbox is on; a connection warning alone is not a reason to reconnect. Authentication, credential and policy errors retain their own instructions, and a switched-off mailbox must be switched back on before it can retry.
|
||||
|
||||
A send safety hold is different from a retryable connection failure. If Warmbly cannot confirm whether the provider accepted a message, sending pauses to avoid retrying a message that might already have been sent. The mailbox is shown as needing attention even without a separate connection warning, and its health details explain the hold. A successful inbox sync does not confirm a send. Campaign and other non-warmup sends still require provider-confirmed resolution. See [send safety](/development/send-safety/).
|
||||
|
||||
Warmup scheduling automatically retires abandoned preparation tasks after ten minutes only when they have no generated message ID, verification token, reservation, dispatch or execution evidence. The normal scheduler can then continue with a new task. Tasks that might have sent a message remain protected; they are not retried just because they are old.
|
||||
|
||||
For an authorized warmup send whose outcome remains unknown after ten minutes, Warmbly can permanently retire that task and resume with a new, distinct message. The old message is never retried or counted as confirmed success or failure; its original charge and send evidence are kept. This recovery runs automatically, including for existing eligible holds. It does not clear sign-in failures, provider restrictions, conflicting results or another unresolved send, and it never overrides a confirmed receipt or a definitive result waiting to be applied. New sending still follows your schedule, daily limits and cooldowns.
|
||||
|
||||
The **Inbox** column is the share of the mailbox's warmup mail that reached partners' inboxes at Google, Microsoft and Yahoo over the last seven days (smaller mail hosts run their own filters and are never counted), and the score is capped at it, so a mailbox landing some mail in spam no longer reads `100`. Click it, or open the drawer's **Deliverability** tab, for the day-by-day history and the split per provider. See [seeing where warmup lands](/guides/warmup/#seeing-where-warmup-lands).
|
||||
|
||||
The detail drawer runs a live `SPF`, `DKIM`, and `DMARC` check on your sending domain. Confirm it is green before sending; major providers require alignment for bulk and cold mail. A domain that stays Failing eventually stops cold sending and warmup from every mailbox on it, so fix the records and press **Check** to clear it straight away; for anyone who can manage mailboxes, the check records its verdict. A lookup DNS did not answer shows the records as Not verified and is never held against the domain. DKIM reads **Not verified** rather than Missing when no key answers, because its selector is not something DNS can be asked for; that never gates sending. See [Deliverability](/guides/deliverability/) for the grace period and exactly what is stopped.
|
||||
|
||||
@ -135,9 +135,16 @@ Some changes ask you to confirm it is you even though you are already signed in:
|
||||
- transferring a workspace to someone else
|
||||
- scheduling a workspace or account for deletion
|
||||
|
||||
Enter your password or a code from your authenticator, whichever your account has. The confirmation lasts five minutes, so a run of related changes only asks once.
|
||||
The prompt offers only what your account can actually use:
|
||||
|
||||
If you signed up with Google, Apple or your company's single sign-on, your account may have neither a password nor 2FA yet. There is nothing to confirm with in that case, so your sign-in is the confirmation: 2FA can be turned on within five minutes of signing in. If more time has passed, sign out, sign in again, and set it up straight away. From then on you confirm with your authenticator.
|
||||
- **a passkey**, if you have added one
|
||||
- **Continue with Google, Apple or your single sign-on**, if you sign in that way. A small window opens at your provider, and the account you pick there has to be the one linked to Warmbly. Allow pop-ups for the dashboard if your browser blocks the window
|
||||
- **your password**, if your account has one
|
||||
- **a code from your authenticator** or a recovery code, if two-factor authentication is on
|
||||
|
||||
The confirmation lasts five minutes, so a run of related changes only asks once. In the iOS app the same prompt offers Continue with Apple and Continue with Google alongside your password and authenticator code.
|
||||
|
||||
An account with none of these (for example one created through a sign-in provider this server no longer offers) is told so instead of being shown a field it cannot fill. Your sign-in is the confirmation in that case: sign in again and repeat the change within five minutes, then add a passkey or turn on 2FA so it does not happen again.
|
||||
|
||||
This exists because the things on that list either hand out a credential that outlives the session that made it, reveal a secret, change who can get in, or cannot be undone from your side. Someone who got hold of a signed-in browser should not be able to do any of them without knowing something you know.
|
||||
|
||||
|
||||
@ -42,6 +42,8 @@ Check today's rolled workday at the top of the tab to see when the mailbox will
|
||||
- Times are local to the mailbox's timezone, or the workspace timezone when it has none.
|
||||
- Editing the ranges does not change today. The new ranges apply from the next roll.
|
||||
- Warmup keeps the same working days and hours, but does not count against the cold limits.
|
||||
- Warmup waits for the workday too, so a mailbox connected after its hours end sends its first warmup email at the start of the next working day. The mailbox list shows when the next one is queued.
|
||||
- Switching to Fixed schedule, back again, or changing the working days moves the mailbox's queued warmup send, and the next send of every active campaign it backs, straight away onto whichever schedule now applies.
|
||||
|
||||
<Fold>
|
||||
|
||||
|
||||
@ -313,6 +313,8 @@ Only inbound mail from that campaign contact to the sending mailbox counts. A th
|
||||
|
||||
When a campaign rotates between sender accounts, a real inbound reply still links to the campaign if the mail client references an earlier step from another mailbox in the same workspace.
|
||||
|
||||
A new email from a contact that answers no thread of yours counts toward the last campaign email you sent them before it arrived. Mail that arrived before any of your emails to them counts toward none, so importing older mail never stops a sequence.
|
||||
|
||||
A reply that lands in the mailbox a send's reply-to named counts the same way, although that mailbox never wrote to the contact. That is what a [shared reply inbox](/guides/mailboxes/#a-shared-reply-inbox) relies on: the reply stops the sequence, runs the reply branch, and is credited to the mailbox that sent the email.
|
||||
|
||||
When an upgrade repairs a reply marker that was provably created by a synchronized Sent message, the contact resumes at the next due step and the campaign reply totals update with it. An upgrade also restores provable human replies that were received by one campaign mailbox but referenced a thread started by another mailbox in the workspace. Warmbly recalculates address verification from the corrected delivery, open, click, reply, and bounce evidence. It leaves ambiguous historical replies untouched.
|
||||
|
||||
@ -167,6 +167,8 @@ Counts come from the server in one request, so they always match the rows. The c
|
||||
|
||||
Search (press `/`) covers the sender and recipients, the subject, and the text of the messages themselves, so a name, an address, or a phrase from the middle of an email all find the conversation. A half-typed word matches as a prefix, so results narrow as you type rather than appearing only once a word is finished. Quoted phrases match exactly, `OR` widens, and a `-word` excludes.
|
||||
|
||||
Participant substring matching starts once the query contains at least three consecutive letters or numbers. Shorter queries still search subject and message words, including prefixes. Search is case-insensitive, and `%`, `_`, and `\` are literal characters, not participant wildcards. If a search cannot finish, the list shows an error instead of claiming there are no matches.
|
||||
|
||||
It searches the scope you are in. When that finds nothing, the empty state offers **Search all mail**, which widens to every folder and keeps what you typed. Changing view yourself clears the box, so a query typed for one scope never silently filters the next.
|
||||
|
||||
The filter button next to the title opens a small panel that narrows the list on the spot: read state, a date range (today, the last 7 or 30 days, or two dates), sender, labels, mailboxes (or a whole tag of them), and sort order. Anything you add shows as a chip under the search box, and each chip removes itself with one click. A view that already fixes one of these, such as Unread or a single mailbox, does not offer that control again.
|
||||
@ -307,7 +309,7 @@ A reply from a different mailbox keeps the conversation's subject and its In-Rep
|
||||
|
||||
**Forward** sends the message you picked to someone new. The subject gets `Fwd:` and the To line starts empty. What you type is an optional note; you can send a forward without one. The recipient gets your note, then the sending mailbox's signature (when signature sync is on), then a `Forwarded message` block with the original From, Date, Subject, To and Cc lines and the original message underneath, formatting included. The Date line uses the sending mailbox's time zone.
|
||||
|
||||
The composer shows the message it will forward under your note; expand **Forwarded message** to read it before you send. Attachments on the original are not forwarded. If only a preview of the original is stored (the thread shows "Only a preview of this message is stored"), the composer says so, and the forward carries that preview. Forwarding one of your own tracked sends puts its links back to their real destinations, so the new recipient's clicks do not count against the original send.
|
||||
The [suppression list](/guides/unsubscribe/#the-suppression-list) does not apply to a forward, so you can forward to an address or domain it keeps out of campaigns. The composer shows the message it will forward under your note; expand **Forwarded message** to read it before you send. Attachments on the original are not forwarded. If only a preview of the original is stored (the thread shows "Only a preview of this message is stored"), the composer says so, and the forward carries that preview. Forwarding one of your own tracked sends puts its links back to their real destinations, so the new recipient's clicks do not count against the original send.
|
||||
|
||||
Once sent, the forward shows in the conversation you forwarded from, as the next outgoing message with its forwarded block, whichever mailbox sent it. While it waits to go out (the undo window, or a scheduled time) it is listed with the conversation's scheduled sends. It appears once the sending mailbox's Sent folder syncs, and that sync files the copy into this conversation rather than adding a second one. Only Warmbly files it there: the recipient gets an ordinary forward that starts a conversation of its own, with no In-Reply-To or References pointing at the original, and a later reply in the conversation still answers its last message from the other side, never the forward.
|
||||
|
||||
@ -347,7 +349,7 @@ For drafts written automatically on every inbound reply and held for approval, t
|
||||
|
||||
The window title bar carries **Full screen**, which lifts the same composer to the centre of the screen over a dimmed backdrop for writing something longer. Click the button again, click the backdrop, or press **Escape** to drop it back to the corner with the draft intact. A second **Escape** closes the window, saving to Drafts as usual. **Minimize** always docks to the bottom right.
|
||||
|
||||
**To** searches contacts as you type and accepts raw addresses. Suppressed recipients (bounced, complained, unsubscribed) are flagged before you write a word, and the send is blocked.
|
||||
**To** searches contacts as you type and accepts raw addresses. Suppressed recipients (bounced, complained, unsubscribed, or on a domain you listed) are flagged before you write a word, and the send is blocked.
|
||||
|
||||
**Seeds**, beside Cc and Bcc, adds your workspace's own [seed inboxes](/guides/placement-tests/#your-own-seed-inboxes) to To, for a quick look by hand at where a message lands. Tick seeds one by one, a whole provider at once, or all of them; the picker remembers your last choice in this browser. It appears once the workspace has a seed inbox and you have the **View campaigns** permission. This sends one message addressed to every seed you add, so each seed sees the others. For a separate copy per seed and a verdict read back automatically, run a [placement test](/guides/placement-tests/#choosing-which-of-your-seeds-get-a-copy) on the seeds you choose.
|
||||
|
||||
@ -383,6 +385,37 @@ With a recipient set, **History** slides out every conversation you have had wit
|
||||
|
||||
Scheduled and smart-queued sends skip the undo window, since they already wait. Cancel them from the **Scheduled** view, or via `DELETE /unibox/scheduled/:task_id`.
|
||||
|
||||
Replies, forwards and new emails from the unibox are conversations, not outreach, so they never count toward a mailbox's daily campaign limit or its warmup volume, and a mailbox that has sent its campaign emails for the day can still answer a lead. They are held to fair use instead, counted in recipients (every To, Cc and Bcc address).
|
||||
|
||||
Each mailbox gets a daily allowance sized below what its provider accepts:
|
||||
|
||||
| Mailbox | Recipients in 24 hours |
|
||||
|---------|------------------------|
|
||||
| Google Workspace or Microsoft 365 | 1,500 |
|
||||
| Other IMAP/SMTP hosts | 750 |
|
||||
| Personal Gmail (gmail.com) | 400 |
|
||||
| Personal Outlook (outlook.com, hotmail.com, live.com) | 250 |
|
||||
|
||||
What Warmbly knows about the mailbox can lower it:
|
||||
|
||||
- connected less than 2 days ago: a quarter; less than 14 days: half
|
||||
- throttled in warmup: half; quarantined or blocked: a quarter
|
||||
- a workspace under restriction: a quarter
|
||||
|
||||
However low that goes, every mailbox keeps at least 50 recipients a day, so a conversation never stalls. Within the day:
|
||||
|
||||
- any one hour takes at most a fifth of it, so a burst cannot spend the whole day
|
||||
- one email goes to at most 100 recipients, or the hourly allowance if that is smaller. A longer list belongs in a campaign
|
||||
- new emails that start a conversation take at most a quarter of it. Replies and forwards inside a conversation do not count toward this part, so a mailbox can always keep answering
|
||||
|
||||
Your plan sets a second, workspace-wide allowance across all your mailboxes together: twice the plan's daily email limit in any 24 hours, and never less than 100. The free trial's 20 emails a day gives 100 unibox recipients, Starter's 150 gives 300, and Grow's 3,000 gives 6,000. An approved increase to your daily limit raises it too. A plan with no daily limit has no workspace allowance, and neither does a self-hosted instance without billing.
|
||||
|
||||
A mailbox that is paused or reconnecting, held after a send problem, or cooling down after a provider limit sends nothing from the unibox either.
|
||||
|
||||
A new email or a reply to an address on your [suppression list](/guides/unsubscribe/#the-suppression-list), directly or through its domain, is refused before it is queued, with a message naming the address and the entry that matched it. A forward is not: it passes on mail you already received to people you picked by hand, so you can forward a lead to a client or colleague whose domain you keep out of campaigns. Fair use still counts a forward's recipients.
|
||||
|
||||
When a send is refused for one of those reasons, it is not sent and the mailbox owner sees an error naming the cause and the limit it reached, so they can send it later or from another mailbox. If Warmbly cannot confirm the email reached the mailbox's sending worker, the error says so; check **Sent** before sending it again.
|
||||
|
||||
</Fold>
|
||||
|
||||
<Fold>
|
||||
|
||||
@ -126,6 +126,7 @@ Only a person answering you can opt out this way: someone replying in one of you
|
||||
- a bounce or delivery failure, which returns your own email (and its unsubscribe header) inside the notice
|
||||
- an out-of-office or other automatic reply
|
||||
- a newsletter or notification sent to a list, whose "unsubscribe" footer is its own sender's
|
||||
- mail a contact sent before they became a contact, for example older messages from an address you later import, which cannot be an answer to anything you sent them. It stays in the unibox but opts nobody out, opens no follow-up task and records no reply. A reply in one of your campaign threads always counts
|
||||
|
||||
Entries written under an earlier, broader reading of replies are read again once. When neither the message that wrote an entry nor any other message from that address asks to stop under this rule, the entry is lifted, the contact's **Subscribed** flag is restored, and the lift is recorded in the audit log. An entry that began with a link or one-click unsubscribe, or whose mail can no longer be found, stays.
|
||||
|
||||
@ -149,7 +150,7 @@ Turn this off with **Settings > Sending > Honour replies that ask to stop** if y
|
||||
| Added by hand | Someone on the team added it |
|
||||
| Imported | Pasted in as part of a list |
|
||||
|
||||
The list is workspace-wide. An entry stops the address in every campaign, in new campaigns created later, and in the unibox composer, which refuses to send to it. Import a list containing a suppressed address and the contact is created but never mailed. The [launch check](/guides/campaigns/#the-launch-check-on-your-list) counts suppressed leads out of a campaign's deliverable total, so what the campaign says it will send is what it sends.
|
||||
The list is workspace-wide. An entry stops the address in every campaign, in new campaigns created later, and in unibox new emails and replies, which are refused with the address named. A unibox [forward](/guides/unibox/#forwarding) is the one exception: it passes on mail already received to recipients picked by hand, so a domain listed to keep clients out of outreach does not stop you forwarding them a lead. Import a list containing a suppressed address and the contact is created but never mailed. The [launch check](/guides/campaigns/#the-launch-check-on-your-list) counts suppressed leads out of a campaign's deliverable total, so what the campaign says it will send is what it sends.
|
||||
|
||||
**Adding entries.** Press **Add** and paste addresses or domains, one per line or as a column from a spreadsheet. A bare domain (`acme.com` or `@acme.com`) suppresses every address at it, which is the usual way to keep customers, partners and your own company out of outreach. An optional reason is kept with each entry.
|
||||
|
||||
|
||||
@ -58,12 +58,12 @@ Only warmup moves to the cloud. Campaigns, contacts, the unibox, tracking and ev
|
||||
New connections link one local workspace to one Warmbly Cloud workspace. A Cloud workspace can have only one active self-hosted connection, including any older instance-wide link. New connections for different local workspaces need different Cloud workspaces, each with its own plan. Existing server-wide connections keep serving their local workspaces without reconnecting or buying another plan. Self-hosted workspace creation itself is unlimited; hosted creation limits and request-rate protection are unchanged. When you enroll a mailbox, the instance sends that mailbox's SMTP/IMAP credential to the cloud, sealed in transit and at rest with the same envelope encryption the hosted product uses for every customer mailbox. From then on:
|
||||
|
||||
- Warmbly Cloud sends the mailbox's warmup mail, replies to partners, and rescues, opens and stars the warmup mail it receives, on Warmbly's own workers
|
||||
- A mailbox enrolled from your instance follows the warmup days, hours and timezone sent by that instance. Cloud does not automatically enable its weekday sending-behavior profile for these warmup-only mailboxes. A Cloud profile you explicitly configure still takes precedence while enabled.
|
||||
- the mailbox's warmup ramp, hours and days belong to Warmbly Cloud from then on, starting from the ones it had on your instance, and its timezone stays your instance's (see [warmup settings](#warmup-settings)). Cloud does not automatically enable its weekday sending-behavior profile for these warmup-only mailboxes. A Cloud profile you explicitly configure still takes precedence while enabled.
|
||||
- the cloud reads only verified warmup mail from the mailbox. Everything else that arrives is dropped unread; no history is imported and nothing is stored
|
||||
- your instance stops its own warmup for that mailbox and takes it out of the local warmup pool, so no mailbox on your server writes to it any more, and keeps sending campaigns from it as usual
|
||||
- your instance recognises both received warmup and the mailbox's own Sent copies and leaves them out of your unibox, asking the cloud to vouch for anything it cannot identify on its own
|
||||
|
||||
Verification accepts a cloud token only for the mailbox that sent or received that warmup message. It also recognises recorded message identifiers when a provider strips the header, including older messages and mail the cloud has already processed. If the cloud cannot answer, inbox ingestion retries verification before publishing the message. This can delay ordinary inbox arrivals for enrolled mailboxes during a cloud outage. Existing Unibox entries are checked automatically in background batches, and confirmed warmup is removed locally without deleting provider mail.
|
||||
Verification accepts a cloud token only for the mailbox that sent or received that warmup message. It also recognises recorded message identifiers when a provider strips the header, including older messages and mail the cloud has already processed. If the cloud cannot answer, inbox ingestion retries verification before publishing the message. This can delay ordinary inbox arrivals for enrolled mailboxes during a cloud outage. Existing Unibox entries are checked automatically in background batches, and confirmed warmup is removed locally without deleting provider mail. When the cloud cannot answer for one mailbox, the rest of that mailbox's entries wait while every other mailbox's entries keep being checked. Once the current check has been through every mailbox, the waiting one is retried from where it stopped: straight away if it got through some entries before failing, otherwise 15 minutes later and then at doubling intervals until the next daily check. An entry that runs out of time in two background batches in a row holds its mailbox the same way.
|
||||
|
||||
Arrivals awaiting verification are stored in your instance's database and retried automatically, including after a consumer restart. Once verification succeeds, ordinary messages enter Unibox and warmup messages are discarded from the pending queue. Pending verification belongs to the instance and is [not included in a workspace export](/guides/workspace-export-import/).
|
||||
- health, daily volume and 7 day totals show up in your instance's settings as the cloud reports them, and the health also gates the mailbox's campaigns here (see [safety and enforcement](#safety-and-enforcement))
|
||||
@ -158,7 +158,21 @@ Once linked, the **Mailboxes** page marks every enrolled mailbox with a blue **C
|
||||
| Today | Warmup mail sent today against today's ramp target |
|
||||
| 7 days | Total sent, and how many landed in spam at partners |
|
||||
|
||||
Pause and resume per mailbox from the same row. Warmup settings (start volume, daily increase, ceiling, reply rate, window, days) are taken from the mailbox's own settings on your instance at enrollment time.
|
||||
Pause and resume per mailbox from the same row. Once a mailbox is enrolled, its warmup settings belong to Warmbly Cloud; see [warmup settings](#warmup-settings).
|
||||
|
||||
### Warmup settings
|
||||
|
||||
Warmbly Cloud owns an enrolled mailbox's starting volume, daily increase, maximum, reply rate, sending window and sending days. Your instance keeps a copy of them on the mailbox, so the Warmup tab, the API and campaign estimates show what Warmbly Cloud actually runs.
|
||||
|
||||
- The first enrollment proposes the settings the mailbox has on your instance. Enrolling it again, or refreshing its credential after a password change, never sends them again, so neither can undo a change made in Warmbly Cloud.
|
||||
- Changing a setting on the Warmup tab, or through `PATCH /emails/:id`, sends it to Warmbly Cloud first. You can change the starting volume and other ramp settings while warmup is active without removing or enrolling the mailbox again. It is saved only once Warmbly Cloud has applied it, and the mailbox then shows the values Warmbly Cloud reports. If Warmbly Cloud cannot be reached, nothing changes. If it answers but does not apply a value, the rest of your change is not saved, the error names what was not applied, and the mailbox shows the values Warmbly Cloud reported.
|
||||
- The copy is refreshed whenever the Cloud mailbox list loads, after every change, and every 15 minutes in the background, so a change made in Warmbly Cloud shows up here within 15 minutes. The Warmup tab shows when it was last refreshed.
|
||||
- The timezone stays your instance's, because campaigns use it too. It travels with every change, and the background refresh sends it again whenever Warmbly Cloud's differs.
|
||||
- Warmbly Cloud does not take 0 from an instance for the starting volume, maximum, daily increase or reply rate, so on an enrolled mailbox those fields start at 1. Sending every day is sent as all seven days.
|
||||
- The content segment, warmup placement, folder and retention on your instance do not apply while Warmbly Cloud warms the mailbox, and the Warmup tab hides them.
|
||||
- Participation and the shared daily and recipient limits are changed with the mailbox's Warmbly Cloud controls. Starting, pausing and resuming warmup are sent to Warmbly Cloud; to stop warmup, remove the mailbox from Warmbly Cloud.
|
||||
|
||||
The background refresh runs in the consumer, so upgrade the backend and consumer together.
|
||||
|
||||
Removing one Cloud-managed mailbox, through its Cloud warmup controls or **Disconnect mailbox**, stops both warmup sending and receiving on Cloud before releasing its link. Cloud keeps the mailbox, sign-in and history; the instance removes its mirror. If the stop fails, removal fails and keeps the link for retry. If releasing the link fails after the stop, warmup remains stopped while you retry. This also applies to existing managed enrollments and older self-hosted clients once Cloud is updated, without disconnecting or re-enrolling other mailboxes. No blanket change is made to disconnected or idle instances, whose Cloud mailboxes may still be intentionally in use. On an older Cloud, removal may only release the mirror; stop participation directly on Cloud until it is updated.
|
||||
|
||||
@ -209,6 +223,8 @@ The same standing feeds the mailbox's health on the **Mailboxes** page, the **Wa
|
||||
|
||||
Cloud standing is observed evidence, not a permanent permission. Healthy, watch and throttled observations have a 15-minute freshness limit. Missing, unrecognized or stale observations stop new control-plane admission with `cloud_evidence_unavailable` until a recognized cloud observation arrives. This availability hold does not claim the provider blocked the mailbox. Existing enrollments start with unknown freshness after upgrading, then the next successful standing sync establishes it.
|
||||
|
||||
An unavailable status holds only two things: campaign sends from the mailbox, and the access tokens your instance draws for a mailbox signed in through Warmbly Cloud. Both wait for a current status. Nothing else reads it as a verdict. The mailbox is not [rested](/guides/mailboxes/#resting-a-tired-mailbox), putting it back into campaigns is not refused, its health score is not lowered, and it counts as a warning rather than an error on the **Mailboxes** page and in the health summary. A rest that is already running stays in place and counts only time with a current status toward its three days. The mailbox drawer shows it as **Status unavailable**, **Settings > Warmbly Cloud** as **Unavailable**, and a campaign's sending plan as **Waiting for Warmbly Cloud status**. A real quarantine or block, whether the cloud reported it or the local reputation ledger still holds it, applies in full while the status is unavailable.
|
||||
|
||||
The Advisor reports an unavailable Cloud status separately from a real pool quarantine or block. A mailbox may still be warming on Cloud while its self-hosted instance lacks a current status. Open **Settings > Warmbly Cloud** and refresh the mailbox list. If the status remains unavailable, check Cloud connectivity and the consumer's `cloud_standing_sync` job, and keep the backend and consumer on the same release. Do not disconnect or re-enroll the mailbox to refresh its status. The **Warming** count indicates warmup is enabled, not that a provider has confirmed a send; the warmup count and send warnings show progress and failures.
|
||||
|
||||
An active quarantine or block remains in force even when the cloud cannot be reached. A bounded hold still ends on its own date; an unbounded block needs review. Removing a mailbox from the cloud does not lift an active hold. Restrictive cloud evidence is retained by workspace and address in the local reputation ledger, so leaving or changing pools, reconnecting, or [exporting and importing the workspace](/guides/workspace-export-import/) does not reset it. The cloud link itself is instance-local and does not travel with the archive. Fresh positive observations from a new link do not erase an unexpired restriction from the previous link. Appeals go through the mailbox on Warmbly Cloud, where the verdict was reached.
|
||||
@ -261,6 +277,8 @@ On your instance, **Settings > Warmbly Cloud > Disconnect** removes only the sel
|
||||
|
||||
SMTP enrollment, mailbox removal and instance disconnect first record a durable local intent. While enrollment or removal is pending, the mailbox is not admitted to local warmup. The regular standing sync retries pending work after an outage or restart, using the same mailbox identity rather than creating another. Remote removal must be acknowledged, or confirmed already absent, before local enrollment is dropped. A failed local cleanup stays pending and is retried.
|
||||
|
||||
Each mailbox's pending work is retried independently. A failed enrollment or removal does not stop the other mailboxes' retries or Cloud status refreshes. The failed mailbox stays pending and unavailable, and the sync still reports the error. When the address is already connected in the linked Cloud workspace, for example because you used hosted Warmbly before self-hosting, Cloud attaches that copy instead of creating a second one, as long as it is an active SMTP/IMAP mailbox, your server presents exactly the SMTP and IMAP host, port, username and password Cloud holds for it, no active Cloud campaign sends from it and no other server has enrolled it. From then on it is your server's mailbox: its warmup settings follow your server, and removing it from your server removes it from Cloud. Any other existing copy answers `pool_link_mailbox_conflict`. Contact support with the connection ID and local mailbox IDs to check the association, without sending credentials or deleting or re-enrolling the mailbox. Cloud never attaches an existing mailbox by address alone.
|
||||
|
||||
Removal is an explicit opt-out. A late enrollment acknowledgment cannot reverse a pending removal. After removal finishes, a new explicit enrollment can create a fresh intent, but it does not reset reputation holds. Disconnect remains pending until remote revocation and local cleanup succeed, and no new enrollment is accepted during it.
|
||||
|
||||
</Fold>
|
||||
|
||||
@ -87,7 +87,7 @@ A worker-loading warning appears only for an active, sending-enabled mailbox aft
|
||||
|
||||
"The sending worker has not loaded this mailbox yet" means that attempt did not send. It is not proof of a deliverability hold, a refused sign-in or an IP allowlist problem. Active mailboxes are reloaded automatically; after a worker restarts, it keeps reporting its restart until the backend acknowledges the request to reload its mailboxes.
|
||||
|
||||
Persistent loading failures and pending warmup tasks at least an hour past their intended schedule are checked every five minutes and reported to configured operator channels and error tracking. The latter is a dispatch problem, not evidence of rejected credentials. The local dispatcher persists a five-minute retry delay after internal failures so an old failing batch cannot indefinitely block newer tasks. Backoff does not alter the intended send time, clear a safety hold, or bypass provider limits.
|
||||
Persistent loading failures and pending warmup tasks at least an hour past their intended schedule are checked every five minutes and reported to configured operator channels and error tracking. The latter is a dispatch problem, not evidence of rejected credentials. The local dispatcher persists a five-minute retry delay after internal failures so an old failing batch cannot indefinitely block newer tasks. Backoff does not alter the intended send time, clear a safety hold, or bypass provider limits. A send whose preparation was interrupted before it was handed to a worker (for example by a restart) is retired after ten minutes, so it never holds up the mailbox's next warmup email.
|
||||
|
||||
A missing mailbox also requests another reload through the worker heartbeat. Existing warmup sends stuck before execution recover automatically: after ten minutes, the scheduler retires a nonce-protected command only when neither worker execution nor a delivery receipt exists, refunds its reserved count, and resumes the mailbox's normal schedule. Delayed copies of the retired command cannot send. Customers do not need to disconnect or re-add their mailboxes. Actually started sends with missing outcomes, conflicting outcomes, provider cooldowns, authentication failures and other safety holds are not cleared by this repair.
|
||||
|
||||
@ -154,6 +154,14 @@ An exclusion is honoured even when it leaves nothing: if every available partner
|
||||
|
||||
A struggling host is downweighted, never excluded: a sender that stops mailing a host entirely can never discover that it recovered there. A host is only judged once at least five of your warmup emails have verifiably arrived there in the window, so one bad delivery out of two is treated as noise rather than a pattern. You can see the numbers this reads under **Deliverability > Warmup inbox placement**, by expanding a provider to its hosts.
|
||||
|
||||
### Mailboxes the pool cannot reach
|
||||
|
||||
A warmup email only counts once the recipient's own sync finds it. Every partner is also weighted by how much of the pool's warmup mail to it arrived over the last seven days, so a partner whose mail half arrives is drawn about half as often.
|
||||
|
||||
A partner that received at least `5` warmup emails from at least `2` different workspaces in that week, and verifiably received none of them, is not offered to anyone. That is a mailbox whose domain has no working mail server, an address that no longer exists, or a server that refuses the mail, and every send to it would only come back to you as a bounce. Emails sent in the last 6 hours are not counted yet, so a slow sync is never mistaken for a dead inbox, and losses from a single workspace never take a partner out, so one sender's problem is not blamed on the recipient.
|
||||
|
||||
Nothing is permanent about it. One warmup email that arrives brings the mailbox straight back, and as the week's evidence ages out it is offered again on its own. If it is your mailbox, the [Advisor](/guides/advisor/) says so and what to check, starting with the domain's MX records.
|
||||
|
||||
### Seeing the diversity you are getting
|
||||
|
||||
The drawer shows confirmed partners/domains/workspaces and verified arrivals over seven days, locally or through Cloud. Pending and failed attempts do not count as confirmed sending. Narrow coverage is a diagnostic limitation, not a reputation verdict.
|
||||
@ -186,7 +194,7 @@ The **inbox rate** is inbox plus category tabs over everything delivered in the
|
||||
|
||||
The rate is taken at Google, Microsoft and Yahoo only, the providers that judge a mailbox. Smaller mail hosts run their own filters, so their spam folders never count toward it: a mailbox whose warmup mail only reached small hosts shows no rate rather than one built from them. Other mail hosts are still shown: their inbox rate sits right under the headline in the drawer and on **Deliverability**, and they appear in the daily charts, the counts and the per-provider breakdown. It only appears once `20` deliveries at those three are in the window, the same sample the pool needs before it acts on spam placement. Until then the mailbox shows how far along it is (`8/20`) rather than a percentage that would swing on every delivery. The mailbox's health score is capped at its inbox rate, so a mailbox with some mail in spam reads `97`, not a flat `100`.
|
||||
|
||||
Filter by provider to distinguish observations from gaps. Historical rescued counts are requests, not confirmed moves or benefits. Receiving-authorized warmup can request spam rescue, but placement remains the originally observed folder, not a rewritten success after a move. Unconfirmed means not observed after timeout, not verified failure. First-folder evidence is separate from housekeeping; absent historical folders stay unknown. Days are UTC. See [warmup placement](/api/reference/analytics/#get-warmup-placement).
|
||||
Filter by provider to distinguish observations from gaps. Confirmed out of spam counts the rescues the recipient's mailbox reported done, beside how many were requested, how many found the message already gone, how many the provider refused and how many have no outcome reported (rescues from before outcomes were recorded stay in that last group). A move is not proof of a reputation benefit. Only mail that arrived in spam is rescued. Receiving-authorized warmup can request spam rescue, but placement remains the originally observed folder, not a rewritten success after a move. Unconfirmed means not observed after timeout, not verified failure. First-folder evidence is separate from housekeeping; absent historical folders stay unknown. Days are UTC. See [warmup placement](/api/reference/analytics/#get-warmup-placement).
|
||||
|
||||
</Fold>
|
||||
|
||||
@ -202,12 +210,16 @@ Filter by provider to distinguish observations from gaps. Historical rescued cou
|
||||
|
||||
Configured diagnostic targets start at 10/day and rise by one toward 40/day before other constraints. These are product pacing defaults, not universal safe volumes or readiness evidence.
|
||||
|
||||
On a self-hosted instance, a mailbox warmed by Warmbly Cloud runs the ramp, hours and days Warmbly Cloud holds for it. The Warmup tab edits those through the cloud, and hides the content segment, placement, folder and retention, which only apply to warmup sent from your instance. See [warmup settings](/guides/warmbly-cloud/#warmup-settings).
|
||||
|
||||
Four things shape the real daily number: sends are spread across your warmup hours with jitter, the target is capped by how many eligible partners exist, a mailbox whose health drops gets reduced volume and wider spacing until it recovers, and a recent spam placement holds the ramp where it is.
|
||||
|
||||
The partner cap exists because a mailbox never writes to the same partner twice in a day. When it applies, the **Warmup** tab of the mailbox drawer says so: it shows the ramp's number next to how many partners are available to the mailbox today, and the target it reports is the lower of the two. The count can fall during the day as partners reach their daily limit, and it recovers the next day. The same tab names the pool the mailbox warms in.
|
||||
|
||||
Warmup hours are read in the mailbox's timezone: its own when one is set on the **Settings** tab of its drawer, otherwise the workspace timezone under **Settings > Profile > Timezones**, and UTC when neither is set. Set the workspace timezone once and every mailbox that follows it reads its warmup hours in the team's day. A mailbox with its own timezone keeps it, and changing the workspace timezone does not change any mailbox's hours.
|
||||
|
||||
While [Sending behavior](/guides/sending-behavior/) is on, which it is for a newly connected mailbox, warmup only sends when both the warmup hours and that workday allow it, and the Warmup tab says so above the hours. A mailbox connected after its workday ends therefore sends its first warmup email the next working morning. Switch the mailbox to **Fixed schedule** on its **Sending** tab to use the warmup hours alone. Changing the warmup hours, days or timezone (the mailbox's own, or the workspace's for a mailbox that follows it), or switching sending behavior on or off, moves the queued warmup send at once. The mailbox list shows when the next warmup send is queued under the day's count.
|
||||
|
||||
Warmup hours must end later than they start on the same day. Equal or reversed hours are rejected, including a change to only one endpoint that would invalidate the stored window. Existing invalid windows are not widened or corrected automatically: pending sends stay unstarted and recheck after five minutes, while admin monitoring shows **Invalid warmup sending hours** until the configured window is corrected. Other send protections still apply after correction; this does not replay uncertain sends or confirm mailbox recovery.
|
||||
|
||||
### Holding the ramp on an early signal
|
||||
@ -240,7 +252,7 @@ Configured diagnostic turns continue disclosed scenarios, not organic relationsh
|
||||
|
||||
Replies thread properly with a real `Re:` subject and `In-Reply-To` header, and candidates must be between 45 minutes and seven days old, so nothing is answered instantly or revived indefinitely.
|
||||
|
||||
Receiving warmup mail can also prompt an answer directly. When a verified warmup email arrives, the recipient sometimes points its next scheduled send back at whoever wrote, 25 minutes to 5 hours later and inside its own warmup hours. That is a re-pointing, not extra work: each mailbox has one warmup send queued at a time, so a reply-back moves that send earlier and aims it, and can never push a send the mailbox had already planned sooner. The chance is the recipient's own reply rate, drawn once when the reply is scheduled rather than again when it sends, and it stops before a thread reaches its message cap so replies cannot answer replies indefinitely.
|
||||
Receiving warmup mail can also prompt an answer directly. When a verified warmup email arrives, the recipient sometimes points its next scheduled send back at whoever wrote, 45 minutes to 5 hours later and inside its own warmup hours. That is a re-pointing, not extra work: each mailbox has one warmup send queued at a time, so a reply-back moves that send earlier and aims it, and can never push a send the mailbox had already planned sooner. When that send is already answering someone, the new email is not answered, so replies never crowd out new threads. The chance is the recipient's own reply rate (raised for a sender in [recovery](#recovery-engagement)), drawn once when the reply is scheduled rather than again when it sends, and it stops before a thread reaches its message cap so replies cannot answer replies indefinitely. A reply crosses the free and paid tiers like the mail it answers: a paid mailbox answers a free partner that wrote to it, and a free mailbox answers a paid one while it is healthy and its workspace is in good standing.
|
||||
|
||||
Scheduling respects windows, provider feedback, shared limits and spacing. Jitter distributes load, not human identities. Receiving participation allows probabilistic spam rescue, reads, importance and stars or flags on verified warmup messages. Filing and selected spam rescues are requested immediately; reads, importance and stars are durably scheduled after a randomized dwell. Current receiving permission and worker assignment are checked again at execution time. These actions are automation, not human feedback or a guaranteed reputation improvement.
|
||||
|
||||
@ -266,7 +278,7 @@ The filing is per mailbox, on the **Warmup** tab of its drawer:
|
||||
|
||||
The folder is created the first time it is needed, and it is also the answer to "what is this message?": nothing but warmup is filed there, so the folder itself is what identifies warmup traffic in a mail client. If another tool already made a folder you use for this, name that one instead and warmup joins it rather than adding a second.
|
||||
|
||||
Spam rescue is provider-specific: Gmail removes Spam and supports Important and Starred labels; Microsoft moves verified Junk messages to Inbox and supports high importance and a follow-up flag; IMAP moves messages out of recognized Spam/Junk folders and uses flags plus best-effort not-junk keywords. IMAP does not have a universal Gmail-style Important label, and SMTP-only mailboxes cannot perform recipient-side actions without mailbox access. A move is not proof that a provider learned a positive reputation signal. Filing remains housekeeping, not engagement evidence. Historical NULL/legacy retains existing behavior until explicit action. Upgrade the consumer, backend and workers together; previously skipped engagement is not replayed onto old mail.
|
||||
Spam rescue is provider-specific: Gmail removes Spam and supports Important and Starred labels; Microsoft moves verified Junk messages to Inbox and supports high importance and a follow-up flag; IMAP moves messages out of recognized Spam/Junk folders and uses flags plus best-effort not-junk keywords. After a rescue runs, the mailbox reports whether the message moved, was already out of spam, was no longer there or was refused, and that is what the placement history shows. IMAP does not have a universal Gmail-style Important label, and SMTP-only mailboxes cannot perform recipient-side actions without mailbox access. A move is not proof that a provider learned a positive reputation signal. Filing remains housekeeping, not engagement evidence. Historical NULL/legacy retains existing behavior until explicit action. Upgrade the consumer, backend and workers together; previously skipped engagement is not replayed onto old mail.
|
||||
|
||||
Mail that arrives somewhere unexpected is still handled. A server-side rule, or another warmup tool's filter, can put warmup mail in a folder of its own choosing before Warmbly sees it; the message is found there and filed from there, and one already sitting in the destination is left alone rather than moved onto itself.
|
||||
|
||||
@ -322,20 +334,28 @@ Pools enforce permission and restrictive complaint/bounce/authentication evidenc
|
||||
| State | Meaning | Effect |
|
||||
|-------|---------|--------|
|
||||
| `healthy` | Normal | Full volume |
|
||||
| `watch` | Mild concern | Spacing stretched modestly |
|
||||
| `throttled` | Probation | Volume cut, spacing widened until recovery |
|
||||
| `watch` | Mild concern | Warmup at full volume with recovery engagement; cold volume eased |
|
||||
| `throttled` | Probation | Volume cut, spacing widened, recovery engagement until recovery |
|
||||
| `quarantined` | Clear problem | Out of the shared pool for a cooldown |
|
||||
| `blocked` | Serious or repeated abuse | Out for longer, review required |
|
||||
|
||||
The thresholds for each signal are in the [health bands](/guides/deliverability/#health-bands) table.
|
||||
|
||||
Watch and throttled standing are warnings, not connection failures. They do not subtract a fixed number of points from mailbox health: the score is still capped by the measured inbox rate, and separate account errors can lower it further. The score is overall health out of 100, not another inbox percentage.
|
||||
|
||||
### How spam placement is judged
|
||||
|
||||
Spam placement is a folder outcome, not misconduct or evidence of diagnostic reputation benefit. It slows sending but does not alone quarantine or block a recipient:
|
||||
|
||||
- `10%` puts the mailbox on watch: warmup and cold sending are spaced out a little more.
|
||||
- `10%` puts the mailbox on watch: warmup keeps its volume and pace (its ramp holds while spam landings are recent), the pool engages harder with its mail (see [recovery engagement](#recovery-engagement)), and cold volume is eased to `70%`.
|
||||
- `20%` throttles it: warmup keeps running at half volume with double spacing, and cold volume is halved.
|
||||
- A watch lifts below `7.5%` and a throttle below `15%`, on their own. They lift below the line that set them rather than at it, so a mailbox sitting near a line is not flipped in and out of a band (and announced to your integrations) with every delivery.
|
||||
- A throttle steps down to watch below `15%`, then returns to healthy below `7.5%`, on its own. They lift below the line that set them rather than at it, so a mailbox sitting near a line is not flipped in and out of a band (and announced to your integrations) with every delivery. The separate [early-signal ramp pause](#holding-the-ramp-on-an-early-signal) still applies to fresh spam landings.
|
||||
|
||||
### Recovery engagement
|
||||
|
||||
While a mailbox is on watch or throttled, the pool works harder for it instead of only slowing it down. Recipients that take part in engagement move nearly every copy of its warmup mail that lands in spam back out of spam (each mailbox at its own rate, so none does it every time) and mark more of it important, and each email has at least a `60%` chance of a reply when the recipient's next warmup send is free to answer it (unless the recipient's owner turned warmup replies off). Replies take the place of the recipient's next scheduled warmup send, so the extra engagement adds no volume. It stops on its own when the band lifts.
|
||||
|
||||
Placement uses today and the preceding six UTC dates, matching the seven-day inbox rate on Deliverability and Emails. Older placement results stop affecting volume at the same boundary as the displayed rate. Complaints and mailbox tampering keep their separate rolling windows.
|
||||
|
||||
Only Google, Microsoft and Yahoo recipients judge a mailbox. They filter on sender reputation, which is what your cold email is judged on at those same providers. Every other mail host runs its own filter, often a stock rule set tuned for that one server, so a message landing in its spam folder is not evidence that the mail is spam. Those landings are shown in the placement view and never held against the mailbox, in the bands, the ramp or the [Advisor](/guides/advisor/). The rate is taken over verified deliveries at the three providers in the last seven days, and needs `20` of them before it acts.
|
||||
|
||||
|
||||
@ -58,6 +58,8 @@ A workspace archive is a single file holding everything one workspace owns. It e
|
||||
|
||||
Diagnostic participation, shared/rolling limits, restrictive send evidence and audited repair-resolution history travel with their registered mailbox/task/organization data. Source-worker executor handles reset; exact provider-message DKIM authorization/nonces are excluded. Portable minimal receipt proof remains historical evidence, never an execution grant. The reconciliation webhook/notification outbox is source-instance-local and excluded to prevent replay on import; drain pending effects before moving. Whole-instance database backups retain it. See [send-safety retention and rollback](/development/send-safety/#upgrade-rollback-and-portability).
|
||||
|
||||
Permanent unknown-warmup retirement markers travel with send history. Imported retired tasks remain cancelled with their outcome recorded as unknown; they never authorize a retry. Use a destination with the compatible retirement schema and recovery code.
|
||||
|
||||
Campaign task replay intent resets to unverified on workspace import. The destination cannot replay an imported dead letter as a verified non-sending wakeup. A contact or sequence binding remains restrictive send intent even if that record is later removed. Whole-instance backups retain the durable intent; migration rollback refuses to discard retained send intent or unverified pending campaign dead letters.
|
||||
|
||||
<Callout title="Moving a whole self-hosted instance is a different tool">
|
||||
@ -246,6 +248,8 @@ A few things belong to the instance rather than the workspace and are left out o
|
||||
|
||||
[Contact imports](/guides/contacts-crm/#importing) do not travel, finished or running, for the same reason: an import is work the source instance is doing. The contacts it created are ordinary contacts and move with the Contacts group, and the saved column mappings come with them. Let a running import finish before exporting, or the rows it has not reached are not in the archive.
|
||||
|
||||
The list of past exports and imports does not travel either. Each job is a record of work the instance that ran it did, and an export's archive lives in that instance's storage until it expires. The destination lists only the exports and imports run there.
|
||||
|
||||
A [community directory](/guides/integrations/#community-apps) listing does not travel. The OAuth app itself moves with the archive, but its listing is a publication on the source instance, featured or hidden by its team. Publish it again on the destination, where it starts as link only and is featured or not by that instance's team. An app suspended on the source arrives suspended, and so does every app imported into a workspace whose members may not build apps on the destination; the destination's operators decide whether to lift it. An import never clears a suspension the destination already applied. Each app passes the same checks as registering it: a name that breaks the [naming rules](/api/error-codes/#name-refusals) becomes "Imported app", and a website, redirect URI or webhook address the destination would refuse is dropped. Its logo arrives empty, so upload it again.
|
||||
|
||||
[Slack](/guides/slack/) account links, the record of which Slack thread belongs to which assistant conversation, and which Slack thread mirrors which inbox conversation in the [inbox channel](/guides/slack/#inbox-in-slack) do not travel. They name the Slack install on the source instance, so members link again after Slack is reconnected on the destination, and inbox conversations start new Slack threads there. The assistant conversations themselves move with the Assistant group.
|
||||
|
||||
@ -123,6 +123,7 @@ That renders as "Hi Alex," when the first name is set and "Hi there," when it's
|
||||
A handful of helpers let you reshape text and work with numbers. They go inside any `{{ }}` or condition:
|
||||
|
||||
- Text: `upper`, `lower`, `title` (change case), `trim`, `contains`, `hasPrefix`, `hasSuffix`
|
||||
- Lists: `has` (holds a whole item), `join` (one line of text)
|
||||
- Numbers: `add`, `sub`, `mul`, `div`, `mod`, `num`, and the coercing comparisons `gtf`, `ltf`, `gef`, `lef`, which treat a value as a number even if it arrives as text
|
||||
|
||||
```
|
||||
|
||||
@ -66,6 +66,14 @@ It does _not_:
|
||||
- Bypass per-mailbox health rules.
|
||||
- Let you ignore complaint or bounce thresholds.
|
||||
|
||||
## Dedicated warmup pool mode
|
||||
|
||||
Reserved sending infrastructure (above) is about which worker authenticates your mailboxes to their providers. It does not change who your mailboxes warm with: by default every workspace draws warmup partners from its plan's shared pool (free or premium), alongside every other customer on that tier.
|
||||
|
||||
Setting a workspace's warmup pool mode to **dedicated** changes that: its mailboxes only ever warm against its own other mailboxes, never another workspace's. Nothing else about warmup changes. The same health rules, spacing, and reply behavior apply; only the pool of possible partners shrinks to one workspace.
|
||||
|
||||
This trades pool size for isolation. A workspace with many mailboxes gets a self-contained warmup loop with no shared-pool reputation risk. A workspace with only one or two mailboxes gets very few (sometimes zero) partners in dedicated mode, since there's no one else to warm with; shared stays the better default for most workspaces. Switch it from workspace settings; the default (shared) keeps every existing workspace's behavior unchanged.
|
||||
|
||||
## Re-entry after quarantine
|
||||
|
||||
A quarantined mailbox shouldn't automatically re-enter the pool just because time elapsed. Re-entry should require:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user