feat: ship skill and MCP workflows with signed updater for v0.0.7
This commit is contained in:
xiaohan.yuan committed
1 parent
1b52df0603
commit
a01085c656
136 files changed
+18646
-217
No files matched your search
@@ -0,0 +1,408 @@
|
||||
import crypto from 'node:crypto'
|
||||
import fsp from 'node:fs/promises'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import {
|
||||
DEFAULT_BASE_URL,
|
||||
SERVER_RELEASES_ROOT,
|
||||
SERVER_STAGING_ROOT,
|
||||
assert,
|
||||
compareVersions,
|
||||
normalizeBaseUrl,
|
||||
parseVersion,
|
||||
readJson,
|
||||
readPublicKey,
|
||||
renderDownloadPage,
|
||||
serializeSignedManifest,
|
||||
sha256File,
|
||||
verifyReleaseInstaller,
|
||||
verifySignedManifest,
|
||||
runProcess,
|
||||
runSsh,
|
||||
validateSshHost,
|
||||
writeFileAtomic
|
||||
} from './lib.mjs'
|
||||
|
||||
const MAX_MANIFEST_BYTES = 1024 * 1024
|
||||
|
||||
export function parseCurrentManifest(raw, publicKeyPem, baseUrl) {
|
||||
assert(Buffer.byteLength(raw, 'utf8') <= MAX_MANIFEST_BYTES, 'Current latest.json exceeds 1 MiB')
|
||||
let document
|
||||
try {
|
||||
document = JSON.parse(raw)
|
||||
} catch {
|
||||
throw new Error('Current latest.json is not valid JSON')
|
||||
}
|
||||
verifySignedManifest(document, publicKeyPem, { baseUrl })
|
||||
return document
|
||||
}
|
||||
|
||||
export function assertMonotonicTransition(current, target) {
|
||||
if (!current) return
|
||||
const compared = compareVersions(target.payload.version, current.payload.version)
|
||||
assert(compared >= 0, `Refusing to publish ${target.payload.version}; server latest is newer (${current.payload.version})`)
|
||||
if (compared === 0) {
|
||||
assert(
|
||||
JSON.stringify(current) === JSON.stringify(target),
|
||||
`Refusing to replace ${target.payload.version} with a different manifest`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export async function validateReleaseBundle({
|
||||
manifestPath,
|
||||
installerPath,
|
||||
publicKeyPath,
|
||||
baseUrl = DEFAULT_BASE_URL
|
||||
}) {
|
||||
const normalizedBaseUrl = normalizeBaseUrl(baseUrl)
|
||||
const publicKeyPem = await readPublicKey(publicKeyPath)
|
||||
let document
|
||||
try {
|
||||
document = await readJson(manifestPath)
|
||||
} catch (error) {
|
||||
throw new Error(`Unable to read release manifest: ${error.message}`)
|
||||
}
|
||||
verifySignedManifest(document, publicKeyPem, { baseUrl: normalizedBaseUrl })
|
||||
const expected = document.payload.platforms['windows-x64']
|
||||
const installer = await verifyReleaseInstaller(installerPath, document.payload.version)
|
||||
assert(installer.size === expected.size, 'Installer size does not match the signed manifest')
|
||||
assert(installer.sha256 === expected.sha256, 'Installer SHA-256 does not match the signed manifest')
|
||||
assert(path.posix.basename(new URL(expected.url).pathname) === installer.filename, 'Signed URL filename does not match the installer')
|
||||
|
||||
return {
|
||||
document,
|
||||
manifestText: serializeSignedManifest(document, { baseUrl: normalizedBaseUrl }),
|
||||
pageHtml: renderDownloadPage(document.payload, { baseUrl: normalizedBaseUrl }),
|
||||
installer,
|
||||
publicKeyPem,
|
||||
baseUrl: normalizedBaseUrl
|
||||
}
|
||||
}
|
||||
|
||||
export async function readLocalLatest(root, publicKeyPem, baseUrl) {
|
||||
const filename = path.join(root, 'releases', 'latest.json')
|
||||
try {
|
||||
const raw = await fsp.readFile(filename, 'utf8')
|
||||
return { raw, document: parseCurrentManifest(raw, publicKeyPem, baseUrl) }
|
||||
} catch (error) {
|
||||
if (error && error.code === 'ENOENT') return { raw: null, document: null }
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function installVersionDirectory({ root, version, installerPath, expectedSha256, runId }) {
|
||||
const releases = path.join(root, 'releases')
|
||||
const versions = path.join(releases, 'versions')
|
||||
const target = path.join(versions, version)
|
||||
const filename = path.basename(installerPath)
|
||||
await fsp.mkdir(versions, { recursive: true })
|
||||
|
||||
try {
|
||||
const entries = await fsp.readdir(target)
|
||||
assert(entries.length === 1 && entries[0] === filename, `Version directory ${version} already conflicts with this artifact`)
|
||||
const existingHash = await sha256File(path.join(target, filename))
|
||||
assert(existingHash === expectedSha256, `Version directory ${version} contains a conflicting artifact`)
|
||||
return
|
||||
} catch (error) {
|
||||
if (!error || error.code !== 'ENOENT') throw error
|
||||
}
|
||||
|
||||
const staging = path.join(root, '.staging', runId)
|
||||
const stagedVersion = path.join(staging, 'version')
|
||||
const stagedFile = path.join(stagedVersion, filename)
|
||||
await fsp.mkdir(stagedVersion, { recursive: true })
|
||||
try {
|
||||
await fsp.copyFile(installerPath, stagedFile)
|
||||
const copiedHash = await sha256File(stagedFile)
|
||||
assert(copiedHash === expectedSha256, 'Staged installer hash changed')
|
||||
await fsp.rename(stagedVersion, target)
|
||||
} finally {
|
||||
await fsp.rm(staging, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
export async function publishLocalRelease({
|
||||
root,
|
||||
release,
|
||||
current,
|
||||
beforeLatestSwap
|
||||
}) {
|
||||
const { document, installer, manifestText, publicKeyPem, baseUrl } = release
|
||||
assertMonotonicTransition(current?.document, document)
|
||||
await installVersionDirectory({
|
||||
root,
|
||||
version: document.payload.version,
|
||||
installerPath: installer.path,
|
||||
expectedSha256: document.payload.platforms['windows-x64'].sha256,
|
||||
runId: crypto.randomBytes(12).toString('hex')
|
||||
})
|
||||
|
||||
const latestPath = path.join(root, 'releases', 'latest.json')
|
||||
const indexPath = path.join(root, 'releases', 'index.html')
|
||||
let previousPage = null
|
||||
try {
|
||||
previousPage = await fsp.readFile(indexPath)
|
||||
} catch (error) {
|
||||
if (!error || error.code !== 'ENOENT') throw error
|
||||
}
|
||||
|
||||
const assertCurrentLatest = async () => {
|
||||
if (current?.raw != null) {
|
||||
const liveRaw = await fsp.readFile(latestPath, 'utf8')
|
||||
assert(liveRaw === current.raw, 'latest.json changed during publication')
|
||||
return
|
||||
}
|
||||
try {
|
||||
await fsp.access(latestPath)
|
||||
throw new Error('latest.json appeared during publication')
|
||||
} catch (error) {
|
||||
if (error && error.code !== 'ENOENT') throw error
|
||||
}
|
||||
}
|
||||
|
||||
await assertCurrentLatest()
|
||||
try {
|
||||
await writeFileAtomic(
|
||||
indexPath,
|
||||
release.pageHtml || renderDownloadPage(document.payload, { baseUrl }),
|
||||
0o644
|
||||
)
|
||||
if (beforeLatestSwap) await beforeLatestSwap()
|
||||
await assertCurrentLatest()
|
||||
await writeFileAtomic(latestPath, manifestText, 0o644)
|
||||
} catch (error) {
|
||||
try {
|
||||
if (previousPage == null) {
|
||||
await fsp.rm(indexPath, { force: true })
|
||||
} else {
|
||||
await writeFileAtomic(indexPath, previousPage, 0o644)
|
||||
}
|
||||
} catch (rollbackError) {
|
||||
throw new Error(`${error.message}; failed to restore download page: ${rollbackError.message}`)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
return { latestPath, document }
|
||||
}
|
||||
|
||||
export async function readLatestViaSsh(host, publicKeyPem, baseUrl) {
|
||||
const script = `set -euo pipefail
|
||||
if [ -f '${SERVER_RELEASES_ROOT}/latest.json' ]; then
|
||||
cat '${SERVER_RELEASES_ROOT}/latest.json'
|
||||
fi
|
||||
`
|
||||
const raw = runSsh(host, script)
|
||||
if (!raw) return { raw: null, document: null }
|
||||
assert(Buffer.byteLength(raw, 'utf8') <= MAX_MANIFEST_BYTES, 'Server latest.json exceeds 1 MiB')
|
||||
return { raw, document: parseCurrentManifest(raw, publicKeyPem, baseUrl) }
|
||||
}
|
||||
|
||||
function sha256Text(value) {
|
||||
return crypto.createHash('sha256').update(value, 'utf8').digest('hex')
|
||||
}
|
||||
|
||||
export async function publishViaSsh({
|
||||
host,
|
||||
release,
|
||||
current
|
||||
}) {
|
||||
validateSshHost(host)
|
||||
const { document, installer, manifestText, publicKeyPem, baseUrl } = release
|
||||
assertMonotonicTransition(current?.document, document)
|
||||
|
||||
const runId = crypto.randomBytes(12).toString('hex')
|
||||
const version = document.payload.version
|
||||
const filename = installer.filename
|
||||
parseVersion(version)
|
||||
assert(/^[A-Za-z0-9._-]+\.exe$/.test(filename), 'Unsafe installer filename')
|
||||
const staging = `${SERVER_STAGING_ROOT}/${runId}`
|
||||
const manifestSha256 = sha256Text(manifestText)
|
||||
const pageHtml = renderDownloadPage(document.payload, { baseUrl })
|
||||
const pageSha256 = sha256Text(pageHtml)
|
||||
const metadata = {
|
||||
runId,
|
||||
version,
|
||||
filename,
|
||||
sha256: document.payload.platforms['windows-x64'].sha256,
|
||||
size: document.payload.platforms['windows-x64'].size,
|
||||
manifestSha256,
|
||||
pageSha256,
|
||||
expectedLatestSha256: current?.raw == null ? null : sha256Text(current.raw)
|
||||
}
|
||||
|
||||
runSsh(host, `set -euo pipefail
|
||||
install -d -o root -g root -m 0700 '${SERVER_STAGING_ROOT}'
|
||||
install -d -o root -g root -m 0700 '${staging}'
|
||||
`)
|
||||
let publicationError
|
||||
try {
|
||||
runProcess('scp', ['-q', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', installer.path, `${host}:${staging}/${filename}`])
|
||||
|
||||
const temporary = await fsp.mkdtemp(path.join(os.tmpdir(), 'ycode-update-deploy-'))
|
||||
const localManifest = path.join(temporary, 'latest.json')
|
||||
const localPage = path.join(temporary, 'index.html')
|
||||
await fsp.writeFile(localManifest, manifestText, { mode: 0o600 })
|
||||
await fsp.writeFile(localPage, pageHtml, { mode: 0o600 })
|
||||
try {
|
||||
runProcess('scp', ['-q', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', localManifest, `${host}:${staging}/latest.json`])
|
||||
runProcess('scp', ['-q', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', localPage, `${host}:${staging}/index.html`])
|
||||
} finally {
|
||||
await fsp.rm(temporary, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
const metadataBase64 = Buffer.from(JSON.stringify(metadata), 'utf8').toString('base64')
|
||||
const remoteScript = `set -euo pipefail
|
||||
umask 022
|
||||
metadata_file="$(mktemp /tmp/ycode-release.XXXXXX.json)"
|
||||
cleanup() {
|
||||
rm -f "$metadata_file"
|
||||
rm -rf '${staging}'
|
||||
}
|
||||
trap cleanup EXIT
|
||||
printf '%s' '${metadataBase64}' | base64 -d > "$metadata_file"
|
||||
python3 - "$metadata_file" <<'PY'
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
meta = json.load(open(sys.argv[1], "r", encoding="utf-8"))
|
||||
run_id = meta["runId"]
|
||||
version = meta["version"]
|
||||
filename = meta["filename"]
|
||||
expected_hash = meta["sha256"]
|
||||
expected_size = meta["size"]
|
||||
manifest_hash = meta["manifestSha256"]
|
||||
page_hash = meta["pageSha256"]
|
||||
expected_latest_hash = meta["expectedLatestSha256"]
|
||||
|
||||
if not re.fullmatch(r"[0-9A-Fa-f]{24}", run_id):
|
||||
raise SystemExit("invalid staging id")
|
||||
if not re.fullmatch(r"[0-9A-Za-z._-]+\\.exe", filename):
|
||||
raise SystemExit("invalid installer filename")
|
||||
if not re.fullmatch(r"[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?", version):
|
||||
raise SystemExit("invalid version")
|
||||
|
||||
public_root = "${path.posix.dirname(SERVER_RELEASES_ROOT)}"
|
||||
releases = "${SERVER_RELEASES_ROOT}"
|
||||
staging = "${SERVER_STAGING_ROOT}/" + run_id
|
||||
versions = os.path.join(releases, "versions")
|
||||
target_dir = os.path.join(versions, version)
|
||||
staged_artifact = os.path.join(staging, filename)
|
||||
staged_manifest = os.path.join(staging, "latest.json")
|
||||
staged_page = os.path.join(staging, "index.html")
|
||||
latest = os.path.join(releases, "latest.json")
|
||||
index = os.path.join(releases, "index.html")
|
||||
|
||||
def digest(pathname):
|
||||
value = hashlib.sha256()
|
||||
with open(pathname, "rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
value.update(chunk)
|
||||
return value.hexdigest()
|
||||
|
||||
if os.stat(staging).st_dev != os.stat(public_root).st_dev:
|
||||
raise SystemExit("staging and release roots are on different filesystems")
|
||||
|
||||
os.makedirs(versions, mode=0o755, exist_ok=True)
|
||||
os.makedirs("${SERVER_STAGING_ROOT}", mode=0o700, exist_ok=True)
|
||||
lock_path = "/var/lock/ycode-release.lock"
|
||||
with open(lock_path, "a+", encoding="utf-8") as lock:
|
||||
fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
|
||||
|
||||
live_latest_hash = digest(latest) if os.path.isfile(latest) else None
|
||||
if live_latest_hash != expected_latest_hash:
|
||||
raise SystemExit("latest.json changed during publication")
|
||||
|
||||
if os.path.getsize(staged_artifact) != expected_size or digest(staged_artifact) != expected_hash:
|
||||
raise SystemExit("staged installer does not match signed metadata")
|
||||
if os.path.getsize(staged_manifest) > 1024 * 1024 or digest(staged_manifest) != manifest_hash:
|
||||
raise SystemExit("staged manifest does not match expected bytes")
|
||||
if os.path.getsize(staged_page) > 512 * 1024 or digest(staged_page) != page_hash:
|
||||
raise SystemExit("staged download page does not match expected bytes")
|
||||
|
||||
if os.path.exists(target_dir):
|
||||
entries = sorted(os.listdir(target_dir))
|
||||
if entries != [filename]:
|
||||
raise SystemExit("version directory conflicts with this artifact")
|
||||
if digest(os.path.join(target_dir, filename)) != expected_hash:
|
||||
raise SystemExit("version directory contains a conflicting artifact")
|
||||
else:
|
||||
version_stage = os.path.join(staging, "version")
|
||||
os.makedirs(version_stage, mode=0o755, exist_ok=False)
|
||||
os.replace(staged_artifact, os.path.join(version_stage, filename))
|
||||
os.chmod(os.path.join(version_stage, filename), 0o644)
|
||||
os.replace(version_stage, target_dir)
|
||||
|
||||
previous_page = None
|
||||
if os.path.isfile(index):
|
||||
with open(index, "rb") as stream:
|
||||
previous_page = stream.read()
|
||||
if len(previous_page) > 512 * 1024:
|
||||
raise SystemExit("existing download page is unexpectedly large")
|
||||
|
||||
next_latest = os.path.join(releases, f".latest.json.{run_id}.tmp")
|
||||
next_page = os.path.join(releases, f".index.html.{run_id}.tmp")
|
||||
os.replace(staged_manifest, next_latest)
|
||||
os.chmod(next_latest, 0o644)
|
||||
with open(next_latest, "rb") as stream:
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(staged_page, next_page)
|
||||
os.chmod(next_page, 0o644)
|
||||
with open(next_page, "rb") as stream:
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
page_swapped = False
|
||||
try:
|
||||
os.replace(next_page, index)
|
||||
page_swapped = True
|
||||
os.replace(next_latest, latest)
|
||||
except Exception:
|
||||
if page_swapped:
|
||||
if previous_page is None:
|
||||
try:
|
||||
os.unlink(index)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
else:
|
||||
rollback_page = os.path.join(releases, f".index.html.{run_id}.rollback")
|
||||
with open(rollback_page, "wb") as stream:
|
||||
stream.write(previous_page)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.chmod(rollback_page, 0o644)
|
||||
os.replace(rollback_page, index)
|
||||
raise
|
||||
directory_fd = os.open(releases, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
|
||||
print(version)
|
||||
PY
|
||||
`
|
||||
runSsh(host, remoteScript)
|
||||
} catch (error) {
|
||||
publicationError = error
|
||||
throw error
|
||||
} finally {
|
||||
try {
|
||||
runSsh(host, `set -euo pipefail
|
||||
rm -rf '${staging}'
|
||||
`)
|
||||
} catch (cleanupError) {
|
||||
if (publicationError) {
|
||||
console.error(`Unable to remove remote staging path after failure: ${cleanupError.message}`)
|
||||
} else {
|
||||
console.warn(`Publication succeeded, but remote staging cleanup failed: ${cleanupError.message}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { version }
|
||||
}
|
||||
Reference in new issue
Block a user