ycode/scripts/updates/transport.mjs

import crypto from 'node:crypto'
import fsp from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import {
  DEFAULT_BASE_URL,
  SERVER_RELEASES_ROOT,
  SERVER_STAGING_ROOT,
  assert,
  compareVersions,
  normalizeBaseUrl,
  parseVersion,
  readJson,
  readPublicKey,
  renderDownloadPage,
  serializeSignedManifest,
  sha256File,
  verifyReleaseInstaller,
  verifySignedManifest,
  runProcess,
  runSsh,
  validateSshHost,
  writeFileAtomic
} from './lib.mjs'

const MAX_MANIFEST_BYTES = 1024 * 1024

export function parseCurrentManifest(raw, publicKeyPem, baseUrl) {
  assert(Buffer.byteLength(raw, 'utf8') <= MAX_MANIFEST_BYTES, 'Current latest.json exceeds 1 MiB')
  let document
  try {
    document = JSON.parse(raw)
  } catch {
    throw new Error('Current latest.json is not valid JSON')
  }
  verifySignedManifest(document, publicKeyPem, { baseUrl })
  return document
}

export function assertMonotonicTransition(current, target) {
  if (!current) return
  const compared = compareVersions(target.payload.version, current.payload.version)
  assert(compared >= 0, `Refusing to publish ${target.payload.version}; server latest is newer (${current.payload.version})`)
  if (compared === 0) {
    assert(
      JSON.stringify(current) === JSON.stringify(target),
      `Refusing to replace ${target.payload.version} with a different manifest`
    )
  }
}

export async function validateReleaseBundle({
  manifestPath,
  installerPath,
  publicKeyPath,
  baseUrl = DEFAULT_BASE_URL
}) {
  const normalizedBaseUrl = normalizeBaseUrl(baseUrl)
  const publicKeyPem = await readPublicKey(publicKeyPath)
  let document
  try {
    document = await readJson(manifestPath)
  } catch (error) {
    throw new Error(`Unable to read release manifest: ${error.message}`)
  }
  verifySignedManifest(document, publicKeyPem, { baseUrl: normalizedBaseUrl })
  const expected = document.payload.platforms['windows-x64']
  const installer = await verifyReleaseInstaller(installerPath, document.payload.version)
  assert(installer.size === expected.size, 'Installer size does not match the signed manifest')
  assert(installer.sha256 === expected.sha256, 'Installer SHA-256 does not match the signed manifest')
  assert(path.posix.basename(new URL(expected.url).pathname) === installer.filename, 'Signed URL filename does not match the installer')

  return {
    document,
    manifestText: serializeSignedManifest(document, { baseUrl: normalizedBaseUrl }),
    pageHtml: renderDownloadPage(document.payload, { baseUrl: normalizedBaseUrl }),
    installer,
    publicKeyPem,
    baseUrl: normalizedBaseUrl
  }
}

export async function readLocalLatest(root, publicKeyPem, baseUrl) {
  const filename = path.join(root, 'releases', 'latest.json')
  try {
    const raw = await fsp.readFile(filename, 'utf8')
    return { raw, document: parseCurrentManifest(raw, publicKeyPem, baseUrl) }
  } catch (error) {
    if (error && error.code === 'ENOENT') return { raw: null, document: null }
    throw error
  }
}

async function installVersionDirectory({ root, version, installerPath, expectedSha256, runId }) {
  const releases = path.join(root, 'releases')
  const versions = path.join(releases, 'versions')
  const target = path.join(versions, version)
  const filename = path.basename(installerPath)
  await fsp.mkdir(versions, { recursive: true })

  try {
    const entries = await fsp.readdir(target)
    assert(entries.length === 1 && entries[0] === filename, `Version directory ${version} already conflicts with this artifact`)
    const existingHash = await sha256File(path.join(target, filename))
    assert(existingHash === expectedSha256, `Version directory ${version} contains a conflicting artifact`)
    return
  } catch (error) {
    if (!error || error.code !== 'ENOENT') throw error
  }

  const staging = path.join(root, '.staging', runId)
  const stagedVersion = path.join(staging, 'version')
  const stagedFile = path.join(stagedVersion, filename)
  await fsp.mkdir(stagedVersion, { recursive: true })
  try {
    await fsp.copyFile(installerPath, stagedFile)
    const copiedHash = await sha256File(stagedFile)
    assert(copiedHash === expectedSha256, 'Staged installer hash changed')
    await fsp.rename(stagedVersion, target)
  } finally {
    await fsp.rm(staging, { recursive: true, force: true })
  }
}

export async function publishLocalRelease({
  root,
  release,
  current,
  beforeLatestSwap
}) {
  const { document, installer, manifestText, publicKeyPem, baseUrl } = release
  assertMonotonicTransition(current?.document, document)
  await installVersionDirectory({
    root,
    version: document.payload.version,
    installerPath: installer.path,
    expectedSha256: document.payload.platforms['windows-x64'].sha256,
    runId: crypto.randomBytes(12).toString('hex')
  })

  const latestPath = path.join(root, 'releases', 'latest.json')
  const indexPath = path.join(root, 'releases', 'index.html')
  let previousPage = null
  try {
    previousPage = await fsp.readFile(indexPath)
  } catch (error) {
    if (!error || error.code !== 'ENOENT') throw error
  }

  const assertCurrentLatest = async () => {
    if (current?.raw != null) {
      const liveRaw = await fsp.readFile(latestPath, 'utf8')
      assert(liveRaw === current.raw, 'latest.json changed during publication')
      return
    }
    try {
      await fsp.access(latestPath)
      throw new Error('latest.json appeared during publication')
    } catch (error) {
      if (error && error.code !== 'ENOENT') throw error
    }
  }

  await assertCurrentLatest()
  try {
    await writeFileAtomic(
      indexPath,
      release.pageHtml || renderDownloadPage(document.payload, { baseUrl }),
      0o644
    )
    if (beforeLatestSwap) await beforeLatestSwap()
    await assertCurrentLatest()
    await writeFileAtomic(latestPath, manifestText, 0o644)
  } catch (error) {
    try {
      if (previousPage == null) {
        await fsp.rm(indexPath, { force: true })
      } else {
        await writeFileAtomic(indexPath, previousPage, 0o644)
      }
    } catch (rollbackError) {
      throw new Error(`${error.message}; failed to restore download page: ${rollbackError.message}`)
    }
    throw error
  }
  return { latestPath, document }
}

export async function readLatestViaSsh(host, publicKeyPem, baseUrl) {
  const script = `set -euo pipefail
if [ -f '${SERVER_RELEASES_ROOT}/latest.json' ]; then
  cat '${SERVER_RELEASES_ROOT}/latest.json'
fi
`
  const raw = runSsh(host, script)
  if (!raw) return { raw: null, document: null }
  assert(Buffer.byteLength(raw, 'utf8') <= MAX_MANIFEST_BYTES, 'Server latest.json exceeds 1 MiB')
  return { raw, document: parseCurrentManifest(raw, publicKeyPem, baseUrl) }
}

function sha256Text(value) {
  return crypto.createHash('sha256').update(value, 'utf8').digest('hex')
}

export async function publishViaSsh({
  host,
  release,
  current
}) {
  validateSshHost(host)
  const { document, installer, manifestText, publicKeyPem, baseUrl } = release
  assertMonotonicTransition(current?.document, document)

  const runId = crypto.randomBytes(12).toString('hex')
  const version = document.payload.version
  const filename = installer.filename
  parseVersion(version)
  assert(/^[A-Za-z0-9._-]+\.exe$/.test(filename), 'Unsafe installer filename')
  const staging = `${SERVER_STAGING_ROOT}/${runId}`
  const manifestSha256 = sha256Text(manifestText)
  const pageHtml = renderDownloadPage(document.payload, { baseUrl })
  const pageSha256 = sha256Text(pageHtml)
  const metadata = {
    runId,
    version,
    filename,
    sha256: document.payload.platforms['windows-x64'].sha256,
    size: document.payload.platforms['windows-x64'].size,
    manifestSha256,
    pageSha256,
    expectedLatestSha256: current?.raw == null ? null : sha256Text(current.raw)
  }

  runSsh(host, `set -euo pipefail
install -d -o root -g root -m 0700 '${SERVER_STAGING_ROOT}'
install -d -o root -g root -m 0700 '${staging}'
`)
  let publicationError
  try {
    runProcess('scp', ['-q', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', installer.path, `${host}:${staging}/${filename}`])

    const temporary = await fsp.mkdtemp(path.join(os.tmpdir(), 'ycode-update-deploy-'))
    const localManifest = path.join(temporary, 'latest.json')
    const localPage = path.join(temporary, 'index.html')
    await fsp.writeFile(localManifest, manifestText, { mode: 0o600 })
    await fsp.writeFile(localPage, pageHtml, { mode: 0o600 })
    try {
      runProcess('scp', ['-q', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', localManifest, `${host}:${staging}/latest.json`])
      runProcess('scp', ['-q', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', localPage, `${host}:${staging}/index.html`])
    } finally {
      await fsp.rm(temporary, { recursive: true, force: true })
    }

    const metadataBase64 = Buffer.from(JSON.stringify(metadata), 'utf8').toString('base64')
    const remoteScript = `set -euo pipefail
umask 022
metadata_file="$(mktemp /tmp/ycode-release.XXXXXX.json)"
cleanup() {
  rm -f "$metadata_file"
  rm -rf '${staging}'
}
trap cleanup EXIT
printf '%s' '${metadataBase64}' | base64 -d > "$metadata_file"
python3 - "$metadata_file" <<'PY'
import fcntl
import hashlib
import json
import os
import re
import shutil
import sys

meta = json.load(open(sys.argv[1], "r", encoding="utf-8"))
run_id = meta["runId"]
version = meta["version"]
filename = meta["filename"]
expected_hash = meta["sha256"]
expected_size = meta["size"]
manifest_hash = meta["manifestSha256"]
page_hash = meta["pageSha256"]
expected_latest_hash = meta["expectedLatestSha256"]

if not re.fullmatch(r"[0-9A-Fa-f]{24}", run_id):
    raise SystemExit("invalid staging id")
if not re.fullmatch(r"[0-9A-Za-z._-]+\\.exe", filename):
    raise SystemExit("invalid installer filename")
if not re.fullmatch(r"[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?", version):
    raise SystemExit("invalid version")

public_root = "${path.posix.dirname(SERVER_RELEASES_ROOT)}"
releases = "${SERVER_RELEASES_ROOT}"
staging = "${SERVER_STAGING_ROOT}/" + run_id
versions = os.path.join(releases, "versions")
target_dir = os.path.join(versions, version)
staged_artifact = os.path.join(staging, filename)
staged_manifest = os.path.join(staging, "latest.json")
staged_page = os.path.join(staging, "index.html")
latest = os.path.join(releases, "latest.json")
index = os.path.join(releases, "index.html")

def digest(pathname):
    value = hashlib.sha256()
    with open(pathname, "rb") as stream:
        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
            value.update(chunk)
    return value.hexdigest()

if os.stat(staging).st_dev != os.stat(public_root).st_dev:
    raise SystemExit("staging and release roots are on different filesystems")

os.makedirs(versions, mode=0o755, exist_ok=True)
os.makedirs("${SERVER_STAGING_ROOT}", mode=0o700, exist_ok=True)
lock_path = "/var/lock/ycode-release.lock"
with open(lock_path, "a+", encoding="utf-8") as lock:
    fcntl.flock(lock.fileno(), fcntl.LOCK_EX)

    live_latest_hash = digest(latest) if os.path.isfile(latest) else None
    if live_latest_hash != expected_latest_hash:
        raise SystemExit("latest.json changed during publication")

    if os.path.getsize(staged_artifact) != expected_size or digest(staged_artifact) != expected_hash:
        raise SystemExit("staged installer does not match signed metadata")
    if os.path.getsize(staged_manifest) > 1024 * 1024 or digest(staged_manifest) != manifest_hash:
        raise SystemExit("staged manifest does not match expected bytes")
    if os.path.getsize(staged_page) > 512 * 1024 or digest(staged_page) != page_hash:
        raise SystemExit("staged download page does not match expected bytes")

    if os.path.exists(target_dir):
        entries = sorted(os.listdir(target_dir))
        if entries != [filename]:
            raise SystemExit("version directory conflicts with this artifact")
        if digest(os.path.join(target_dir, filename)) != expected_hash:
            raise SystemExit("version directory contains a conflicting artifact")
    else:
        version_stage = os.path.join(staging, "version")
        os.makedirs(version_stage, mode=0o755, exist_ok=False)
        os.replace(staged_artifact, os.path.join(version_stage, filename))
        os.chmod(os.path.join(version_stage, filename), 0o644)
        os.replace(version_stage, target_dir)

    previous_page = None
    if os.path.isfile(index):
        with open(index, "rb") as stream:
            previous_page = stream.read()
        if len(previous_page) > 512 * 1024:
            raise SystemExit("existing download page is unexpectedly large")

    next_latest = os.path.join(releases, f".latest.json.{run_id}.tmp")
    next_page = os.path.join(releases, f".index.html.{run_id}.tmp")
    os.replace(staged_manifest, next_latest)
    os.chmod(next_latest, 0o644)
    with open(next_latest, "rb") as stream:
        os.fsync(stream.fileno())
    os.replace(staged_page, next_page)
    os.chmod(next_page, 0o644)
    with open(next_page, "rb") as stream:
        os.fsync(stream.fileno())

    page_swapped = False
    try:
        os.replace(next_page, index)
        page_swapped = True
        os.replace(next_latest, latest)
    except Exception:
        if page_swapped:
            if previous_page is None:
                try:
                    os.unlink(index)
                except FileNotFoundError:
                    pass
            else:
                rollback_page = os.path.join(releases, f".index.html.{run_id}.rollback")
                with open(rollback_page, "wb") as stream:
                    stream.write(previous_page)
                    stream.flush()
                    os.fsync(stream.fileno())
                os.chmod(rollback_page, 0o644)
                os.replace(rollback_page, index)
        raise
    directory_fd = os.open(releases, os.O_RDONLY)
    try:
        os.fsync(directory_fd)
    finally:
        os.close(directory_fd)

print(version)
PY
`
    runSsh(host, remoteScript)
  } catch (error) {
    publicationError = error
    throw error
  } finally {
    try {
      runSsh(host, `set -euo pipefail
rm -rf '${staging}'
`)
    } catch (cleanupError) {
      if (publicationError) {
        console.error(`Unable to remove remote staging path after failure: ${cleanupError.message}`)
      } else {
        console.warn(`Publication succeeded, but remote staging cleanup failed: ${cleanupError.message}`)
      }
    }
  }

  return { version }
}