feat: complete WorkBuddy parity and Windows release workflow
This commit is contained in:
xiaohan.yuan committed
1 parent
143516b3a8
commit
d5512df549
434 files changed
+54977
-1854
No files matched your search
@@ -0,0 +1,156 @@
|
||||
import { promises as fs } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url'
|
||||
import { build } from 'esbuild'
|
||||
import { unzipSync } from 'fflate'
|
||||
|
||||
const project = fileURLToPath(new URL('../', import.meta.url))
|
||||
const lockPath = path.join(project, 'resources/sandbox/git-runtime.lock.json')
|
||||
const defaults = path.join(project, 'resources/sandbox/win32-x64/git')
|
||||
const sha256 = bytes => createHash('sha256').update(bytes).digest('hex')
|
||||
const hashPattern = /^[a-f0-9]{64}$/
|
||||
const uuidPattern = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/
|
||||
const within = (root, target) => { const rel = path.relative(root, target); return rel !== '..' && !rel.startsWith(`..${path.sep}`) && !path.isAbsolute(rel) }
|
||||
const safePath = name => typeof name === 'string' && name.length > 0 && name.length <= 2048 && !path.isAbsolute(name) && !/[\\:\x00-\x1f]/.test(name)
|
||||
&& !name.split('/').some(part => !part || part === '.' || part === '..' || /[. ]$/.test(part) || /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(part))
|
||||
async function regular(file, maximum = 128 * 1024 * 1024) {
|
||||
const stat = await fs.lstat(file)
|
||||
if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > maximum) throw new Error(`Unsafe bundled Git file: ${file}`)
|
||||
const bytes = await fs.readFile(file)
|
||||
if (bytes.length !== stat.size) throw new Error('Bundled Git file changed during reading')
|
||||
return bytes
|
||||
}
|
||||
async function directory(file) {
|
||||
const stat = await fs.lstat(file)
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error(`Bundled Git directory is a link or special file: ${file}`)
|
||||
}
|
||||
export async function readGitRuntimeLock() {
|
||||
const lock = JSON.parse((await regular(lockPath, 64 * 1024)).toString('utf8'))
|
||||
if (lock.format !== 1 || lock.provider !== 'git-for-windows' || lock.platform !== 'win32-x64' || !/^\d+\.\d+\.\d+\.windows\.\d+$/.test(lock.version)
|
||||
|| !hashPattern.test(lock.runtimeManifestSha256) || !hashPattern.test(lock.archive?.sha256) || !Number.isSafeInteger(lock.archive.bytes) || lock.archive.bytes < 1 || lock.archive.bytes > 128 * 1024 * 1024
|
||||
|| new URL(lock.archive.url).origin !== 'https://github.com' || !lock.archive.url.startsWith('https://github.com/git-for-windows/git/releases/download/')
|
||||
|| lock.archive.name !== path.basename(lock.archive.url) || !Number.isSafeInteger(Date.parse(lock.publishedAt))) throw new Error('Invalid pinned Git runtime lock')
|
||||
return lock
|
||||
}
|
||||
export async function verifyBundledGit(outputDirectory = defaults) {
|
||||
const lock = await readGitRuntimeLock()
|
||||
await directory(outputDirectory); await directory(path.join(outputDirectory, 'files'))
|
||||
const bytes = await regular(path.join(outputDirectory, 'manifest.json'), 4 * 1024 * 1024), manifest = JSON.parse(bytes.toString('utf8'))
|
||||
if (sha256(bytes) !== lock.runtimeManifestSha256) throw new Error('Bundled Git manifest digest mismatch against release lock')
|
||||
if (manifest.version !== 1 || manifest.kind !== 'git' || !uuidPattern.test(manifest.id) || manifest.distribution?.version !== lock.version
|
||||
|| manifest.distribution.archiveSha256 !== lock.archive.sha256 || manifest.distribution.releaseUrl !== lock.releaseUrl
|
||||
|| !Array.isArray(manifest.files) || manifest.files.length < 7 || manifest.files.length > 12_000) throw new Error('Bundled Git does not match the pinned release')
|
||||
const filesRoot = path.join(outputDirectory, 'files'), checked = new Set([filesRoot]), seen = new Set()
|
||||
let total = 0
|
||||
for (const file of manifest.files) {
|
||||
if (!safePath(file.path) || seen.has(file.path.toLowerCase()) || !hashPattern.test(file.sha256) || !Number.isSafeInteger(file.bytes) || file.bytes < 0 || file.bytes > 128 * 1024 * 1024) throw new Error('Invalid bundled Git file manifest')
|
||||
seen.add(file.path.toLowerCase())
|
||||
let parent = path.dirname(path.join(filesRoot, file.path))
|
||||
while (!checked.has(parent)) { if (!within(filesRoot, parent)) throw new Error('Bundled Git path escaped'); await directory(parent); checked.add(parent); parent = path.dirname(parent) }
|
||||
const content = await regular(path.join(filesRoot, file.path))
|
||||
if (content.length !== file.bytes || sha256(content) !== file.sha256) throw new Error(`Bundled Git digest mismatch: ${file.path}`)
|
||||
total += content.length
|
||||
}
|
||||
if (total !== manifest.bytes || total > 512 * 1024 * 1024 || !['git.exe', 'license.txt', 'third_party_notices.txt', 'sources.json', 'package-versions.txt'].every(name => seen.has(name))) throw new Error('Bundled Git license inventory or size is invalid')
|
||||
return { version: lock.version, archiveSha256: lock.archive.sha256, manifestSha256: sha256(bytes), files: manifest.files.length, bytes: total }
|
||||
}
|
||||
|
||||
export async function vendorSandboxGit({ archivePath, outputDirectory = defaults, offline = false, rebuild = false } = {}) {
|
||||
const lock = await readGitRuntimeLock(), target = path.resolve(outputDirectory)
|
||||
if (path.basename(target) !== 'git' || !within(project, target) || target === project) throw new Error('Git output must be a project-local generated git directory')
|
||||
if (await fs.lstat(target).catch(() => null)) {
|
||||
await directory(target)
|
||||
const previous = await regular(path.join(target, 'manifest.json'), 4 * 1024 * 1024).then(bytes => JSON.parse(bytes.toString('utf8'))).catch(() => null)
|
||||
if (previous?.version !== 1 || previous.kind !== 'git' || !hashPattern.test(previous.distribution?.archiveSha256)) throw new Error('Refusing to replace an existing directory that is not a generated Git bundle')
|
||||
}
|
||||
if (!rebuild && await fs.stat(path.join(target, 'manifest.json')).catch(() => null)) {
|
||||
try { const current = await verifyBundledGit(target); console.log(`SANDBOX_GIT_BUNDLE_READY ${current.version} ${current.bytes}`); return current } catch { /* Rebuild explicitly requested vendored resources from the pinned archive. */ }
|
||||
}
|
||||
const cache = path.join(project, '.tmp/vendor-git')
|
||||
await fs.mkdir(cache, { recursive: true })
|
||||
const archive = archivePath ? path.resolve(archivePath) : path.join(cache, lock.archive.name)
|
||||
if (!await fs.lstat(archive).catch(() => null)) {
|
||||
if (archivePath || offline) throw new Error('Pinned MinGit archive is missing; provide --archive or allow this explicit vendor command to download it')
|
||||
const response = await fetch(lock.archive.url, { signal: AbortSignal.timeout(180_000) })
|
||||
if (!response.ok || Number(response.headers.get('content-length') || 0) > lock.archive.bytes) throw new Error(`MinGit download failed: ${response.status}`)
|
||||
const bytes = Buffer.from(await response.arrayBuffer())
|
||||
if (bytes.length !== lock.archive.bytes || sha256(bytes) !== lock.archive.sha256) throw new Error('Official MinGit archive digest mismatch')
|
||||
await fs.writeFile(archive, bytes, { flag: 'wx', mode: 0o600 })
|
||||
}
|
||||
const archiveBytes = await regular(archive)
|
||||
if (archiveBytes.length !== lock.archive.bytes || sha256(archiveBytes) !== lock.archive.sha256) throw new Error('Official MinGit archive digest mismatch')
|
||||
const unpacked = unzipSync(archiveBytes, { filter: file => /^(?:ucrt64|mingw64)\/bin\/(?:git(?:-remote-https?|-upload-pack|-receive-pack)?\.exe|[^/]+\.dll)$/i.test(file.name)
|
||||
|| /^usr\/bin\/(?:(?:sh|ssh)\.exe|[^/]+\.dll)$/i.test(file.name)
|
||||
|| /^usr\/share\/licenses\//.test(file.name)
|
||||
|| /^(?:ucrt64|mingw64)\/share\/licenses\//.test(file.name) || ['LICENSE.txt', 'etc/package-versions.txt'].includes(file.name) })
|
||||
const work = await fs.mkdtemp(path.join(project, '.tmp/vendor-git-build-'))
|
||||
const extraction = path.join(work, 'archive'), stateRoot = path.join(work, 'state'), bundle = path.join(work, 'bundle')
|
||||
for (const [name, bytes] of Object.entries(unpacked)) {
|
||||
if (name.endsWith('/')) continue
|
||||
if (!safePath(name) || bytes.length > 128 * 1024 * 1024) throw new Error('Unsafe MinGit archive entry')
|
||||
const destination = path.join(extraction, name)
|
||||
await fs.mkdir(path.dirname(destination), { recursive: true })
|
||||
await fs.writeFile(destination, bytes, { flag: 'wx', mode: 0o600 })
|
||||
}
|
||||
const prefix = unpacked['ucrt64/bin/git.exe'] ? 'ucrt64' : 'mingw64'
|
||||
const loaderPath = path.join(work, 'loader.mjs')
|
||||
await build({ absWorkingDir: project, stdin: { contents: "export { importSandboxToolchain } from './src/main/sandbox/toolchains'; export { configureSandbox } from './src/main/sandbox/configuration';", resolveDir: project, sourcefile: 'git-vendor-loader.ts', loader: 'ts' }, bundle: true, platform: 'node', format: 'esm', outfile: loaderPath, logLevel: 'warning' })
|
||||
const loader = await import(pathToFileURL(loaderPath).href)
|
||||
loader.configureSandbox({ stateRoot, resourceDirectory: path.join(work, 'resources'), useBundledGit: false })
|
||||
const imported = await loader.importSandboxToolchain('git', path.join(extraction, prefix, 'bin/git.exe'))
|
||||
const importedObject = path.join(stateRoot, 'toolchains/objects', imported.id)
|
||||
await fs.cp(importedObject, bundle, { recursive: true, errorOnExist: true, force: false })
|
||||
const manifest = JSON.parse(await fs.readFile(path.join(bundle, 'manifest.json'), 'utf8'))
|
||||
const put = async (name, bytes) => {
|
||||
if (!safePath(name)) throw new Error('Invalid third-party notice path')
|
||||
const destination = path.join(bundle, 'files', name)
|
||||
await fs.mkdir(path.dirname(destination), { recursive: true })
|
||||
await fs.writeFile(destination, bytes, { flag: 'wx', mode: 0o600 })
|
||||
manifest.files.push({ path: name, bytes: bytes.length, sha256: sha256(bytes) }); manifest.bytes += bytes.length
|
||||
}
|
||||
for (const [name, bytes] of Object.entries(unpacked)) {
|
||||
if (name.endsWith('/')) continue
|
||||
if (name === 'LICENSE.txt') await put('LICENSE.txt', bytes)
|
||||
else if (name === 'etc/package-versions.txt') await put('package-versions.txt', bytes)
|
||||
else if (/^(?:ucrt64|mingw64)\/share\/licenses\//.test(name)) await put(name.replace(/^(?:ucrt64|mingw64)\/share\//, ''), bytes)
|
||||
else if (/^usr\/share\/licenses\//.test(name)) await put(name.replace(/^usr\/share\//, 'usr/'), bytes)
|
||||
}
|
||||
await put('SOURCES.json', Buffer.from(JSON.stringify({ provider: lock.provider, version: lock.version, releaseUrl: lock.releaseUrl, archive: lock.archive, sources: lock.sources, components: manifest.files.filter(file => /\.(?:exe|dll)$/i.test(file.path)).map(file => file.path) }, null, 2) + '\n'))
|
||||
await put('THIRD_PARTY_NOTICES.txt', Buffer.from(`YCode bundled Git runtime\n\nUnmodified Git for Windows ${lock.version} and its required DLLs, extracted from the official MinGit distribution.\nRelease: ${lock.releaseUrl}\nArchive SHA-256: ${lock.archive.sha256}\n\nGit is licensed under GPL-2.0; see LICENSE.txt. Dependency license/copyright notices from the upstream archive are preserved under licenses/. Some notices refer to upstream components not included in this runtime subset. See SOURCES.json and package-versions.txt for provenance and source/build references.\n\nGit executes as a separate process. No host Git configuration or credentials are bundled.\n`))
|
||||
const idBytes = Buffer.from(lock.archive.sha256.slice(0, 32), 'hex'); idBytes[6] = (idBytes[6] & 15) | 64; idBytes[8] = (idBytes[8] & 63) | 128
|
||||
const hex = idBytes.toString('hex')
|
||||
manifest.id = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
|
||||
manifest.createdAt = Date.parse(lock.publishedAt)
|
||||
manifest.distribution = { version: lock.version, archiveSha256: lock.archive.sha256, releaseUrl: lock.releaseUrl }
|
||||
manifest.features.push('bundled')
|
||||
manifest.files.sort((a, b) => Buffer.compare(Buffer.from(a.path), Buffer.from(b.path)))
|
||||
const manifestBytes = Buffer.from(JSON.stringify(manifest) + '\n')
|
||||
await fs.writeFile(path.join(bundle, 'manifest.json'), manifestBytes)
|
||||
if (sha256(manifestBytes) !== lock.runtimeManifestSha256) throw new Error(`Generated Git manifest needs review: ${path.join(bundle, 'manifest.json')} SHA-256 ${sha256(manifestBytes)}. Update the reviewed source lock before publishing resources.`)
|
||||
const verified = await verifyBundledGit(bundle)
|
||||
const parent = path.dirname(target)
|
||||
await fs.mkdir(parent, { recursive: true })
|
||||
if (!within(await fs.realpath(project), await fs.realpath(parent))) throw new Error('Git bundle output parent escaped the project')
|
||||
const current = await fs.lstat(target).catch(() => null), backup = path.join(work, 'previous-bundle')
|
||||
if (current) {
|
||||
await directory(target)
|
||||
if (path.dirname(await fs.realpath(target)) !== await fs.realpath(parent)) throw new Error('Git bundle replacement target escaped its parent')
|
||||
await fs.rename(target, backup)
|
||||
}
|
||||
try { await fs.rename(bundle, target) } catch (error) { if (current) await fs.rename(backup, target); throw error }
|
||||
console.log(`SANDBOX_GIT_BUNDLE_PASS ${verified.version} ${verified.files} files ${verified.bytes} bytes`)
|
||||
return verified
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
const args = process.argv.slice(2), options = {}
|
||||
while (args.length) {
|
||||
const name = args.shift()
|
||||
if (name === '--offline') options.offline = true
|
||||
else if (name === '--rebuild') options.rebuild = true
|
||||
else if (name === '--archive' || name === '--output') { const value = args.shift(); if (!value) throw new Error(`Missing ${name}`); options[name === '--archive' ? 'archivePath' : 'outputDirectory'] = value }
|
||||
else throw new Error(`Unknown option: ${name}`)
|
||||
}
|
||||
await vendorSandboxGit(options)
|
||||
}
|
||||
Reference in new issue
Block a user