summaryrefslogtreecommitdiffstats
path: root/scripts/cache
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/cache')
-rw-r--r--scripts/cache/files.ts138
-rw-r--r--scripts/cache/lock.ts220
-rw-r--r--scripts/cache/logger.ts152
-rw-r--r--scripts/cache/manifest.ts76
-rw-r--r--scripts/cache/metadata.ts96
-rw-r--r--scripts/cache/s3.ts117
-rw-r--r--scripts/cache/utils.ts85
7 files changed, 884 insertions, 0 deletions
diff --git a/scripts/cache/files.ts b/scripts/cache/files.ts
new file mode 100644
index 0000000..e2bdf8b
--- /dev/null
+++ b/scripts/cache/files.ts
@@ -0,0 +1,138 @@
+import { $ } from 'bun';
+import { mkdir, readdir, rm, exists } from 'fs/promises';
+import { join, relative, sep } from 'path';
+import type { CacheMetadata, FileMetadata } from './utils';
+
+export async function calculateFileChecksum(filePath: string): Promise<string> {
+ const fileBlob = Bun.file(filePath);
+ const size = fileBlob.size;
+
+ const hasher = new Bun.CryptoHasher('sha256');
+ if (size <= 10 * 1024 * 1024 /** 10MB */)
+ return hasher.update(await fileBlob.arrayBuffer()).digest('hex');
+
+ const reader = fileBlob.stream().getReader();
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ if (value) hasher.update(value);
+ }
+
+ return hasher.digest('hex');
+}
+
+export async function calculateDirectoryChecksums(
+ paths: string[]
+): Promise<Record<string, FileMetadata>> {
+ const files: Record<string, FileMetadata> = {};
+
+ for (const path of paths) {
+ const entries = await readdir(path, {
+ recursive: true,
+ withFileTypes: true
+ });
+
+ await Promise.all(
+ entries
+ .filter((entry) => entry.isFile())
+ .map(async (entry) => {
+ const fullPath = join(entry.parentPath, entry.name);
+ const relativePath = relative('.', fullPath).split(sep).join('/');
+
+ const size = Bun.file(fullPath).size;
+ const checksum = await calculateFileChecksum(fullPath);
+
+ files[relativePath] = { checksum, size };
+ })
+ );
+ }
+
+ return files;
+}
+
+export async function validateCache(metadata: CacheMetadata): Promise<boolean> {
+ console.log('Validating cache...');
+ let valid = 0;
+ let invalid = 0;
+ let missing = 0;
+
+ const totalFiles = Object.keys(metadata.files).length;
+
+ for (const [filePath, fileInfo] of Object.entries(metadata.files)) {
+ const fullPath = join('.', filePath);
+
+ if (!(await exists(fullPath))) {
+ missing++;
+ continue;
+ }
+
+ try {
+ const actualChecksum = await calculateFileChecksum(fullPath);
+ if (actualChecksum === fileInfo.checksum) valid++;
+ else invalid++;
+ } catch (e) {
+ invalid++;
+ }
+ }
+
+ const isValid = invalid === 0 && missing === 0;
+
+ if (isValid) {
+ console.log(`Cache is valid: ${valid} files matched`);
+ } else {
+ console.log(
+ `Cache validation failed: ${valid} valid, ${invalid} invalid, ${missing} missing (total: ${totalFiles})`
+ );
+ }
+
+ return isValid;
+}
+
+export async function extractTar(tarPath: string): Promise<void> {
+ const compressedData = await Bun.file(tarPath).arrayBuffer();
+ const decompressed = Bun.zstdDecompressSync(new Uint8Array(compressedData));
+
+ // Write decompressed tar to temp file
+ const tempTarPath = tarPath + '.tmp';
+ await Bun.write(tempTarPath, decompressed);
+
+ await $`tar -xf ${tempTarPath}`.quiet().finally(async () => {
+ await rm(tempTarPath).catch(() => {});
+ });
+}
+
+export async function compressToTar(
+ paths: string[],
+ outputPath: string
+): Promise<Record<string, FileMetadata>> {
+ const checksums = await calculateDirectoryChecksums(paths);
+
+ const tempTarPath = outputPath + '.tmp';
+ await $`tar -cf ${tempTarPath} ${paths}`.quiet();
+
+ try {
+ const tarData = await Bun.file(tempTarPath).arrayBuffer();
+ const compressed = Bun.zstdCompressSync(new Uint8Array(tarData));
+ await Bun.write(outputPath, compressed);
+ } finally {
+ await rm(tempTarPath).catch(() => {});
+ }
+
+ return checksums;
+}
+
+export async function ensureDir(dir: string): Promise<void> {
+ if (!(await exists(dir))) {
+ await mkdir(dir, { recursive: true });
+ }
+}
+
+export async function cleanupDir(dir: string): Promise<void> {
+ try {
+ await rm(dir, { recursive: true, force: true });
+ } catch (e: any) {
+ if (e.code !== 'EBUSY' && e.code !== 'ENOTEMPTY') {
+ throw e;
+ }
+ }
+}
diff --git a/scripts/cache/lock.ts b/scripts/cache/lock.ts
new file mode 100644
index 0000000..674f4be
--- /dev/null
+++ b/scripts/cache/lock.ts
@@ -0,0 +1,220 @@
+import type { S3Client } from 'bun';
+import { hostname } from 'os';
+import type { CacheLock } from './utils';
+import {
+ LOCK_DOUBLE_CHECK_MS,
+ LOCK_KEY,
+ LOCK_MAX_RETRIES,
+ LOCK_RETRY_MAX_MS,
+ LOCK_RETRY_START_MS,
+ LOCK_TIMEOUT_MS,
+ writeJsonToS3
+} from './utils';
+
+export function generateInstanceId(): string {
+ return `${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
+}
+
+/**
+ * Checks if a process is still running (only works on same machine).
+ * Uses Node.js process.kill(pid, 0) which doesn't actually kill but checks existence.
+ */
+function isProcessRunning(pid: number): boolean {
+ try {
+ // Signal 0 doesn't kill, just checks if process exists
+ process.kill(pid, 0);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+/**
+ * Checks if a lock is stale based on Restic's algorithm:
+ * 1. If timestamp is older than LOCK_TIMEOUT_MS (30 min), it's stale
+ * 2. If on same machine and process doesn't exist, it's stale
+ */
+function isLockStale(lock: CacheLock): boolean {
+ const lockAge = Date.now() - lock.timestamp;
+ const timeSinceRenewal = lock.renewedAt ? Date.now() - lock.renewedAt : lockAge;
+
+ // Check 1: Timestamp-based staleness (30 minutes)
+ if (timeSinceRenewal > LOCK_TIMEOUT_MS) {
+ return true;
+ }
+
+ // Check 2: Process-based staleness (only on same machine)
+ if (lock.hostname === hostname()) {
+ if (!isProcessRunning(lock.pid)) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/**
+ * Acquires a distributed lock using Restic's double-check pattern.
+ * This is simpler and more reliable than the lease-based approach.
+ *
+ * Algorithm (inspired by Restic):
+ * 1. Check for existing locks
+ * 2. If lock exists and is NOT stale, retry with exponential backoff
+ * 3. If no lock or stale lock found, create our lock
+ * 4. Wait 200ms (LOCK_DOUBLE_CHECK_MS)
+ * 5. Re-check: verify we still own the lock
+ * 6. If verification fails, we lost the race - retry
+ */
+export async function acquireLock(s3: S3Client, instanceId: string): Promise<string | null> {
+ const lockFile = s3.file(LOCK_KEY);
+ let retryDelay = LOCK_RETRY_START_MS;
+
+ for (let attempt = 0; attempt < LOCK_MAX_RETRIES; attempt++) {
+ try {
+ // Step 1: Check for existing lock
+ if (await lockFile.exists()) {
+ const lockContent = await lockFile.text();
+ const existingLock: CacheLock = JSON.parse(lockContent);
+
+ // Check if lock is stale
+ if (isLockStale(existingLock)) {
+ console.log('Stale lock detected, removing...');
+ await lockFile.delete().catch(() => {});
+ } else {
+ // Lock is valid, need to retry
+ console.log(
+ `Lock busy, retrying in ${retryDelay / 1000}s (${attempt + 1}/${LOCK_MAX_RETRIES})...`
+ );
+ await Bun.sleep(retryDelay);
+
+ // Exponential backoff: double delay each time, up to max
+ retryDelay = Math.min(retryDelay * 2, LOCK_RETRY_MAX_MS);
+ continue;
+ }
+ }
+
+ // Step 2: Create new lock
+ const newLock: CacheLock = {
+ locked: true,
+ timestamp: Date.now(),
+ instance: instanceId,
+ ttl: LOCK_TIMEOUT_MS,
+ renewedAt: Date.now(),
+ pid: process.pid,
+ hostname: hostname()
+ };
+
+ await writeJsonToS3(s3, LOCK_KEY, newLock);
+
+ // Step 3: Wait for double-check delay (Restic's waitBeforeLockCheck pattern)
+ // This allows any racing processes to also write their locks
+ await Bun.sleep(LOCK_DOUBLE_CHECK_MS);
+
+ // Step 4: Verify we still own the lock (detect race conditions)
+ if (await lockFile.exists()) {
+ const verifyContent = await lockFile.text();
+ const verifyLock: CacheLock = JSON.parse(verifyContent);
+
+ if (verifyLock.instance === instanceId) {
+ // Successfully acquired lock
+ console.log('Lock acquired');
+ return instanceId;
+ }
+ }
+
+ // Lost the race - another process overwrote our lock
+ // Retry with exponential backoff
+ console.log(`Lost lock race, retrying in ${retryDelay / 1000}s...`);
+ await Bun.sleep(retryDelay);
+ retryDelay = Math.min(retryDelay * 2, LOCK_RETRY_MAX_MS);
+ } catch (e) {
+ console.error(`Lock error: ${e}`);
+ await Bun.sleep(retryDelay);
+ retryDelay = Math.min(retryDelay * 2, LOCK_RETRY_MAX_MS);
+ }
+ }
+
+ console.error('Failed to acquire lock');
+ return null;
+}
+
+/**
+ * Renews the lock to extend its TTL. Should be called periodically during long operations.
+ */
+export async function renewLock(s3: S3Client, instanceId: string): Promise<boolean> {
+ const lockFile = s3.file(LOCK_KEY);
+
+ try {
+ if (!(await lockFile.exists())) {
+ return false;
+ }
+
+ const lock: CacheLock = JSON.parse(await lockFile.text());
+
+ if (lock.instance !== instanceId) {
+ return false;
+ }
+
+ // Update renewal time
+ lock.renewedAt = Date.now();
+ await writeJsonToS3(s3, LOCK_KEY, lock);
+
+ console.log('Lock renewed');
+ return true;
+ } catch (e) {
+ console.error('Failed to renew lock:', e);
+ return false;
+ }
+}
+
+/**
+ * Releases the lock if owned by this instance.
+ */
+export async function releaseLock(s3: S3Client, instanceId: string): Promise<void> {
+ const lockFile = s3.file(LOCK_KEY);
+
+ try {
+ if (!(await lockFile.exists())) {
+ return;
+ }
+
+ const lock: CacheLock = JSON.parse(await lockFile.text());
+
+ if (lock.instance === instanceId) {
+ await lockFile.delete();
+ console.log('Lock released');
+ }
+ } catch (e) {
+ console.error('Failed to release lock:', e);
+ }
+}
+
+/**
+ * Executes a callback while holding the lock, with automatic renewal.
+ */
+export async function withLock<T>(
+ s3: S3Client,
+ callback: (instanceId: string) => Promise<T>
+): Promise<T | null> {
+ const instanceId = generateInstanceId();
+ const lockToken = await acquireLock(s3, instanceId);
+
+ if (!lockToken) {
+ return null;
+ }
+
+ // Setup automatic lock renewal every 2 minutes
+ const renewalInterval = setInterval(
+ async () => {
+ await renewLock(s3, instanceId);
+ },
+ 2 * 60 * 1000
+ );
+
+ try {
+ return await callback(instanceId);
+ } finally {
+ clearInterval(renewalInterval);
+ await releaseLock(s3, instanceId);
+ }
+}
diff --git a/scripts/cache/logger.ts b/scripts/cache/logger.ts
new file mode 100644
index 0000000..1743370
--- /dev/null
+++ b/scripts/cache/logger.ts
@@ -0,0 +1,152 @@
+/**
+ * Checks if the current environment supports interactive terminal features
+ * like carriage return (\r) for progress updates.
+ *
+ * Returns false for:
+ * - Non-TTY environments (CI/CD logs, file redirects)
+ * - Dumb terminals
+ * - Environments without cursor control support
+ */
+function isInteractiveTerminal(): boolean {
+ // Check if stdout is a TTY (interactive terminal)
+ if (!process.stdout.isTTY) return false;
+ // Check for dumb terminal
+ if (process.env.TERM === 'dumb') return false;
+ // Check for CI environments (most set CI=true)
+ if (process.env.CI === 'true' || process.env.CI === '1') return false;
+
+ // Check for common CI environment variables
+ const ciEnvVars = [
+ 'GITHUB_ACTIONS',
+ 'GITLAB_CI',
+ 'CIRCLECI',
+ 'TRAVIS',
+ 'JENKINS_HOME',
+ 'BUILDKITE',
+ 'DRONE',
+ 'RENDER', // Render.com
+ 'CF_PAGES', // Cloudflare Pages
+ 'VERCEL' // Vercel
+ ];
+
+ for (const envVar of ciEnvVars) {
+ if (process.env[envVar]) return false;
+ }
+
+ return true;
+}
+
+/**
+ * Formats transfer statistics (size and speed).
+ */
+function formatTransferStats(bytes: number, elapsedSeconds: number): string {
+ const sizeMB = (bytes / (1024 * 1024)).toFixed(2);
+ const speedMBps = (bytes / (1024 * 1024) / elapsedSeconds).toFixed(2);
+ return `${sizeMB} MB (${speedMBps} MB/s)`;
+}
+
+class TimerLogger {
+ private isInteractive: boolean;
+ private startTime: number;
+ private lastLogTime: number;
+ private prefix: string;
+
+ constructor(prefix: string) {
+ this.isInteractive = isInteractiveTerminal();
+ this.startTime = Date.now();
+ this.lastLogTime = this.startTime;
+ this.prefix = prefix;
+ }
+
+ /**
+ * Logs timer progress at regular intervals (throttled to 1 second).
+ */
+ progress(): this {
+ const now = Date.now();
+ if (now - this.lastLogTime >= 1000) {
+ const elapsed = (now - this.startTime) / 1000;
+ const message = `${this.prefix} (${elapsed.toFixed(0)}s)...`;
+
+ if (this.isInteractive) process.stdout.write(`\r${message}`);
+ else console.log(message);
+
+ this.lastLogTime = now;
+ }
+ return this;
+ }
+
+ /**
+ * Logs final timer completion message.
+ */
+ complete(): void {
+ const elapsed = (Date.now() - this.startTime) / 1000;
+ const message = `${this.prefix} (${elapsed.toFixed(0)}s)`;
+
+ if (this.isInteractive) process.stdout.write(`\r\x1b[K${message}\n`);
+ else console.log(message);
+ }
+}
+
+class TransferLogger {
+ private isInteractive: boolean;
+ private startTime: number;
+ private lastLogTime: number;
+ private prefix: string;
+
+ constructor(prefix: string) {
+ this.isInteractive = isInteractiveTerminal();
+ this.startTime = Date.now();
+ this.lastLogTime = this.startTime;
+ this.prefix = prefix;
+ }
+
+ /**
+ * Logs transfer progress at regular intervals (throttled to 1 second).
+ */
+ progress(bytes: number): this {
+ const now = Date.now();
+ if (now - this.lastLogTime >= 1000) {
+ const elapsed = (now - this.startTime) / 1000;
+ const message = `${this.prefix} ${formatTransferStats(bytes, elapsed)}...`;
+
+ if (this.isInteractive) process.stdout.write(`\r${message}`);
+ else console.log(message);
+
+ this.lastLogTime = now;
+ }
+ return this;
+ }
+
+ /**
+ * Logs final transfer completion message.
+ */
+ complete(bytes: number): void {
+ if (bytes > 0) {
+ const elapsed = (Date.now() - this.startTime) / 1000;
+ const message = `${this.prefix} ${formatTransferStats(bytes, elapsed)}`;
+
+ if (this.isInteractive) process.stdout.write(`\r\x1b[K${message}\n`);
+ else console.log(message);
+ }
+ }
+}
+
+class Logger {
+ /**
+ * Creates a timer progress logger.
+ * Usage: log.timer('Uploading cache').progress().complete()
+ */
+ timer(prefix: string): TimerLogger {
+ return new TimerLogger(prefix);
+ }
+
+ /**
+ * Creates a transfer progress logger.
+ * Usage: log.transfer('Received').progress(bytes).complete(bytes)
+ */
+ transfer(prefix: string): TransferLogger {
+ return new TransferLogger(prefix);
+ }
+}
+
+export const log = new Logger();
diff --git a/scripts/cache/manifest.ts b/scripts/cache/manifest.ts
new file mode 100644
index 0000000..7558047
--- /dev/null
+++ b/scripts/cache/manifest.ts
@@ -0,0 +1,76 @@
+import type { S3Client } from 'bun';
+import type { CacheEntry, CacheManifest } from './utils';
+import { writeJsonToS3 } from './utils';
+
+const MANIFEST_KEY = 'manifest.json';
+const MANIFEST_VERSION = 1;
+
+export async function loadManifest(s3: S3Client): Promise<CacheManifest> {
+ const manifestFile = s3.file(MANIFEST_KEY);
+
+ try {
+ if (await manifestFile.exists()) {
+ const data = await manifestFile.text();
+ const manifest: CacheManifest = JSON.parse(data);
+
+ if (manifest.version === MANIFEST_VERSION) {
+ return manifest;
+ }
+ }
+ } catch (e) {
+ console.error('Failed to load manifest:', e);
+ }
+
+ // Return empty manifest if not found or invalid
+ return {
+ version: MANIFEST_VERSION,
+ caches: []
+ };
+}
+
+export async function saveManifest(s3: S3Client, manifest: CacheManifest): Promise<void> {
+ await writeJsonToS3(s3, MANIFEST_KEY, manifest);
+}
+
+export async function addCacheEntry(
+ s3: S3Client,
+ key: string,
+ hash: string,
+ timestamp: number
+): Promise<void> {
+ const manifest = await loadManifest(s3);
+
+ // Remove existing entry with same key if exists
+ manifest.caches = manifest.caches.filter((entry) => entry.key !== key);
+
+ // Add new entry
+ manifest.caches.push({
+ key,
+ hash,
+ timestamp,
+ lastAccessed: timestamp
+ });
+
+ await saveManifest(s3, manifest);
+}
+
+export async function removeCacheEntry(s3: S3Client, key: string): Promise<void> {
+ const manifest = await loadManifest(s3);
+ manifest.caches = manifest.caches.filter((entry) => entry.key !== key);
+ await saveManifest(s3, manifest);
+}
+
+export function findCacheByKey(manifest: CacheManifest, key: string): CacheEntry | null {
+ return manifest.caches.find((entry) => entry.key === key) || null;
+}
+
+export function findCacheByPrefix(manifest: CacheManifest, prefix: string): CacheEntry | null {
+ const matching = manifest.caches.filter((entry) => entry.key.startsWith(prefix));
+
+ if (matching.length === 0) {
+ return null;
+ }
+
+ // Return most recently created cache
+ return matching.sort((a, b) => b.timestamp - a.timestamp)[0];
+}
diff --git a/scripts/cache/metadata.ts b/scripts/cache/metadata.ts
new file mode 100644
index 0000000..2326d08
--- /dev/null
+++ b/scripts/cache/metadata.ts
@@ -0,0 +1,96 @@
+import type { S3Client } from 'bun';
+import { METADATA_VERSION, writeJsonToS3 } from './utils';
+import type { CacheMetadata, FileMetadata } from './utils';
+
+function getMetadataKey(cacheKey: string): string {
+ return `${cacheKey}.meta.json`;
+}
+
+export async function saveMetadata(
+ s3: S3Client,
+ key: string,
+ files: Record<string, FileMetadata>,
+ cacheFilePath: string
+): Promise<string> {
+ const content = await Bun.file(cacheFilePath).arrayBuffer();
+ const hash = new Bun.CryptoHasher('sha256').update(content).digest('hex');
+
+ const metadata: CacheMetadata = {
+ key,
+ hash,
+ timestamp: Date.now(),
+ lastAccessed: Date.now(),
+ files,
+ version: METADATA_VERSION
+ };
+
+ const metadataKey = getMetadataKey(key);
+ await writeJsonToS3(s3, metadataKey, metadata);
+
+ console.log(`Metadata saved: ${metadataKey}`);
+ return hash;
+}
+
+export async function loadMetadata(s3: S3Client, cacheKey: string): Promise<CacheMetadata | null> {
+ const metadataKey = getMetadataKey(cacheKey);
+ const metadataFile = s3.file(metadataKey);
+
+ try {
+ if (!(await metadataFile.exists())) {
+ return null;
+ }
+
+ const metadata: CacheMetadata = JSON.parse(await metadataFile.text());
+
+ if (metadata.version !== METADATA_VERSION) {
+ return null;
+ }
+
+ return metadata;
+ } catch (e) {
+ console.error('Failed to load metadata:', e);
+ return null;
+ }
+}
+
+async function updateMetadataAccessTime(
+ s3: S3Client,
+ cacheKey: string,
+ metadata: CacheMetadata
+): Promise<void> {
+ metadata.lastAccessed = Date.now();
+
+ const metadataKey = getMetadataKey(cacheKey);
+ await writeJsonToS3(s3, metadataKey, metadata);
+}
+
+export async function updateBothAccessTimes(
+ s3: S3Client,
+ cacheKey: string,
+ metadata: CacheMetadata
+): Promise<void> {
+ await updateMetadataAccessTime(s3, cacheKey, metadata);
+
+ // Also update manifest
+ const { loadManifest, saveManifest } = await import('./manifest');
+ const manifest = await loadManifest(s3);
+ const entry = manifest.caches.find((e) => e.key === cacheKey);
+
+ if (entry) {
+ entry.lastAccessed = Date.now();
+ await saveManifest(s3, manifest);
+ }
+}
+
+export async function deleteMetadata(s3: S3Client, cacheKey: string): Promise<void> {
+ const metadataKey = getMetadataKey(cacheKey);
+ const metadataFile = s3.file(metadataKey);
+
+ try {
+ if (await metadataFile.exists()) {
+ await metadataFile.delete();
+ }
+ } catch (e) {
+ console.error(`Failed to delete metadata: ${e}`);
+ }
+}
diff --git a/scripts/cache/s3.ts b/scripts/cache/s3.ts
new file mode 100644
index 0000000..ffbb6ac
--- /dev/null
+++ b/scripts/cache/s3.ts
@@ -0,0 +1,117 @@
+import { S3Client } from 'bun';
+import { MAX_CACHE_AGE_DAYS, MAX_CACHE_FILES } from './utils';
+import { findCacheByKey, findCacheByPrefix, loadManifest, removeCacheEntry } from './manifest';
+import { deleteMetadata } from './metadata';
+
+const s3Config = {
+ ENDPOINT: process.env.S3_ENDPOINT,
+ ACCESS_KEY_ID: process.env.S3_ACCESS_KEY_ID,
+ SECRET_ACCESS_KEY: process.env.S3_SECRET_ACCESS_KEY,
+ BUCKET_NAME: process.env.S3_BUCKET_NAME,
+ REGION: process.env.S3_REGION
+};
+
+export const ENABLED =
+ !!s3Config.ENDPOINT &&
+ !!s3Config.ACCESS_KEY_ID &&
+ !!s3Config.SECRET_ACCESS_KEY &&
+ !!s3Config.BUCKET_NAME;
+
+let client: S3Client | null = null;
+
+export function getClient(): S3Client | null {
+ if (!ENABLED || client) return client;
+
+ client = new S3Client({
+ endpoint: s3Config.ENDPOINT,
+ accessKeyId: s3Config.ACCESS_KEY_ID,
+ secretAccessKey: s3Config.SECRET_ACCESS_KEY,
+ bucket: s3Config.BUCKET_NAME,
+ region: s3Config.REGION
+ });
+ return client;
+}
+
+const cacheExists = async (s3: S3Client, key: string) =>
+ await s3
+ .file(key)
+ .exists()
+ .catch(() => false);
+
+const cleanupStaleCache = async (s3: S3Client, key: string): Promise<void> => {
+ console.log(`Cleaning stale cache from manifest (cache missing): ${key}`);
+ await deleteMetadata(s3, key);
+ await removeCacheEntry(s3, key);
+};
+
+export async function resolveCacheKey(
+ s3: S3Client,
+ key: string,
+ restoreKeys?: string[]
+): Promise<string | null> {
+ const manifest = await loadManifest(s3);
+
+ // Try exact match first
+ const exactMatch = findCacheByKey(manifest, key);
+ if (exactMatch) {
+ if (await cacheExists(s3, exactMatch.key)) {
+ return exactMatch.key;
+ }
+ await cleanupStaleCache(s3, exactMatch.key);
+ }
+
+ // Try restore keys in order (prefix matching), preferring most recent
+ if (restoreKeys && restoreKeys.length > 0) {
+ for (const prefix of restoreKeys) {
+ const match = findCacheByPrefix(manifest, prefix);
+ if (match) {
+ if (await cacheExists(s3, match.key)) {
+ return match.key;
+ }
+ await cleanupStaleCache(s3, match.key);
+ }
+ }
+ }
+
+ return null;
+}
+
+export async function cleanupOldCaches(s3: S3Client, prefix: string): Promise<void> {
+ const manifest = await loadManifest(s3);
+
+ // Filter caches by prefix
+ const filesWithMetadata = manifest.caches
+ .filter((entry) => entry.key.startsWith(prefix))
+ .map((entry) => ({
+ key: entry.key,
+ lastAccessed: entry.lastAccessed,
+ timestamp: entry.timestamp
+ }));
+
+ // Sort by lastAccessed (most recently accessed first)
+ const files = filesWithMetadata.sort((a, b) => b.lastAccessed - a.lastAccessed);
+
+ const now = Date.now();
+ const maxAge = MAX_CACHE_AGE_DAYS * 24 * 60 * 60 * 1000;
+ let manifestUpdated = false;
+
+ for (let i = 0; i < files.length; i++) {
+ const entry = files[i];
+ const age = now - entry.lastAccessed;
+ const shouldDelete = i >= MAX_CACHE_FILES || age > maxAge;
+
+ if (shouldDelete) {
+ console.log(
+ `Deleting cache: ${entry.key} (age: ${Math.floor(age / (24 * 60 * 60 * 1000))} days, position: ${i + 1})`
+ );
+ await s3.file(entry.key).delete();
+ await deleteMetadata(s3, entry.key);
+ await removeCacheEntry(s3, entry.key);
+ manifestUpdated = true;
+ }
+ }
+
+ if (manifestUpdated) {
+ console.log('Manifest updated after cleanup');
+ }
+}
diff --git a/scripts/cache/utils.ts b/scripts/cache/utils.ts
new file mode 100644
index 0000000..c095000
--- /dev/null
+++ b/scripts/cache/utils.ts
@@ -0,0 +1,85 @@
+import type { S3Client } from 'bun';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface FileMetadata {
+ checksum: string;
+ size: number;
+}
+
+export interface CacheMetadata {
+ key: string;
+ hash: string;
+ timestamp: number;
+ lastAccessed: number;
+ files: Record<string, FileMetadata>;
+ version: number;
+}
+
+export interface CacheLock {
+ locked: boolean;
+ timestamp: number;
+ instance: string;
+ ttl: number;
+ renewedAt?: number;
+ pid: number;
+ hostname: string;
+}
+
+export interface S3ListObject {
+ key: string;
+ lastModified?: string;
+}
+
+export interface CacheEntry {
+ key: string;
+ timestamp: number;
+ lastAccessed: number;
+ hash: string;
+}
+
+export interface CacheManifest {
+ version: number;
+ caches: CacheEntry[];
+}
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+export const LOCK_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes (matches Restic)
+export const LOCK_RETRY_START_MS = 5000; // 5 seconds (initial retry delay)
+export const LOCK_RETRY_MAX_MS = 60000; // 60 seconds (max retry delay)
+export const LOCK_MAX_RETRIES = 6; // With exponential backoff: 5s, 10s, 20s, 40s, 60s, 60s
+export const LOCK_DOUBLE_CHECK_MS = 200; // 200ms delay for double-check pattern (matches Restic)
+
+export const METADATA_VERSION = 1;
+export const METADATA_KEY = 'metadata.json';
+export const LOCK_KEY = 'cache.lock';
+
+export const MAX_CACHE_FILES = 7;
+export const MAX_CACHE_AGE_DAYS = 7;
+
+export const TMP_DIR = 'tmp';
+export const STATIC_DIR = 'static';
+export const CACHE_FILE_NAME = 'extensions-cache.tzst';
+
+// Cache configuration
+export const CACHE_PATHS = ['static'];
+export const CACHE_KEY_PREFIX = 'extensions-';
+export const CACHE_RESTORE_KEYS = ['extensions-'];
+export const EXTENSIONS_CONFIG_FILE = 'extensions.json';
+
+// Helper to generate cache key from extensions.json
+export async function generateCacheKey(): Promise<string> {
+ const content = await Bun.file(EXTENSIONS_CONFIG_FILE).arrayBuffer();
+ const hash = new Bun.CryptoHasher('sha256').update(content).digest('hex');
+ return `${CACHE_KEY_PREFIX}${hash}.tzst`;
+}
+
+// Helper to write JSON to S3 file
+export async function writeJsonToS3(s3: S3Client, key: string, data: any): Promise<void> {
+ await Bun.write(s3.file(key), JSON.stringify(data, null, 2));
+}