52 lines
2.9 KiB
TypeScript
52 lines
2.9 KiB
TypeScript
// Run inside the existing worker, piping the two Spaces credentials as JSON on stdin.
|
|
// Copies without deleting or overwriting objects; verifies every destination byte.
|
|
import { createHash } from "node:crypto";
|
|
import { S3Client, ListObjectsV2Command, GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
|
|
const credentials = JSON.parse(await Bun.stdin.text());
|
|
const source = new S3Client({ endpoint: process.env.S3_ENDPOINT, region: process.env.S3_REGION, forcePathStyle: true,
|
|
credentials: { accessKeyId: process.env.S3_ACCESS_KEY!, secretAccessKey: process.env.S3_SECRET_KEY! } });
|
|
const destination = new S3Client({ endpoint: "https://nyc3.digitaloceanspaces.com", region: "nyc3",
|
|
credentials: { accessKeyId: credentials.SPACES_ACCESS_KEY, secretAccessKey: credentials.SPACES_SECRET_KEY },
|
|
requestChecksumCalculation: "WHEN_REQUIRED" });
|
|
const digest = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex");
|
|
let token: string | undefined;
|
|
let verified = 0;
|
|
let copied = 0;
|
|
let bytes = 0;
|
|
try {
|
|
do {
|
|
const page = await source.send(new ListObjectsV2Command({ Bucket: process.env.S3_BUCKET!, ContinuationToken: token }));
|
|
for (const object of page.Contents ?? []) {
|
|
if (!object.Key) continue;
|
|
const original = await source.send(new GetObjectCommand({ Bucket: process.env.S3_BUCKET!, Key: object.Key }));
|
|
const body = await original.Body!.transformToByteArray();
|
|
let existing: Uint8Array | undefined;
|
|
try {
|
|
const response = await destination.send(new GetObjectCommand({ Bucket: "hltma", Key: object.Key }));
|
|
existing = await response.Body!.transformToByteArray();
|
|
} catch (error) {
|
|
if ((error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode !== 404) throw error;
|
|
}
|
|
if (!existing) {
|
|
await destination.send(new PutObjectCommand({ Bucket: "hltma", Key: object.Key, Body: body,
|
|
ContentType: original.ContentType, CacheControl: original.CacheControl, ContentDisposition: original.ContentDisposition,
|
|
Metadata: original.Metadata }));
|
|
copied++;
|
|
const response = await destination.send(new GetObjectCommand({ Bucket: "hltma", Key: object.Key }));
|
|
existing = await response.Body!.transformToByteArray();
|
|
}
|
|
if (digest(body) !== digest(existing)) throw new Error("Content mismatch; no objects overwritten or deleted");
|
|
verified++;
|
|
bytes += body.length;
|
|
if (verified % 20 === 0) console.log(JSON.stringify({ verified, copied, bytes }));
|
|
}
|
|
token = page.NextContinuationToken;
|
|
} while (token);
|
|
console.log(JSON.stringify({ complete: true, verified, copied, bytes }));
|
|
} catch (error) {
|
|
// Do not print SDK request details, object keys, or credentials.
|
|
console.error(JSON.stringify({ complete: false, verified, copied, error: error instanceof Error ? error.name : "MigrationError" }));
|
|
process.exitCode = 1;
|
|
}
|