Migrate Manyangles object storage to DigitalOcean Spaces

compose.coolify.yml

@@ -6,13 +6,13 @@ x-environment: &environment
   NEXT_PUBLIC_APP_URL: ${SERVICE_URL_WEB}
   BETTER_AUTH_URL: ${SERVICE_URL_WEB}
   BETTER_AUTH_SECRET: ${SERVICE_PASSWORD_64_AUTH}
-  S3_ENDPOINT: http://garage:3900
+  S3_ENDPOINT: https://nyc3.digitaloceanspaces.com
-  S3_PUBLIC_ENDPOINT: ${SERVICE_URL_GARAGE}
+  S3_PUBLIC_ENDPOINT: https://nyc3.digitaloceanspaces.com
-  S3_REGION: garage
+  S3_REGION: nyc3
-  S3_BUCKET: manyangles
+  S3_BUCKET: hltma
-  S3_ACCESS_KEY: GK${SERVICE_HEX_24_S3KEY}
+  S3_ACCESS_KEY: ${SPACES_ACCESS_KEY:?Set Spaces access key}
-  S3_SECRET_KEY: ${SERVICE_HEX_64_S3SECRET}
+  S3_SECRET_KEY: ${SPACES_SECRET_KEY:?Set Spaces secret key}
-  S3_FORCE_PATH_STYLE: "true"
+  S3_FORCE_PATH_STYLE: "false"
   EMAIL_PROVIDER: resend
   EMAIL_FROM: ${EMAIL_FROM}
   RESEND_API_KEY: ${RESEND_API_KEY}
@@ -67,7 +67,15 @@ services:
   storage-init:
     build: {context: ., target: worker}
     command: ["bun", "packages/storage/src/configure.ts"]
-    environment: *environment
+    # Keep the previous store available for rollback. Spaces CORS/lifecycle
+    # are managed separately; its scoped object key cannot change bucket rules.
+    environment:
+      NEXT_PUBLIC_APP_URL: ${SERVICE_URL_WEB}
+      S3_ENDPOINT: http://garage:3900
+      S3_REGION: garage
+      S3_BUCKET: manyangles
+      S3_ACCESS_KEY: GK${SERVICE_HEX_24_S3KEY}
+      S3_SECRET_KEY: ${SERVICE_HEX_64_S3SECRET}
     restart: "no"
     depends_on:
       garage: {condition: service_healthy}

docs/deployment.md

@@ -2,6 +2,30 @@
 
 ## Coolify on one VM
 
+### Current Manyangles storage
+
+`compose.coolify.yml` uses the private Spaces bucket `hltma` in `nyc3` for
+web/worker storage. Set `SPACES_ACCESS_KEY` and `SPACES_SECRET_KEY` in Coolify;
+never commit their values. The endpoint is `https://nyc3.digitaloceanspaces.com`
+and virtual-host addressing is enabled. Configure bucket CORS separately for
+`https://ma.hadlock.tech` (GET/PUT/HEAD, allowed headers `*`, exposed headers
+ETag/Content-Length). The bucket-scoped object key cannot administer these rules.
+
+Garage and its initializer/volumes remain in the stack for migration rollback.
+Do not remove them until the migration and a recovery path are verified. Do not
+blindly roll back after new uploads reach Spaces: synchronize those objects back
+first. Bucket lifecycle must also be configured separately (expire `exports/`
+after one day; never expire originals). The worker performs export cleanup too.
+
+For the initial migration, `scripts/copy-to-spaces.ts` runs in the old worker's
+storage package directory, receiving the two Spaces secrets as JSON on stdin.
+It preserves object paths and verifies SHA-256, refusing differing existing
+objects. It does not delete anything or perform the live cutover. Pause web
+writes, allow the ten-minute signed-upload lifetime to expire and any in-flight
+transfers to finish, drain workers, then repeat the copy before deploying.
+
+### Original Garage setup (retained for reference)
+
 Use the repository Docker Compose build pack and `/compose.coolify.yml`.
 This adds isolated Postgres and Garage volumes, migrations, and a one-shot
 storage initializer (exact-origin CORS, one-day export expiry and abandoned

scripts/copy-to-spaces.ts

@@ -0,0 +1,51 @@
+// Run inside the existing worker, piping the two Spaces credentials as JSON on stdin.
+// Copies without deleting or overwriting objects; verifies every destination byte.
+import { createHash } from "node:crypto";
+import { S3Client, ListObjectsV2Command, GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
+
+const credentials = JSON.parse(await Bun.stdin.text());
+const source = new S3Client({ endpoint: process.env.S3_ENDPOINT, region: process.env.S3_REGION, forcePathStyle: true,
+  credentials: { accessKeyId: process.env.S3_ACCESS_KEY!, secretAccessKey: process.env.S3_SECRET_KEY! } });
+const destination = new S3Client({ endpoint: "https://nyc3.digitaloceanspaces.com", region: "nyc3",
+  credentials: { accessKeyId: credentials.SPACES_ACCESS_KEY, secretAccessKey: credentials.SPACES_SECRET_KEY },
+  requestChecksumCalculation: "WHEN_REQUIRED" });
+const digest = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex");
+let token: string | undefined;
+let verified = 0;
+let copied = 0;
+let bytes = 0;
+try {
+  do {
+    const page = await source.send(new ListObjectsV2Command({ Bucket: process.env.S3_BUCKET!, ContinuationToken: token }));
+    for (const object of page.Contents ?? []) {
+      if (!object.Key) continue;
+      const original = await source.send(new GetObjectCommand({ Bucket: process.env.S3_BUCKET!, Key: object.Key }));
+      const body = await original.Body!.transformToByteArray();
+      let existing: Uint8Array | undefined;
+      try {
+        const response = await destination.send(new GetObjectCommand({ Bucket: "hltma", Key: object.Key }));
+        existing = await response.Body!.transformToByteArray();
+      } catch (error) {
+        if ((error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode !== 404) throw error;
+      }
+      if (!existing) {
+        await destination.send(new PutObjectCommand({ Bucket: "hltma", Key: object.Key, Body: body,
+          ContentType: original.ContentType, CacheControl: original.CacheControl, ContentDisposition: original.ContentDisposition,
+          Metadata: original.Metadata }));
+        copied++;
+        const response = await destination.send(new GetObjectCommand({ Bucket: "hltma", Key: object.Key }));
+        existing = await response.Body!.transformToByteArray();
+      }
+      if (digest(body) !== digest(existing)) throw new Error("Content mismatch; no objects overwritten or deleted");
+      verified++;
+      bytes += body.length;
+      if (verified % 20 === 0) console.log(JSON.stringify({ verified, copied, bytes }));
+    }
+    token = page.NextContinuationToken;
+  } while (token);
+  console.log(JSON.stringify({ complete: true, verified, copied, bytes }));
+} catch (error) {
+  // Do not print SDK request details, object keys, or credentials.
+  console.error(JSON.stringify({ complete: false, verified, copied, error: error instanceof Error ? error.name : "MigrationError" }));
+  process.exitCode = 1;
+}