/** * Directus → MongoDB migration script * * Run from the backend directory: * npx tsx scripts/migrate-directus.ts * * What it does: * - Fetches all records from db.lad1908.org (updates, events, minutes, deaffocus) * - Downloads publicly-accessible PDFs into uploads/documents/ * - Downloads publicly-accessible images into uploads/images/ * - Inserts records into MongoDB (skips existing directusId to allow re-runs) * - Writes scripts/migration-report.json with a full manifest */ import 'dotenv/config'; import mongoose from 'mongoose'; import fs from 'fs-extra'; import path from 'path'; import https from 'https'; import http from 'http'; import { fileURLToPath } from 'url'; import Update from '../src/models/Update.js'; import Article from '../src/models/Article.js'; import Event from '../src/models/Event.js'; import Document from '../src/models/Document.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const BACKEND_ROOT = path.resolve(__dirname, '..'); const DIRECTUS = 'https://db.lad1908.org'; // ── helpers ────────────────────────────────────────────────────────────────── async function connectDB() { const user = process.env.MONGO_USER; const pass = process.env.MONGO_PASSWORD; const host = process.env.MONGO_HOST || 'localhost'; const port = process.env.MONGO_PORT || '27017'; const db = process.env.MONGO_DB || 'lad_db'; const uri = user && pass ? `mongodb://${user}:${pass}@${host}:${port}/${db}?authSource=${db}` : `mongodb://${host}:${port}/${db}`; await mongoose.connect(uri); console.log('✓ MongoDB connected'); } async function fetchAll(collection: string): Promise { const results: any[] = []; const limit = 100; let offset = 0; while (true) { const url = `${DIRECTUS}/items/${collection}?limit=${limit}&offset=${offset}`; const res = await fetch(url); if (!res.ok) throw new Error(`Directus ${collection} fetch failed: ${res.status}`); const json: any = await res.json(); const batch: any[] = json.data ?? []; results.push(...batch); if (batch.length < limit) break; offset += limit; } return results; } function downloadFile(url: string, dest: string): Promise<{ ok: boolean; size: number }> { return new Promise((resolve) => { const proto = url.startsWith('https') ? https : http; const req = proto.get(url, (res) => { if (res.statusCode === 301 || res.statusCode === 302) { return downloadFile(res.headers.location!, dest).then(resolve); } if (res.statusCode !== 200) { res.resume(); return resolve({ ok: false, size: 0 }); } const stream = fs.createWriteStream(dest); let size = 0; res.on('data', (chunk: Buffer) => { size += chunk.length; }); res.pipe(stream); stream.on('finish', () => resolve({ ok: true, size })); stream.on('error', () => resolve({ ok: false, size: 0 })); }); req.on('error', () => resolve({ ok: false, size: 0 })); }); } function safeFilename(prefix: string, id: number, date: string, ext: string): string { return `${prefix}-${date.replace(/[^0-9-]/g, '')}-${id}${ext}`; } function mapMinutesType(type: string | null): string { switch (type) { case 'Board Meeting': return 'board_meeting_minutes'; case 'Committee Meeting': return 'committee_meeting_minutes'; case 'Quarterly Meeting': return 'quarterly_meeting_minutes'; case 'Conference Minutes': return 'conference_minutes'; case 'Financial Report': return 'financial_report'; case 'Special Meeting': return 'special_meeting_minutes'; default: return 'meeting_minutes'; } } // ── main ───────────────────────────────────────────────────────────────────── interface Report { startedAt: string; finishedAt?: string; inserted: Record; skipped: Record; downloadedFiles: number; failedDownloads: { collection: string; directusId: number; assetId: string; reason: string }[]; membersOnlyPDFs: { directusId: number; date: string; type: string; assetId: string }[]; errors: { collection: string; directusId: number; message: string }[]; } async function main() { await connectDB(); await fs.ensureDir(path.join(BACKEND_ROOT, 'uploads/documents')); await fs.ensureDir(path.join(BACKEND_ROOT, 'uploads/images')); const report: Report = { startedAt: new Date().toISOString(), inserted: { updates: 0, events: 0, minutes: 0, articles: 0 }, skipped: { updates: 0, events: 0, minutes: 0, articles: 0 }, downloadedFiles: 0, failedDownloads: [], membersOnlyPDFs: [], errors: [] }; // ── UPDATES ──────────────────────────────────────────────────────────────── console.log('\n── Fetching updates…'); const directusUpdates = await fetchAll('updates'); console.log(` Found ${directusUpdates.length} records`); for (const r of directusUpdates) { if (!r.id) continue; const exists = await Update.findOne({ directusId: r.id }); if (exists) { report.skipped.updates++; continue; } let photoUrl: string | undefined; if (r.main_photo) { const assetUrl = `${DIRECTUS}/assets/${r.main_photo}`; const fname = safeFilename('update', r.id, r.date ?? 'undated', '.jpg'); const dest = path.join(BACKEND_ROOT, 'uploads/images', fname); const dl = await downloadFile(assetUrl, dest); if (dl.ok) { photoUrl = `uploads/images/${fname}`; report.downloadedFiles++; } else { photoUrl = assetUrl; // fall back to Directus URL; replace when images are provided report.failedDownloads.push({ collection: 'updates', directusId: r.id, assetId: r.main_photo, reason: '403 or unreachable' }); } } try { await new Update({ directusId: r.id, title: r.title ?? '(untitled)', date: r.date ? new Date(r.date) : new Date(r.created_on), body: r.info ?? '', status: r.status === 'published' ? 'published' : 'draft', photoUrl, photoCaption: r.main_photo_caption ?? undefined, videoUrl: r.video ?? undefined, gallery: Array.isArray(r.gallery) ? r.gallery : [] }).save(); report.inserted.updates++; } catch (e: any) { report.errors.push({ collection: 'updates', directusId: r.id, message: e.message }); } } console.log(` Inserted ${report.inserted.updates}, skipped ${report.skipped.updates}`); // ── EVENTS ───────────────────────────────────────────────────────────────── console.log('\n── Fetching events…'); const directusEvents = await fetchAll('events'); console.log(` Found ${directusEvents.length} records`); for (const r of directusEvents) { if (!r.id) continue; const exists = await Event.findOne({ directusId: r.id }); if (exists) { report.skipped.events++; continue; } let photoUrl: string | undefined; if (r.main_photo) { const assetUrl = `${DIRECTUS}/assets/${r.main_photo}`; const fname = safeFilename('event', r.id, r.date ?? 'undated', '.jpg'); const dest = path.join(BACKEND_ROOT, 'uploads/images', fname); const dl = await downloadFile(assetUrl, dest); if (dl.ok) { photoUrl = `uploads/images/${fname}`; report.downloadedFiles++; } else { photoUrl = assetUrl; report.failedDownloads.push({ collection: 'events', directusId: r.id, assetId: r.main_photo, reason: '403 or unreachable' }); } } try { await new Event({ directusId: r.id, title: r.title ?? '(untitled)', description: r.info?.trim() || 'No description provided', date: r.date ? new Date(r.date) : new Date(r.created_on), time: 'TBD', location: 'TBD', category: 'general', status: r.status === 'published' ? 'published' : 'draft', registrationRequired: false, registeredCount: 0, photoUrl, photoCaption: r.main_photo_caption ?? undefined }).save(); report.inserted.events++; } catch (e: any) { report.errors.push({ collection: 'events', directusId: r.id, message: e.message }); } } console.log(` Inserted ${report.inserted.events}, skipped ${report.skipped.events}`); // ── MINUTES ──────────────────────────────────────────────────────────────── console.log('\n── Fetching minutes…'); const directusMinutes = await fetchAll('minutes'); console.log(` Found ${directusMinutes.length} records`); for (const r of directusMinutes) { if (!r.id) continue; const exists = await Document.findOne({ directusId: r.id }); if (exists) { report.skipped.minutes++; continue; } if (r.members_only) { report.membersOnlyPDFs.push({ directusId: r.id, date: r.date ?? '', type: r.type ?? '', assetId: r.attachment ?? '' }); report.skipped.minutes++; continue; } if (!r.attachment) { report.errors.push({ collection: 'minutes', directusId: r.id, message: 'No attachment UUID' }); continue; } const assetUrl = `${DIRECTUS}/assets/${r.attachment}`; const dateStr = (r.date ?? 'undated').substring(0, 10); const fname = safeFilename('minutes', r.id, dateStr, '.pdf'); const dest = path.join(BACKEND_ROOT, 'uploads/documents', fname); const dl = await downloadFile(assetUrl, dest); if (!dl.ok) { report.failedDownloads.push({ collection: 'minutes', directusId: r.id, assetId: r.attachment, reason: '403 or unreachable' }); continue; } report.downloadedFiles++; const title = `${r.type ?? 'Meeting'} — ${dateStr}`; try { await new Document({ directusId: r.id, directusAssetId: r.attachment, title, description: title, filePath: `uploads/documents/${fname}`, originalFilename: fname, fileType: 'pdf', fileSize: dl.size, documentType: mapMinutesType(r.type), isPublic: true, meetingDate: r.date ? new Date(r.date) : undefined, uploadDate: new Date(r.created_on), lastModified: new Date(r.created_on), accessibilityChecked: false, hasTextVersion: false }).save(); report.inserted.minutes++; } catch (e: any) { report.errors.push({ collection: 'minutes', directusId: r.id, message: e.message }); } } console.log(` Inserted ${report.inserted.minutes}, skipped ${report.skipped.minutes} (${report.membersOnlyPDFs.length} members-only)`); // ── DEAF FOCUS ARTICLES ──────────────────────────────────────────────────── console.log('\n── Fetching Deaf Focus articles…'); const directusArticles = await fetchAll('deaffocus'); console.log(` Found ${directusArticles.length} records`); for (const r of directusArticles) { if (!r.id) continue; const exists = await Article.findOne({ directusId: r.id }); if (exists) { report.skipped.articles++; continue; } try { await new Article({ directusId: r.id, title: r.title ?? '(untitled)', body: r.body ?? '', order: r.order ?? 0, status: 'published' }).save(); report.inserted.articles++; } catch (e: any) { report.errors.push({ collection: 'articles', directusId: r.id, message: e.message }); } } console.log(` Inserted ${report.inserted.articles}, skipped ${report.skipped.articles}`); // ── REPORT ───────────────────────────────────────────────────────────────── report.finishedAt = new Date().toISOString(); const reportPath = path.join(__dirname, 'migration-report.json'); await fs.writeJSON(reportPath, report, { spaces: 2 }); console.log('\n══════════════════════════════════════════'); console.log('Migration complete'); console.log(` Updates inserted: ${report.inserted.updates}`); console.log(` Events inserted: ${report.inserted.events}`); console.log(` Minutes inserted: ${report.inserted.minutes}`); console.log(` Articles inserted: ${report.inserted.articles}`); console.log(` Files downloaded: ${report.downloadedFiles}`); console.log(` Members-only PDFs: ${report.membersOnlyPDFs.length} (manual upload needed)`); console.log(` Failed downloads: ${report.failedDownloads.length}`); console.log(` Errors: ${report.errors.length}`); console.log(` Full report: scripts/migration-report.json`); console.log('══════════════════════════════════════════\n'); if (report.membersOnlyPDFs.length > 0) { console.log('Members-only PDFs that need manual upload via the admin panel:'); for (const m of report.membersOnlyPDFs) { console.log(` [${m.directusId}] ${m.date} — ${m.type} (asset: ${m.assetId})`); } } await mongoose.disconnect(); } main().catch((err) => { console.error('Migration failed:', err); process.exit(1); });