Chris Haulmark
c9e042fa98
Backend: - New models: Update, Article (Deaf Focus) - Expanded Event + Document models: directusId, photoUrl, new minute types - New routes/controllers: /api/updates, /api/articles (full CRUD) - Migration script: pulls all data from db.lad1908.org Directus API, downloads public PDFs into uploads/documents/, idempotent re-runs - .env.example for production deployment reference - Rename ocd_db → lad_db throughout Frontend: - Contact: LAD contact info, removed Olathe address and map - Membership: LAD copy, membership fees replaced with contact-us prompt - Donate: LAD name in metadata, hero, and FAQs Infrastructure: - mongo-init: simplified, uses MONGO_USER/MONGO_PASSWORD env vars, lad_db - docker-compose: rename volumes ocd-* → lad-*, image names updated - Upload directory structure committed via .gitkeep files
343 lines
14 KiB
TypeScript
343 lines
14 KiB
TypeScript
/**
|
|
* Directus → MongoDB migration script
|
|
*
|
|
* Run from the backend directory:
|
|
* npx tsx scripts/migrate-directus.ts
|
|
*
|
|
* What it does:
|
|
* - Fetches all records from db.lad1908.org (updates, events, minutes, deaffocus)
|
|
* - Downloads publicly-accessible PDFs into uploads/documents/
|
|
* - Downloads publicly-accessible images into uploads/images/
|
|
* - Inserts records into MongoDB (skips existing directusId to allow re-runs)
|
|
* - Writes scripts/migration-report.json with a full manifest
|
|
*/
|
|
|
|
import 'dotenv/config';
|
|
import mongoose from 'mongoose';
|
|
import fs from 'fs-extra';
|
|
import path from 'path';
|
|
import https from 'https';
|
|
import http from 'http';
|
|
import { fileURLToPath } from 'url';
|
|
|
|
import Update from '../src/models/Update.js';
|
|
import Article from '../src/models/Article.js';
|
|
import Event from '../src/models/Event.js';
|
|
import Document from '../src/models/Document.js';
|
|
|
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
const BACKEND_ROOT = path.resolve(__dirname, '..');
|
|
const DIRECTUS = 'https://db.lad1908.org';
|
|
|
|
// ── helpers ──────────────────────────────────────────────────────────────────
|
|
|
|
async function connectDB() {
|
|
const user = process.env.MONGO_USER;
|
|
const pass = process.env.MONGO_PASSWORD;
|
|
const host = process.env.MONGO_HOST || 'localhost';
|
|
const port = process.env.MONGO_PORT || '27017';
|
|
const db = process.env.MONGO_DB || 'lad_db';
|
|
const uri = user && pass
|
|
? `mongodb://${user}:${pass}@${host}:${port}/${db}?authSource=${db}`
|
|
: `mongodb://${host}:${port}/${db}`;
|
|
await mongoose.connect(uri);
|
|
console.log('✓ MongoDB connected');
|
|
}
|
|
|
|
async function fetchAll(collection: string): Promise<any[]> {
|
|
const results: any[] = [];
|
|
const limit = 100;
|
|
let offset = 0;
|
|
while (true) {
|
|
const url = `${DIRECTUS}/items/${collection}?limit=${limit}&offset=${offset}`;
|
|
const res = await fetch(url);
|
|
if (!res.ok) throw new Error(`Directus ${collection} fetch failed: ${res.status}`);
|
|
const json: any = await res.json();
|
|
const batch: any[] = json.data ?? [];
|
|
results.push(...batch);
|
|
if (batch.length < limit) break;
|
|
offset += limit;
|
|
}
|
|
return results;
|
|
}
|
|
|
|
function downloadFile(url: string, dest: string): Promise<{ ok: boolean; size: number }> {
|
|
return new Promise((resolve) => {
|
|
const proto = url.startsWith('https') ? https : http;
|
|
const req = proto.get(url, (res) => {
|
|
if (res.statusCode === 301 || res.statusCode === 302) {
|
|
return downloadFile(res.headers.location!, dest).then(resolve);
|
|
}
|
|
if (res.statusCode !== 200) {
|
|
res.resume();
|
|
return resolve({ ok: false, size: 0 });
|
|
}
|
|
const stream = fs.createWriteStream(dest);
|
|
let size = 0;
|
|
res.on('data', (chunk: Buffer) => { size += chunk.length; });
|
|
res.pipe(stream);
|
|
stream.on('finish', () => resolve({ ok: true, size }));
|
|
stream.on('error', () => resolve({ ok: false, size: 0 }));
|
|
});
|
|
req.on('error', () => resolve({ ok: false, size: 0 }));
|
|
});
|
|
}
|
|
|
|
function safeFilename(prefix: string, id: number, date: string, ext: string): string {
|
|
return `${prefix}-${date.replace(/[^0-9-]/g, '')}-${id}${ext}`;
|
|
}
|
|
|
|
function mapMinutesType(type: string | null): string {
|
|
switch (type) {
|
|
case 'Board Meeting': return 'board_meeting_minutes';
|
|
case 'Committee Meeting': return 'committee_meeting_minutes';
|
|
case 'Quarterly Meeting': return 'quarterly_meeting_minutes';
|
|
case 'Conference Minutes': return 'conference_minutes';
|
|
case 'Financial Report': return 'financial_report';
|
|
case 'Special Meeting': return 'special_meeting_minutes';
|
|
default: return 'meeting_minutes';
|
|
}
|
|
}
|
|
|
|
// ── main ─────────────────────────────────────────────────────────────────────
|
|
|
|
interface Report {
|
|
startedAt: string;
|
|
finishedAt?: string;
|
|
inserted: Record<string, number>;
|
|
skipped: Record<string, number>;
|
|
downloadedFiles: number;
|
|
failedDownloads: { collection: string; directusId: number; assetId: string; reason: string }[];
|
|
membersOnlyPDFs: { directusId: number; date: string; type: string; assetId: string }[];
|
|
errors: { collection: string; directusId: number; message: string }[];
|
|
}
|
|
|
|
async function main() {
|
|
await connectDB();
|
|
|
|
await fs.ensureDir(path.join(BACKEND_ROOT, 'uploads/documents'));
|
|
await fs.ensureDir(path.join(BACKEND_ROOT, 'uploads/images'));
|
|
|
|
const report: Report = {
|
|
startedAt: new Date().toISOString(),
|
|
inserted: { updates: 0, events: 0, minutes: 0, articles: 0 },
|
|
skipped: { updates: 0, events: 0, minutes: 0, articles: 0 },
|
|
downloadedFiles: 0,
|
|
failedDownloads: [],
|
|
membersOnlyPDFs: [],
|
|
errors: []
|
|
};
|
|
|
|
// ── UPDATES ────────────────────────────────────────────────────────────────
|
|
console.log('\n── Fetching updates…');
|
|
const directusUpdates = await fetchAll('updates');
|
|
console.log(` Found ${directusUpdates.length} records`);
|
|
|
|
for (const r of directusUpdates) {
|
|
if (!r.id) continue;
|
|
const exists = await Update.findOne({ directusId: r.id });
|
|
if (exists) { report.skipped.updates++; continue; }
|
|
|
|
let photoUrl: string | undefined;
|
|
if (r.main_photo) {
|
|
const assetUrl = `${DIRECTUS}/assets/${r.main_photo}`;
|
|
const fname = safeFilename('update', r.id, r.date ?? 'undated', '.jpg');
|
|
const dest = path.join(BACKEND_ROOT, 'uploads/images', fname);
|
|
const dl = await downloadFile(assetUrl, dest);
|
|
if (dl.ok) {
|
|
photoUrl = `uploads/images/${fname}`;
|
|
report.downloadedFiles++;
|
|
} else {
|
|
photoUrl = assetUrl; // fall back to Directus URL; replace when images are provided
|
|
report.failedDownloads.push({ collection: 'updates', directusId: r.id, assetId: r.main_photo, reason: '403 or unreachable' });
|
|
}
|
|
}
|
|
|
|
try {
|
|
await new Update({
|
|
directusId: r.id,
|
|
title: r.title ?? '(untitled)',
|
|
date: r.date ? new Date(r.date) : new Date(r.created_on),
|
|
body: r.info ?? '',
|
|
status: r.status === 'published' ? 'published' : 'draft',
|
|
photoUrl,
|
|
photoCaption: r.main_photo_caption ?? undefined,
|
|
videoUrl: r.video ?? undefined,
|
|
gallery: Array.isArray(r.gallery) ? r.gallery : []
|
|
}).save();
|
|
report.inserted.updates++;
|
|
} catch (e: any) {
|
|
report.errors.push({ collection: 'updates', directusId: r.id, message: e.message });
|
|
}
|
|
}
|
|
console.log(` Inserted ${report.inserted.updates}, skipped ${report.skipped.updates}`);
|
|
|
|
// ── EVENTS ─────────────────────────────────────────────────────────────────
|
|
console.log('\n── Fetching events…');
|
|
const directusEvents = await fetchAll('events');
|
|
console.log(` Found ${directusEvents.length} records`);
|
|
|
|
for (const r of directusEvents) {
|
|
if (!r.id) continue;
|
|
const exists = await Event.findOne({ directusId: r.id });
|
|
if (exists) { report.skipped.events++; continue; }
|
|
|
|
let photoUrl: string | undefined;
|
|
if (r.main_photo) {
|
|
const assetUrl = `${DIRECTUS}/assets/${r.main_photo}`;
|
|
const fname = safeFilename('event', r.id, r.date ?? 'undated', '.jpg');
|
|
const dest = path.join(BACKEND_ROOT, 'uploads/images', fname);
|
|
const dl = await downloadFile(assetUrl, dest);
|
|
if (dl.ok) {
|
|
photoUrl = `uploads/images/${fname}`;
|
|
report.downloadedFiles++;
|
|
} else {
|
|
photoUrl = assetUrl;
|
|
report.failedDownloads.push({ collection: 'events', directusId: r.id, assetId: r.main_photo, reason: '403 or unreachable' });
|
|
}
|
|
}
|
|
|
|
try {
|
|
await new Event({
|
|
directusId: r.id,
|
|
title: r.title ?? '(untitled)',
|
|
description: r.info?.trim() || 'No description provided',
|
|
date: r.date ? new Date(r.date) : new Date(r.created_on),
|
|
time: 'TBD',
|
|
location: 'TBD',
|
|
category: 'general',
|
|
status: r.status === 'published' ? 'published' : 'draft',
|
|
registrationRequired: false,
|
|
registeredCount: 0,
|
|
photoUrl,
|
|
photoCaption: r.main_photo_caption ?? undefined
|
|
}).save();
|
|
report.inserted.events++;
|
|
} catch (e: any) {
|
|
report.errors.push({ collection: 'events', directusId: r.id, message: e.message });
|
|
}
|
|
}
|
|
console.log(` Inserted ${report.inserted.events}, skipped ${report.skipped.events}`);
|
|
|
|
// ── MINUTES ────────────────────────────────────────────────────────────────
|
|
console.log('\n── Fetching minutes…');
|
|
const directusMinutes = await fetchAll('minutes');
|
|
console.log(` Found ${directusMinutes.length} records`);
|
|
|
|
for (const r of directusMinutes) {
|
|
if (!r.id) continue;
|
|
const exists = await Document.findOne({ directusId: r.id });
|
|
if (exists) { report.skipped.minutes++; continue; }
|
|
|
|
if (r.members_only) {
|
|
report.membersOnlyPDFs.push({
|
|
directusId: r.id,
|
|
date: r.date ?? '',
|
|
type: r.type ?? '',
|
|
assetId: r.attachment ?? ''
|
|
});
|
|
report.skipped.minutes++;
|
|
continue;
|
|
}
|
|
|
|
if (!r.attachment) {
|
|
report.errors.push({ collection: 'minutes', directusId: r.id, message: 'No attachment UUID' });
|
|
continue;
|
|
}
|
|
|
|
const assetUrl = `${DIRECTUS}/assets/${r.attachment}`;
|
|
const dateStr = (r.date ?? 'undated').substring(0, 10);
|
|
const fname = safeFilename('minutes', r.id, dateStr, '.pdf');
|
|
const dest = path.join(BACKEND_ROOT, 'uploads/documents', fname);
|
|
const dl = await downloadFile(assetUrl, dest);
|
|
|
|
if (!dl.ok) {
|
|
report.failedDownloads.push({ collection: 'minutes', directusId: r.id, assetId: r.attachment, reason: '403 or unreachable' });
|
|
continue;
|
|
}
|
|
report.downloadedFiles++;
|
|
|
|
const title = `${r.type ?? 'Meeting'} — ${dateStr}`;
|
|
|
|
try {
|
|
await new Document({
|
|
directusId: r.id,
|
|
directusAssetId: r.attachment,
|
|
title,
|
|
description: title,
|
|
filePath: `uploads/documents/${fname}`,
|
|
originalFilename: fname,
|
|
fileType: 'pdf',
|
|
fileSize: dl.size,
|
|
documentType: mapMinutesType(r.type),
|
|
isPublic: true,
|
|
meetingDate: r.date ? new Date(r.date) : undefined,
|
|
uploadDate: new Date(r.created_on),
|
|
lastModified: new Date(r.created_on),
|
|
accessibilityChecked: false,
|
|
hasTextVersion: false
|
|
}).save();
|
|
report.inserted.minutes++;
|
|
} catch (e: any) {
|
|
report.errors.push({ collection: 'minutes', directusId: r.id, message: e.message });
|
|
}
|
|
}
|
|
console.log(` Inserted ${report.inserted.minutes}, skipped ${report.skipped.minutes} (${report.membersOnlyPDFs.length} members-only)`);
|
|
|
|
// ── DEAF FOCUS ARTICLES ────────────────────────────────────────────────────
|
|
console.log('\n── Fetching Deaf Focus articles…');
|
|
const directusArticles = await fetchAll('deaffocus');
|
|
console.log(` Found ${directusArticles.length} records`);
|
|
|
|
for (const r of directusArticles) {
|
|
if (!r.id) continue;
|
|
const exists = await Article.findOne({ directusId: r.id });
|
|
if (exists) { report.skipped.articles++; continue; }
|
|
|
|
try {
|
|
await new Article({
|
|
directusId: r.id,
|
|
title: r.title ?? '(untitled)',
|
|
body: r.body ?? '',
|
|
order: r.order ?? 0,
|
|
status: 'published'
|
|
}).save();
|
|
report.inserted.articles++;
|
|
} catch (e: any) {
|
|
report.errors.push({ collection: 'articles', directusId: r.id, message: e.message });
|
|
}
|
|
}
|
|
console.log(` Inserted ${report.inserted.articles}, skipped ${report.skipped.articles}`);
|
|
|
|
// ── REPORT ─────────────────────────────────────────────────────────────────
|
|
report.finishedAt = new Date().toISOString();
|
|
const reportPath = path.join(__dirname, 'migration-report.json');
|
|
await fs.writeJSON(reportPath, report, { spaces: 2 });
|
|
|
|
console.log('\n══════════════════════════════════════════');
|
|
console.log('Migration complete');
|
|
console.log(` Updates inserted: ${report.inserted.updates}`);
|
|
console.log(` Events inserted: ${report.inserted.events}`);
|
|
console.log(` Minutes inserted: ${report.inserted.minutes}`);
|
|
console.log(` Articles inserted: ${report.inserted.articles}`);
|
|
console.log(` Files downloaded: ${report.downloadedFiles}`);
|
|
console.log(` Members-only PDFs: ${report.membersOnlyPDFs.length} (manual upload needed)`);
|
|
console.log(` Failed downloads: ${report.failedDownloads.length}`);
|
|
console.log(` Errors: ${report.errors.length}`);
|
|
console.log(` Full report: scripts/migration-report.json`);
|
|
console.log('══════════════════════════════════════════\n');
|
|
|
|
if (report.membersOnlyPDFs.length > 0) {
|
|
console.log('Members-only PDFs that need manual upload via the admin panel:');
|
|
for (const m of report.membersOnlyPDFs) {
|
|
console.log(` [${m.directusId}] ${m.date} — ${m.type} (asset: ${m.assetId})`);
|
|
}
|
|
}
|
|
|
|
await mongoose.disconnect();
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error('Migration failed:', err);
|
|
process.exit(1);
|
|
});
|