feat(aep): carte AEP — push Gitea 2026-04-28
This commit is contained in:
Jules Neny
234
worker/daily-digest.js
Normal file
234
worker/daily-digest.js
Normal file
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env node
|
||||
// worker/daily-digest.js — Email récap quotidien nouvelles fiches AEP
|
||||
// Cron : 0 8 * * * (tous les jours à 8h)
|
||||
// Usage : node worker/daily-digest.js
|
||||
|
||||
import 'dotenv/config'
|
||||
|
||||
// ─── CONFIG DEPUIS .env ───────────────────────────────────────────────────────
|
||||
const NOCODB_URL = process.env.NOCODB_URL || 'http://localhost:8070';
|
||||
const NOCODB_TOKEN = process.env.NOCODB_TOKEN;
|
||||
const NOCODB_BASE = process.env.NOCODB_BASE;
|
||||
const NOCODB_TABLE_ORGAS = process.env.NOCODB_TABLE_ORGAS;
|
||||
const RESEND_API_KEY = process.env.RESEND_API_KEY;
|
||||
const RESEND_FROM = process.env.RESEND_FROM || 'AEP Digest <noreply@trans-former.fr>';
|
||||
const EMAIL_DEST = process.env.EMAIL_JULES || 'transformationsresilientes@gmail.com';
|
||||
const NOCODB_ADMIN_URL = process.env.NOCODB_ADMIN_URL || 'http://localhost:8070';
|
||||
|
||||
// ─── UTILITAIRES LOG ─────────────────────────────────────────────────────────
|
||||
function log(...args) {
|
||||
const ts = new Date().toISOString();
|
||||
console.log(`[${ts}]`, ...args);
|
||||
}
|
||||
|
||||
// ─── NOCODB — FETCH FICHES DERNIÈRES 24H ─────────────────────────────────────
|
||||
async function fetchNewFiches() {
|
||||
const since = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
|
||||
|
||||
// NocoDB ne supporte pas bien le filtre datetime sur CreatedAt — on récupère
|
||||
// les 200 dernières fiches triées par date décroissante et on filtre en JS
|
||||
// (même pattern que getBudgetMoisCourant dans enrich.js)
|
||||
const url = `${NOCODB_URL}/api/v1/db/data/noco/${NOCODB_BASE}/${NOCODB_TABLE_ORGAS}?limit=200&sort=-CreatedAt`;
|
||||
|
||||
const res = await fetch(url, {
|
||||
headers: { 'xc-token': NOCODB_TOKEN }
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`NocoDB GET fiches → ${res.status}: ${await res.text()}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const rows = data.list || [];
|
||||
|
||||
// Filtre JS sur les 24 dernières heures
|
||||
const recent = rows.filter(row => {
|
||||
const created = new Date(row.CreatedAt || row.created_at || 0);
|
||||
return created >= new Date(since);
|
||||
});
|
||||
|
||||
return recent;
|
||||
}
|
||||
|
||||
// ─── CONSTRUCTION EMAIL HTML ─────────────────────────────────────────────────
|
||||
function buildEmailHtml(fiches) {
|
||||
const count = fiches.length;
|
||||
const date = new Date().toLocaleDateString('fr-FR', {
|
||||
weekday: 'long', day: 'numeric', month: 'long', year: 'numeric'
|
||||
});
|
||||
|
||||
const lignes = fiches.map(f => {
|
||||
const nom = f.nom || '(sans nom)';
|
||||
const url = f.url || null;
|
||||
const echelle = f.echelle || '—';
|
||||
const statut = f.moderation_status || 'pending';
|
||||
const ficheUrl = `https://aep.trans-former.fr/fiche/${f.Id}`;
|
||||
|
||||
const nomHtml = url
|
||||
? `<a href="${url}" style="color:#1a56db;text-decoration:none;">${escHtml(nom)}</a>`
|
||||
: escHtml(nom);
|
||||
|
||||
const statutColor = statut === 'approved' ? '#057a55'
|
||||
: statut === 'rejected' ? '#c81e1e'
|
||||
: statut === 'ai_processed' ? '#1a56db'
|
||||
: '#9ca3af'; // pending / autre
|
||||
|
||||
return `
|
||||
<tr style="border-bottom:1px solid #f3f4f6;">
|
||||
<td style="padding:10px 12px;font-size:14px;">${nomHtml}</td>
|
||||
<td style="padding:10px 12px;font-size:13px;color:#6b7280;">${escHtml(echelle)}</td>
|
||||
<td style="padding:10px 12px;">
|
||||
<span style="display:inline-block;padding:2px 8px;border-radius:4px;font-size:12px;
|
||||
font-weight:600;background:#f3f4f6;color:${statutColor};">
|
||||
${escHtml(statut)}
|
||||
</span>
|
||||
</td>
|
||||
<td style="padding:10px 12px;font-size:12px;">
|
||||
<a href="${ficheUrl}" style="color:#6b7280;">fiche #${f.Id}</a>
|
||||
</td>
|
||||
</tr>`;
|
||||
}).join('');
|
||||
|
||||
return `<!DOCTYPE html>
|
||||
<html lang="fr">
|
||||
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"></head>
|
||||
<body style="margin:0;padding:0;background:#f9fafb;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;">
|
||||
<table width="100%" cellpadding="0" cellspacing="0" style="background:#f9fafb;padding:32px 0;">
|
||||
<tr><td align="center">
|
||||
<table width="600" cellpadding="0" cellspacing="0" style="background:#ffffff;border-radius:8px;overflow:hidden;box-shadow:0 1px 3px rgba(0,0,0,0.1);">
|
||||
|
||||
<!-- Header -->
|
||||
<tr>
|
||||
<td style="background:#1a56db;padding:24px 32px;">
|
||||
<p style="margin:0;font-size:12px;color:#bfdbfe;text-transform:uppercase;letter-spacing:1px;">AEP — Agences En Pratique</p>
|
||||
<h1 style="margin:4px 0 0;font-size:20px;color:#ffffff;font-weight:700;">
|
||||
${count} nouvelle${count > 1 ? 's' : ''} fiche${count > 1 ? 's' : ''} soumise${count > 1 ? 's' : ''}
|
||||
</h1>
|
||||
<p style="margin:4px 0 0;font-size:13px;color:#bfdbfe;">${date}</p>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- Corps -->
|
||||
<tr>
|
||||
<td style="padding:24px 32px;">
|
||||
<p style="margin:0 0 16px;font-size:14px;color:#374151;">
|
||||
${count} fiche${count > 1 ? 's ont été soumises' : ' a été soumise'} dans les dernières 24 heures.
|
||||
</p>
|
||||
|
||||
<!-- Tableau fiches -->
|
||||
<table width="100%" cellpadding="0" cellspacing="0" style="border-collapse:collapse;border:1px solid #e5e7eb;border-radius:6px;overflow:hidden;">
|
||||
<thead>
|
||||
<tr style="background:#f9fafb;">
|
||||
<th style="padding:10px 12px;text-align:left;font-size:12px;font-weight:600;color:#6b7280;text-transform:uppercase;letter-spacing:0.5px;">Organisation</th>
|
||||
<th style="padding:10px 12px;text-align:left;font-size:12px;font-weight:600;color:#6b7280;text-transform:uppercase;letter-spacing:0.5px;">Échelle</th>
|
||||
<th style="padding:10px 12px;text-align:left;font-size:12px;font-weight:600;color:#6b7280;text-transform:uppercase;letter-spacing:0.5px;">Statut</th>
|
||||
<th style="padding:10px 12px;text-align:left;font-size:12px;font-weight:600;color:#6b7280;text-transform:uppercase;letter-spacing:0.5px;">Lien</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
${lignes}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<!-- CTA modération -->
|
||||
<div style="margin-top:24px;text-align:center;">
|
||||
<a href="${NOCODB_ADMIN_URL}"
|
||||
style="display:inline-block;padding:10px 20px;background:#1a56db;color:#ffffff;
|
||||
border-radius:6px;font-size:14px;font-weight:600;text-decoration:none;">
|
||||
Ouvrir NocoDB pour modérer
|
||||
</a>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- Footer -->
|
||||
<tr>
|
||||
<td style="padding:16px 32px;border-top:1px solid #f3f4f6;">
|
||||
<p style="margin:0;font-size:11px;color:#9ca3af;text-align:center;">
|
||||
Digest automatique AEP — envoyé chaque matin à 8h
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
</table>
|
||||
</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
// Échappement HTML minimal
|
||||
function escHtml(str) {
|
||||
return String(str ?? '')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"');
|
||||
}
|
||||
|
||||
// ─── ENVOI VIA RESEND ─────────────────────────────────────────────────────────
|
||||
async function sendDigest(fiches) {
|
||||
const count = fiches.length;
|
||||
const subject = `AEP — ${count} nouvelle${count > 1 ? 's' : ''} fiche${count > 1 ? 's' : ''} soumise${count > 1 ? 's' : ''}`;
|
||||
const html = buildEmailHtml(fiches);
|
||||
|
||||
const res = await fetch('https://api.resend.com/emails', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${RESEND_API_KEY}`,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
from: RESEND_FROM,
|
||||
to: [EMAIL_DEST],
|
||||
subject,
|
||||
html
|
||||
})
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.text();
|
||||
throw new Error(`Resend API ${res.status}: ${err}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
log(`Email digest envoyé → ${EMAIL_DEST} | id: ${data.id} | sujet: "${subject}"`);
|
||||
}
|
||||
|
||||
// ─── MAIN ─────────────────────────────────────────────────────────────────────
|
||||
async function run() {
|
||||
log('=== daily-digest démarré ===');
|
||||
|
||||
// Vérification variables obligatoires
|
||||
if (!NOCODB_TOKEN || !NOCODB_BASE || !NOCODB_TABLE_ORGAS) {
|
||||
log('ERREUR : Variables NocoDB manquantes (NOCODB_TOKEN, NOCODB_BASE, NOCODB_TABLE_ORGAS)');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!RESEND_API_KEY) {
|
||||
log('ERREUR : RESEND_API_KEY manquante');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
// Fetch fiches des 24 dernières heures
|
||||
const fiches = await fetchNewFiches();
|
||||
log(`${fiches.length} fiche(s) trouvée(s) dans les dernières 24h`);
|
||||
|
||||
if (fiches.length === 0) {
|
||||
log('Aucune nouvelle fiche — digest non envoyé.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Envoi email
|
||||
await sendDigest(fiches);
|
||||
log('=== daily-digest terminé avec succès ===');
|
||||
|
||||
} catch (e) {
|
||||
log('ERREUR daily-digest:', e.message);
|
||||
console.error(e.stack);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
run();
|
||||
495
worker/enrich.js
Normal file
495
worker/enrich.js
Normal file
@@ -0,0 +1,495 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* NAV V2 — Worker enrichissement IA
|
||||
* Lancé via systemd timer toutes les 5 minutes
|
||||
* Pipeline : fetch pending → scrape crawl4ai → Mistral Nemo → update NocoDB → log stats
|
||||
*/
|
||||
|
||||
import { spawnSync, execSync } from 'child_process';
|
||||
import { existsSync, writeFileSync, unlinkSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
// ─── CONFIG DEPUIS .env ───────────────────────────────────────────────────────
|
||||
const NOCODB_URL = process.env.NOCODB_URL || 'http://localhost:8070';
|
||||
const NOCODB_TOKEN = process.env.NOCODB_TOKEN;
|
||||
const NOCODB_BASE = process.env.NOCODB_BASE;
|
||||
const NOCODB_TABLE_ORGAS = process.env.NOCODB_TABLE_ORGAS;
|
||||
const NOCODB_TABLE_STATS = process.env.NOCODB_TABLE_STATS;
|
||||
const MISTRAL_API_KEY = process.env.MISTRAL_API_KEY;
|
||||
const RESEND_API_KEY = process.env.RESEND_API_KEY;
|
||||
const RESEND_FROM = process.env.RESEND_FROM || 'contact@trans-former.fr';
|
||||
const EMAIL_JULES = process.env.EMAIL_JULES || 'jules@trans-former.fr';
|
||||
const BUDGET_MAX_EUR = parseFloat(process.env.BUDGET_MAX_EUR || '20');
|
||||
const WORKER_LIMIT = parseInt(process.env.WORKER_LIMIT || '5');
|
||||
const LOCK_FILE = '/tmp/nav-worker.lock';
|
||||
|
||||
// ─── PRIX MISTRAL NEMO (USD → EUR) ───────────────────────────────────────────
|
||||
const NEMO_PRICE_IN = 0.02 / 1_000_000; // $0.02 / 1M tokens input
|
||||
const NEMO_PRICE_OUT = 0.04 / 1_000_000; // $0.04 / 1M tokens output
|
||||
const USD_TO_EUR = 0.93;
|
||||
|
||||
// ─── TAXONOMIE VALIDE (apostrophe typographique U+2019 comme NocoDB) ─────────
|
||||
const VALID_FONCTIONS = [
|
||||
'Juridique', 'Technique', 'Économique', 'Administratif', 'Chantier',
|
||||
'Comptabilité', 'Développement', 'Formation', 'Gestion d\u2019agence', 'Santé mentale'
|
||||
];
|
||||
|
||||
// ─── MAPPING NORMALISATION TAGS ───────────────────────────────────────────────
|
||||
const TAG_MAP = [
|
||||
[['juridique', 'droit', 'litige', 'contrat', 'déontologie', 'décennale', 'médiation', 'pi ', 'propriété intellectuelle', 'ccag', 'marchés publics droit'], 'Juridique'],
|
||||
[['technique', 're2020', 'thermique', 'structure', 'bim', 'dtu', 'acoustique', 'matériaux', 'simulation', 'opr', 'réserves', 'acv', 'pcd', 'eurocodes', 'pmc'], 'Technique'],
|
||||
[['économique', 'prix', 'tarif', 'honoraire', 'devis', 'roi', 'financement', 'subvention', 'cee', 'maprimerenov', 'anah', 'business plan', 'pricing'], 'Économique'],
|
||||
[['administratif', 'permis', 'plu', 'plui', 'erp', 'autorisation travaux', 'abf', 'patrimoine', 'marchés publics procédure', 'cctp', 'dpgf', 'concours', 'urbanisme'], 'Administratif'],
|
||||
[['chantier', 'coordination chantier', 'det', 'suivi travaux', 'sps', 'sécurité chantier', 'planning chantier', 'entreprise', 'sous-traitance', 'réception travaux'], 'Chantier'],
|
||||
[['comptabilité', 'fiscal', 'tva', 'bnc', 'bic', 'expert-comptable', 'bilan', 'trésorerie', 'transmission agence', 'création agence', 'micro'], 'Comptabilité'],
|
||||
[['développement', 'prospection', 'commercial', 'client', 'réseau', 'candidature', 'consultation', 'acquisition', 'marketing', 'notoriété', 'ao '], 'Développement'],
|
||||
[['formation', 'école', 'mooc', 'organisme', 'formation continue', 'cpf', 'dpc', 'cfaa'], 'Formation'],
|
||||
[['gestion d\u2019agence', 'gestion d\'agence', 'rh', 'recrutement', 'emploi', 'salaire', 'ccn', 'convention collective', 'idcc', 'temps de travail', 'management'], 'Gestion d\u2019agence'],
|
||||
[['santé mentale', 'burn-out', 'épuisement', 'souffrance', 'bien-être', 'harcèlement', 'stress', 'psychologique', 'équilibre'], 'Santé mentale'],
|
||||
];
|
||||
|
||||
function normalizeTag(raw) {
|
||||
const t = raw.toLowerCase().trim();
|
||||
// D'abord chercher correspondance exacte dans les valeurs valides
|
||||
const exact = VALID_FONCTIONS.find(v => v.toLowerCase() === t);
|
||||
if (exact) return exact;
|
||||
// Sinon chercher par mots-clés
|
||||
for (const [patterns, normalized] of TAG_MAP) {
|
||||
if (patterns.some(p => t.includes(p))) return normalized;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── UTILITAIRES LOG ─────────────────────────────────────────────────────────
|
||||
function log(...args) {
|
||||
const ts = new Date().toISOString();
|
||||
console.log(`[${ts}]`, ...args);
|
||||
}
|
||||
|
||||
// ─── LOCK ANTI-OVERLAP ────────────────────────────────────────────────────────
|
||||
function acquireLock() {
|
||||
if (existsSync(LOCK_FILE)) {
|
||||
const content = execSync(`cat ${LOCK_FILE}`).toString().trim();
|
||||
const pid = parseInt(content);
|
||||
try {
|
||||
execSync(`kill -0 ${pid} 2>/dev/null`);
|
||||
return false; // Process encore vivant
|
||||
} catch {
|
||||
log('Lock orphelin détecté, suppression');
|
||||
unlinkSync(LOCK_FILE);
|
||||
}
|
||||
}
|
||||
writeFileSync(LOCK_FILE, process.pid.toString());
|
||||
return true;
|
||||
}
|
||||
|
||||
function releaseLock() {
|
||||
try { unlinkSync(LOCK_FILE); } catch {}
|
||||
}
|
||||
|
||||
// ─── NOCODB API ──────────────────────────────────────────────────────────────
|
||||
async function nocodbGet(path) {
|
||||
const res = await fetch(`${NOCODB_URL}/api/v1/db/data/noco/${NOCODB_BASE}/${path}`, {
|
||||
headers: { 'xc-token': NOCODB_TOKEN }
|
||||
});
|
||||
if (!res.ok) throw new Error(`NocoDB GET ${path} → ${res.status}: ${await res.text()}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async function nocodbPatch(tableId, rowId, data) {
|
||||
const res = await fetch(`${NOCODB_URL}/api/v1/db/data/noco/${NOCODB_BASE}/${tableId}/${rowId}`, {
|
||||
method: 'PATCH',
|
||||
headers: { 'xc-token': NOCODB_TOKEN, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(data)
|
||||
});
|
||||
if (!res.ok) throw new Error(`NocoDB PATCH ${tableId}/${rowId} → ${res.status}: ${await res.text()}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async function nocodbPost(tableId, data) {
|
||||
const res = await fetch(`${NOCODB_URL}/api/v1/db/data/noco/${NOCODB_BASE}/${tableId}`, {
|
||||
method: 'POST',
|
||||
headers: { 'xc-token': NOCODB_TOKEN, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(data)
|
||||
});
|
||||
if (!res.ok) throw new Error(`NocoDB POST ${tableId} → ${res.status}: ${await res.text()}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
// ─── BUDGET CIRCUIT BREAKER ──────────────────────────────────────────────────
|
||||
async function getBudgetMoisCourant() {
|
||||
const now = new Date();
|
||||
const year = now.getFullYear();
|
||||
const month = now.getMonth(); // 0-indexed
|
||||
// NocoDB ne supporte pas bien le filtre datetime — on récupère tout et filtre en JS
|
||||
try {
|
||||
const data = await nocodbGet(`${NOCODB_TABLE_STATS}?limit=1000&sort=-timestamp`);
|
||||
const total = (data.list || []).reduce((sum, row) => {
|
||||
const ts = new Date(row.timestamp || row.CreatedAt || 0);
|
||||
if (ts.getFullYear() === year && ts.getMonth() === month) {
|
||||
return sum + (parseFloat(row.cout_eur) || 0);
|
||||
}
|
||||
return sum;
|
||||
}, 0);
|
||||
return total;
|
||||
} catch (e) {
|
||||
log('Erreur lecture budget:', e.message);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async function logUsage(usage, model, endpoint, orgaId) {
|
||||
const tokensIn = usage?.prompt_tokens || 0;
|
||||
const tokensOut = usage?.completion_tokens || 0;
|
||||
const coutEur = ((tokensIn * NEMO_PRICE_IN) + (tokensOut * NEMO_PRICE_OUT)) * USD_TO_EUR;
|
||||
|
||||
await nocodbPost(NOCODB_TABLE_STATS, {
|
||||
model,
|
||||
endpoint,
|
||||
tokens_in: tokensIn,
|
||||
tokens_out: tokensOut,
|
||||
cout_eur: parseFloat(coutEur.toFixed(6)),
|
||||
timestamp: new Date().toISOString(),
|
||||
orga_id: orgaId || null
|
||||
});
|
||||
|
||||
log(`Usage log: ${tokensIn}in + ${tokensOut}out = €${coutEur.toFixed(6)} (${model})`);
|
||||
return coutEur;
|
||||
}
|
||||
|
||||
// ─── FETCH FICHES PENDING ────────────────────────────────────────────────────
|
||||
async function fetchPendingRows() {
|
||||
const data = await nocodbGet(
|
||||
`${NOCODB_TABLE_ORGAS}?where=(moderation_status,eq,pending)~and(ai_processed,eq,false)&limit=${WORKER_LIMIT}&sort=submitted_at`
|
||||
);
|
||||
return data.list || [];
|
||||
}
|
||||
|
||||
// ─── SCRAPING CRAWL4AI (mode HTTP statique, sans Playwright) ─────────────────
|
||||
async function scrapeWithCrawl4ai(url) {
|
||||
log(`Scraping: ${url}`);
|
||||
|
||||
// Script Python temporaire pour crawl4ai
|
||||
const urlSafe = url.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
|
||||
const script = `
|
||||
import asyncio, sys
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
|
||||
|
||||
async def scrape():
|
||||
strategy = AsyncHTTPCrawlerStrategy()
|
||||
run_cfg = CrawlerRunConfig(
|
||||
word_count_threshold=20,
|
||||
excluded_tags=['nav', 'footer', 'script', 'style', 'head'],
|
||||
remove_overlay_elements=True
|
||||
)
|
||||
async with AsyncWebCrawler(crawler_strategy=strategy, verbose=False) as crawler:
|
||||
result = await crawler.arun(url='${urlSafe}', config=run_cfg)
|
||||
if result.success and result.markdown:
|
||||
content = result.markdown[:16000]
|
||||
sys.stdout.buffer.write(content.encode('utf-8'))
|
||||
else:
|
||||
sys.stderr.write(f"Scrape failed: success={result.success}\\n")
|
||||
sys.exit(1)
|
||||
|
||||
asyncio.run(scrape())
|
||||
`;
|
||||
|
||||
const scriptPath = join(tmpdir(), `nav-scrape-${Date.now()}.py`);
|
||||
writeFileSync(scriptPath, script, 'utf-8');
|
||||
|
||||
try {
|
||||
const result = spawnSync('python3', [scriptPath], {
|
||||
timeout: 180_000,
|
||||
maxBuffer: 20 * 1024 * 1024,
|
||||
env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1', PYTHONIOENCODING: 'utf-8' }
|
||||
});
|
||||
|
||||
if (result.error) throw result.error;
|
||||
|
||||
const stdout = result.stdout?.toString('utf-8') || '';
|
||||
const stderr = result.stderr?.toString('utf-8') || '';
|
||||
|
||||
if (result.status === 0 && stdout.length > 50) {
|
||||
log(`Scrape OK: ${stdout.length} chars`);
|
||||
return stdout.trim();
|
||||
} else {
|
||||
throw new Error(`Scrape failed (code ${result.status}): ${stderr.slice(0, 300)}`);
|
||||
}
|
||||
} finally {
|
||||
try { unlinkSync(scriptPath); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── APPEL MISTRAL NEMO ───────────────────────────────────────────────────────
|
||||
const SYSTEM_PROMPT = `Tu es un assistant spécialisé dans l'écosystème professionnel de l'architecture en France. Tu reçois des informations sur une organisation ou ressource liée au secteur de l'architecture, et tu dois les enrichir pour alimenter une cartographie collaborative.
|
||||
|
||||
RÈGLES ABSOLUES :
|
||||
1. Tu ne dois JAMAIS inventer d'informations non présentes dans les sources fournies.
|
||||
2. Si une information est absente ou incertaine, retourne \`null\` pour ce champ.
|
||||
3. Tu dois retourner UNIQUEMENT un objet JSON valide, sans texte avant ou après.
|
||||
4. La description_enrichie doit être neutre, factuelle, en français, sans jugement de valeur.
|
||||
5. Les points_cles sont des phrases courtes (max 12 mots chacune), actionnables pour un architecte.
|
||||
6. Pour les tags_fonction, ne propose que des valeurs parmi la liste autorisée.
|
||||
|
||||
TAXONOMIE AUTORISÉE :
|
||||
- Échelle (une seule valeur) : "National" | "Régional" | "Départemental" | "Local"
|
||||
- Territoire (une seule valeur) : "Métropole" | "Guadeloupe" | "Martinique" | "Guyane" | "Réunion" | "Mayotte" | null
|
||||
- Tags fonction (1 à 5 valeurs) : "Juridique" | "Technique" | "Économique" | "Administratif" | "Chantier" | "Comptabilité" | "Développement" | "Formation" | "Gestion d\u2019agence" | "Santé mentale"
|
||||
|
||||
FORMAT DE SORTIE JSON :
|
||||
{
|
||||
"description_enrichie": "string (max 300 chars, français, neutre, factuel)",
|
||||
"points_cles": ["string", "string", "string"],
|
||||
"tags_fonction": ["Valeur1", "Valeur2"],
|
||||
"echelle": "National" | "Régional" | "Départemental" | "Local" | null,
|
||||
"territoire": "Métropole" | ... | null,
|
||||
"localisation_ville": "string" | null,
|
||||
"confiance": "haute" | "moyenne" | "faible"
|
||||
}
|
||||
|
||||
Le champ "confiance" reflète ta certitude globale sur l'enrichissement :
|
||||
- "haute" : URL scrapée avec contenu riche, informations claires
|
||||
- "moyenne" : URL scrapée mais contenu partiel, ou description_user seule suffisante
|
||||
- "faible" : URL non disponible et description_user vague, inférences importantes`;
|
||||
|
||||
function buildUserPrompt(row, scrapeContent) {
|
||||
return `ORGANISATION À ENRICHIR :
|
||||
|
||||
Nom : ${row.nom}
|
||||
URL : ${row.url || 'non fournie'}
|
||||
Description soumise par l'utilisateur : ${row.description_user || row.description || 'non fournie'}
|
||||
|
||||
CONTENU DU SITE WEB (extrait par scraping) :
|
||||
${scrapeContent || 'Site non accessible ou URL non fournie.'}
|
||||
|
||||
---
|
||||
|
||||
Enrichis cette fiche selon les règles du system prompt. Retourne uniquement le JSON.`;
|
||||
}
|
||||
|
||||
async function callMistralWithRetry(row, scrapeContent, maxRetries = 2) {
|
||||
const userPrompt = buildUserPrompt(row, scrapeContent);
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
if (attempt > 0) {
|
||||
log(`Retry ${attempt}/${maxRetries} pour fiche ${row.Id}`);
|
||||
await new Promise(r => setTimeout(r, 2000 * attempt));
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await fetch('https://api.mistral.ai/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${MISTRAL_API_KEY}`,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'open-mistral-nemo',
|
||||
temperature: 0.2,
|
||||
max_tokens: 800,
|
||||
response_format: { type: 'json_object' },
|
||||
messages: [
|
||||
{ role: 'system', content: SYSTEM_PROMPT },
|
||||
{ role: 'user', content: userPrompt }
|
||||
]
|
||||
}),
|
||||
signal: AbortSignal.timeout(60_000)
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.text();
|
||||
throw new Error(`Mistral API ${res.status}: ${err}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
if (!content) throw new Error('Réponse Mistral vide');
|
||||
|
||||
const parsed = JSON.parse(content);
|
||||
// Attacher usage pour logging
|
||||
parsed._usage = data.usage;
|
||||
parsed._raw = content;
|
||||
return parsed;
|
||||
|
||||
} catch (e) {
|
||||
log(`Erreur Mistral (tentative ${attempt + 1}): ${e.message}`);
|
||||
if (attempt === maxRetries) return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── EMAIL JULES VIA RESEND ───────────────────────────────────────────────────
|
||||
async function sendEmailJules(subject, body) {
|
||||
if (!RESEND_API_KEY) {
|
||||
log('RESEND_API_KEY absent, email skippé');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const res = await fetch('https://api.resend.com/emails', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${RESEND_API_KEY}`,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
from: RESEND_FROM,
|
||||
to: [EMAIL_JULES],
|
||||
subject,
|
||||
text: body
|
||||
})
|
||||
});
|
||||
if (!res.ok) log('Email error:', await res.text());
|
||||
else log('Email envoyé à Jules:', subject);
|
||||
} catch (e) {
|
||||
log('Email exception:', e.message);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── VÉRIFICATION SEUIL 5 FICHES PENDING MODÉRATION ─────────────────────────
|
||||
async function checkModerationQueue() {
|
||||
const data = await nocodbGet(
|
||||
`${NOCODB_TABLE_ORGAS}?where=(moderation_status,eq,ai_processed)&limit=100`
|
||||
);
|
||||
const count = data.pageInfo?.totalRows || 0;
|
||||
if (count >= 5) {
|
||||
log(`Seuil modération atteint: ${count} fiches en attente`);
|
||||
await sendEmailJules(
|
||||
`NAV — ${count} fiches à modérer`,
|
||||
`Bonjour Jules,\n\n${count} fiches ont été enrichies par l'IA et attendent ta validation dans NocoDB.\n\nLien NocoDB : http://localhost:8070\nFiltre : moderation_status = ai_processed\n\nBonne modération !`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── MAIN ─────────────────────────────────────────────────────────────────────
|
||||
async function run() {
|
||||
if (!acquireLock()) {
|
||||
log('Worker déjà en cours, skip');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
log('=== Worker NAV enrichissement démarré ===');
|
||||
|
||||
try {
|
||||
// Vérification clés obligatoires
|
||||
if (!NOCODB_TOKEN || !NOCODB_BASE || !NOCODB_TABLE_ORGAS || !MISTRAL_API_KEY) {
|
||||
throw new Error('Variables .env manquantes (NOCODB_TOKEN, NOCODB_BASE, NOCODB_TABLE_ORGAS, MISTRAL_API_KEY)');
|
||||
}
|
||||
|
||||
// Check budget global
|
||||
const budgetMois = await getBudgetMoisCourant();
|
||||
log(`Budget mois courant: €${budgetMois.toFixed(4)} / €${BUDGET_MAX_EUR}`);
|
||||
|
||||
if (budgetMois >= BUDGET_MAX_EUR) {
|
||||
log('Budget épuisé pour ce mois. Worker en pause.');
|
||||
await sendEmailJules(
|
||||
'NAV — Budget IA épuisé ce mois',
|
||||
`Le budget IA de ${BUDGET_MAX_EUR}€ a été atteint. Le worker est en pause jusqu'au 1er du mois prochain.\n\nConsommation actuelle : €${budgetMois.toFixed(4)}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Fetch fiches pending
|
||||
const rows = await fetchPendingRows();
|
||||
log(`${rows.length} fiche(s) à traiter`);
|
||||
|
||||
if (rows.length === 0) {
|
||||
log('Rien à traiter.');
|
||||
return;
|
||||
}
|
||||
|
||||
let processedCount = 0;
|
||||
|
||||
for (const row of rows) {
|
||||
const rowStart = Date.now();
|
||||
log(`--- Traitement fiche ${row.Id}: ${row.nom} ---`);
|
||||
|
||||
// Re-check budget avant chaque fiche
|
||||
const budgetCheck = await getBudgetMoisCourant();
|
||||
if (budgetCheck >= BUDGET_MAX_EUR) {
|
||||
log('Budget atteint mid-pipeline, arrêt.');
|
||||
break;
|
||||
}
|
||||
|
||||
// Scraping
|
||||
let scrapeContent = null;
|
||||
const hasUrl = row.url && row.url.trim().length > 0;
|
||||
const shouldScrape = hasUrl && (row.scrape_status === 'pending' || !row.scrape_status);
|
||||
|
||||
if (shouldScrape) {
|
||||
try {
|
||||
scrapeContent = await scrapeWithCrawl4ai(row.url);
|
||||
await nocodbPatch(NOCODB_TABLE_ORGAS, row.Id, {
|
||||
scrape_status: 'scraped',
|
||||
scrape_content: scrapeContent
|
||||
});
|
||||
} catch (e) {
|
||||
log(`Scrape échoué: ${e.message}`);
|
||||
await nocodbPatch(NOCODB_TABLE_ORGAS, row.Id, { scrape_status: 'failed' });
|
||||
// Continue avec l'IA sans contenu scrape
|
||||
}
|
||||
} else if (!hasUrl) {
|
||||
await nocodbPatch(NOCODB_TABLE_ORGAS, row.Id, { scrape_status: 'no_link' });
|
||||
}
|
||||
|
||||
// Appel Mistral Nemo
|
||||
const enriched = await callMistralWithRetry(row, scrapeContent);
|
||||
|
||||
if (!enriched) {
|
||||
log(`Échec Mistral sur fiche ${row.Id}, flag ai_error`);
|
||||
await nocodbPatch(NOCODB_TABLE_ORGAS, row.Id, {
|
||||
moderation_status: 'ai_error',
|
||||
ai_processed: true
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Normalisation tags
|
||||
const rawTags = enriched.tags_fonction || [];
|
||||
const normalizedTags = [...new Set(rawTags.map(normalizeTag).filter(Boolean))];
|
||||
|
||||
// Update NocoDB
|
||||
const updateData = {
|
||||
description_enrichie: enriched.description_enrichie || null,
|
||||
points_cles: enriched.points_cles ? JSON.stringify(enriched.points_cles) : null,
|
||||
tags_fonction: normalizedTags.join(','),
|
||||
moderation_status: 'ai_processed',
|
||||
ai_processed: true,
|
||||
ai_raw_output: JSON.stringify({ output: enriched, confiance: enriched.confiance })
|
||||
};
|
||||
|
||||
// Conserver echelle/territoire/localisation si l'IA les a enrichis
|
||||
if (enriched.echelle && !row.echelle) updateData.echelle = enriched.echelle;
|
||||
if (enriched.territoire && !row.territoire) updateData.territoire = enriched.territoire;
|
||||
if (enriched.localisation_ville && !row.localisation_ville) {
|
||||
updateData.localisation_ville = enriched.localisation_ville;
|
||||
}
|
||||
|
||||
await nocodbPatch(NOCODB_TABLE_ORGAS, row.Id, updateData);
|
||||
|
||||
// Log usage tokens
|
||||
await logUsage(enriched._usage, 'open-mistral-nemo', 'enrichissement', row.Id);
|
||||
|
||||
const elapsed = ((Date.now() - rowStart) / 1000).toFixed(1);
|
||||
log(`Fiche ${row.Id} traitée en ${elapsed}s — confiance: ${enriched.confiance || 'nc'}`);
|
||||
processedCount++;
|
||||
}
|
||||
|
||||
log(`=== Run terminé: ${processedCount}/${rows.length} fiches traitées en ${((Date.now() - startTime) / 1000).toFixed(1)}s ===`);
|
||||
|
||||
// Check seuil modération
|
||||
await checkModerationQueue();
|
||||
|
||||
} catch (e) {
|
||||
log('ERREUR WORKER:', e.message);
|
||||
console.error(e.stack);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
run();
|
||||
13
worker/package.json
Normal file
13
worker/package.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "nav-worker",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"description": "NAV V2 — Worker enrichissement IA (Mistral Nemo + crawl4ai)",
|
||||
"main": "enrich.js",
|
||||
"scripts": {
|
||||
"start": "node -r dotenv/config enrich.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"dotenv": "^16.4.5"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user