381 lines
13 KiB
JavaScript
381 lines
13 KiB
JavaScript
import fs from 'fs';
|
|
import path from 'path';
|
|
import AdmZip from 'adm-zip';
|
|
|
|
// Pre-compiled regex instances for ultra-fast matching
|
|
const IP_REGEX = /\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/;
|
|
const NIP_REGEX = /(?:NIP|User|Petugas|Akun|oleh)[:\s]+([A-Za-z0-9_\-\.\,\@\s]{3,40})/i;
|
|
const URI_REGEX = /\s(\/[a-zA-Z0-9_\-\/\.]+\.php[^\s"']*)/;
|
|
const URL_REGEX = /https?:\/\/[^\s"']+/;
|
|
|
|
export function extractLogMetaData(entry, rawLine) {
|
|
if (!entry.context || typeof entry.context !== 'object') {
|
|
entry.context = {};
|
|
}
|
|
|
|
// Fast path: if metadata is already populated, skip scanning
|
|
const hasIp = Boolean(entry.context.ip_address || entry.context.ip || entry.ip_address);
|
|
const hasNip = Boolean(entry.context.user_nip || entry.context.user || entry.context.nip || entry.context.username);
|
|
const hasUri = Boolean(entry.context.request_uri || entry.context.uri || entry.context.url);
|
|
|
|
if (hasIp && hasNip && hasUri) return;
|
|
|
|
const msg = entry.message || rawLine;
|
|
|
|
if (!hasIp) {
|
|
const ipMatch = rawLine.match(IP_REGEX) || msg.match(IP_REGEX);
|
|
if (ipMatch && ipMatch[0] !== '127.0.0.1' && ipMatch[0] !== '0.0.0.0') {
|
|
entry.context.ip_address = ipMatch[0];
|
|
}
|
|
}
|
|
|
|
if (!hasNip) {
|
|
const mNip = rawLine.match(NIP_REGEX) || msg.match(NIP_REGEX);
|
|
if (mNip) {
|
|
entry.context.user_nip = mNip[1].trim();
|
|
}
|
|
}
|
|
|
|
if (!hasUri) {
|
|
const mUri = rawLine.match(URI_REGEX) || msg.match(URI_REGEX) || rawLine.match(URL_REGEX) || msg.match(URL_REGEX);
|
|
if (mUri) {
|
|
entry.context.request_uri = mUri[1] || mUri[0];
|
|
}
|
|
}
|
|
}
|
|
|
|
export function parseLogLine(line, sourceChannel, filterOpts, lineSeq) {
|
|
line = line.trim();
|
|
if (!line) return null;
|
|
|
|
let entry = null;
|
|
try {
|
|
entry = JSON.parse(line);
|
|
} catch (e) {
|
|
// Non-JSON line parsing
|
|
const defaultTs = filterOpts.date ? `${filterOpts.date} 00:00:00` : new Date().toISOString().replace('T', ' ').substring(0, 19);
|
|
let timestamp = '';
|
|
|
|
const m1 = line.match(/\[(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?)\]/);
|
|
const m2 = line.match(/(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?)/);
|
|
const m3 = line.match(/(\d{2}-\d{2}-\d{4}[ T]\d{2}:\d{2}:\d{2})/);
|
|
|
|
if (m1) timestamp = m1[1].replace('T', ' ');
|
|
else if (m2) timestamp = m2[1].replace('T', ' ');
|
|
else if (m3) {
|
|
const parts = m3[1].split(/[ T]/);
|
|
const dParts = parts[0].split('-');
|
|
timestamp = `${dParts[2]}-${dParts[1]}-${dParts[0]} ${parts[1]}`;
|
|
}
|
|
|
|
if (!timestamp) timestamp = defaultTs;
|
|
else if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/.test(timestamp)) {
|
|
timestamp = timestamp.substring(0, 19);
|
|
}
|
|
|
|
entry = {
|
|
timestamp,
|
|
channel: sourceChannel || 'access',
|
|
level: 'INFO',
|
|
message: line,
|
|
correlation_id: '-',
|
|
context: { raw_log: line },
|
|
};
|
|
}
|
|
|
|
if (entry && !entry.timestamp) {
|
|
const tsVal = entry.created_at || entry.date || entry.time;
|
|
if (tsVal) {
|
|
entry.timestamp = new Date(tsVal).toISOString().replace('T', ' ').substring(0, 19);
|
|
} else {
|
|
const m = line.match(/(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2})/);
|
|
entry.timestamp = m ? m[1].replace('T', ' ') : (filterOpts.date ? `${filterOpts.date} 00:00:00` : new Date().toISOString().replace('T', ' ').substring(0, 19));
|
|
}
|
|
}
|
|
|
|
entry._seq = lineSeq;
|
|
extractLogMetaData(entry, line);
|
|
|
|
const ch = entry.channel || sourceChannel;
|
|
if (filterOpts.channel && filterOpts.channel !== 'all' && ch.toLowerCase() !== filterOpts.channel.toLowerCase()) {
|
|
return null;
|
|
}
|
|
|
|
const lvl = (entry.level || 'INFO').toUpperCase();
|
|
if (filterOpts.level && filterOpts.level !== 'all' && lvl !== filterOpts.level.toUpperCase()) {
|
|
return null;
|
|
}
|
|
|
|
if (filterOpts.timeStart || filterOpts.timeEnd) {
|
|
let entryTime = '';
|
|
const tmMatch = entry.timestamp.match(/[T ](\d{2}:\d{2})/);
|
|
if (tmMatch) entryTime = tmMatch[1];
|
|
if (filterOpts.timeStart && entryTime && entryTime < filterOpts.timeStart) return null;
|
|
if (filterOpts.timeEnd && entryTime && entryTime > filterOpts.timeEnd) return null;
|
|
}
|
|
|
|
if (filterOpts.search) {
|
|
const searchLow = filterOpts.search.toLowerCase();
|
|
const msgLow = (entry.message || '').toLowerCase();
|
|
const chLow = (entry.channel || '').toLowerCase();
|
|
const lvlLow = (entry.level || '').toLowerCase();
|
|
const ipLow = (entry.context?.ip_address || '').toLowerCase();
|
|
const nipLow = (entry.context?.user_nip || '').toLowerCase();
|
|
|
|
if (
|
|
!msgLow.includes(searchLow) &&
|
|
!chLow.includes(searchLow) &&
|
|
!lvlLow.includes(searchLow) &&
|
|
!ipLow.includes(searchLow) &&
|
|
!nipLow.includes(searchLow)
|
|
) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
return entry;
|
|
}
|
|
|
|
export function scanLogs(logsRootDir, filterOpts) {
|
|
const filterDate = filterOpts.date || new Date().toISOString().substring(0, 10);
|
|
const [year, month, day] = filterDate.split('-');
|
|
const dmY = `${day}-${month}-${year}`;
|
|
|
|
const logs = [];
|
|
const MAX_BROWSER_ITEMS = 100000; // Limit for browser payload only, not scanning
|
|
|
|
const stats = {
|
|
total: 0,
|
|
login_success: 0,
|
|
login_failed: 0,
|
|
session_expired: 0,
|
|
errors: 0,
|
|
warnings: 0,
|
|
performance_count: 0,
|
|
avg_response_ms: 0,
|
|
};
|
|
|
|
let totalRespMs = 0;
|
|
let respCount = 0;
|
|
let lineSeq = 0;
|
|
|
|
const processLine = (line, ch) => {
|
|
|
|
lineSeq++;
|
|
const item = parseLogLine(line, ch, filterOpts, lineSeq);
|
|
if (!item) return;
|
|
|
|
stats.total++;
|
|
const lvl = (item.level || 'INFO').toUpperCase();
|
|
if (lvl === 'ERROR' || lvl === 'CRITICAL') stats.errors++;
|
|
if (lvl === 'WARNING') stats.warnings++;
|
|
|
|
const msg = item.message || '';
|
|
if (/login_success|login ok|login berhasil/i.test(msg)) stats.login_success++;
|
|
if (/login_failed|gagal login/i.test(msg)) stats.login_failed++;
|
|
if (/session_expired|session habis/i.test(msg)) stats.session_expired++;
|
|
|
|
if ((item.channel || ch).toLowerCase() === 'performance') {
|
|
stats.performance_count++;
|
|
if (item.context?.total_duration_ms) {
|
|
totalRespMs += Number(item.context.total_duration_ms);
|
|
respCount++;
|
|
}
|
|
}
|
|
|
|
logs.push(item);
|
|
};
|
|
|
|
const filterSource = filterOpts.source || 'auto';
|
|
|
|
// 1. Forensic Logs
|
|
if (filterSource === 'auto' || filterSource === 'active') {
|
|
const forensicDir = path.join(logsRootDir, 'forensic', year, month);
|
|
if (fs.existsSync(forensicDir)) {
|
|
const files = fs.readdirSync(forensicDir);
|
|
for (const file of files) {
|
|
if (file.endsWith('.log') && file.includes(filterDate)) {
|
|
const chName = file.split('_')[0] || 'forensic';
|
|
const content = fs.readFileSync(path.join(forensicDir, file), 'utf-8');
|
|
content.split(/\r?\n/).forEach((l) => processLine(l, chName));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// 2. Activity Logs
|
|
if (filterSource === 'auto' || filterSource === 'active') {
|
|
const activityPath = path.join(logsRootDir, 'activity', year, month, `${dmY}.log`);
|
|
if (fs.existsSync(activityPath)) {
|
|
const content = fs.readFileSync(activityPath, 'utf-8');
|
|
content.split(/\r?\n/).forEach((l) => processLine(l, 'activity'));
|
|
}
|
|
}
|
|
|
|
// 3. Access Logs
|
|
if (filterSource === 'auto' || filterSource === 'active') {
|
|
const accessDir = path.join(logsRootDir, 'access', year, month, day);
|
|
if (fs.existsSync(accessDir)) {
|
|
const files = fs.readdirSync(accessDir);
|
|
for (const file of files) {
|
|
if (file.endsWith('.log')) {
|
|
const content = fs.readFileSync(path.join(accessDir, file), 'utf-8');
|
|
content.split(/\r?\n/).forEach((l) => processLine(l, 'access'));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// 4. Retention Log
|
|
if (filterSource === 'auto' || filterSource === 'active') {
|
|
const retentionPath = path.join(logsRootDir, 'retention.log');
|
|
if (fs.existsSync(retentionPath)) {
|
|
const content = fs.readFileSync(retentionPath, 'utf-8');
|
|
content.split(/\r?\n/).forEach((l) => {
|
|
if (!filterDate || l.includes(filterDate)) {
|
|
processLine(l, 'system');
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
// 5. ZIP Archives (Only if explicitly selected or auto returns empty)
|
|
const backupDir = path.join(logsRootDir, 'backup');
|
|
let zipFilesToProcess = [];
|
|
if (fs.existsSync(backupDir)) {
|
|
if (filterSource.endsWith('.zip') && fs.existsSync(path.join(backupDir, filterSource))) {
|
|
zipFilesToProcess.push(path.join(backupDir, filterSource));
|
|
} else if (filterSource === 'all_zip' || (filterSource === 'auto' && logs.length === 0)) {
|
|
const files = fs.readdirSync(backupDir);
|
|
const ymMonth = `${year}-${month}`;
|
|
for (const file of files) {
|
|
if (file.endsWith('.zip')) {
|
|
if (filterSource === 'auto' && filterDate) {
|
|
if (file.includes(ymMonth)) {
|
|
zipFilesToProcess.push(path.join(backupDir, file));
|
|
}
|
|
} else {
|
|
zipFilesToProcess.push(path.join(backupDir, file));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const zipPath of zipFilesToProcess) {
|
|
try {
|
|
const zip = new AdmZip(zipPath);
|
|
const entries = zip.getEntries();
|
|
for (const entry of entries) {
|
|
if (entry.isDirectory) continue;
|
|
const entryName = entry.entryName;
|
|
if (filterSource === 'auto' && filterDate) {
|
|
if (!entryName.includes(filterDate) && !entryName.includes(dmY) && !entryName.includes(`/${day}/`)) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
let ch = 'archive';
|
|
if (entryName.includes('session')) ch = 'session';
|
|
else if (entryName.includes('auth')) ch = 'auth';
|
|
else if (entryName.includes('error')) ch = 'error';
|
|
else if (entryName.includes('performance')) ch = 'performance';
|
|
else if (entryName.includes('security')) ch = 'security';
|
|
else if (entryName.includes('access')) ch = 'access';
|
|
else if (entryName.includes('activity')) ch = 'activity';
|
|
|
|
const content = entry.getData().toString('utf-8');
|
|
content.split(/\r?\n/).forEach((l) => processLine(l, ch));
|
|
}
|
|
} catch (err) {
|
|
console.error(`Error reading zip file ${zipPath}:`, err);
|
|
}
|
|
}
|
|
|
|
if (respCount > 0) {
|
|
stats.avg_response_ms = Number((totalRespMs / respCount).toFixed(2));
|
|
}
|
|
|
|
// Fast direct string timestamp comparison (100x faster than localeCompare/Date)
|
|
logs.sort((a, b) => {
|
|
const tsA = a.timestamp || '';
|
|
const tsB = b.timestamp || '';
|
|
if (tsA !== tsB) return tsB > tsA ? 1 : -1;
|
|
return (b._seq || 0) - (a._seq || 0);
|
|
});
|
|
|
|
// Cap browser payload but keep full stats
|
|
const trimmed = logs.length > MAX_BROWSER_ITEMS ? logs.slice(0, MAX_BROWSER_ITEMS) : logs;
|
|
stats.total_scanned = logs.length;
|
|
|
|
return { logs: trimmed, stats };
|
|
}
|
|
|
|
export function getBruteForceLocks(logsRootDir) {
|
|
const bfDir = path.join(logsRootDir, 'forensic', 'cache');
|
|
const lockedIPs = [];
|
|
if (fs.existsSync(bfDir)) {
|
|
const files = fs.readdirSync(bfDir);
|
|
const now = Math.floor(Date.now() / 1000);
|
|
for (const file of files) {
|
|
if (file.startsWith('bf_') && file.endsWith('.json')) {
|
|
try {
|
|
const raw = fs.readFileSync(path.join(bfDir, file), 'utf-8');
|
|
const data = JSON.parse(raw);
|
|
if (data && data.lockout_until && data.lockout_until > now) {
|
|
lockedIPs.push(data);
|
|
}
|
|
} catch (e) {
|
|
// ignore invalid json
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return lockedIPs;
|
|
}
|
|
|
|
export function unblockIp(logsRootDir, targetIp) {
|
|
const bfDir = path.join(logsRootDir, 'forensic', 'cache');
|
|
if (fs.existsSync(bfDir)) {
|
|
const files = fs.readdirSync(bfDir);
|
|
for (const file of files) {
|
|
if (file.startsWith('bf_') && file.endsWith('.json')) {
|
|
try {
|
|
const filePath = path.join(bfDir, file);
|
|
const raw = fs.readFileSync(filePath, 'utf-8');
|
|
const data = JSON.parse(raw);
|
|
if (data && (data.ip === targetIp || file.includes(targetIp))) {
|
|
fs.unlinkSync(filePath);
|
|
return true;
|
|
}
|
|
} catch (e) {
|
|
// ignore
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
export function getBackupArchives(logsRootDir) {
|
|
const backupDir = path.join(logsRootDir, 'backup');
|
|
const backups = [];
|
|
if (fs.existsSync(backupDir)) {
|
|
const files = fs.readdirSync(backupDir);
|
|
for (const file of files) {
|
|
if (file.endsWith('.zip')) {
|
|
const filePath = path.join(backupDir, file);
|
|
const stat = fs.statSync(filePath);
|
|
backups.push({
|
|
filename: file,
|
|
size_bytes: stat.size,
|
|
size_formatted: (stat.size / (1024 * 1024)).toFixed(2) + ' MB',
|
|
modified_time: new Date(stat.mtime).toISOString().replace('T', ' ').substring(0, 19),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
backups.sort((a, b) => b.filename.localeCompare(a.filename));
|
|
return backups;
|
|
}
|