scripts/js/momentum_regime_server.js

Repository

BOT_EU

Original path

scripts/js/momentum_regime_server.js

Role

SOURCE

Size

82265 bytes

Lines

2565

SHA-256

996c68e49cf2fc80fc8a5b8fd856b7740f159b65278bca47f087e6e983b50b2a

Displayed range

501–1000

    if (Number.isFinite(ts) && ts < minTsMs) {
      removed += 1;
      continue;
    }
    keptRows.push(row);
  }

  if (removed > 0) {
    const out = [header, ...keptRows].join("\n") + "\n";
    await fs.writeFile(filePath, out, "utf8");
  }

  return { removed, kept: keptRows.length, total: Math.max(0, lines.length - 1) };
}

async function pruneJsonlByRetention(filePath, minTsMs) {
  let raw = "";
  try {
    raw = await fs.readFile(filePath, "utf8");
  } catch {
    return { removed: 0, kept: 0, total: 0 };
  }

  const lines = String(raw || "").split(/\r?\n/).filter((l) => l.trim().length > 0);
  let removed = 0;
  const kept = [];

  for (const line of lines) {
    try {
      const obj = JSON.parse(line);
      const ts = parseTsMsLoose(obj?.tsMs) || parseTsMsLoose(obj?.ts);
      if (Number.isFinite(ts) && ts < minTsMs) {
        removed += 1;
        continue;
      }
      kept.push(JSON.stringify(obj));
    } catch {
      kept.push(line);
    }
  }

  if (removed > 0) {
    const out = kept.length ? `${kept.join("\n")}\n` : "";
    await fs.writeFile(filePath, out, "utf8");
  }

  return { removed, kept: kept.length, total: lines.length };
}

async function pruneIndicatorHistoryByRetention(filePath, minTsMs) {
  let raw = "";
  try {
    raw = await fs.readFile(filePath, "utf8");
  } catch {
    return { removed: 0, kept: 0, total: 0 };
  }

  let arr = [];
  try {
    arr = JSON.parse(raw);
  } catch {
    arr = [];
  }

  if (!Array.isArray(arr)) arr = [];
  const total = arr.length;
  const kept = arr.filter((r) => {
    const ts = parseTsMsLoose(r?.tsMs) || parseTsMsLoose(r?.ts);
    if (!Number.isFinite(ts)) return true;
    return ts >= minTsMs;
  });
  const removed = total - kept.length;

  if (removed > 0) {
    await fs.writeFile(filePath, JSON.stringify(kept, null, 2), "utf8");
  }

  return { removed, kept: kept.length, total };
}

async function cleanupOldFilesInDir(dirPath, minTsMs, matcher) {
  let entries = [];
  try {
    entries = await fs.readdir(dirPath, { withFileTypes: true });
  } catch {
    return { removed: 0, scanned: 0 };
  }

  let scanned = 0;
  let removed = 0;
  for (const ent of entries) {
    if (!ent.isFile()) continue;
    if (!matcher(ent.name)) continue;
    const abs = path.resolve(dirPath, ent.name);
    scanned += 1;
    try {
      const st = await fs.stat(abs);
      if (Number(st?.mtimeMs || 0) < minTsMs) {
        await fs.unlink(abs);
        removed += 1;
      }
    } catch {
      // ignore single-file errors
    }
  }
  return { removed, scanned };
}

async function cleanupOldRunDirs(minTsMs) {
  let entries = [];
  try {
    entries = await fs.readdir(RUNS_DIR, { withFileTypes: true });
  } catch {
    return { removed: 0, scanned: 0 };
  }

  let scanned = 0;
  let removed = 0;
  for (const ent of entries) {
    if (!ent.isDirectory()) continue;
    const abs = path.resolve(RUNS_DIR, ent.name);
    scanned += 1;
    try {
      const st = await fs.stat(abs);
      if (Number(st?.mtimeMs || 0) < minTsMs) {
        await fs.rm(abs, { recursive: true, force: true });
        removed += 1;
      }
    } catch {
      // ignore single-dir errors
    }
  }
  return { removed, scanned };
}

async function runFilesystemCleanup() {
  const minTsMs = nowMs() - FILE_RETENTION_MS;

  await ensureDir(LOGS_DIR);
  await ensureDir(CSV_DIR);

  const [csvAlias, jsonlAlias, indicator, candleFiles, tempFiles, runDirs, csvExports] = await Promise.all([
    pruneCsvByRetention(HISTORY_ALIAS_PATH, minTsMs),
    pruneJsonlByRetention(HISTORY_ALIAS_JSONL_PATH, minTsMs),
    pruneIndicatorHistoryByRetention(INDICATOR_HISTORY_JSON_PATH, minTsMs),
    cleanupOldFilesInDir(
      LOGS_DIR,
      minTsMs,
      (name) => name.toLowerCase().endsWith(".csv") && name !== path.basename(HISTORY_ALIAS_PATH)
    ),
    cleanupOldFilesInDir(
      LOGS_DIR,
      minTsMs,
      (name) => /\.(tmp|temp|lock)$/i.test(name)
    ),
    cleanupOldRunDirs(minTsMs),
    cleanupOldFilesInDir(
      CSV_DIR,
      minTsMs,
      (name) => /\.(csv|tsv|jsonl|tmp|temp)$/i.test(name)
    ),
  ]);

  const removedTotal =
    Number(csvAlias.removed || 0) +
    Number(jsonlAlias.removed || 0) +
    Number(indicator.removed || 0) +
    Number(candleFiles.removed || 0) +
    Number(tempFiles.removed || 0) +
    Number(runDirs.removed || 0) +
    Number(csvExports.removed || 0);

  if (removedTotal > 0) {
    console.log(
      `[cleanup] retention=${FILE_RETENTION_DAYS}d removed=${removedTotal} ` +
      `(historyCsv=${csvAlias.removed} historyJsonl=${jsonlAlias.removed} indicator=${indicator.removed} ` +
      `logCsv=${candleFiles.removed} logTmp=${tempFiles.removed} runs=${runDirs.removed} csvDir=${csvExports.removed})`
    );
  }
}

function startFilesystemCleanup() {
  if (state.cleanupTimer) clearInterval(state.cleanupTimer);
  runFilesystemCleanup().catch((e) => {
    console.log(`[cleanup] startup error: ${e?.message || e}`);
  });
  state.cleanupTimer = setInterval(() => {
    runFilesystemCleanup().catch((e) => {
      console.log(`[cleanup] periodic error: ${e?.message || e}`);
    });
  }, FILE_CLEANUP_EVERY_MS);
}

function stopFilesystemCleanup() {
  if (state.cleanupTimer) {
    clearInterval(state.cleanupTimer);
    state.cleanupTimer = null;
  }
}

function normalizeSymbol(s) {
  return String(s || "")
    .trim()
    .toUpperCase()
    .replace(/[^A-Z0-9]/g, "");
}

async function chooseHistoryFileViaExplorer() {
  const script = [
    "Add-Type -AssemblyName System.Windows.Forms",
    "$dlg = New-Object System.Windows.Forms.OpenFileDialog",
    "$dlg.Filter = 'History files (*.csv;*.jsonl;*.json)|*.csv;*.jsonl;*.json|All files (*.*)|*.*'",
    "$dlg.Multiselect = $false",
    "$dlg.Title = 'Wybierz plik historii (CSV/JSONL/JSON)'",
    "if($dlg.ShowDialog() -eq [System.Windows.Forms.DialogResult]::OK){ Write-Output $dlg.FileName }",
  ].join("; ");

  try {
    const { stdout } = await execFileAsync("powershell", ["-NoProfile", "-STA", "-Command", script], {
      windowsHide: true,
      maxBuffer: 1024 * 1024,
    });

    const selected = String(stdout || "").trim();
    return selected || null;
  } catch {
    return null;
  }
}

async function askStartupMode() {
  const rl = readline.createInterface({ input, output });
  try {
    console.log("\n=== momentum_regime server ===");
    const modeRaw = await rl.question("Tryb [live/historia]: ");
    const mode = String(modeRaw || "").trim().toLowerCase();

    if (mode.startsWith("h")) {
      state.mode = "history";

      let selected = await chooseHistoryFileViaExplorer();
      if (!selected) {
        const fallback = await rl.question("Nie wybrano pliku w eksploratorze. Podaj ścieżkę ręcznie: ");
        selected = String(fallback || "").trim();
      }

      if (!selected) {
        throw new Error("Nie wskazano pliku historii.");
      }

      state.selectedHistoryPath = selected;
      state.selectedHistoryRows = await loadHistoryRowsFromFile(selected);
      if (!state.selectedHistoryRows.length) {
        throw new Error("Wybrany plik historii nie zawiera poprawnych rekordów.");
      }

      state.symbol = normalizeSymbol(state.selectedHistoryRows.at(-1)?.symbol || "BTCUSDT") || "BTCUSDT";
      state.category = String(state.selectedHistoryRows.at(-1)?.category || "linear").toLowerCase();
      state.selectedIndicatorRows = buildIndicatorRowsFromHistoryRows(state.selectedHistoryRows, state.symbol, state.category);

      await syncHistoryAliasCsv();

      console.log(`Tryb: historia | rekordy: ${state.selectedHistoryRows.length} | symbol: ${state.symbol}`);
      console.log(`Plik: ${state.selectedHistoryPath}`);
      return;
    }

    state.mode = "live";
    const symbolRaw = await rl.question("Podaj walutę/symbol (np. BTCUSDT): ");
    const symbol = normalizeSymbol(symbolRaw) || "BTCUSDT";
    state.symbol = symbol;
    state.category = "linear";

    await ensureSessionCsv(symbol);

    await startMarketSignal({
      symbol,
      category: state.category,
      pollMs: 10_000,
      silent: true,
      debug: false,
    });

    startLiveCsvCapture();
    console.log(`Tryb: live | symbol: ${symbol}`);
    console.log(`CSV sesji: ${state.sessionCsvPath}`);
  } finally {
    rl.close();
  }
}

async function setupFromArgsIfProvided() {
  const modeRaw = String(parseArg("mode") || "").trim().toLowerCase();
  if (!modeRaw) return false;

  if (modeRaw.startsWith("h")) {
    const historyFile = String(parseArg("historyFile") || "").trim();
    if (!historyFile) {
      throw new Error("Dla --mode history podaj --historyFile <path>");
    }

    state.mode = "history";
    state.selectedHistoryPath = historyFile;
    state.selectedHistoryRows = await loadHistoryRowsFromFile(historyFile);
    if (!state.selectedHistoryRows.length) {
      throw new Error("Wybrany plik historii nie zawiera poprawnych rekordów.");
    }

    state.symbol = normalizeSymbol(parseArg("symbol") || state.selectedHistoryRows.at(-1)?.symbol || "BTCUSDT") || "BTCUSDT";
    state.category = String(parseArg("category") || state.selectedHistoryRows.at(-1)?.category || "linear").toLowerCase();
    state.selectedIndicatorRows = buildIndicatorRowsFromHistoryRows(state.selectedHistoryRows, state.symbol, state.category);

    await syncHistoryAliasCsv();
    console.log(`Tryb: historia | rekordy: ${state.selectedHistoryRows.length} | symbol: ${state.symbol}`);
    console.log(`Plik: ${state.selectedHistoryPath}`);
    return true;
  }

  state.mode = "live";
  state.symbol = normalizeSymbol(parseArg("symbol") || "BTCUSDT") || "BTCUSDT";
  state.category = String(parseArg("category") || "linear").toLowerCase();

  await ensureSessionCsv(state.symbol);
  await startMarketSignal({
    symbol: state.symbol,
    category: state.category,
    pollMs: 10_000,
    silent: true,
    debug: false,
  });

  startLiveCsvCapture();
  console.log(`Tryb: live | symbol: ${state.symbol}`);
  console.log(`CSV sesji: ${state.sessionCsvPath}`);
  return true;
}

function rowsFromCsv(raw) {
  const lines = raw.split(/\r?\n/).filter((line) => line.trim());
  if (!lines.length) return [];

  const header = lines[0].split(",").map((h) => h.trim());
  const idxTs = header.indexOf("ts");
  const idxTsMs = header.indexOf("tsMs");
  const idxSymbol = header.indexOf("symbol");
  const idxCategory = header.indexOf("category");
  const idxPrice = header.indexOf("price");
  const idxRegime = header.indexOf("regime");
  const idxCci = header.indexOf("cci");
  const idxAdx14 = header.indexOf("adx14");
  const idxAtrPct = header.indexOf("atrPct");
  const idxEmaSpreadPct = header.indexOf("emaSpreadPct");
  const idxMomentumScore = header.indexOf("momentumScore");
  const idxPressureScore = header.indexOf("pressureScore");
  const idxShockScore = header.indexOf("shockScore");
  const idxShockDir = header.indexOf("shockDir");
  const idxBotSignal = header.indexOf("botSignal");

  if (idxPrice < 0) return [];

  const rows = [];
  for (let i = 1; i < lines.length; i++) {
    const cols = lines[i].split(",");
    const tsMs = safeNum(cols[idxTsMs], 0);
    const ts = (idxTs >= 0 ? cols[idxTs] : null) || (tsMs > 0 ? toIso(tsMs) : null);
    const price = safeNum(cols[idxPrice], NaN);
    if (!Number.isFinite(price)) continue;

    rows.push({
      ts: ts || toIso(nowMs()),
      tsMs: tsMs > 0 ? tsMs : new Date(ts || Date.now()).getTime(),
      symbol: idxSymbol >= 0 ? String(cols[idxSymbol] || "").trim() : state.symbol,
      category: idxCategory >= 0 ? String(cols[idxCategory] || "").trim() : state.category,
      price,
      regime: idxRegime >= 0 ? String(cols[idxRegime] || "").trim() : null,
      cci: idxCci >= 0 ? numOrNull(cols[idxCci]) : null,
      adx14: idxAdx14 >= 0 ? numOrNull(cols[idxAdx14]) : null,
      atrPct: idxAtrPct >= 0 ? numOrNull(cols[idxAtrPct]) : null,
      emaSpreadPct: idxEmaSpreadPct >= 0 ? numOrNull(cols[idxEmaSpreadPct]) : null,
      momentumScore: idxMomentumScore >= 0 ? numOrNull(cols[idxMomentumScore]) : null,
      pressureScore: idxPressureScore >= 0 ? numOrNull(cols[idxPressureScore]) : null,
      shockScore: idxShockScore >= 0 ? numOrNull(cols[idxShockScore]) : null,
      shockDir: idxShockDir >= 0 ? numOrNull(cols[idxShockDir]) : null,
      botSignal: idxBotSignal >= 0 ? numOrNull(cols[idxBotSignal]) : null,
    });
  }

  rows.sort((a, b) => a.tsMs - b.tsMs);
  return rows;
}

function rowsFromJsonl(raw) {
  const out = [];
  for (const line of raw.split(/\r?\n/)) {
    if (!line.trim()) continue;
    try {
      const r = JSON.parse(line);
      const tsMs = safeNum(r.tsMs, 0);
      const price = safeNum(r.price, NaN);
      if (!Number.isFinite(tsMs) || !Number.isFinite(price)) continue;
      out.push({
        ts: r.ts || toIso(tsMs),
        tsMs,
        symbol: String(r.symbol || state.symbol),
        category: String(r.category || state.category),
        price,
        regime: r.regime ?? null,
        cci: numOrNull(r.cci),
        adx14: numOrNull(r.adx14),
        atrPct: numOrNull(r.atrPct),
        emaSpreadPct: numOrNull(r.emaSpreadPct),
        momentumScore: numOrNull(r.momentumScore),
        pressureScore: numOrNull(r.pressureScore),
        shockScore: numOrNull(r.shockScore),
        shockDir:          numOrNull(r.shockDir),
        botSignal:         numOrNull(r.botSignal),
        impulseDepthScore: numOrNull(r.impulseDepthScore),
        volRatio:          numOrNull(r.volRatio),
        obImbalance:       numOrNull(r.obImbalance),
        ob3Imbalance:      numOrNull(r.ob3Imbalance),
        obSpread:          numOrNull(r.obSpread),
        obBidConc:         numOrNull(r.obBidConc),
        obAskConc:         numOrNull(r.obAskConc),
        obBidWallPct:      numOrNull(r.obBidWallPct),
        obAskWallPct:      numOrNull(r.obAskWallPct),
        obMicroAdj:        numOrNull(r.obMicroAdj),
      });
    } catch {
    }
  }
  out.sort((a, b) => a.tsMs - b.tsMs);
  return out;
}

async function listTradeHistoryCsvFiles() {
  try {
    const entries = await fs.readdir(RUNS_DIR, { withFileTypes: true });
    const out = [];
    for (const ent of entries) {
      if (!ent.isDirectory()) continue;
      const name = String(ent.name || "");
      // Check for fill_events.tsv directly in this directory
      const abs = path.resolve(RUNS_DIR, name, "fill_events.tsv");
      let st;
      try {
        st = await fs.stat(abs);
        out.push({ name, mtimeMs: Number(st.mtimeMs || 0), size: Number(st.size || 0) });
      } catch {
        // Not found at top level — scan one level deeper (handles "BTC/USDT_PAPER_..." nested runs)
        try {
          const subEntries = await fs.readdir(path.resolve(RUNS_DIR, name), { withFileTypes: true });
          for (const sub of subEntries) {
            if (!sub.isDirectory()) continue;
            const subName = `${name}/${String(sub.name || "")}`;
            const subAbs = path.resolve(RUNS_DIR, subName, "fill_events.tsv");
            let subSt;
            try { subSt = await fs.stat(subAbs); } catch { continue; }
            out.push({ name: subName, mtimeMs: Number(subSt.mtimeMs || 0), size: Number(subSt.size || 0) });
          }
        } catch { /* ignore */ }
      }
    }
    out.sort((a, b) => b.mtimeMs - a.mtimeMs);
    return out;
  } catch {
    return [];
  }
}

function splitTsvLine(line) {
  return String(line || "").split("\t").map((v) => String(v || "").trim());
}

function parseTradeHistoryCsv(raw) {
  const lines = String(raw || "").split(/\r?\n/).filter((l) => String(l).trim().length > 0);
  if (!lines.length) return { summary: {}, headers: [], rows: [] };

  const headers = splitTsvLine(lines[0]);
  const rows = [];
  for (let i = 1; i < lines.length; i += 1) {
    const cols = splitTsvLine(lines[i]);
    if (!cols.length) continue;
    const row = {};
    for (let k = 0; k < headers.length; k += 1) {
      const key = String(headers[k] || "").trim();
      if (!key) continue;
      row[key] = cols[k] ?? "";
    }
    rows.push(row);
  }

  const wanted = [
    "ts",
    "market",
    "side",
    "kind",
    "price",
    "qty_base",
    "quote_amount",