src/meta/offerFamilyClassifier.js

Repository

Demand-Radar

Original path

src/meta/offerFamilyClassifier.js

Role

SOURCE

Size

27192 bytes

Lines

925

SHA-256

8e6a6229b4d651e598884af1b184d18eaf9b0081bfc0b4bb937d012c87283c5c

Displayed range

1–500

const crypto = require('crypto');

const DAY_MS = 24 * 60 * 60 * 1000;

const OFFER_FAMILY_THRESHOLDS = {
  TEST_MAX_DAYS: 7,
  REPEATED_TEST_MAX_DAYS: 14,
  ESTABLISHED_MIN_DAYS: 30,
  EVERGREEN_MIN_DAYS: 90,
  DIRECT_SUCCESSOR_MAX_GAP: 7,
  SUCCESSOR_AFTER_GAP_MAX: 29,
  RELAUNCH_MIN_GAP: 30,
  EDGE_LINK_MIN_CONFIDENCE: 70,
};

const FAMILY_CLASS_ORDER = {
  EVERGREEN: 1,
  ESTABLISHED: 2,
  PROMISING: 3,
  REPEATED_TEST: 4,
  TEST_ONLY: 5,
  UNCLASSIFIED: 6,
};

const TRACKING_PARAM_PREFIXES = [
  'utm_',
  'fb_',
  'ga_',
  'mc_',
];

const TRACKING_PARAM_EXACT = new Set([
  'utm_source',
  'utm_medium',
  'utm_campaign',
  'utm_content',
  'utm_term',
  'fbclid',
  'gclid',
  'mc_cid',
  'mc_eid',
  'ref',
  'ref_src',
  'igshid',
  'hsa_cam',
  'hsa_grp',
  'hsa_ad',
  'hsa_src',
  'hsa_net',
  'hsa_kw',
  'hsa_acc',
  's',
  'si',
]);

function toDate(value) {
  if (!value) return null;
  const date = new Date(value);
  if (Number.isNaN(date.getTime())) return null;
  return date;
}

function toIso(value) {
  if (!value) return null;
  const date = toDate(value);
  return date ? date.toISOString() : null;
}

function diffDays(startDate, endDate) {
  if (!startDate || !endDate) return null;
  const diff = Math.floor((endDate.getTime() - startDate.getTime()) / DAY_MS);
  return diff >= 0 ? diff : 0;
}

function normalizePath(pathname) {
  let path = pathname || '/';
  if (!path.startsWith('/')) {
    path = `/${path}`;
  }

  if (path.length > 1 && path.endsWith('/')) {
    path = path.slice(0, -1);
  }

  return path;
}

function isTrackingParam(name) {
  if (!name) return false;
  const lower = name.toLowerCase();
  if (TRACKING_PARAM_EXACT.has(lower)) return true;
  return TRACKING_PARAM_PREFIXES.some(prefix => lower.startsWith(prefix));
}

function canonicalizeUrl(urlText) {
  if (!urlText || typeof urlText !== 'string') return null;

  try {
    const parsed = new URL(urlText.trim());
    parsed.hash = '';

    const protocol = parsed.protocol.toLowerCase();
    const host = parsed.hostname.toLowerCase().replace(/^www\./, '');
    const path = normalizePath(parsed.pathname);

    const params = [];
    for (const [name, value] of parsed.searchParams.entries()) {
      if (isTrackingParam(name)) continue;
      params.push([name, value]);
    }

    params.sort((left, right) => {
      if (left[0] === right[0]) {
        return left[1].localeCompare(right[1]);
      }
      return left[0].localeCompare(right[0]);
    });

    const query = params.length
      ? `?${params.map(([name, value]) => `${encodeURIComponent(name)}=${encodeURIComponent(value)}`).join('&')}`
      : '';

    return `${protocol}//${host}${path}${query}`;
  } catch {
    return null;
  }
}

function normalizeText(value) {
  return String(value || '')
    .toLowerCase()
    .trim()
    .replace(/[\u2018\u2019\u201c\u201d]/g, ' ')
    .replace(/[^\p{L}\p{N}\s]+/gu, ' ')
    .replace(/\s+/g, ' ');
}

function toTokenSet(value) {
  const normalized = normalizeText(value);
  if (!normalized) return new Set();
  return new Set(normalized.split(' ').filter(Boolean));
}

function jaccardSimilarity(leftText, rightText) {
  const left = toTokenSet(leftText);
  const right = toTokenSet(rightText);
  if (!left.size && !right.size) return 0;

  let intersection = 0;
  for (const token of left) {
    if (right.has(token)) intersection += 1;
  }

  const union = left.size + right.size - intersection;
  if (!union) return 0;
  return intersection / union;
}

function combinedTextSimilarity(leftAd, rightAd) {
  const titleSimilarity = jaccardSimilarity(leftAd.title, rightAd.title);
  const bodySimilarity = jaccardSimilarity(leftAd.bodyText, rightAd.bodyText);

  const leftHasBody = normalizeText(leftAd.bodyText).length > 0;
  const rightHasBody = normalizeText(rightAd.bodyText).length > 0;

  const combinedSimilarity = leftHasBody && rightHasBody
    ? (titleSimilarity * 0.45) + (bodySimilarity * 0.55)
    : titleSimilarity;

  return {
    titleSimilarity,
    bodySimilarity,
    combinedSimilarity,
  };
}

function getDomainAndPath(urlText) {
  const canonical = canonicalizeUrl(urlText);
  if (!canonical) {
    return {
      canonical: null,
      domain: null,
      path: null,
    };
  }

  try {
    const parsed = new URL(canonical);
    return {
      canonical,
      domain: parsed.hostname,
      path: normalizePath(parsed.pathname),
    };
  } catch {
    return {
      canonical,
      domain: null,
      path: null,
    };
  }
}

function pathSimilarity(leftPath, rightPath) {
  if (!leftPath || !rightPath) return 0;
  if (leftPath === rightPath) return 1;

  const leftParts = leftPath.split('/').filter(Boolean);
  const rightParts = rightPath.split('/').filter(Boolean);
  if (!leftParts.length || !rightParts.length) return 0;

  let samePrefix = 0;
  const max = Math.min(leftParts.length, rightParts.length);
  for (let index = 0; index < max; index += 1) {
    if (leftParts[index] === rightParts[index]) {
      samePrefix += 1;
    } else {
      break;
    }
  }

  return samePrefix / Math.max(leftParts.length, rightParts.length);
}

function buildPairKey(leftId, rightId) {
  return leftId < rightId ? `${leftId}::${rightId}` : `${rightId}::${leftId}`;
}

function buildFamilyId(projectId, memberIds) {
  const hash = crypto
    .createHash('sha1')
    .update(`${projectId}|${memberIds.join('|')}`)
    .digest('hex')
    .slice(0, 24);
  return `fam_${hash}`;
}

class UnionFind {
  constructor(ids) {
    this.parent = new Map();
    this.rank = new Map();
    for (const id of ids) {
      this.parent.set(id, id);
      this.rank.set(id, 0);
    }
  }

  find(id) {
    const parent = this.parent.get(id);
    if (parent === id) return id;
    const root = this.find(parent);
    this.parent.set(id, root);
    return root;
  }

  union(leftId, rightId) {
    const leftRoot = this.find(leftId);
    const rightRoot = this.find(rightId);
    if (leftRoot === rightRoot) return;

    const leftRank = this.rank.get(leftRoot) || 0;
    const rightRank = this.rank.get(rightRoot) || 0;

    if (leftRank < rightRank) {
      this.parent.set(leftRoot, rightRoot);
      return;
    }

    if (leftRank > rightRank) {
      this.parent.set(rightRoot, leftRoot);
      return;
    }

    this.parent.set(rightRoot, leftRoot);
    this.rank.set(leftRoot, leftRank + 1);
  }
}

function buildOfferMatch(leftAd, rightAd) {
  const reasons = [];

  if (!leftAd.pageId || !rightAd.pageId || leftAd.pageId !== rightAd.pageId) {
    return {
      shouldLink: false,
      confidence: 0,
      matchType: 'PAGE_MISMATCH',
      reasons: ['inne page_id'],
      titleSimilarity: 0,
      bodySimilarity: 0,
      combinedSimilarity: 0,
    };
  }

  reasons.push('same page_id');

  const leftUrl = getDomainAndPath(leftAd.destinationUrl);
  const rightUrl = getDomainAndPath(rightAd.destinationUrl);

  const text = combinedTextSimilarity(leftAd, rightAd);

  if (leftUrl.canonical && rightUrl.canonical && leftUrl.canonical === rightUrl.canonical) {
    reasons.push('same canonical destination URL');
    if (text.titleSimilarity >= 0.7) {
      reasons.push(`title similarity ${text.titleSimilarity.toFixed(2)}`);
      return {
        shouldLink: true,
        confidence: 100,
        matchType: 'PAGE_CANONICAL_URL_EXACT',
        reasons,
        ...text,
      };
    }

    return {
      shouldLink: true,
      confidence: 98,
      matchType: 'PAGE_CANONICAL_URL_EXACT',
      reasons,
      ...text,
    };
  }

  if (leftAd.collationId && rightAd.collationId && leftAd.collationId === rightAd.collationId) {
    reasons.push('same collation_id');
    return {
      shouldLink: true,
      confidence: 95,
      matchType: 'PAGE_COLLATION_ID_EXACT',
      reasons,
      ...text,
    };
  }

  if (
    leftUrl.canonical
    && rightUrl.canonical
    && leftUrl.canonical === rightUrl.canonical
    && text.titleSimilarity >= 0.62
  ) {
    reasons.push('same canonical URL');
    reasons.push(`title similarity ${text.titleSimilarity.toFixed(2)}`);
    return {
      shouldLink: true,
      confidence: 90,
      matchType: 'PAGE_CANONICAL_TITLE_SIMILAR',
      reasons,
      ...text,
    };
  }

  const sameDomain = leftUrl.domain && rightUrl.domain && leftUrl.domain === rightUrl.domain;
  const urlPathSimilarity = pathSimilarity(leftUrl.path, rightUrl.path);

  if (sameDomain && text.combinedSimilarity >= 0.72 && urlPathSimilarity >= 0.34) {
    const confidence = Math.min(
      89,
      Math.round(78 + ((text.combinedSimilarity - 0.72) * 20) + (urlPathSimilarity * 6)),
    );

    reasons.push('same domain');
    reasons.push(`title similarity ${text.titleSimilarity.toFixed(2)}`);
    reasons.push(`body similarity ${text.bodySimilarity.toFixed(2)}`);
    reasons.push(`path similarity ${urlPathSimilarity.toFixed(2)}`);

    return {
      shouldLink: confidence >= OFFER_FAMILY_THRESHOLDS.EDGE_LINK_MIN_CONFIDENCE,
      confidence,
      matchType: 'PAGE_TEXT_DOMAIN_SIMILARITY',
      reasons,
      ...text,
    };
  }

  if (text.combinedSimilarity >= 0.9) {
    const confidence = Math.min(84, Math.round(70 + ((text.combinedSimilarity - 0.9) * 40)));
    reasons.push(`title similarity ${text.titleSimilarity.toFixed(2)}`);
    reasons.push(`body similarity ${text.bodySimilarity.toFixed(2)}`);
    reasons.push('destination URL changed');

    return {
      shouldLink: confidence >= OFFER_FAMILY_THRESHOLDS.EDGE_LINK_MIN_CONFIDENCE,
      confidence,
      matchType: 'PAGE_TEXT_HIGH_URL_CHANGED',
      reasons,
      ...text,
    };
  }

  return {
    shouldLink: false,
    confidence: Math.round(text.combinedSimilarity * 60),
    matchType: 'INSUFFICIENT_EVIDENCE',
    reasons: [...reasons, `combined similarity ${text.combinedSimilarity.toFixed(2)}`],
    ...text,
  };
}

function buildIntervals(ads, now) {
  const ranges = [];

  for (const ad of ads) {
    const start = toDate(ad.startDate || ad.firstSeenAt);
    if (!start) continue;

    const stop = toDate(ad.endDate) || (ad.isActive ? now : toDate(ad.lastSeenAt)) || start;
    const end = stop.getTime() < start.getTime() ? start : stop;

    ranges.push({
      start,
      end,
    });
  }

  ranges.sort((left, right) => left.start.getTime() - right.start.getTime());
  return ranges;
}

function unionIntervalDays(intervals) {
  if (!intervals.length) return 0;

  let covered = 0;
  let currentStart = intervals[0].start;
  let currentEnd = intervals[0].end;

  for (let index = 1; index < intervals.length; index += 1) {
    const next = intervals[index];
    if (next.start.getTime() <= currentEnd.getTime()) {
      if (next.end.getTime() > currentEnd.getTime()) {
        currentEnd = next.end;
      }
      continue;
    }

    covered += diffDays(currentStart, currentEnd) + 1;
    currentStart = next.start;
    currentEnd = next.end;
  }

  covered += diffDays(currentStart, currentEnd) + 1;
  return covered;
}

function median(values) {
  if (!values.length) return 0;
  const sorted = [...values].sort((left, right) => left - right);
  const middle = Math.floor(sorted.length / 2);
  if (sorted.length % 2 === 1) return sorted[middle];
  return Math.round((sorted[middle - 1] + sorted[middle]) / 2);
}

function classifyTemporalRelationship(previousAd, currentAd, now) {
  if (!previousAd) {
    return {
      relationshipType: 'FIRST',
      gapDays: null,
    };
  }

  const previousStart = toDate(previousAd.startDate || previousAd.firstSeenAt);
  const previousEnd = toDate(previousAd.endDate)
    || (previousAd.isActive ? now : toDate(previousAd.lastSeenAt))
    || previousStart;

  const currentStart = toDate(currentAd.startDate || currentAd.firstSeenAt);
  if (!previousEnd || !currentStart) {
    return {
      relationshipType: 'UNKNOWN',
      gapDays: null,
    };
  }

  if (currentStart.getTime() === previousEnd.getTime()) {
    return {
      relationshipType: 'DIRECT_SUCCESSOR',
      gapDays: 0,
    };
  }

  if (currentStart.getTime() < previousEnd.getTime()) {
    const overlapDays = Math.abs(diffDays(currentStart, previousEnd));
    return {
      relationshipType: 'PARALLEL',
      gapDays: overlapDays,
    };
  }

  const gapDays = diffDays(previousEnd, currentStart);

  if (gapDays <= OFFER_FAMILY_THRESHOLDS.DIRECT_SUCCESSOR_MAX_GAP) {
    return {
      relationshipType: 'DIRECT_SUCCESSOR',
      gapDays,
    };
  }

  if (gapDays <= OFFER_FAMILY_THRESHOLDS.SUCCESSOR_AFTER_GAP_MAX) {
    return {
      relationshipType: 'SUCCESSOR_AFTER_GAP',
      gapDays,
    };
  }