gadget/gadget-drone/src/tools/network/web-fetcher.ts

// Copyright (C) 2026 Rob Colbert <rob.colbert@openplatform.us>
// Licensed under the Apache License, Version 2.0

import crypto from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";

import { Readability } from "@mozilla/readability";
import { JSDOM } from "jsdom";
import { chromium, type Browser, type Page } from "playwright";
import TurndownService from "turndown";

export interface FetchResult {
  url: string;
  title: string;
  markdown: string;
  totalLineCount: number;
  shownLineCount: number;
  cacheStatus: "hit" | "miss" | "disabled";
}

export class WebFetcher {
  private turndown: TurndownService;

  constructor(private cacheDir: string) {
    this.turndown = new TurndownService({
      headingStyle: "atx",
      codeBlockStyle: "fenced",
      hr: "---",
    });
    this.turndown.remove([
      "script",
      "style",
      "noscript",
      "nav",
      "footer",
      "header",
      "button",
      "input",
      "form",
    ]);
  }

  async fetchUrlWithRange(
    url: string,
    startLine: number = 1,
    endLine?: number,
    useCache: boolean = true,
  ): Promise<FetchResult> {
    const result = await this.fetchUrl(url, useCache);

    if (startLine === 1 && endLine === undefined) {
      return result;
    }

    const lines = result.markdown.split("\n");
    const startIdx = Math.max(0, startLine - 1);
    const endIdx = endLine !== undefined ? Math.min(endLine, lines.length) : lines.length;
    const selectedLines = lines.slice(startIdx, endIdx);

    return {
      ...result,
      markdown: selectedLines.join("\n"),
      shownLineCount: selectedLines.length,
    };
  }

  private async fetchUrl(url: string, useCache: boolean): Promise<FetchResult> {
    if (useCache) {
      const cached = await this.readFromCache(url);
      if (cached) {
        return { ...cached, cacheStatus: "hit" };
      }
    }

    const fetched = await this.fetchFresh(url, useCache ? "miss" : "disabled");
    if (useCache) {
      await this.writeToCache(url, fetched.title, fetched.markdown);
    }
    return fetched;
  }

  private async fetchFresh(
    url: string,
    cacheStatus: FetchResult["cacheStatus"],
  ): Promise<FetchResult> {
    const browser: Browser = await chromium.launch({ headless: true });
    const context = await browser.newContext();
    const page: Page = await context.newPage();

    try {
      await page.goto(url, { waitUntil: "networkidle", timeout: 30000 });
      const title = await page.title();
      const rawHtml = await page.content();
      const doc = new JSDOM(rawHtml, { url });
      const article = new Readability(doc.window.document).parse();

      let htmlContent: string;
      let extractedTitle = title;

      if (article) {
        htmlContent = article.content || "";
        extractedTitle = article.title || title;
      } else {
        htmlContent = await page.evaluate(() => {
          const main = document.querySelector("main") || document.body;
          return main.innerHTML;
        });
      }

      const markdown = this.addLineNumbers(this.turndown.turndown(htmlContent));
      const lineCount = markdown.split("\n").length;

      return {
        url,
        title: extractedTitle,
        markdown,
        totalLineCount: lineCount,
        shownLineCount: lineCount,
        cacheStatus,
      };
    } finally {
      await browser.close();
    }
  }

  private getCacheFilePath(url: string): string {
    const hash = crypto.createHash("sha256").update(url).digest("hex");
    return path.join(this.cacheDir, `fetch-url-${hash}.md`);
  }

  private async ensureCacheDir(): Promise<void> {
    await fs.mkdir(this.cacheDir, { recursive: true });
  }

  private async readFromCache(url: string): Promise<Omit<FetchResult, "cacheStatus"> | null> {
    try {
      const content = await fs.readFile(this.getCacheFilePath(url), "utf-8");
      const lines = content.split("\n");
      const titleLine = lines.find((line) => line.startsWith("## Title:"));
      const fetchedLineIndex = lines.findIndex((line) => line.startsWith("## Fetched:"));
      const markdownStartIndex = fetchedLineIndex === -1 ? 0 : fetchedLineIndex + 2;
      const markdown = lines.slice(markdownStartIndex).join("\n");
      const lineCount = markdown.split("\n").length;

      return {
        url,
        title: titleLine?.replace("## Title:", "").trim() || "Unknown",
        markdown,
        totalLineCount: lineCount,
        shownLineCount: lineCount,
      };
    } catch {
      return null;
    }
  }

  private async writeToCache(url: string, title: string, markdown: string): Promise<void> {
    await this.ensureCacheDir();
    const content = [
      `## URL: ${url}`,
      `## Title: ${title}`,
      `## Fetched: ${new Date().toISOString()}`,
      "",
      markdown,
    ].join("\n");
    await fs.writeFile(this.getCacheFilePath(url), content, "utf-8");
  }

  private addLineNumbers(text: string): string {
    return text
      .split("\n")
      .map((line, index) => `${index + 1}: ${line}`)
      .join("\n");
  }
}