Code Coverage |
||||||||||
Lines |
Functions and Methods |
Classes and Traits |
||||||||
| Total | |
0.00% |
0 / 42 |
|
0.00% |
0 / 2 |
CRAP | |
0.00% |
0 / 1 |
| WebScraperService | |
0.00% |
0 / 42 |
|
0.00% |
0 / 2 |
110 | |
0.00% |
0 / 1 |
| scrape | |
0.00% |
0 / 18 |
|
0.00% |
0 / 1 |
6 | |||
| extractText | |
0.00% |
0 / 24 |
|
0.00% |
0 / 1 |
72 | |||
| 1 | <?php |
| 2 | |
| 3 | namespace App\Http\Services; |
| 4 | |
| 5 | use GuzzleHttp\Client; |
| 6 | use Illuminate\Support\Facades\Log; |
| 7 | use Symfony\Component\DomCrawler\Crawler; |
| 8 | |
| 9 | /** |
| 10 | * Service for scraping website content and extracting meaningful text. |
| 11 | * |
| 12 | * Used by the roleplay auto-populate feature to extract company/product |
| 13 | * information from a given URL. Returns empty string on failure so the |
| 14 | * caller can fall back to AI grounding. |
| 15 | */ |
| 16 | class WebScraperService |
| 17 | { |
| 18 | /** |
| 19 | * Maximum number of characters to return from scraped content. |
| 20 | */ |
| 21 | private const MAX_CONTENT_LENGTH = 15000; |
| 22 | |
| 23 | /** |
| 24 | * Scrape a website URL and extract meaningful text content. |
| 25 | * |
| 26 | * Fetches the HTML, removes non-content elements (scripts, styles, nav, footer), |
| 27 | * and extracts text from headings, paragraphs, and list items. |
| 28 | * |
| 29 | * Returns an empty string if the URL cannot be fetched or yields no content, |
| 30 | * allowing the caller to fall back to AI grounding. |
| 31 | * |
| 32 | * @param string $url The URL to scrape |
| 33 | * @return string The extracted text content (truncated to 15,000 chars), or empty string on failure |
| 34 | */ |
| 35 | public function scrape(string $url): string |
| 36 | { |
| 37 | try { |
| 38 | $client = new Client([ |
| 39 | 'timeout' => 15, |
| 40 | 'connect_timeout' => 10, |
| 41 | 'headers' => [ |
| 42 | 'User-Agent' => 'Mozilla/5.0 (compatible; FlyMSG/1.0)', |
| 43 | 'Accept' => 'text/html,application/xhtml+xml', |
| 44 | ], |
| 45 | 'verify' => true, |
| 46 | ]); |
| 47 | |
| 48 | $response = $client->request('GET', $url); |
| 49 | $html = $response->getBody()->getContents(); |
| 50 | |
| 51 | return $this->extractText($html); |
| 52 | } catch (\Exception $e) { |
| 53 | Log::warning('[WebScraperService] Failed to scrape URL', [ |
| 54 | 'url' => $url, |
| 55 | 'error' => $e->getMessage(), |
| 56 | ]); |
| 57 | |
| 58 | return ''; |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | /** |
| 63 | * Extract meaningful text from raw HTML content. |
| 64 | * |
| 65 | * @param string $html Raw HTML content |
| 66 | * @return string Cleaned text content |
| 67 | */ |
| 68 | private function extractText(string $html): string |
| 69 | { |
| 70 | $crawler = new Crawler($html); |
| 71 | |
| 72 | // Remove non-content elements |
| 73 | $crawler->filter('script, style, nav, footer, header, iframe, noscript, svg')->each(function ($node) { |
| 74 | $domNode = $node->getNode(0); |
| 75 | if ($domNode && $domNode->parentNode) { |
| 76 | $domNode->parentNode->removeChild($domNode); |
| 77 | } |
| 78 | }); |
| 79 | |
| 80 | $texts = []; |
| 81 | |
| 82 | // Meta description |
| 83 | $metaDesc = $crawler->filter('meta[name="description"]'); |
| 84 | if ($metaDesc->count() > 0) { |
| 85 | $content = $metaDesc->attr('content'); |
| 86 | if ($content) { |
| 87 | $texts[] = $content; |
| 88 | } |
| 89 | } |
| 90 | |
| 91 | // Page title |
| 92 | $title = $crawler->filter('title'); |
| 93 | if ($title->count() > 0) { |
| 94 | $texts[] = $title->text(''); |
| 95 | } |
| 96 | |
| 97 | // Headings and body content |
| 98 | $crawler->filter('h1, h2, h3, p, li, td, blockquote')->each(function ($node) use (&$texts) { |
| 99 | $text = trim($node->text('')); |
| 100 | if (strlen($text) > 10) { |
| 101 | $texts[] = $text; |
| 102 | } |
| 103 | }); |
| 104 | |
| 105 | $cleanedText = implode("\n", array_filter($texts)); |
| 106 | |
| 107 | if (strlen($cleanedText) > self::MAX_CONTENT_LENGTH) { |
| 108 | $cleanedText = substr($cleanedText, 0, self::MAX_CONTENT_LENGTH); |
| 109 | } |
| 110 | |
| 111 | return $cleanedText; |
| 112 | } |
| 113 | } |