// Initial input validation

if (empty($url) || !filter_var($url, FILTER_VALIDATE_URL)) {

error_log("Invalid or empty URL provided to summarize_article: " . $url);

return false;

}

// Placeholder for fetching article content

$htmlContent = fetch_article_content($url);

if ($htmlContent === false) {

error_log("Failed to fetch content for URL: " . $url);

return false;

}

// Placeholder for extracting main text

$mainText = extract_main_text($htmlContent, $url);

if (empty($mainText)) {

error_log("Failed to extract main text from URL: " . $url);

return false;

}

// Placeholder for summarizing text

$summary = generate_summary($mainText);

if (empty($summary)) {

error_log("Failed to generate summary for URL: " . $url);

return false;

}

return $summary;

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, $url);

curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); // Follow redirects

curl_setopt($ch, CURLOPT_MAXREDIRS, 5); // Limit redirects

curl_setopt($ch, CURLOPT_TIMEOUT, 30); // Timeout in seconds

curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; ArticleSummarizerBot/1.0; +http://example.com/bot)'); // Set a user agent

curl_setopt($ch, CURLOPT_FAILONERROR, true); // Fail on HTTP errors (4xx or 5xx)

// It's good practice to verify SSL certificates.

// For local testing, you might need to point to a CA bundle if not configured in php.ini

// curl_setopt($ch, CURLOPT_CAINFO, '/path/to/cacert.pem');

// curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, true);

// curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);

// However, for general robustness in varied environments, sometimes VERIFYPEER is set to false.

// This is a security risk for production if you don't trust the source.

// For this example, I'll leave it to PHP's default or server config.

$htmlContent = curl_exec($ch);

$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);

if (curl_errno($ch)) {

error_log("cURL error for " . $url . ": " . curl_error($ch));

curl_close($ch);

return false;

}

// Check for HTTP success codes (2xx)

// CURLOPT_FAILONERROR handles 4xx/5xx, but it's good to be explicit or handle other 2xx codes if needed.

// For simplicity, we assume FAILONERROR has done its job if no curl_errno.

// if ($httpCode >= 200 && $httpCode < 300) {

// // Content fetched successfully

// } else {

// error_log("HTTP error for " . $url . ": Status code " . $httpCode);

// curl_close($ch);

// return false;

// }

curl_close($ch);

if ($htmlContent === false) {

// This case might be redundant if curl_errno is checked, but good for safety.

error_log("curl_exec returned false without a specific cURL error for URL: " . $url);

return false;

}

return $htmlContent;

if (empty($htmlContent)) {

return "";

}

$doc = new DOMDocument();

// Suppress errors from invalid HTML

@$doc->loadHTML($htmlContent);

if ($doc === false) {

error_log("Failed to parse HTML for main text extraction from URL: " . $url);

return "";

}

$xpath = new DOMXPath($doc);

// 1. Remove script and style elements

foreach ($xpath->query('//script | //style') as $node) {

$node->parentNode->removeChild($node);

}

// 2. Attempt to find common article containers

// This is a simplified list. More sophisticated selectors would be needed for robustness.

$queries = [

'//article',

'//*[contains(@class, "article-content")]',

'//*[contains(@class, "post-content")]',

'//*[contains(@class, "main-content")]',

'//*[contains(@id, "article-body")]',

'//*[contains(@id, "content")]',

'//*[contains(@class, "story")]',

// Add more specific selectors if known for target sites

];

$mainContentNode = null;

foreach ($queries as $query) {

$nodes = $xpath->query($query);

if ($nodes->length > 0) {

// For simplicity, take the first one found.

// A better approach might score them based on text length or p tags.

$mainContentNode = $nodes->item(0);

break;

}

}

$extractedText = "";

if ($mainContentNode) {

// If a specific container is found, try to get text only from it.

// Remove common noise tags from within this container too.

$noiseSelectors = ['.//nav', './/header', './/footer', './/aside', './/*[contains(@class, "sidebar")]', './/*[contains(@class, "comments")]', './/*[contains(@id, "comments")]', './/*[contains(@class, "advertisement")]', './/*[contains(@class, "ad-")]'];

foreach ($noiseSelectors as $selector) {

foreach ($xpath->query($selector, $mainContentNode) as $node) {

$node->parentNode->removeChild($node);

}

}

$paragraphs = $xpath->query('.//p', $mainContentNode);

foreach ($paragraphs as $p) {

$extractedText .= trim($p->textContent) . "\n\n";

}

// If no p tags, get all text content of the node

if (empty(trim($extractedText))) {

$extractedText = trim($mainContentNode->textContent);

}

} else {

// Fallback: If no specific container found, try to get all text from body,

// excluding common noise tags from the whole body.

// This is less precise.

$bodyNode = $xpath->query('//body')->item(0);

if ($bodyNode) {

$noiseSelectors = ['//nav', '//header', '//footer', '//aside', '//*[contains(@class, "sidebar")]', '//*[contains(@class, "comments")]', '//*[contains(@id, "comments")]', '//*[contains(@class, "advertisement")]', '//*[contains(@class, "ad-")]'];

foreach ($noiseSelectors as $selector) {

foreach ($xpath->query($selector) as $node) { // Query from root

if ($node->parentNode) { // Check if node still exists

$node->parentNode->removeChild($node);

}

}

}

// Get text from all paragraph tags in the body

$paragraphs = $xpath->query('//body//p');

if ($paragraphs->length > 0) {

foreach ($paragraphs as $p) {

$extractedText .= trim($p->textContent) . "\n\n";

}

} else {

// If no p tags, get all text from body (might be noisy)

$extractedText = trim($bodyNode->textContent);

}

}

}

// Normalize whitespace and clean up

$extractedText = preg_replace('/\s\s+/', ' ', $extractedText); // Replace multiple spaces with single

$extractedText = preg_replace('/\n\s*\n/', "\n\n", $extractedText); // Normalize multiple newlines

return trim($extractedText);

if (empty(trim($text))) {

return "Could not generate summary: Input text is empty.";

}

// 1. Sentence Segmentation (basic)

// This regex might not be perfect for all cases (e.g., Mr. Jones) but is a common start.

$sentences = preg_split('/(?<=[.?!])\s+(?=[A-Z"\'])/', trim($text));

if (empty($sentences) || count($sentences) <= $numSentences) {

return $text; // Return original text if it's short or sentence splitting fails

}

// 2. Stop Words (a small, common list)

$stopWords = [

'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', "aren't", 'as', 'at',

'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by',

'can', "can't", 'cannot', 'could', "couldn't", 'did', "didn't", 'do', 'does', "doesn't", 'doing', "don't", 'down', 'during',

'each', 'few', 'for', 'from', 'further', 'had', "hadn't", 'has', "hasn't", 'have', "haven't", 'having', 'he', "he'd", "he'll", "he's", 'her', 'here', "here's", 'hers', 'herself', 'him', 'himself', 'his', 'how', "how's",

'i', "i'd", "i'll", "i'm", "i've", 'if', 'in', 'into', 'is', "isn't", 'it', "it's", 'its', 'itself',

"let's", 'me', 'more', 'most', "mustn't", 'my', 'myself',

'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours', 'ourselves', 'out', 'over', 'own',

'same', "shan't", 'she', "she'd", "she'll", "she's", 'should', "shouldn't", 'so', 'some', 'such',

'than', 'that', "that's", 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', "there's", 'these', 'they', "they'd", "they'll", "they're", "they've", 'this', 'those', 'through', 'to', 'too',

'under', 'until', 'up', 'very', 'was', "wasn't", 'we', "we'd", "we'll", "we're", "we've", 'were', "weren't", 'what', "what's", 'when', "when's", 'where', "where's", 'which', 'while', 'who', "who's", 'whom', 'why', "why's", 'with', "won't", 'would', "wouldn't",

'you', "you'd", "you'll", "you're", "you've", 'your', 'yours', 'yourself', 'yourselves',

// Common for news

'said', 'mr', 'mrs', 'ms', 'dr', 'prof'

];

$stopWordsRegex = '/\b(' . implode('|', $stopWords) . ')\b/i';

// 3. Word Frequency Calculation

$wordFrequencies = [];

foreach ($sentences as $sentence) {

$cleanedSentence = preg_replace($stopWordsRegex, '', strtolower($sentence));

$words = str_word_count($cleanedSentence, 1); // Get words as an array

if (is_array($words)) {

foreach ($words as $word) {

if (strlen($word) > 2) { // Ignore very short words (like 's from possession)

$wordFrequencies[$word] = ($wordFrequencies[$word] ?? 0) + 1;

}

}

}

}

if(empty($wordFrequencies)) { // If all words were stop words or text was very short

return implode('. ', array_slice($sentences, 0, $numSentences)) . '.';

}

// 4. Sentence Scoring

$sentenceScores = [];

foreach ($sentences as $index => $sentence) {

$cleanedSentence = preg_replace($stopWordsRegex, '', strtolower($sentence));

$wordsInSentence = str_word_count($cleanedSentence, 1);

$score = 0;

if (is_array($wordsInSentence)) {

foreach ($wordsInSentence as $word) {

if (strlen($word) > 2 && isset($wordFrequencies[$word])) {

$score += $wordFrequencies[$word];

}

}

}

// Normalize score by sentence length to avoid bias towards longer sentences

$sentenceScores[$index] = count($wordsInSentence) > 0 ? $score / count($wordsInSentence) : 0;

}

// 5. Summary Selection

arsort($sentenceScores); // Sort sentences by score in descending order

$topSentenceIndices = array_slice(array_keys($sentenceScores), 0, $numSentences);

sort($topSentenceIndices); // Sort back by original order

$summary = "";

foreach ($topSentenceIndices as $index) {

$summary .= trim($sentences[$index]) . ". "; // Add period if missing, then a space

}

return rtrim($summary); // Remove trailing space