Skip to main content

pacsea/sources/news/
fetch.rs

1//! News fetching functionality with HTTP client and error handling.
2
3use crate::integrations::arch_toolkit::ToolkitContext;
4use crate::sources::news::cache::{ARTICLE_CACHE, ARTICLE_CACHE_TTL_SECONDS, ArticleCacheEntry};
5use crate::sources::news::parse::parse_arch_news_html;
6use crate::sources::news::utils::is_archlinux_url;
7use crate::sources::news::{
8    aur::extract_aur_pkg_from_url,
9    cache::{load_article_entry_from_disk_cache, save_article_to_disk_cache},
10    utils::is_arch_package_url,
11};
12use crate::state::NewsItem;
13use reqwest;
14use std::sync::LazyLock;
15use std::time::{Duration, Instant};
16use tracing::{info, warn};
17
18/// Result type alias for Arch Linux news fetching operations.
19type Result<T> = super::Result<T>;
20
21/// What: Extract cache path from an official package URL.
22///
23/// Inputs:
24/// - `url`: The official package URL.
25///
26/// Output:
27/// - `Some(PathBuf)` if URL is valid; `None` otherwise.
28///
29/// Details:
30/// - Parses URL format: `https://archlinux.org/packages/{repo}/{arch}/{name}/`
31/// - Handles query parameters and fragments in the name.
32fn extract_official_package_cache_path(url: &str) -> Option<std::path::PathBuf> {
33    let lower = url.to_ascii_lowercase();
34    let pos = lower.find("archlinux.org/packages/")?;
35    let after = &url[pos + "archlinux.org/packages/".len()..];
36    let parts: Vec<&str> = after.split('/').filter(|s| !s.is_empty()).collect();
37    if parts.len() >= 3 {
38        let repo = parts[0];
39        let arch = parts[1];
40        let name = parts[2]
41            .split('?')
42            .next()
43            .unwrap_or(parts[2])
44            .split('#')
45            .next()
46            .unwrap_or(parts[2]);
47        Some(crate::sources::official_json_cache_path(repo, arch, name))
48    } else {
49        None
50    }
51}
52
53/// What: Prepend official package JSON changes to content if available.
54///
55/// Inputs:
56/// - `url`: The official package URL.
57/// - `content`: The content to prepend changes to.
58///
59/// Output:
60/// - Content with changes prepended if available; original content otherwise.
61///
62/// Details:
63/// - Only modifies content if changes are detected and not already present.
64fn prepend_official_package_changes(url: &str, content: &str) -> String {
65    let Some(cache_path) = extract_official_package_cache_path(url) else {
66        return content.to_string();
67    };
68
69    let Some(cached_json) = crate::sources::load_official_json_cache(&cache_path) else {
70        return content.to_string();
71    };
72
73    let pkg_obj = cached_json.get("pkg").unwrap_or(&cached_json);
74
75    let Some(pkg_name) = pkg_obj.get("pkgname").and_then(serde_json::Value::as_str) else {
76        return content.to_string();
77    };
78
79    let Some(changes) = crate::sources::get_official_json_changes(pkg_name) else {
80        return content.to_string();
81    };
82
83    if content.starts_with("Changes detected") {
84        content.to_string()
85    } else {
86        format!("{changes}\n\n─── Package Info ───\n\n{content}")
87    }
88}
89
90/// Shared HTTP client with connection pooling for news content fetching.
91/// Connection pooling is enabled by default in `reqwest::Client`.
92/// Uses browser-like headers to work with archlinux.org's `DDoS` protection.
93static HTTP_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
94    use reqwest::header::{ACCEPT, ACCEPT_LANGUAGE, HeaderMap, HeaderValue};
95    let mut headers = HeaderMap::new();
96    // Browser-like Accept header
97    headers.insert(
98        ACCEPT,
99        HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
100    );
101    // Accept-Language header for completeness
102    headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5"));
103    reqwest::Client::builder()
104        .connect_timeout(Duration::from_secs(15))
105        .timeout(Duration::from_secs(30))
106        // Firefox-like User-Agent with Pacsea identifier for transparency
107        .user_agent(format!(
108            "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0 Pacsea/{}",
109            env!("CARGO_PKG_VERSION")
110        ))
111        .default_headers(headers)
112        .build()
113        .expect("Failed to create HTTP client")
114});
115
116/// What: Fetch recent Arch Linux news items with optional early date filtering.
117///
118/// Input:
119/// - `limit`: Maximum number of items to return (best-effort)
120/// - `cutoff_date`: Optional date string (YYYY-MM-DD) for early filtering
121///
122/// Output: `Ok(Vec<NewsItem>)` with date/title/url; `Err` on network or parse failures
123///
124/// # Errors
125/// - Returns `Err` when network request fails (curl execution error)
126/// - Returns `Err` when RSS feed cannot be fetched from Arch Linux website
127/// - Returns `Err` when response body cannot be decoded as UTF-8
128///
129/// Details: Downloads the Arch Linux news RSS feed and iteratively parses `<item>` blocks,
130/// extracting `<title>`, `<link>`, and `<pubDate>`. The `pubDate` value is normalized to a
131/// date-only form via `strip_time_and_tz`. If `cutoff_date` is provided, stops fetching when
132/// items exceed the date limit.
133pub async fn fetch_arch_news(limit: usize, cutoff_date: Option<&str>) -> Result<Vec<NewsItem>> {
134    let context = ToolkitContext::new()?;
135    fetch_arch_news_with_context(&context, limit, cutoff_date).await
136}
137
138/// What: Fetch bounded Arch news with a caller-owned toolkit context.
139///
140/// Inputs:
141/// - `context`: Runtime integration context.
142/// - `limit`: Maximum returned rows.
143/// - `cutoff_date`: Optional oldest accepted date.
144///
145/// Output:
146/// - Pacsea news rows or an actionable error.
147///
148/// Details:
149/// - arch-toolkit owns bounded feed transport and parsing; Pacsea owns aggregation and caching.
150pub async fn fetch_arch_news_with_context(
151    context: &ToolkitContext,
152    limit: usize,
153    cutoff_date: Option<&str>,
154) -> Result<Vec<NewsItem>> {
155    crate::integrations::arch_toolkit::news::fetch_arch_news(context, limit, cutoff_date)
156        .await
157        .map_err(Into::into)
158}
159
160/// What: Fetch the full article content from an Arch news URL.
161///
162/// Inputs:
163/// - `url`: The news article URL (e.g., `https://archlinux.org/news/...`)
164///
165/// Output:
166/// - `Ok(String)` with the article text content; `Err` on network/parse failure.
167///
168/// # Errors
169/// - Network fetch failures
170/// - HTML parsing failures
171///
172/// Details:
173/// - For AUR package URLs, fetches and renders AUR comments instead.
174/// - For Arch news URLs, checks cache first (15-minute in-memory, 14-day disk TTL).
175/// - Applies rate limiting for archlinux.org URLs to prevent aggressive fetching.
176/// - Fetches the HTML page and extracts content from the article body.
177/// - Strips HTML tags and normalizes whitespace.
178/// - Caches successful fetches in both in-memory and disk caches.
179pub async fn fetch_news_content(url: &str) -> Result<String> {
180    use crate::sources::news::aur::render_aur_comments;
181
182    if let Some(pkg) = extract_aur_pkg_from_url(url) {
183        // Check for JSON changes first
184        let changes = crate::sources::get_aur_json_changes(&pkg);
185        let comments = crate::sources::fetch_aur_comments(pkg.clone()).await?;
186        let mut rendered = render_aur_comments(&pkg, &comments);
187
188        // Prepend JSON changes if available
189        if let Some(changes_text) = changes {
190            rendered = format!("{changes_text}\n\n─── AUR Comments ───\n\n{rendered}");
191        }
192
193        return Ok(rendered);
194    }
195
196    // Check for official package URL and load cached JSON to get package name and changes
197    if is_arch_package_url(url)
198        && let Ok(cache) = ARTICLE_CACHE.lock()
199        && let Some(entry) = cache.get(url)
200        && entry.timestamp.elapsed().as_secs() < ARTICLE_CACHE_TTL_SECONDS
201    {
202        let content = prepend_official_package_changes(url, &entry.content);
203        return Ok(content);
204    }
205
206    // 1. Check in-memory cache first (fastest, 15-minute TTL)
207    let cached_entry: Option<ArticleCacheEntry> = if let Ok(cache) = ARTICLE_CACHE.lock()
208        && let Some(entry) = cache.get(url)
209        && entry.timestamp.elapsed().as_secs() < ARTICLE_CACHE_TTL_SECONDS
210    {
211        info!(url, "using in-memory cached article content");
212        return Ok(entry.content.clone());
213    } else {
214        None
215    };
216
217    // 2. Check disk cache (14-day TTL) - useful after app restart
218    let disk_entry = load_article_entry_from_disk_cache(url);
219    if let Some(ref entry) = disk_entry {
220        // Populate in-memory cache from disk
221        if let Ok(mut cache) = ARTICLE_CACHE.lock() {
222            cache.insert(
223                url.to_string(),
224                ArticleCacheEntry {
225                    content: entry.content.clone(),
226                    timestamp: Instant::now(),
227                    etag: entry.etag.clone(),
228                    last_modified: entry.last_modified.clone(),
229                },
230            );
231        }
232        // Check for official package changes and prepend if available
233        if is_arch_package_url(url) {
234            let content = prepend_official_package_changes(url, &entry.content);
235            return Ok(content);
236        }
237        return Ok(entry.content.clone());
238    }
239
240    // 3. Check circuit breaker before making request (no network call)
241    let endpoint_pattern = crate::sources::feeds::extract_endpoint_pattern(url);
242    if let Err(e) = crate::sources::feeds::check_circuit_breaker(&endpoint_pattern) {
243        warn!(url, endpoint_pattern, error = %e, "circuit breaker blocking request");
244        // Try to return cached content if available
245        if let Some(cached) = cached_entry {
246            return Ok(cached.content);
247        }
248        if let Some(disk) = disk_entry {
249            return Ok(disk.content);
250        }
251        return Err(e);
252    }
253
254    // 4. Fetch from network with conditional requests
255    // Get cached ETag/Last-Modified for conditional request
256    let cached_etag = cached_entry
257        .as_ref()
258        .and_then(|e: &ArticleCacheEntry| e.etag.as_ref())
259        .or_else(|| disk_entry.as_ref().and_then(|e| e.etag.as_ref()))
260        .cloned();
261    let cached_last_modified = cached_entry
262        .as_ref()
263        .and_then(|e: &ArticleCacheEntry| e.last_modified.as_ref())
264        .or_else(|| disk_entry.as_ref().and_then(|e| e.last_modified.as_ref()))
265        .cloned();
266
267    // Fetch from network
268    let (body, etag, last_modified) =
269        match fetch_from_network(url, cached_etag, cached_last_modified, &endpoint_pattern).await {
270            Ok(result) => result,
271            Err(e) if e.to_string() == "304 Not Modified" => {
272                // Return cached content on 304
273                if let Some(cached) = cached_entry {
274                    return Ok(cached.content);
275                }
276                if let Some(disk) = disk_entry {
277                    return Ok(disk.content);
278                }
279                warn!(url, "304 response but no cached content available");
280                return Err("304 Not Modified but no cache available".into());
281            }
282            Err(e) => return Err(e),
283        };
284
285    // Use the toolkit's pure article parser for ordinary Arch news, while preserving Pacsea's
286    // richer official-package and advisory presentation paths.
287    let content = if !is_arch_package_url(url) && url.contains("archlinux.org/news/") {
288        crate::integrations::arch_toolkit::news::extract_article_text(&body, url)
289            .unwrap_or_else(|_| parse_arch_news_html(&body, Some(url)))
290    } else {
291        parse_arch_news_html(&body, Some(url))
292    };
293
294    // Prepend official package JSON changes if available
295    let content = if is_arch_package_url(url) {
296        prepend_official_package_changes(url, &content)
297    } else {
298        content
299    };
300
301    let parsed_len = content.len();
302    if parsed_len == 0 {
303        warn!(url, "parsed news content is empty");
304    } else {
305        info!(url, parsed_len, "parsed news content");
306    }
307
308    // 5. Cache the result with ETag/Last-Modified
309    // Save to in-memory cache
310    if let Ok(mut cache) = ARTICLE_CACHE.lock() {
311        cache.insert(
312            url.to_string(),
313            ArticleCacheEntry {
314                content: content.clone(),
315                timestamp: Instant::now(),
316                etag: etag.clone(),
317                last_modified: last_modified.clone(),
318            },
319        );
320    }
321    // Save to disk cache for persistence across restarts
322    save_article_to_disk_cache(url, &content, etag, last_modified);
323
324    Ok(content)
325}
326
327/// What: Fetch content from network with conditional requests.
328///
329/// Inputs:
330/// - `url`: The URL to fetch.
331/// - `cached_etag`: Optional `ETag` from cache.
332/// - `cached_last_modified`: Optional `Last-Modified` from cache.
333/// - `endpoint_pattern`: Endpoint pattern for circuit breaker.
334///
335/// Output:
336/// - `Ok((body, etag, last_modified))` on success.
337/// - `Err` on network or HTTP errors.
338///
339/// Details:
340/// - Applies rate limiting for archlinux.org URLs.
341/// - Uses conditional requests if `ETag`/`Last-Modified` available.
342/// - Handles 304 Not Modified responses.
343async fn fetch_from_network(
344    url: &str,
345    cached_etag: Option<String>,
346    cached_last_modified: Option<String>,
347    endpoint_pattern: &str,
348) -> Result<(String, Option<String>, Option<String>)> {
349    // Apply rate limiting and acquire semaphore for archlinux.org URLs
350    let _permit = if is_archlinux_url(url) {
351        Some(crate::sources::feeds::rate_limit_archlinux().await)
352    } else {
353        None
354    };
355
356    // Fetch from network with conditional requests using reqwest (connection pooling)
357    let client = HTTP_CLIENT.clone();
358    let mut request = client.get(url);
359
360    // Add conditional request headers if we have cached ETag/Last-Modified
361    if let Some(ref etag) = cached_etag {
362        request = request.header("If-None-Match", etag);
363    }
364    if let Some(ref last_mod) = cached_last_modified {
365        request = request.header("If-Modified-Since", last_mod);
366    }
367
368    let http_response = request.send().await.map_err(|e| {
369        warn!(error = %e, url, "failed to fetch news content");
370        crate::sources::feeds::record_circuit_breaker_outcome(endpoint_pattern, false);
371        Box::<dyn std::error::Error + Send + Sync>::from(format!("Network error: {e}"))
372    })?;
373
374    let status = http_response.status();
375    let status_code = status.as_u16();
376
377    // Handle 304 Not Modified
378    if status_code == 304 {
379        info!(
380            url,
381            "server returned 304 Not Modified, using cached content"
382        );
383        return Err("304 Not Modified".into());
384    }
385
386    // Extract ETag and Last-Modified from response headers before consuming body
387    let etag = http_response
388        .headers()
389        .get("etag")
390        .and_then(|h| h.to_str().ok())
391        .map(ToString::to_string);
392    let last_modified = http_response
393        .headers()
394        .get("last-modified")
395        .and_then(|h| h.to_str().ok())
396        .map(ToString::to_string);
397
398    // Check for HTTP errors
399    if status.is_client_error() || status.is_server_error() {
400        crate::sources::feeds::record_circuit_breaker_outcome(endpoint_pattern, false);
401        return Err(handle_http_error(status, status_code, &http_response).into());
402    }
403
404    let body = http_response.text().await.map_err(|e| {
405        warn!(error = %e, url, "failed to read response body");
406        Box::<dyn std::error::Error + Send + Sync>::from(format!("Failed to read response: {e}"))
407    })?;
408
409    info!(url, bytes = body.len(), "fetched news page");
410    crate::sources::feeds::record_circuit_breaker_outcome(endpoint_pattern, true);
411
412    Ok((body, etag, last_modified))
413}
414
415/// What: Handle HTTP error responses and format error messages.
416///
417/// Inputs:
418/// - `status`: HTTP status code object.
419/// - `status_code`: HTTP status code as u16.
420/// - `http_response`: HTTP response object to extract headers.
421///
422/// Output:
423/// - Formatted error message string.
424///
425/// Details:
426/// - Handles 429 (Too Many Requests) and 503 (Service Unavailable) with Retry-After headers.
427/// - Formats generic error messages for other HTTP errors.
428fn handle_http_error(
429    status: reqwest::StatusCode,
430    status_code: u16,
431    http_response: &reqwest::Response,
432) -> String {
433    if status_code == 429 {
434        let mut msg = "HTTP 429 Too Many Requests - rate limited by server".to_string();
435        if let Some(retry_after) = http_response.headers().get("retry-after")
436            && let Ok(retry_str) = retry_after.to_str()
437        {
438            msg.push_str(" (Retry-After: ");
439            msg.push_str(retry_str);
440            msg.push(')');
441        }
442        msg
443    } else if status_code == 503 {
444        let mut msg = "HTTP 503 Service Unavailable".to_string();
445        if let Some(retry_after) = http_response.headers().get("retry-after")
446            && let Ok(retry_str) = retry_after.to_str()
447        {
448            msg.push_str(" (Retry-After: ");
449            msg.push_str(retry_str);
450            msg.push(')');
451        }
452        msg
453    } else {
454        format!("HTTP error: {status}")
455    }
456}