1use crate::integrations::arch_toolkit::ToolkitContext;
4use crate::sources::news::cache::{ARTICLE_CACHE, ARTICLE_CACHE_TTL_SECONDS, ArticleCacheEntry};
5use crate::sources::news::parse::parse_arch_news_html;
6use crate::sources::news::utils::is_archlinux_url;
7use crate::sources::news::{
8 aur::extract_aur_pkg_from_url,
9 cache::{load_article_entry_from_disk_cache, save_article_to_disk_cache},
10 utils::is_arch_package_url,
11};
12use crate::state::NewsItem;
13use reqwest;
14use std::sync::LazyLock;
15use std::time::{Duration, Instant};
16use tracing::{info, warn};
17
18type Result<T> = super::Result<T>;
20
21fn extract_official_package_cache_path(url: &str) -> Option<std::path::PathBuf> {
33 let lower = url.to_ascii_lowercase();
34 let pos = lower.find("archlinux.org/packages/")?;
35 let after = &url[pos + "archlinux.org/packages/".len()..];
36 let parts: Vec<&str> = after.split('/').filter(|s| !s.is_empty()).collect();
37 if parts.len() >= 3 {
38 let repo = parts[0];
39 let arch = parts[1];
40 let name = parts[2]
41 .split('?')
42 .next()
43 .unwrap_or(parts[2])
44 .split('#')
45 .next()
46 .unwrap_or(parts[2]);
47 Some(crate::sources::official_json_cache_path(repo, arch, name))
48 } else {
49 None
50 }
51}
52
53fn prepend_official_package_changes(url: &str, content: &str) -> String {
65 let Some(cache_path) = extract_official_package_cache_path(url) else {
66 return content.to_string();
67 };
68
69 let Some(cached_json) = crate::sources::load_official_json_cache(&cache_path) else {
70 return content.to_string();
71 };
72
73 let pkg_obj = cached_json.get("pkg").unwrap_or(&cached_json);
74
75 let Some(pkg_name) = pkg_obj.get("pkgname").and_then(serde_json::Value::as_str) else {
76 return content.to_string();
77 };
78
79 let Some(changes) = crate::sources::get_official_json_changes(pkg_name) else {
80 return content.to_string();
81 };
82
83 if content.starts_with("Changes detected") {
84 content.to_string()
85 } else {
86 format!("{changes}\n\n─── Package Info ───\n\n{content}")
87 }
88}
89
90static HTTP_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
94 use reqwest::header::{ACCEPT, ACCEPT_LANGUAGE, HeaderMap, HeaderValue};
95 let mut headers = HeaderMap::new();
96 headers.insert(
98 ACCEPT,
99 HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
100 );
101 headers.insert(ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.5"));
103 reqwest::Client::builder()
104 .connect_timeout(Duration::from_secs(15))
105 .timeout(Duration::from_secs(30))
106 .user_agent(format!(
108 "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0 Pacsea/{}",
109 env!("CARGO_PKG_VERSION")
110 ))
111 .default_headers(headers)
112 .build()
113 .expect("Failed to create HTTP client")
114});
115
116pub async fn fetch_arch_news(limit: usize, cutoff_date: Option<&str>) -> Result<Vec<NewsItem>> {
134 let context = ToolkitContext::new()?;
135 fetch_arch_news_with_context(&context, limit, cutoff_date).await
136}
137
138pub async fn fetch_arch_news_with_context(
151 context: &ToolkitContext,
152 limit: usize,
153 cutoff_date: Option<&str>,
154) -> Result<Vec<NewsItem>> {
155 crate::integrations::arch_toolkit::news::fetch_arch_news(context, limit, cutoff_date)
156 .await
157 .map_err(Into::into)
158}
159
160pub async fn fetch_news_content(url: &str) -> Result<String> {
180 use crate::sources::news::aur::render_aur_comments;
181
182 if let Some(pkg) = extract_aur_pkg_from_url(url) {
183 let changes = crate::sources::get_aur_json_changes(&pkg);
185 let comments = crate::sources::fetch_aur_comments(pkg.clone()).await?;
186 let mut rendered = render_aur_comments(&pkg, &comments);
187
188 if let Some(changes_text) = changes {
190 rendered = format!("{changes_text}\n\n─── AUR Comments ───\n\n{rendered}");
191 }
192
193 return Ok(rendered);
194 }
195
196 if is_arch_package_url(url)
198 && let Ok(cache) = ARTICLE_CACHE.lock()
199 && let Some(entry) = cache.get(url)
200 && entry.timestamp.elapsed().as_secs() < ARTICLE_CACHE_TTL_SECONDS
201 {
202 let content = prepend_official_package_changes(url, &entry.content);
203 return Ok(content);
204 }
205
206 let cached_entry: Option<ArticleCacheEntry> = if let Ok(cache) = ARTICLE_CACHE.lock()
208 && let Some(entry) = cache.get(url)
209 && entry.timestamp.elapsed().as_secs() < ARTICLE_CACHE_TTL_SECONDS
210 {
211 info!(url, "using in-memory cached article content");
212 return Ok(entry.content.clone());
213 } else {
214 None
215 };
216
217 let disk_entry = load_article_entry_from_disk_cache(url);
219 if let Some(ref entry) = disk_entry {
220 if let Ok(mut cache) = ARTICLE_CACHE.lock() {
222 cache.insert(
223 url.to_string(),
224 ArticleCacheEntry {
225 content: entry.content.clone(),
226 timestamp: Instant::now(),
227 etag: entry.etag.clone(),
228 last_modified: entry.last_modified.clone(),
229 },
230 );
231 }
232 if is_arch_package_url(url) {
234 let content = prepend_official_package_changes(url, &entry.content);
235 return Ok(content);
236 }
237 return Ok(entry.content.clone());
238 }
239
240 let endpoint_pattern = crate::sources::feeds::extract_endpoint_pattern(url);
242 if let Err(e) = crate::sources::feeds::check_circuit_breaker(&endpoint_pattern) {
243 warn!(url, endpoint_pattern, error = %e, "circuit breaker blocking request");
244 if let Some(cached) = cached_entry {
246 return Ok(cached.content);
247 }
248 if let Some(disk) = disk_entry {
249 return Ok(disk.content);
250 }
251 return Err(e);
252 }
253
254 let cached_etag = cached_entry
257 .as_ref()
258 .and_then(|e: &ArticleCacheEntry| e.etag.as_ref())
259 .or_else(|| disk_entry.as_ref().and_then(|e| e.etag.as_ref()))
260 .cloned();
261 let cached_last_modified = cached_entry
262 .as_ref()
263 .and_then(|e: &ArticleCacheEntry| e.last_modified.as_ref())
264 .or_else(|| disk_entry.as_ref().and_then(|e| e.last_modified.as_ref()))
265 .cloned();
266
267 let (body, etag, last_modified) =
269 match fetch_from_network(url, cached_etag, cached_last_modified, &endpoint_pattern).await {
270 Ok(result) => result,
271 Err(e) if e.to_string() == "304 Not Modified" => {
272 if let Some(cached) = cached_entry {
274 return Ok(cached.content);
275 }
276 if let Some(disk) = disk_entry {
277 return Ok(disk.content);
278 }
279 warn!(url, "304 response but no cached content available");
280 return Err("304 Not Modified but no cache available".into());
281 }
282 Err(e) => return Err(e),
283 };
284
285 let content = if !is_arch_package_url(url) && url.contains("archlinux.org/news/") {
288 crate::integrations::arch_toolkit::news::extract_article_text(&body, url)
289 .unwrap_or_else(|_| parse_arch_news_html(&body, Some(url)))
290 } else {
291 parse_arch_news_html(&body, Some(url))
292 };
293
294 let content = if is_arch_package_url(url) {
296 prepend_official_package_changes(url, &content)
297 } else {
298 content
299 };
300
301 let parsed_len = content.len();
302 if parsed_len == 0 {
303 warn!(url, "parsed news content is empty");
304 } else {
305 info!(url, parsed_len, "parsed news content");
306 }
307
308 if let Ok(mut cache) = ARTICLE_CACHE.lock() {
311 cache.insert(
312 url.to_string(),
313 ArticleCacheEntry {
314 content: content.clone(),
315 timestamp: Instant::now(),
316 etag: etag.clone(),
317 last_modified: last_modified.clone(),
318 },
319 );
320 }
321 save_article_to_disk_cache(url, &content, etag, last_modified);
323
324 Ok(content)
325}
326
327async fn fetch_from_network(
344 url: &str,
345 cached_etag: Option<String>,
346 cached_last_modified: Option<String>,
347 endpoint_pattern: &str,
348) -> Result<(String, Option<String>, Option<String>)> {
349 let _permit = if is_archlinux_url(url) {
351 Some(crate::sources::feeds::rate_limit_archlinux().await)
352 } else {
353 None
354 };
355
356 let client = HTTP_CLIENT.clone();
358 let mut request = client.get(url);
359
360 if let Some(ref etag) = cached_etag {
362 request = request.header("If-None-Match", etag);
363 }
364 if let Some(ref last_mod) = cached_last_modified {
365 request = request.header("If-Modified-Since", last_mod);
366 }
367
368 let http_response = request.send().await.map_err(|e| {
369 warn!(error = %e, url, "failed to fetch news content");
370 crate::sources::feeds::record_circuit_breaker_outcome(endpoint_pattern, false);
371 Box::<dyn std::error::Error + Send + Sync>::from(format!("Network error: {e}"))
372 })?;
373
374 let status = http_response.status();
375 let status_code = status.as_u16();
376
377 if status_code == 304 {
379 info!(
380 url,
381 "server returned 304 Not Modified, using cached content"
382 );
383 return Err("304 Not Modified".into());
384 }
385
386 let etag = http_response
388 .headers()
389 .get("etag")
390 .and_then(|h| h.to_str().ok())
391 .map(ToString::to_string);
392 let last_modified = http_response
393 .headers()
394 .get("last-modified")
395 .and_then(|h| h.to_str().ok())
396 .map(ToString::to_string);
397
398 if status.is_client_error() || status.is_server_error() {
400 crate::sources::feeds::record_circuit_breaker_outcome(endpoint_pattern, false);
401 return Err(handle_http_error(status, status_code, &http_response).into());
402 }
403
404 let body = http_response.text().await.map_err(|e| {
405 warn!(error = %e, url, "failed to read response body");
406 Box::<dyn std::error::Error + Send + Sync>::from(format!("Failed to read response: {e}"))
407 })?;
408
409 info!(url, bytes = body.len(), "fetched news page");
410 crate::sources::feeds::record_circuit_breaker_outcome(endpoint_pattern, true);
411
412 Ok((body, etag, last_modified))
413}
414
415fn handle_http_error(
429 status: reqwest::StatusCode,
430 status_code: u16,
431 http_response: &reqwest::Response,
432) -> String {
433 if status_code == 429 {
434 let mut msg = "HTTP 429 Too Many Requests - rate limited by server".to_string();
435 if let Some(retry_after) = http_response.headers().get("retry-after")
436 && let Ok(retry_str) = retry_after.to_str()
437 {
438 msg.push_str(" (Retry-After: ");
439 msg.push_str(retry_str);
440 msg.push(')');
441 }
442 msg
443 } else if status_code == 503 {
444 let mut msg = "HTTP 503 Service Unavailable".to_string();
445 if let Some(retry_after) = http_response.headers().get("retry-after")
446 && let Ok(retry_str) = retry_after.to_str()
447 {
448 msg.push_str(" (Retry-After: ");
449 msg.push_str(retry_str);
450 msg.push(')');
451 }
452 msg
453 } else {
454 format!("HTTP error: {status}")
455 }
456}