1mod cache;
3mod helpers;
4mod news_fetch;
5mod rate_limit;
6mod updates;
7
8use std::collections::{HashMap, HashSet};
9use std::hash::BuildHasher;
10use std::path::PathBuf;
11
12use crate::state::types::{NewsFeedItem, NewsSortMode, severity_rank};
13use tracing::{info, warn};
14
15use helpers::load_update_versions;
16use news_fetch::fetch_slow_sources;
17use updates::{fetch_installed_aur_comments, fetch_installed_updates};
18
19type Result<T> = super::Result<T>;
21
22#[must_use]
40pub fn optimize_max_age_for_startup(
41 last_startup: Option<&str>,
42 default_max_age: Option<u32>,
43) -> Option<u32> {
44 let Some(ts) = last_startup else {
45 return default_max_age;
47 };
48
49 let parsed = chrono::NaiveDateTime::parse_from_str(ts, "%Y%m%d:%H%M%S").ok();
51 let Some(last_dt) = parsed else {
52 tracing::debug!(timestamp = %ts, "failed to parse last startup timestamp");
53 return default_max_age;
54 };
55
56 let now = chrono::Local::now().naive_local();
57 let elapsed = now.signed_duration_since(last_dt);
58
59 if elapsed.num_hours() < 1 {
60 info!(
62 hours_since_last = elapsed.num_hours(),
63 "recent startup detected, using minimal fetch window"
64 );
65 Some(1)
66 } else if elapsed.num_hours() < 24 {
67 info!(
69 hours_since_last = elapsed.num_hours(),
70 "startup within 24h, using 2-day fetch window"
71 );
72 Some(2)
73 } else if elapsed.num_days() < 7 {
74 let optimized = default_max_age.map_or(7, |d| d.min(7));
76 info!(
77 days_since_last = elapsed.num_days(),
78 optimized_max_age = optimized,
79 "startup within 7 days, using optimized fetch window"
80 );
81 Some(optimized)
82 } else {
83 default_max_age
85 }
86}
87
88#[allow(clippy::struct_excessive_bools)]
107pub struct NewsFeedContext<'a, HS, HV, HC>
108where
109 HS: BuildHasher + Send + Sync + 'static,
110 HV: BuildHasher + Send + Sync + 'static,
111 HC: BuildHasher + Send + Sync + 'static,
112{
113 pub force_emit_all: bool,
115 pub updates_list_path: Option<PathBuf>,
117 pub limit: usize,
119 pub include_arch_news: bool,
121 pub include_advisories: bool,
123 pub include_pkg_updates: bool,
125 pub include_aur_comments: bool,
127 pub installed_filter: Option<&'a HashSet<String, HS>>,
129 pub installed_only: bool,
131 pub sort_mode: NewsSortMode,
133 pub seen_pkg_versions: &'a mut HashMap<String, String, HV>,
135 pub seen_aur_comments: &'a mut HashMap<String, String, HC>,
137 pub max_age_days: Option<u32>,
139}
140
141struct FastSourcesConfig<'a, HS, HV, HC> {
143 include_pkg_updates: bool,
145 include_aur_comments: bool,
147 installed_filter: Option<&'a HashSet<String, HS>>,
149 limit: usize,
151 seen_pkg_versions: &'a mut HashMap<String, String, HV>,
153 seen_aur_comments: &'a mut HashMap<String, String, HC>,
155 force_emit_all: bool,
157 updates_versions: Option<&'a HashMap<String, (String, String)>>,
159}
160
161async fn fetch_fast_sources<HS, HV, HC>(
173 config: FastSourcesConfig<'_, HS, HV, HC>,
174) -> (
175 std::result::Result<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>,
176 std::result::Result<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>,
177)
178where
179 HS: BuildHasher + Send + Sync + 'static,
180 HV: BuildHasher + Send + Sync + 'static,
181 HC: BuildHasher + Send + Sync + 'static,
182{
183 tokio::join!(
184 async {
185 if config.include_pkg_updates {
186 if let Some(installed) = config.installed_filter {
187 if installed.is_empty() {
188 warn!(
189 "include_pkg_updates set but installed set is empty; skipping updates"
190 );
191 Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
192 } else {
193 info!(
194 "fetching package updates: installed_count={}, limit={}",
195 installed.len(),
196 config.limit
197 );
198 let result = fetch_installed_updates(
199 installed,
200 config.limit,
201 config.seen_pkg_versions,
202 config.force_emit_all,
203 config.updates_versions,
204 )
205 .await;
206 match &result {
207 Ok(updates) => {
208 info!("package updates fetch completed: items={}", updates.len());
209 }
210 Err(e) => {
211 warn!(error = %e, "installed package updates fetch failed");
212 }
213 }
214 match result {
215 Ok(updates) => Ok(updates),
216 Err(_e) => Ok::<
217 Vec<NewsFeedItem>,
218 Box<dyn std::error::Error + Send + Sync>,
219 >(Vec::new()),
220 }
221 }
222 } else {
223 warn!("include_pkg_updates set but installed_filter missing; skipping updates");
224 Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
225 }
226 } else {
227 Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
228 }
229 },
230 async {
231 if config.include_aur_comments {
232 if let Some(installed) = config.installed_filter {
233 if installed.is_empty() {
234 warn!(
235 "include_aur_comments set but installed set is empty; skipping comments"
236 );
237 Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
238 } else {
239 info!(
240 "fetching AUR comments: installed_count={}, limit={}",
241 installed.len(),
242 config.limit
243 );
244 let result = fetch_installed_aur_comments(
245 installed,
246 config.limit,
247 config.seen_aur_comments,
248 config.force_emit_all,
249 )
250 .await;
251 match &result {
252 Ok(comments) => {
253 info!("AUR comments fetch completed: items={}", comments.len());
254 }
255 Err(e) => {
256 warn!(error = %e, "installed AUR comments fetch failed");
257 }
258 }
259 match result {
260 Ok(comments) => Ok(comments),
261 Err(_e) => Ok::<
262 Vec<NewsFeedItem>,
263 Box<dyn std::error::Error + Send + Sync>,
264 >(Vec::new()),
265 }
266 }
267 } else {
268 warn!(
269 "include_aur_comments set but installed_filter missing; skipping comments"
270 );
271 Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
272 }
273 } else {
274 Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
275 }
276 }
277 )
278}
279
280fn combine_feed_results(
296 arch_result: std::result::Result<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>,
297 advisories_result: std::result::Result<
298 Vec<NewsFeedItem>,
299 Box<dyn std::error::Error + Send + Sync>,
300 >,
301 updates_result: std::result::Result<
302 Vec<NewsFeedItem>,
303 Box<dyn std::error::Error + Send + Sync>,
304 >,
305 comments_result: std::result::Result<
306 Vec<NewsFeedItem>,
307 Box<dyn std::error::Error + Send + Sync>,
308 >,
309 sort_mode: NewsSortMode,
310) -> Vec<NewsFeedItem> {
311 let mut items: Vec<NewsFeedItem> = Vec::new();
312 match arch_result {
313 Ok(mut arch_items) => items.append(&mut arch_items),
314 Err(e) => warn!(error = %e, "arch news fetch failed; continuing without Arch news"),
315 }
316 match advisories_result {
317 Ok(mut adv_items) => items.append(&mut adv_items),
318 Err(e) => warn!(error = %e, "advisories fetch failed; continuing without advisories"),
319 }
320 match updates_result {
321 Ok(mut upd_items) => items.append(&mut upd_items),
322 Err(e) => warn!(error = %e, "updates fetch failed; continuing without updates"),
323 }
324 match comments_result {
325 Ok(mut cmt_items) => items.append(&mut cmt_items),
326 Err(e) => warn!(error = %e, "comments fetch failed; continuing without comments"),
327 }
328 sort_news_items(&mut items, sort_mode);
329 items
330}
331
332type PrepareFetchContextReturn<'a, HS, HV, HC> = (
334 Option<String>,
335 Option<HashMap<String, (String, String)>>,
336 usize,
337 bool,
338 bool,
339 bool,
340 bool,
341 Option<&'a HashSet<String, HS>>,
342 bool,
343 NewsSortMode,
344 &'a mut HashMap<String, String, HV>,
345 &'a mut HashMap<String, String, HC>,
346 bool,
347);
348
349fn prepare_fetch_context<HS, HV, HC>(
360 ctx: NewsFeedContext<'_, HS, HV, HC>,
361) -> PrepareFetchContextReturn<'_, HS, HV, HC>
362where
363 HS: BuildHasher + Send + Sync + 'static,
364 HV: BuildHasher + Send + Sync + 'static,
365 HC: BuildHasher + Send + Sync + 'static,
366{
367 let NewsFeedContext {
368 limit,
369 include_arch_news,
370 include_advisories,
371 include_pkg_updates,
372 include_aur_comments,
373 installed_filter,
374 installed_only,
375 sort_mode,
376 seen_pkg_versions,
377 seen_aur_comments,
378 force_emit_all,
379 updates_list_path,
380 max_age_days,
381 } = ctx;
382
383 info!(
384 limit,
385 include_arch_news,
386 include_advisories,
387 include_pkg_updates,
388 include_aur_comments,
389 installed_only,
390 installed_filter = installed_filter.is_some(),
391 sort_mode = ?sort_mode,
392 max_age_days,
393 "fetch_news_feed start"
394 );
395
396 let cutoff_date = max_age_days.and_then(|days| {
397 chrono::Utc::now()
398 .checked_sub_signed(chrono::Duration::days(i64::from(days)))
399 .map(|dt| dt.format("%Y-%m-%d").to_string())
400 });
401 let updates_versions = if force_emit_all {
402 load_update_versions(updates_list_path.as_ref())
403 } else {
404 None
405 };
406
407 (
408 cutoff_date,
409 updates_versions,
410 limit,
411 include_arch_news,
412 include_advisories,
413 include_pkg_updates,
414 include_aur_comments,
415 installed_filter,
416 installed_only,
417 sort_mode,
418 seen_pkg_versions,
419 seen_aur_comments,
420 force_emit_all,
421 )
422}
423
424fn sort_news_items(items: &mut [NewsFeedItem], mode: NewsSortMode) {
434 match mode {
435 NewsSortMode::DateDesc => items.sort_by(|a, b| b.date.cmp(&a.date)),
436 NewsSortMode::DateAsc => items.sort_by(|a, b| a.date.cmp(&b.date)),
437 NewsSortMode::Title => {
438 items.sort_by(|a, b| {
439 a.title
440 .to_lowercase()
441 .cmp(&b.title.to_lowercase())
442 .then(b.date.cmp(&a.date))
443 });
444 }
445 NewsSortMode::SourceThenTitle => items.sort_by(|a, b| {
446 a.source
447 .cmp(&b.source)
448 .then(b.date.cmp(&a.date))
449 .then(a.title.to_lowercase().cmp(&b.title.to_lowercase()))
450 }),
451 NewsSortMode::SeverityThenDate => items.sort_by(|a, b| {
452 let sa = severity_rank(a.severity);
453 let sb = severity_rank(b.severity);
454 sb.cmp(&sa)
455 .then(b.date.cmp(&a.date))
456 .then(a.title.to_lowercase().cmp(&b.title.to_lowercase()))
457 }),
458 NewsSortMode::UnreadThenDate => {
459 items.sort_by(|a, b| b.date.cmp(&a.date));
461 }
462 }
463}
464
465pub async fn fetch_news_feed<HS, HV, HC>(
469 ctx: NewsFeedContext<'_, HS, HV, HC>,
470) -> Result<Vec<NewsFeedItem>>
471where
472 HS: BuildHasher + Send + Sync + 'static,
473 HV: BuildHasher + Send + Sync + 'static,
474 HC: BuildHasher + Send + Sync + 'static,
475{
476 let toolkit = crate::integrations::arch_toolkit::ToolkitContext::new()?;
477 fetch_news_feed_with_context(ctx, &toolkit).await
478}
479
480pub async fn fetch_news_feed_with_context<HS, HV, HC>(
493 ctx: NewsFeedContext<'_, HS, HV, HC>,
494 toolkit: &crate::integrations::arch_toolkit::ToolkitContext,
495) -> Result<Vec<NewsFeedItem>>
496where
497 HS: BuildHasher + Send + Sync + 'static,
498 HV: BuildHasher + Send + Sync + 'static,
499 HC: BuildHasher + Send + Sync + 'static,
500{
501 let (
502 cutoff_date,
503 updates_versions,
504 limit,
505 include_arch_news,
506 include_advisories,
507 include_pkg_updates,
508 include_aur_comments,
509 installed_filter,
510 installed_only,
511 sort_mode,
512 seen_pkg_versions,
513 seen_aur_comments,
514 force_emit_all,
515 ) = prepare_fetch_context(ctx);
516
517 info!(
518 "starting fetch: arch_news={include_arch_news}, advisories={include_advisories}, pkg_updates={include_pkg_updates}, aur_comments={include_aur_comments}"
519 );
520 rate_limit::reset_archlinux_backoff();
521
522 let ((updates_result, comments_result), (arch_result, advisories_result)) = tokio::join!(
527 fetch_fast_sources(FastSourcesConfig {
528 include_pkg_updates,
529 include_aur_comments,
530 installed_filter,
531 limit,
532 seen_pkg_versions,
533 seen_aur_comments,
534 force_emit_all,
535 updates_versions: updates_versions.as_ref(),
536 }),
537 fetch_slow_sources(
538 toolkit,
539 include_arch_news,
540 include_advisories,
541 limit,
542 installed_filter,
543 installed_only,
544 cutoff_date.as_deref(),
545 )
546 );
547 info!("fetch completed, combining results...");
548
549 let items = combine_feed_results(
550 arch_result,
551 advisories_result,
552 updates_result,
553 comments_result,
554 sort_mode,
555 );
556 info!(
557 total = items.len(),
558 arch = items
559 .iter()
560 .filter(|i| matches!(i.source, crate::state::types::NewsFeedSource::ArchNews))
561 .count(),
562 advisories = items
563 .iter()
564 .filter(|i| matches!(
565 i.source,
566 crate::state::types::NewsFeedSource::SecurityAdvisory
567 ))
568 .count(),
569 updates = items
570 .iter()
571 .filter(|i| {
572 matches!(
573 i.source,
574 crate::state::types::NewsFeedSource::InstalledPackageUpdate
575 | crate::state::types::NewsFeedSource::AurPackageUpdate
576 )
577 })
578 .count(),
579 aur_comments = items
580 .iter()
581 .filter(|i| matches!(i.source, crate::state::types::NewsFeedSource::AurComment))
582 .count(),
583 "fetch_news_feed success"
584 );
585 Ok(items)
586}
587
588const CONTINUATION_LIMIT: usize = 1000;
590
591pub async fn fetch_continuation_items<HS, HI>(
609 installed: &HashSet<String, HS>,
610 initial_ids: &HashSet<String, HI>,
611) -> Result<Vec<NewsFeedItem>>
612where
613 HS: std::hash::BuildHasher + Send + Sync + 'static,
614 HI: std::hash::BuildHasher + Send + Sync,
615{
616 let toolkit = crate::integrations::arch_toolkit::ToolkitContext::new()?;
617 fetch_continuation_items_with_context(installed, initial_ids, &toolkit).await
618}
619
620pub async fn fetch_continuation_items_with_context<HS, HI>(
633 installed: &HashSet<String, HS>,
634 initial_ids: &HashSet<String, HI>,
635 toolkit: &crate::integrations::arch_toolkit::ToolkitContext,
636) -> Result<Vec<NewsFeedItem>>
637where
638 HS: std::hash::BuildHasher + Send + Sync + 'static,
639 HI: std::hash::BuildHasher + Send + Sync,
640{
641 use crate::state::types::NewsFeedSource;
642
643 info!(
644 installed_count = installed.len(),
645 initial_count = initial_ids.len(),
646 "starting continuation fetch"
647 );
648
649 let ((updates_result, comments_result), (arch_result, advisories_result)) = tokio::join!(
651 async {
652 let mut seen_versions: HashMap<String, String> = HashMap::new();
654 let mut seen_aur_comments: HashMap<String, String> = HashMap::new();
655 let updates = fetch_installed_updates(
656 installed,
657 CONTINUATION_LIMIT,
658 &mut seen_versions,
659 true, None,
661 )
662 .await;
663 let comments = fetch_installed_aur_comments(
664 installed,
665 CONTINUATION_LIMIT,
666 &mut seen_aur_comments,
667 true, )
669 .await;
670 (updates, comments)
671 },
672 fetch_slow_sources(
673 toolkit,
674 true, true, CONTINUATION_LIMIT,
677 Some(installed),
678 false, None, )
681 );
682
683 let mut items = Vec::new();
684
685 if let Ok(arch_items) = arch_result {
687 for item in arch_items {
688 if !initial_ids.contains(&item.id) {
689 items.push(item);
690 }
691 }
692 }
693
694 if let Ok(adv_items) = advisories_result {
696 for item in adv_items {
697 if !initial_ids.contains(&item.id) {
698 items.push(item);
699 }
700 }
701 }
702
703 if let Ok(upd_items) = updates_result {
705 for item in upd_items {
706 if !initial_ids.contains(&item.id) {
707 items.push(item);
708 }
709 }
710 }
711
712 if let Ok(comment_items) = comments_result {
714 for item in comment_items {
715 if !initial_ids.contains(&item.id) {
716 items.push(item);
717 }
718 }
719 }
720
721 sort_news_items(&mut items, NewsSortMode::DateDesc);
723
724 info!(
725 total = items.len(),
726 arch = items
727 .iter()
728 .filter(|i| matches!(i.source, NewsFeedSource::ArchNews))
729 .count(),
730 advisories = items
731 .iter()
732 .filter(|i| matches!(i.source, NewsFeedSource::SecurityAdvisory))
733 .count(),
734 updates = items
735 .iter()
736 .filter(|i| matches!(
737 i.source,
738 NewsFeedSource::InstalledPackageUpdate | NewsFeedSource::AurPackageUpdate
739 ))
740 .count(),
741 "continuation fetch complete"
742 );
743
744 Ok(items)
745}
746
747pub use rate_limit::{
749 check_circuit_breaker, extract_endpoint_pattern, extract_retry_after_from_error,
750 increase_archlinux_backoff, rate_limit_archlinux, record_circuit_breaker_outcome,
751 reset_archlinux_backoff, take_network_error,
752};
753pub use updates::{
754 get_aur_json_changes, get_official_json_changes, load_official_json_cache,
755 official_json_cache_path,
756};
757
758#[cfg(test)]
759mod tests;