Skip to main content

pacsea/sources/feeds/
mod.rs

1//! Aggregated news feed fetcher (Arch news + security advisories).
2mod cache;
3mod helpers;
4mod news_fetch;
5mod rate_limit;
6mod updates;
7
8use std::collections::{HashMap, HashSet};
9use std::hash::BuildHasher;
10use std::path::PathBuf;
11
12use crate::state::types::{NewsFeedItem, NewsSortMode, severity_rank};
13use tracing::{info, warn};
14
15use helpers::load_update_versions;
16use news_fetch::fetch_slow_sources;
17use updates::{fetch_installed_aur_comments, fetch_installed_updates};
18
19/// Result type alias for news feed fetching operations.
20type Result<T> = super::Result<T>;
21
22/// What: Calculate optimal `max_age_days` based on last startup timestamp.
23///
24/// Inputs:
25/// - `last_startup`: Optional timestamp in `YYYYMMDD:HHMMSS` format.
26/// - `default_max_age`: Default max age in days if no optimization applies.
27///
28/// Output:
29/// - Optimized `max_age_days` value, or `None` to fetch all.
30///
31/// Details:
32/// - If last startup was within 1 hour: use 1 day (recent data likely cached)
33/// - If last startup was within 24 hours: use 2 days
34/// - If last startup was within 7 days: use configured `max_age` or 7 days
35/// - Otherwise: use configured `max_age`
36/// - This reduces unnecessary fetching when the app was recently used.
37/// - NOTE: This only affects Arch news and advisories date filtering.
38///   Package updates are ALWAYS fetched fresh to detect new packages and version changes.
39#[must_use]
40pub fn optimize_max_age_for_startup(
41    last_startup: Option<&str>,
42    default_max_age: Option<u32>,
43) -> Option<u32> {
44    let Some(ts) = last_startup else {
45        // No previous startup recorded, use default
46        return default_max_age;
47    };
48
49    // Parse timestamp: YYYYMMDD:HHMMSS
50    let parsed = chrono::NaiveDateTime::parse_from_str(ts, "%Y%m%d:%H%M%S").ok();
51    let Some(last_dt) = parsed else {
52        tracing::debug!(timestamp = %ts, "failed to parse last startup timestamp");
53        return default_max_age;
54    };
55
56    let now = chrono::Local::now().naive_local();
57    let elapsed = now.signed_duration_since(last_dt);
58
59    if elapsed.num_hours() < 1 {
60        // Very recent startup (< 1 hour): minimal fresh fetch needed
61        info!(
62            hours_since_last = elapsed.num_hours(),
63            "recent startup detected, using minimal fetch window"
64        );
65        Some(1)
66    } else if elapsed.num_hours() < 24 {
67        // Within last day: use 2 days to be safe
68        info!(
69            hours_since_last = elapsed.num_hours(),
70            "startup within 24h, using 2-day fetch window"
71        );
72        Some(2)
73    } else if elapsed.num_days() < 7 {
74        // Within last week: use configured or 7 days
75        let optimized = default_max_age.map_or(7, |d| d.min(7));
76        info!(
77            days_since_last = elapsed.num_days(),
78            optimized_max_age = optimized,
79            "startup within 7 days, using optimized fetch window"
80        );
81        Some(optimized)
82    } else {
83        // More than a week: use configured max_age
84        default_max_age
85    }
86}
87
88/// What: Input context for fetching a combined news feed.
89///
90/// Inputs:
91/// - `limit`: Maximum number of items per source.
92/// - `include_*`: Source toggles.
93/// - `installed_filter`: Optional installed-package set for scoping.
94/// - `installed_only`: Whether to restrict advisories to installed packages.
95/// - `sort_mode`: Sort order.
96/// - `seen_pkg_versions`: Last-seen map for package updates.
97/// - `seen_aur_comments`: Last-seen map for AUR comments.
98/// - `max_age_days`: Optional maximum age in days for filtering items (enables early filtering).
99///
100/// Output:
101/// - Mutable references updated in place alongside returned feed items.
102///
103/// Details:
104/// - Hashers are generic to remain compatible with caller-supplied maps.
105/// - `max_age_days` enables early date filtering during fetch to improve performance.
106#[allow(clippy::struct_excessive_bools)]
107pub struct NewsFeedContext<'a, HS, HV, HC>
108where
109    HS: BuildHasher + Send + Sync + 'static,
110    HV: BuildHasher + Send + Sync + 'static,
111    HC: BuildHasher + Send + Sync + 'static,
112{
113    /// Emit all sources even on first run (bypasses baseline gating).
114    pub force_emit_all: bool,
115    /// Optional path to `available_updates.txt` for filtering noisy first-run emissions.
116    pub updates_list_path: Option<PathBuf>,
117    /// Maximum number of items per source.
118    pub limit: usize,
119    /// Whether to include Arch news RSS posts.
120    pub include_arch_news: bool,
121    /// Whether to include security advisories.
122    pub include_advisories: bool,
123    /// Whether to include installed package updates.
124    pub include_pkg_updates: bool,
125    /// Whether to include installed AUR comments.
126    pub include_aur_comments: bool,
127    /// Optional installed-package filter set.
128    pub installed_filter: Option<&'a HashSet<String, HS>>,
129    /// Whether to restrict advisories to installed packages.
130    pub installed_only: bool,
131    /// Sort mode for the resulting feed.
132    pub sort_mode: NewsSortMode,
133    /// Last-seen versions map (updated in place).
134    pub seen_pkg_versions: &'a mut HashMap<String, String, HV>,
135    /// Last-seen AUR comments map (updated in place).
136    pub seen_aur_comments: &'a mut HashMap<String, String, HC>,
137    /// Optional maximum age in days for early date filtering during fetch.
138    pub max_age_days: Option<u32>,
139}
140
141/// Configuration for fetching fast sources.
142struct FastSourcesConfig<'a, HS, HV, HC> {
143    /// Whether to fetch package updates.
144    include_pkg_updates: bool,
145    /// Whether to fetch AUR comments.
146    include_aur_comments: bool,
147    /// Optional set of installed package names.
148    installed_filter: Option<&'a HashSet<String, HS>>,
149    /// Maximum items per source.
150    limit: usize,
151    /// Last-seen versions map (updated in place).
152    seen_pkg_versions: &'a mut HashMap<String, String, HV>,
153    /// Last-seen AUR comments map (updated in place).
154    seen_aur_comments: &'a mut HashMap<String, String, HC>,
155    /// Whether to emit all items regardless of last-seen.
156    force_emit_all: bool,
157    /// Optional pre-loaded update versions.
158    updates_versions: Option<&'a HashMap<String, (String, String)>>,
159}
160
161/// What: Fetch fast sources (package updates and AUR comments) in parallel.
162///
163/// Inputs:
164/// - `config`: Configuration struct containing all fetch parameters.
165///
166/// Output:
167/// - Tuple of (`updates_result`, `comments_result`).
168///
169/// Details:
170/// - Fetches both sources in parallel for better performance.
171/// - Returns empty vectors on errors (graceful degradation).
172async fn fetch_fast_sources<HS, HV, HC>(
173    config: FastSourcesConfig<'_, HS, HV, HC>,
174) -> (
175    std::result::Result<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>,
176    std::result::Result<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>,
177)
178where
179    HS: BuildHasher + Send + Sync + 'static,
180    HV: BuildHasher + Send + Sync + 'static,
181    HC: BuildHasher + Send + Sync + 'static,
182{
183    tokio::join!(
184        async {
185            if config.include_pkg_updates {
186                if let Some(installed) = config.installed_filter {
187                    if installed.is_empty() {
188                        warn!(
189                            "include_pkg_updates set but installed set is empty; skipping updates"
190                        );
191                        Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
192                    } else {
193                        info!(
194                            "fetching package updates: installed_count={}, limit={}",
195                            installed.len(),
196                            config.limit
197                        );
198                        let result = fetch_installed_updates(
199                            installed,
200                            config.limit,
201                            config.seen_pkg_versions,
202                            config.force_emit_all,
203                            config.updates_versions,
204                        )
205                        .await;
206                        match &result {
207                            Ok(updates) => {
208                                info!("package updates fetch completed: items={}", updates.len());
209                            }
210                            Err(e) => {
211                                warn!(error = %e, "installed package updates fetch failed");
212                            }
213                        }
214                        match result {
215                            Ok(updates) => Ok(updates),
216                            Err(_e) => Ok::<
217                                Vec<NewsFeedItem>,
218                                Box<dyn std::error::Error + Send + Sync>,
219                            >(Vec::new()),
220                        }
221                    }
222                } else {
223                    warn!("include_pkg_updates set but installed_filter missing; skipping updates");
224                    Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
225                }
226            } else {
227                Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
228            }
229        },
230        async {
231            if config.include_aur_comments {
232                if let Some(installed) = config.installed_filter {
233                    if installed.is_empty() {
234                        warn!(
235                            "include_aur_comments set but installed set is empty; skipping comments"
236                        );
237                        Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
238                    } else {
239                        info!(
240                            "fetching AUR comments: installed_count={}, limit={}",
241                            installed.len(),
242                            config.limit
243                        );
244                        let result = fetch_installed_aur_comments(
245                            installed,
246                            config.limit,
247                            config.seen_aur_comments,
248                            config.force_emit_all,
249                        )
250                        .await;
251                        match &result {
252                            Ok(comments) => {
253                                info!("AUR comments fetch completed: items={}", comments.len());
254                            }
255                            Err(e) => {
256                                warn!(error = %e, "installed AUR comments fetch failed");
257                            }
258                        }
259                        match result {
260                            Ok(comments) => Ok(comments),
261                            Err(_e) => Ok::<
262                                Vec<NewsFeedItem>,
263                                Box<dyn std::error::Error + Send + Sync>,
264                            >(Vec::new()),
265                        }
266                    }
267                } else {
268                    warn!(
269                        "include_aur_comments set but installed_filter missing; skipping comments"
270                    );
271                    Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
272                }
273            } else {
274                Ok::<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>(Vec::new())
275            }
276        }
277    )
278}
279
280/// What: Combine feed results from all sources into a single sorted vector.
281///
282/// Inputs:
283/// - `arch_result`: Arch news fetch result.
284/// - `advisories_result`: Advisories fetch result.
285/// - `updates_result`: Package updates fetch result.
286/// - `comments_result`: AUR comments fetch result.
287/// - `sort_mode`: Sort mode for the final result.
288///
289/// Output:
290/// - Combined and sorted vector of news feed items.
291///
292/// Details:
293/// - Gracefully handles errors by logging warnings and continuing.
294/// - Sorts items according to the specified sort mode.
295fn combine_feed_results(
296    arch_result: std::result::Result<Vec<NewsFeedItem>, Box<dyn std::error::Error + Send + Sync>>,
297    advisories_result: std::result::Result<
298        Vec<NewsFeedItem>,
299        Box<dyn std::error::Error + Send + Sync>,
300    >,
301    updates_result: std::result::Result<
302        Vec<NewsFeedItem>,
303        Box<dyn std::error::Error + Send + Sync>,
304    >,
305    comments_result: std::result::Result<
306        Vec<NewsFeedItem>,
307        Box<dyn std::error::Error + Send + Sync>,
308    >,
309    sort_mode: NewsSortMode,
310) -> Vec<NewsFeedItem> {
311    let mut items: Vec<NewsFeedItem> = Vec::new();
312    match arch_result {
313        Ok(mut arch_items) => items.append(&mut arch_items),
314        Err(e) => warn!(error = %e, "arch news fetch failed; continuing without Arch news"),
315    }
316    match advisories_result {
317        Ok(mut adv_items) => items.append(&mut adv_items),
318        Err(e) => warn!(error = %e, "advisories fetch failed; continuing without advisories"),
319    }
320    match updates_result {
321        Ok(mut upd_items) => items.append(&mut upd_items),
322        Err(e) => warn!(error = %e, "updates fetch failed; continuing without updates"),
323    }
324    match comments_result {
325        Ok(mut cmt_items) => items.append(&mut cmt_items),
326        Err(e) => warn!(error = %e, "comments fetch failed; continuing without comments"),
327    }
328    sort_news_items(&mut items, sort_mode);
329    items
330}
331
332/// Return type for `prepare_fetch_context` function.
333type PrepareFetchContextReturn<'a, HS, HV, HC> = (
334    Option<String>,
335    Option<HashMap<String, (String, String)>>,
336    usize,
337    bool,
338    bool,
339    bool,
340    bool,
341    Option<&'a HashSet<String, HS>>,
342    bool,
343    NewsSortMode,
344    &'a mut HashMap<String, String, HV>,
345    &'a mut HashMap<String, String, HC>,
346    bool,
347);
348
349/// What: Prepare fetch context and calculate derived values.
350///
351/// Inputs:
352/// - `ctx`: News feed context.
353///
354/// Output:
355/// - Tuple of (`cutoff_date`, `updates_versions`, and extracted context fields).
356///
357/// Details:
358/// - Extracts context fields and calculates cutoff date and update versions.
359fn prepare_fetch_context<HS, HV, HC>(
360    ctx: NewsFeedContext<'_, HS, HV, HC>,
361) -> PrepareFetchContextReturn<'_, HS, HV, HC>
362where
363    HS: BuildHasher + Send + Sync + 'static,
364    HV: BuildHasher + Send + Sync + 'static,
365    HC: BuildHasher + Send + Sync + 'static,
366{
367    let NewsFeedContext {
368        limit,
369        include_arch_news,
370        include_advisories,
371        include_pkg_updates,
372        include_aur_comments,
373        installed_filter,
374        installed_only,
375        sort_mode,
376        seen_pkg_versions,
377        seen_aur_comments,
378        force_emit_all,
379        updates_list_path,
380        max_age_days,
381    } = ctx;
382
383    info!(
384        limit,
385        include_arch_news,
386        include_advisories,
387        include_pkg_updates,
388        include_aur_comments,
389        installed_only,
390        installed_filter = installed_filter.is_some(),
391        sort_mode = ?sort_mode,
392        max_age_days,
393        "fetch_news_feed start"
394    );
395
396    let cutoff_date = max_age_days.and_then(|days| {
397        chrono::Utc::now()
398            .checked_sub_signed(chrono::Duration::days(i64::from(days)))
399            .map(|dt| dt.format("%Y-%m-%d").to_string())
400    });
401    let updates_versions = if force_emit_all {
402        load_update_versions(updates_list_path.as_ref())
403    } else {
404        None
405    };
406
407    (
408        cutoff_date,
409        updates_versions,
410        limit,
411        include_arch_news,
412        include_advisories,
413        include_pkg_updates,
414        include_aur_comments,
415        installed_filter,
416        installed_only,
417        sort_mode,
418        seen_pkg_versions,
419        seen_aur_comments,
420        force_emit_all,
421    )
422}
423
424/// What: Sort news feed items by the specified mode.
425///
426/// Inputs:
427/// - `items`: Mutable slice of news feed items to sort.
428/// - `mode`: Sort mode (date descending, etc.).
429///
430/// Output: Items are sorted in place.
431///
432/// Details: Sorts news items according to the specified sort mode.
433fn sort_news_items(items: &mut [NewsFeedItem], mode: NewsSortMode) {
434    match mode {
435        NewsSortMode::DateDesc => items.sort_by(|a, b| b.date.cmp(&a.date)),
436        NewsSortMode::DateAsc => items.sort_by(|a, b| a.date.cmp(&b.date)),
437        NewsSortMode::Title => {
438            items.sort_by(|a, b| {
439                a.title
440                    .to_lowercase()
441                    .cmp(&b.title.to_lowercase())
442                    .then(b.date.cmp(&a.date))
443            });
444        }
445        NewsSortMode::SourceThenTitle => items.sort_by(|a, b| {
446            a.source
447                .cmp(&b.source)
448                .then(b.date.cmp(&a.date))
449                .then(a.title.to_lowercase().cmp(&b.title.to_lowercase()))
450        }),
451        NewsSortMode::SeverityThenDate => items.sort_by(|a, b| {
452            let sa = severity_rank(a.severity);
453            let sb = severity_rank(b.severity);
454            sb.cmp(&sa)
455                .then(b.date.cmp(&a.date))
456                .then(a.title.to_lowercase().cmp(&b.title.to_lowercase()))
457        }),
458        NewsSortMode::UnreadThenDate => {
459            // Fetch pipeline lacks read-state context; fall back to newest-first.
460            items.sort_by(|a, b| b.date.cmp(&a.date));
461        }
462    }
463}
464
465/// # Errors
466/// - Network failures fetching sources
467/// - JSON parse errors from upstream feeds
468pub async fn fetch_news_feed<HS, HV, HC>(
469    ctx: NewsFeedContext<'_, HS, HV, HC>,
470) -> Result<Vec<NewsFeedItem>>
471where
472    HS: BuildHasher + Send + Sync + 'static,
473    HV: BuildHasher + Send + Sync + 'static,
474    HC: BuildHasher + Send + Sync + 'static,
475{
476    let toolkit = crate::integrations::arch_toolkit::ToolkitContext::new()?;
477    fetch_news_feed_with_context(ctx, &toolkit).await
478}
479
480/// What: Fetch the combined feed with a caller-owned shared toolkit context.
481///
482/// Inputs:
483/// - `ctx`: Pacsea source, filter, sort, and seen-state policy.
484/// - `toolkit`: Runtime integration context.
485///
486/// Output:
487/// - Combined Pacsea feed items.
488///
489/// Details:
490/// - Only list-level Arch news and advisory transport/parsing use toolkit; caches, retries,
491///   aggregation, AUR comments, update detection, and read state remain Pacsea-owned.
492pub async fn fetch_news_feed_with_context<HS, HV, HC>(
493    ctx: NewsFeedContext<'_, HS, HV, HC>,
494    toolkit: &crate::integrations::arch_toolkit::ToolkitContext,
495) -> Result<Vec<NewsFeedItem>>
496where
497    HS: BuildHasher + Send + Sync + 'static,
498    HV: BuildHasher + Send + Sync + 'static,
499    HC: BuildHasher + Send + Sync + 'static,
500{
501    let (
502        cutoff_date,
503        updates_versions,
504        limit,
505        include_arch_news,
506        include_advisories,
507        include_pkg_updates,
508        include_aur_comments,
509        installed_filter,
510        installed_only,
511        sort_mode,
512        seen_pkg_versions,
513        seen_aur_comments,
514        force_emit_all,
515    ) = prepare_fetch_context(ctx);
516
517    info!(
518        "starting fetch: arch_news={include_arch_news}, advisories={include_advisories}, pkg_updates={include_pkg_updates}, aur_comments={include_aur_comments}"
519    );
520    rate_limit::reset_archlinux_backoff();
521
522    // Fetch ALL sources in parallel for best responsiveness:
523    // - Fast sources (AUR comments, package updates) run in parallel and complete quickly
524    // - Slow sources (arch news, advisories from archlinux.org) run sequentially with each other
525    //   but IN PARALLEL with the fast sources, so they don't block everything
526    let ((updates_result, comments_result), (arch_result, advisories_result)) = tokio::join!(
527        fetch_fast_sources(FastSourcesConfig {
528            include_pkg_updates,
529            include_aur_comments,
530            installed_filter,
531            limit,
532            seen_pkg_versions,
533            seen_aur_comments,
534            force_emit_all,
535            updates_versions: updates_versions.as_ref(),
536        }),
537        fetch_slow_sources(
538            toolkit,
539            include_arch_news,
540            include_advisories,
541            limit,
542            installed_filter,
543            installed_only,
544            cutoff_date.as_deref(),
545        )
546    );
547    info!("fetch completed, combining results...");
548
549    let items = combine_feed_results(
550        arch_result,
551        advisories_result,
552        updates_result,
553        comments_result,
554        sort_mode,
555    );
556    info!(
557        total = items.len(),
558        arch = items
559            .iter()
560            .filter(|i| matches!(i.source, crate::state::types::NewsFeedSource::ArchNews))
561            .count(),
562        advisories = items
563            .iter()
564            .filter(|i| matches!(
565                i.source,
566                crate::state::types::NewsFeedSource::SecurityAdvisory
567            ))
568            .count(),
569        updates = items
570            .iter()
571            .filter(|i| {
572                matches!(
573                    i.source,
574                    crate::state::types::NewsFeedSource::InstalledPackageUpdate
575                        | crate::state::types::NewsFeedSource::AurPackageUpdate
576                )
577            })
578            .count(),
579        aur_comments = items
580            .iter()
581            .filter(|i| matches!(i.source, crate::state::types::NewsFeedSource::AurComment))
582            .count(),
583        "fetch_news_feed success"
584    );
585    Ok(items)
586}
587
588/// Limit for continuation fetching (effectively unlimited).
589const CONTINUATION_LIMIT: usize = 1000;
590
591/// What: Fetch continuation items for background loading after initial batch.
592///
593/// Inputs:
594/// - `installed`: Set of installed package names.
595/// - `initial_ids`: IDs of items already fetched in initial batch.
596///
597/// Output:
598/// - `Ok(Vec<NewsFeedItem>)`: Additional items not in initial batch.
599///
600/// # Errors
601/// - Network errors when fetching from any source.
602/// - Parsing errors from upstream feeds.
603///
604/// Details:
605/// - Fetches items from all sources with a high limit (1000).
606/// - Filters out items already in `initial_ids`.
607/// - Used by background continuation worker to stream additional items to UI.
608pub async fn fetch_continuation_items<HS, HI>(
609    installed: &HashSet<String, HS>,
610    initial_ids: &HashSet<String, HI>,
611) -> Result<Vec<NewsFeedItem>>
612where
613    HS: std::hash::BuildHasher + Send + Sync + 'static,
614    HI: std::hash::BuildHasher + Send + Sync,
615{
616    let toolkit = crate::integrations::arch_toolkit::ToolkitContext::new()?;
617    fetch_continuation_items_with_context(installed, initial_ids, &toolkit).await
618}
619
620/// What: Fetch continuation feed rows with a shared toolkit context.
621///
622/// Inputs:
623/// - `installed`: Installed package names.
624/// - `initial_ids`: IDs already emitted.
625/// - `toolkit`: Runtime integration context.
626///
627/// Output:
628/// - Additional feed rows not already emitted.
629///
630/// Details:
631/// - Preserves Pacsea continuation, update, comment, and deduplication policy.
632pub async fn fetch_continuation_items_with_context<HS, HI>(
633    installed: &HashSet<String, HS>,
634    initial_ids: &HashSet<String, HI>,
635    toolkit: &crate::integrations::arch_toolkit::ToolkitContext,
636) -> Result<Vec<NewsFeedItem>>
637where
638    HS: std::hash::BuildHasher + Send + Sync + 'static,
639    HI: std::hash::BuildHasher + Send + Sync,
640{
641    use crate::state::types::NewsFeedSource;
642
643    info!(
644        installed_count = installed.len(),
645        initial_count = initial_ids.len(),
646        "starting continuation fetch"
647    );
648
649    // Fetch from all sources in parallel
650    let ((updates_result, comments_result), (arch_result, advisories_result)) = tokio::join!(
651        async {
652            // Package updates - use fresh seen maps (continuation doesn't track seen state)
653            let mut seen_versions: HashMap<String, String> = HashMap::new();
654            let mut seen_aur_comments: HashMap<String, String> = HashMap::new();
655            let updates = fetch_installed_updates(
656                installed,
657                CONTINUATION_LIMIT,
658                &mut seen_versions,
659                true, // force_emit_all
660                None,
661            )
662            .await;
663            let comments = fetch_installed_aur_comments(
664                installed,
665                CONTINUATION_LIMIT,
666                &mut seen_aur_comments,
667                true, // force_emit_all
668            )
669            .await;
670            (updates, comments)
671        },
672        fetch_slow_sources(
673            toolkit,
674            true, // include_arch_news
675            true, // include_advisories
676            CONTINUATION_LIMIT,
677            Some(installed),
678            false, // installed_only
679            None,  // cutoff_date
680        )
681    );
682
683    let mut items = Vec::new();
684
685    // Add Arch news (filter out already-sent items)
686    if let Ok(arch_items) = arch_result {
687        for item in arch_items {
688            if !initial_ids.contains(&item.id) {
689                items.push(item);
690            }
691        }
692    }
693
694    // Add advisories (filter out already-sent items)
695    if let Ok(adv_items) = advisories_result {
696        for item in adv_items {
697            if !initial_ids.contains(&item.id) {
698                items.push(item);
699            }
700        }
701    }
702
703    // Add package updates (filter out already-sent items)
704    if let Ok(upd_items) = updates_result {
705        for item in upd_items {
706            if !initial_ids.contains(&item.id) {
707                items.push(item);
708            }
709        }
710    }
711
712    // Add AUR comments (filter out already-sent items)
713    if let Ok(comment_items) = comments_result {
714        for item in comment_items {
715            if !initial_ids.contains(&item.id) {
716                items.push(item);
717            }
718        }
719    }
720
721    // Sort by date descending
722    sort_news_items(&mut items, NewsSortMode::DateDesc);
723
724    info!(
725        total = items.len(),
726        arch = items
727            .iter()
728            .filter(|i| matches!(i.source, NewsFeedSource::ArchNews))
729            .count(),
730        advisories = items
731            .iter()
732            .filter(|i| matches!(i.source, NewsFeedSource::SecurityAdvisory))
733            .count(),
734        updates = items
735            .iter()
736            .filter(|i| matches!(
737                i.source,
738                NewsFeedSource::InstalledPackageUpdate | NewsFeedSource::AurPackageUpdate
739            ))
740            .count(),
741        "continuation fetch complete"
742    );
743
744    Ok(items)
745}
746
747// Re-export public functions from submodules
748pub use rate_limit::{
749    check_circuit_breaker, extract_endpoint_pattern, extract_retry_after_from_error,
750    increase_archlinux_backoff, rate_limit_archlinux, record_circuit_breaker_outcome,
751    reset_archlinux_backoff, take_network_error,
752};
753pub use updates::{
754    get_aur_json_changes, get_official_json_changes, load_official_json_cache,
755    official_json_cache_path,
756};
757
758#[cfg(test)]
759mod tests;