test: added more tests (#16)

Co-authored-by: Sharang Parnerkar <parnerkarsharang@gmail.com> Reviewed-on: #16
2026-02-25 10:01:56 +00:00
parent 9085da9fae
commit 1d7aebf37c
29 changed files with 2243 additions and 22 deletions
@@ -72,7 +72,25 @@ mod inner {
        }

        let html = resp.text().await.ok()?;
-        let document = scraper::Html::parse_document(&html);
+        parse_article_html(&html)
+    }
+
+    /// Parse article text from raw HTML without any network I/O.
+    ///
+    /// Uses a tiered extraction strategy:
+    /// 1. Try content within `<article>`, `<main>`, or `[role="main"]`
+    /// 2. Fall back to all `<p>` tags outside excluded containers
+    ///
+    /// # Arguments
+    ///
+    /// * `html` - Raw HTML string to parse
+    ///
+    /// # Returns
+    ///
+    /// The extracted text, or `None` if extraction yields < 100 chars.
+    /// Output is capped at 8000 characters.
+    pub(crate) fn parse_article_html(html: &str) -> Option<String> {
+        let document = scraper::Html::parse_document(html);

        // Strategy 1: Extract from semantic article containers.
        // Most news sites wrap the main content in <article>, <main>,
@@ -134,7 +152,7 @@ mod inner {
    }

    /// Sum the total character length of all collected text parts.
-    fn joined_len(parts: &[String]) -> usize {
+    pub(crate) fn joined_len(parts: &[String]) -> usize {
        parts.iter().map(|s| s.len()).sum()
    }
 }
@@ -325,3 +343,150 @@ pub async fn chat_followup(
        .map(|choice| choice.message.content.clone())
        .ok_or_else(|| ServerFnError::new("Empty response from Ollama"))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pretty_assertions::assert_eq;
+
+    // -----------------------------------------------------------------------
+    // FollowUpMessage serde tests
+    // -----------------------------------------------------------------------
+
+    #[test]
+    fn followup_message_serde_round_trip() {
+        let msg = FollowUpMessage {
+            role: "assistant".into(),
+            content: "Here is my answer.".into(),
+        };
+        let json = serde_json::to_string(&msg).expect("serialize FollowUpMessage");
+        let back: FollowUpMessage =
+            serde_json::from_str(&json).expect("deserialize FollowUpMessage");
+        assert_eq!(msg, back);
+    }
+
+    #[test]
+    fn followup_message_deserialize_from_json_literal() {
+        let json = r#"{"role":"system","content":"You are helpful."}"#;
+        let msg: FollowUpMessage = serde_json::from_str(json).expect("deserialize literal");
+        assert_eq!(msg.role, "system");
+        assert_eq!(msg.content, "You are helpful.");
+    }
+
+    // -----------------------------------------------------------------------
+    // joined_len and parse_article_html tests (server feature required)
+    // -----------------------------------------------------------------------
+
+    #[cfg(feature = "server")]
+    mod server_tests {
+        use super::super::inner::{joined_len, parse_article_html};
+        use pretty_assertions::assert_eq;
+
+        #[test]
+        fn joined_len_empty_input() {
+            assert_eq!(joined_len(&[]), 0);
+        }
+
+        #[test]
+        fn joined_len_sums_correctly() {
+            let parts = vec!["abc".into(), "de".into(), "fghij".into()];
+            assert_eq!(joined_len(&parts), 10);
+        }
+
+        // -------------------------------------------------------------------
+        // parse_article_html tests
+        // -------------------------------------------------------------------
+
+        // Helper: generate a string of given length from a repeated word.
+        fn lorem(len: usize) -> String {
+            "Lorem ipsum dolor sit amet consectetur adipiscing elit "
+                .repeat((len / 55) + 1)
+                .chars()
+                .take(len)
+                .collect()
+        }
+
+        #[test]
+        fn article_tag_extracts_text() {
+            let body = lorem(250);
+            let html = format!("<html><body><article><p>{body}</p></article></body></html>");
+            let result = parse_article_html(&html);
+            assert!(result.is_some(), "expected Some for article tag");
+            assert!(result.unwrap().contains("Lorem"));
+        }
+
+        #[test]
+        fn main_tag_extracts_text() {
+            let body = lorem(250);
+            let html = format!("<html><body><main><p>{body}</p></main></body></html>");
+            let result = parse_article_html(&html);
+            assert!(result.is_some(), "expected Some for main tag");
+        }
+
+        #[test]
+        fn fallback_to_p_tags_when_article_main_yield_little() {
+            // No <article>/<main>, so falls back to <p> tags
+            let body = lorem(250);
+            let html = format!("<html><body><div><p>{body}</p></div></body></html>");
+            let result = parse_article_html(&html);
+            assert!(result.is_some(), "expected fallback to <p> tags");
+        }
+
+        #[test]
+        fn excludes_nav_footer_aside_content() {
+            // Content only inside excluded containers -- should be excluded
+            let body = lorem(250);
+            let html = format!(
+                "<html><body>\
+                 <nav><p>{body}</p></nav>\
+                 <footer><p>{body}</p></footer>\
+                 <aside><p>{body}</p></aside>\
+                 </body></html>"
+            );
+            let result = parse_article_html(&html);
+            assert!(result.is_none(), "expected None for excluded-only content");
+        }
+
+        #[test]
+        fn returns_none_when_text_too_short() {
+            let html = "<html><body><p>Short.</p></body></html>";
+            let result = parse_article_html(html);
+            assert!(result.is_none(), "expected None for short text");
+        }
+
+        #[test]
+        fn truncates_at_8000_chars() {
+            let body = lorem(10000);
+            let html = format!("<html><body><article><p>{body}</p></article></body></html>");
+            let result = parse_article_html(&html).expect("expected Some");
+            assert!(
+                result.len() <= 8000,
+                "expected <= 8000 chars, got {}",
+                result.len()
+            );
+        }
+
+        #[test]
+        fn skips_fragments_under_30_chars() {
+            // Only fragments < 30 chars -- should yield None
+            let html = "<html><body><article>\
+                         <p>Short frag one</p>\
+                         <p>Another tiny bit</p>\
+                         </article></body></html>";
+            let result = parse_article_html(html);
+            assert!(result.is_none(), "expected None for tiny fragments");
+        }
+
+        #[test]
+        fn extracts_from_role_main_attribute() {
+            let body = lorem(250);
+            let html = format!(
+                "<html><body>\
+                 <div role=\"main\"><p>{body}</p></div>\
+                 </body></html>"
+            );
+            let result = parse_article_html(&html);
+            assert!(result.is_some(), "expected Some for role=main");
+        }
+    }
+}