Make defuddle parse existing local HTML source files

Copilot · pirate · Copilot · commit 7a96232f2e2e · 2026-03-03T20:18:22.000Z
Co-authored-by: pirate &lt;511499+pirate@users.noreply.github.com&gt;
diff --git a/abx_plugins/plugins/defuddle/on_Snapshot__57_defuddle.py b/abx_plugins/plugins/defuddle/on_Snapshot__57_defuddle.py
@@ -57,14 +57,50 @@ def get_env_array(name: str, default: list[str] | None = None) -> list[str]:
         return default if default is not None else []
 
 
+def find_html_source() -> str | None:
+    """Return first non-empty HTML source file from sibling extractor outputs."""
+    search_patterns = [
+        "singlefile/singlefile.html",
+        "*_singlefile/singlefile.html",
+        "singlefile/*.html",
+        "*_singlefile/*.html",
+        "dom/output.html",
+        "*_dom/output.html",
+        "dom/*.html",
+        "*_dom/*.html",
+        "wget/**/*.html",
+        "*_wget/**/*.html",
+        "wget/**/*.htm",
+        "*_wget/**/*.htm",
+    ]
+
+    for base in (Path.cwd(), Path.cwd().parent):
+        for pattern in search_patterns:
+            for match in base.glob(pattern):
+                if match.is_file() and match.stat().st_size > 0:
+                    return str(match)
+    return None
+
+
 def extract_defuddle(url: str, binary: str) -> tuple[bool, str | None, str]:
     timeout = get_env_int("DEFUDDLE_TIMEOUT") or get_env_int("TIMEOUT", 60)
     defuddle_args = get_env_array("DEFUDDLE_ARGS", [])
     defuddle_args_extra = get_env_array("DEFUDDLE_ARGS_EXTRA", [])
     output_dir = Path(OUTPUT_DIR)
+    html_source = find_html_source()
+    if not html_source:
+        return False, None, "No HTML source found (run singlefile, dom, or wget first)"
 
     try:
-        cmd = [binary, *defuddle_args, *defuddle_args_extra, url]
+        cmd = [
+            binary,
+            *defuddle_args,
+            "parse",
+            html_source,
+            *defuddle_args_extra,
+        ]
+        if "--json" not in cmd and "-j" not in cmd:
+            cmd.append("--json")
         result = subprocess.run(
             cmd,
             stdout=subprocess.PIPE,
diff --git a/abx_plugins/plugins/defuddle/tests/test_defuddle.py b/abx_plugins/plugins/defuddle/tests/test_defuddle.py
@@ -27,6 +27,18 @@
 TEST_URL = "https://example.com"
 
 
+def create_example_html(tmpdir: Path) -> Path:
+    """Create a local singlefile HTML fixture used as parser input."""
+    singlefile_dir = tmpdir / "singlefile"
+    singlefile_dir.mkdir(parents=True, exist_ok=True)
+    html_file = singlefile_dir / "singlefile.html"
+    html_file.write_text(
+        "<html><head><title>Example Domain</title></head><body><article><h1>Example Domain</h1><p>Example text body</p></article></body></html>",
+        encoding="utf-8",
+    )
+    return html_file
+
+
 def test_hook_script_exists():
     assert DEFUDDLE_HOOK.exists(), f"Hook script not found: {DEFUDDLE_HOOK}"
 
@@ -57,6 +69,7 @@ def test_reports_missing_dependency_when_not_installed():
         tmpdir = Path(tmpdir)
         snap_dir = tmpdir / "snap"
         snap_dir.mkdir(parents=True, exist_ok=True)
+        create_example_html(snap_dir)
 
         env = {"PATH": "/nonexistent", "HOME": str(tmpdir), "SNAP_DIR": str(snap_dir)}
         result = subprocess.run(
@@ -87,10 +100,20 @@ def test_extracts_article_with_json_output_from_binary():
         tmpdir = Path(tmpdir)
         snap_dir = tmpdir / "snap"
         snap_dir.mkdir(parents=True, exist_ok=True)
+        expected_html = create_example_html(snap_dir)
 
         fake_binary = tmpdir / "fake_defuddle.py"
         fake_binary.write_text(
-            "import json,sys; print(json.dumps({'content':'<article>Example</article>','textContent':'Example text','title':'Example Title'}))"
+            "import json, pathlib, sys\n"
+            "args = sys.argv[1:]\n"
+            "assert 'parse' in args\n"
+            "idx = args.index('parse') + 1\n"
+            "source = pathlib.Path(args[idx])\n"
+            "assert source.is_file()\n"
+            "assert str(source).startswith('/')\n"
+            "assert not str(source).startswith('http')\n"
+            "assert '--json' in args or '-j' in args\n"
+            "print(json.dumps({'content':'<article>Example</article>','textContent':'Example text','title':'Example Title'}))\n"
         )
         fake_binary.chmod(fake_binary.stat().st_mode | stat.S_IXUSR)
 
@@ -126,3 +149,4 @@ def test_extracts_article_with_json_output_from_binary():
         assert "Example text" in (output_dir / "content.txt").read_text(encoding="utf-8")
         metadata = json.loads((output_dir / "article.json").read_text(encoding="utf-8"))
         assert metadata.get("title") == "Example Title"
+        assert expected_html.exists()
