v0.2.0: image size filter, open folder, license (Seth Strickland)

- CLI: --min-image-size / --max-image-size (k/m suffixes); filter by HEAD Content-Length
- Fetcher: head_metadata() for content-type and content-length
- GUI: image size filter (Min KB / Max MB), Open folder button
- README: author and license attribution
- LICENSE: Copyright (c) 2025 Seth Strickland
- Bump to 0.2.0; CHANGELOG updated

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: buzzcauldron

CHANGELOG.md

@@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+## [0.2.0] - 2025-02-04
+
+- GUI (tkinter) with file-type selector, image size filter, and Open folder button.
+- CLI: `--types` (pdf/text/images), `--min-image-size` / `--max-image-size`, `--workers`, `--no-progress`.
+- Auto-install deps via `BASIC_SCRAPER_AUTO_INSTALL_DEPS=1`.
+- Hardware-autodetected parallel crawl; progress bar (optional tqdm).
+- Docker image (slim) and GitHub Actions (PyInstaller + Docker).
+- License: MIT, Copyright (c) 2025 Seth Strickland.
+
 ## [0.1.0]
 
 - Initial package layout, CLI stub, and semver setup.

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2025
+Copyright (c) 2025 Seth Strickland
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -2,6 +2,8 @@
 
 Basic scraper: PDFs, text, and images from websites at high quality, stored locally.
 
+**Author:** Seth Strickland · **License:** [MIT](LICENSE)
+
 ## Versioning
 
 This project follows [Semantic Versioning 2.0.0](https://semver.org/): `MAJOR.MINOR.PATCH`.
@@ -14,7 +16,7 @@ For the **0.y.z** range, the public API is treated as unstable: MINOR may introd
 
 ## License
 
-MIT. See [LICENSE](LICENSE).
+MIT License. Copyright (c) 2025 Seth Strickland. See [LICENSE](LICENSE).
 
 ## Install and run
 
@@ -30,6 +32,8 @@ This installs the package in editable mode and registers the `scrape` and `scrap
 scrape --url https://example.com/page [--out-dir output] [--delay 1] [--crawl] [--max-depth 2] [--same-domain-only]
 ```
 
+Filter images by file size (uses HEAD `Content-Length`): `--min-image-size 50k` and/or `--max-image-size 5m` (suffixes `k`/`m` for KB/MB).
+
 Or open the simple GUI:
 
 ```bash

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "basic-scraper"
-version = "0.1.0"
+version = "0.2.0"
 description = "Basic scraper: PDFs, text, and images from websites at high quality, stored locally."
 readme = "README.md"
 license = { text = "MIT" }

web_scraper/cli.py

@@ -43,6 +43,18 @@
 VALID_TYPES = frozenset({"pdf", "text", "images"})
 
 
+def _parse_size(s: str) -> int:
+    """Parse size string to bytes: 100, 100k, 1m (case-insensitive)."""
+    s = s.strip().lower()
+    if not s:
+        raise ValueError("empty size")
+    if s.endswith("k"):
+        return int(s[:-1]) * 1024
+    if s.endswith("m"):
+        return int(s[:-1]) * 1024 * 1024
+    return int(s)
+
+
 def _scrape_page(
     url: str,
     out_dir: Path,
@@ -54,6 +66,8 @@ def _scrape_page(
     collect_links: bool,
     types: set[str] | None = None,
     progress_callback: Callable[[str], None] | None = None,
+    min_image_size: int | None = None,
+    max_image_size: int | None = None,
 ) -> list[str]:
     """
     Scrape a single page: PDFs, text, images (according to types).
@@ -122,10 +136,14 @@ def _scrape_page(
             if img_url in urls_map:
                 continue
             best_url = get_best_image_url(img_url, None, try_high_res=True)
-            ct = fetcher.head_content_type(best_url, delay=delay)
+            ct, content_length = fetcher.head_metadata(best_url, delay=delay)
             if ct and not ct.startswith("image/"):
                 best_url = img_url
-                ct = fetcher.head_content_type(img_url, delay=delay)
+                ct, content_length = fetcher.head_metadata(img_url, delay=delay)
+            if min_image_size is not None and content_length is not None and content_length < min_image_size:
+                continue
+            if max_image_size is not None and content_length is not None and content_length > max_image_size:
+                continue
             dest = path_for_image(out_dir, domain, best_url, ct)
             if dest.exists():
                 urls_map[img_url] = str(dest)
@@ -195,11 +213,37 @@ def main() -> None:
         action="store_true",
         help="Disable progress bar (e.g. for scripting)",
     )
+    parser.add_argument(
+        "--min-image-size",
+        type=str,
+        default=None,
+        metavar="SIZE",
+        help="Skip images smaller than SIZE (e.g. 50k, 1m). Uses HEAD Content-Length.",
+    )
+    parser.add_argument(
+        "--max-image-size",
+        type=str,
+        default=None,
+        metavar="SIZE",
+        help="Skip images larger than SIZE (e.g. 5m, 10m). Uses HEAD Content-Length.",
+    )
     args = parser.parse_args()
 
     out_dir = Path(args.out_dir)
     limit = args.limit
     types_set = set(args.types) if args.types else None
+    min_image_size = None
+    max_image_size = None
+    if args.min_image_size:
+        try:
+            min_image_size = _parse_size(args.min_image_size)
+        except ValueError as e:
+            parser.error(f"--min-image-size: {e}")
+    if args.max_image_size:
+        try:
+            max_image_size = _parse_size(args.max_image_size)
+        except ValueError as e:
+            parser.error(f"--max-image-size: {e}")
     workers = args.workers if args.workers is not None else default_workers()
     workers = max(1, min(workers, default_workers()))
 
@@ -208,9 +252,13 @@ def main() -> None:
         _crawl_parallel(
             args.url, out_dir, args.delay, args.max_depth,
             args.same_domain_only, limit, types_set, workers, use_progress,
+            min_image_size, max_image_size,
         )
     else:
-        _run_single_or_sequential_crawl(args, out_dir, limit, types_set, workers, use_progress)
+        _run_single_or_sequential_crawl(
+            args, out_dir, limit, types_set, workers, use_progress,
+            min_image_size, max_image_size,
+        )
 
     print("\nDone.", file=sys.stderr)
 
@@ -222,6 +270,8 @@ def _run_single_or_sequential_crawl(
     types_set: set[str] | None,
     workers: int,
     use_progress: bool,
+    min_image_size: int | None,
+    max_image_size: int | None,
 ) -> None:
     """Single-page scrape or sequential crawl (workers=1)."""
     with Fetcher() as fetcher:
@@ -248,6 +298,8 @@ def _run_single_or_sequential_crawl(
                         url, out_dir, args.delay, manifest, fetcher,
                         limit, limit, collect_links=True, types=types_set,
                         progress_callback=None,
+                        min_image_size=min_image_size,
+                        max_image_size=max_image_size,
                     )
                     if use_progress:
                         pbar.update(1)
@@ -276,6 +328,8 @@ def _run_single_or_sequential_crawl(
                     args.url, out_dir, args.delay, manifest, fetcher,
                     limit, limit, collect_links=False, types=types_set,
                     progress_callback=progress_cb,
+                    min_image_size=min_image_size,
+                    max_image_size=max_image_size,
                 )
             finally:
                 if use_progress:
@@ -292,6 +346,8 @@ def _crawl_parallel(
     types_set: set[str] | None,
     workers: int,
     use_progress: bool,
+    min_image_size: int | None,
+    max_image_size: int | None,
 ) -> None:
     """Crawl with a thread pool; each worker uses its own Fetcher, shared manifest lock."""
     start_domain = urlparse(start_url).netloc
@@ -314,6 +370,9 @@ def process_one(url: str, depth: int) -> list[str]:
                     links = _scrape_page(
                         url, out_dir, delay, manifest, fetcher,
                         limit, limit, collect_links=True, types=types_set,
+                        progress_callback=None,
+                        min_image_size=min_image_size,
+                        max_image_size=max_image_size,
                     )
                 except Exception as e:
                     print(f"Error {url}: {e}", file=sys.stderr)

web_scraper/fetcher.py

@@ -98,6 +98,22 @@ def head_content_type(self, url: str, *, timeout: float = 10.0, delay: float = 0
         except Exception:
             return None
 
+    def head_metadata(self, url: str, *, timeout: float = 10.0, delay: float = 0) -> tuple[str | None, int | None]:
+        """HEAD request; returns (content_type, content_length). content_length is None if header missing."""
+        if delay > 0:
+            time.sleep(delay)
+        try:
+            client = self._get_client()
+            resp = client.head(url, timeout=timeout)
+            resp.raise_for_status()
+            ct = resp.headers.get("content-type", "")
+            content_type = ct.split(";")[0].strip().lower() if ct else None
+            cl = resp.headers.get("content-length")
+            content_length = int(cl) if cl is not None and cl.isdigit() else None
+            return content_type, content_length
+        except Exception:
+            return None, None
+
 
 def fetch_html(url: str, *, timeout: float = DEFAULT_TIMEOUT, delay: float = 0) -> tuple[bytes, str]:
     """Standalone fetch (creates temporary client). Prefer Fetcher for multiple requests."""

web_scraper/gui.py

@@ -11,6 +11,23 @@
 from web_scraper._deps import check_required
 
 
+def _open_folder(path: str) -> None:
+    """Open path in the system file manager; create dir if missing."""
+    if not path or not path.strip():
+        return
+    abs_path = os.path.abspath(path.strip())
+    try:
+        os.makedirs(abs_path, exist_ok=True)
+    except OSError:
+        pass
+    if sys.platform == "darwin":
+        subprocess.run(["open", abs_path], check=False)
+    elif sys.platform == "win32":
+        os.startfile(abs_path)  # type: ignore[attr-defined]
+    else:
+        subprocess.run(["xdg-open", abs_path], check=False)
+
+
 def main() -> None:
     check_required()
     root = tk.Tk()
@@ -26,9 +43,13 @@ def main() -> None:
     url_entry.grid(row=1, column=0, columnspan=2, sticky=tk.EW, pady=(0, 8))
 
     ttk.Label(main_frame, text="Output directory").grid(row=2, column=0, sticky=tk.W, pady=(0, 2))
+    out_row = ttk.Frame(main_frame)
+    out_row.grid(row=3, column=0, columnspan=2, sticky=tk.EW, pady=(0, 8))
+    main_frame.columnconfigure(0, weight=1)
     out_var = tk.StringVar(value="output")
-    out_entry = ttk.Entry(main_frame, textvariable=out_var, width=50)
-    out_entry.grid(row=3, column=0, columnspan=2, sticky=tk.EW, pady=(0, 8))
+    out_entry = ttk.Entry(out_row, textvariable=out_var, width=50)
+    out_entry.pack(side=tk.LEFT, fill=tk.X, expand=True, padx=(0, 8))
+    ttk.Button(out_row, text="Open folder", command=lambda: _open_folder(out_var.get())).pack(side=tk.LEFT)
 
     types_frame = ttk.LabelFrame(main_frame, text="File types")
     types_frame.grid(row=4, column=0, columnspan=2, sticky=tk.W, pady=(0, 8))
@@ -39,8 +60,17 @@ def main() -> None:
     ttk.Checkbutton(types_frame, text="Text", variable=type_text_var).pack(side=tk.LEFT, padx=(0, 12))
     ttk.Checkbutton(types_frame, text="Images", variable=type_images_var).pack(side=tk.LEFT)
 
+    size_frame = ttk.LabelFrame(main_frame, text="Image size filter (optional)")
+    size_frame.grid(row=5, column=0, columnspan=2, sticky=tk.W, pady=(0, 8))
+    min_image_var = tk.StringVar(value="")
+    max_image_var = tk.StringVar(value="")
+    ttk.Label(size_frame, text="Min (KB):").pack(side=tk.LEFT, padx=(0, 4))
+    ttk.Entry(size_frame, textvariable=min_image_var, width=8).pack(side=tk.LEFT, padx=(0, 12))
+    ttk.Label(size_frame, text="Max (MB):").pack(side=tk.LEFT, padx=(8, 4))
+    ttk.Entry(size_frame, textvariable=max_image_var, width=8).pack(side=tk.LEFT)
+
     opts_frame = ttk.Frame(main_frame)
-    opts_frame.grid(row=5, column=0, columnspan=2, sticky=tk.W, pady=(0, 8))
+    opts_frame.grid(row=6, column=0, columnspan=2, sticky=tk.W, pady=(0, 8))
     delay_var = tk.DoubleVar(value=1.0)
     ttk.Label(opts_frame, text="Delay (s):").pack(side=tk.LEFT)
     delay_spin = ttk.Spinbox(opts_frame, from_=0.5, to=10, increment=0.5, width=5, textvariable=delay_var)
@@ -55,9 +85,9 @@ def main() -> None:
     ttk.Checkbutton(opts_frame, text="Same domain only", variable=same_domain_var).pack(side=tk.LEFT)
 
     log_frame = ttk.LabelFrame(main_frame, text="Log")
-    log_frame.grid(row=6, column=0, columnspan=2, sticky=tk.NSEW, pady=(0, 8))
+    log_frame.grid(row=7, column=0, columnspan=2, sticky=tk.NSEW, pady=(0, 8))
     main_frame.columnconfigure(0, weight=1)
-    main_frame.rowconfigure(6, weight=1)
+    main_frame.rowconfigure(7, weight=1)
 
     log_text = tk.Text(log_frame, height=8, wrap=tk.WORD, state=tk.DISABLED)
     log_scroll = ttk.Scrollbar(log_frame)
@@ -122,6 +152,12 @@ def run_scrape(scrape_btn_ref: tk.Widget) -> None:
             append_log("Error: Select at least one file type.\n")
             scrape_btn_ref.config(state=tk.NORMAL)
             return
+        min_s = min_image_var.get().strip()
+        if min_s:
+            cmd.extend(["--min-image-size", f"{min_s}k"])
+        max_s = max_image_var.get().strip()
+        if max_s:
+            cmd.extend(["--max-image-size", f"{max_s}m"])
         if crawl_var.get():
             cmd.extend(["--crawl", "--max-depth", str(depth)])
             if same_domain_var.get():
@@ -164,7 +200,7 @@ def poll_queue(btn: tk.Widget) -> None:
         poll_queue(scrape_btn_ref)
 
     btn_frame = ttk.Frame(main_frame)
-    btn_frame.grid(row=7, column=0, columnspan=2)
+    btn_frame.grid(row=8, column=0, columnspan=2)
     scrape_btn = ttk.Button(btn_frame, text="Scrape", command=lambda: run_scrape(scrape_btn))
     scrape_btn.pack(side=tk.LEFT, padx=(0, 8))
     ttk.Button(btn_frame, text="Clear log", command=lambda: (log_text.config(state=tk.NORMAL), log_text.delete("1.0", tk.END), log_text.config(state=tk.DISABLED))).pack(side=tk.LEFT)