Merge pull request #13 from costiash/fix/critical-bugs-handle

feat!: v0.5.0 - Version-aware upgrade mechanism with breaking changes

Author: costiash

.gitattributes

@@ -0,0 +1,9 @@
+# Mark documentation files as generated content
+# This prevents them from appearing in PR diffs by default
+# and excludes them from language statistics
+
+docs/*.md linguist-generated=true
+docs/**/*.md linguist-generated=true
+
+# Also mark search index as generated
+docs/.search_index.json linguist-generated=true

.github/workflows/claude-code-review.yml

@@ -3,12 +3,12 @@ name: Claude Code Review
 on:
   pull_request:
     types: [opened, synchronize]
-    # Optional: Only run on specific file changes
-    # paths:
-    #   - "src/**/*.ts"
-    #   - "src/**/*.tsx"
-    #   - "src/**/*.js"
-    #   - "src/**/*.jsx"
+    # Ignore documentation files (auto-generated, large volume)
+    paths-ignore:
+      - "docs/**"
+      - "docs/*.md"
+      - "*.md"
+      - "enhancements/**"
 
 jobs:
   claude-review:

.github/workflows/update-docs.yml

@@ -29,7 +29,17 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r scripts/requirements.txt
-    
+
+    # =================================================================
+    # SAFEGUARD: Count files BEFORE fetch to detect mass deletions
+    # =================================================================
+    - name: Count files before fetch
+      id: count-before
+      run: |
+        BEFORE=$(find docs/ -name "*.md" 2>/dev/null | wc -l)
+        echo "count=$BEFORE" >> $GITHUB_OUTPUT
+        echo "📊 Files before fetch: $BEFORE"
+
     - name: Fetch latest documentation
       id: fetch-docs
       env:
@@ -38,7 +48,51 @@ jobs:
       run: |
         python scripts/fetch_claude_docs.py || echo "fetch_failed=true" >> $GITHUB_OUTPUT
       continue-on-error: true
-    
+
+    # =================================================================
+    # SAFEGUARD: Validate file count AFTER fetch (FAIL + REVERT on trigger)
+    # =================================================================
+    - name: Validate after fetch (safeguard)
+      id: validate-safeguard
+      run: |
+        AFTER=$(find docs/ -name "*.md" 2>/dev/null | wc -l)
+        BEFORE=${{ steps.count-before.outputs.count }}
+
+        echo "📊 Files after fetch: $AFTER (was: $BEFORE)"
+
+        # SAFEGUARD 1: Check deletion percentage
+        if [ "$BEFORE" -gt 0 ]; then
+          DELETED=$((BEFORE - AFTER))
+          if [ "$DELETED" -lt 0 ]; then
+            DELETED=0
+          fi
+          PERCENT=$((DELETED * 100 / BEFORE))
+
+          if [ "$PERCENT" -gt 10 ]; then
+            echo "::error::🚨 SAFEGUARD TRIGGERED: $PERCENT% of files would be deleted!"
+            echo "::error::   Before: $BEFORE files, After: $AFTER files, Would delete: $DELETED"
+            echo "::error::   This indicates sitemap discovery failure."
+            echo ""
+            echo "Reverting docs/ to previous state..."
+            git checkout -- docs/
+            echo "safeguard_triggered=true" >> $GITHUB_OUTPUT
+            echo "::error::Workflow failed. Manual investigation required."
+            exit 1
+          fi
+        fi
+
+        # SAFEGUARD 2: Check minimum file count
+        if [ "$AFTER" -lt 250 ]; then
+          echo "::error::🚨 SAFEGUARD TRIGGERED: Only $AFTER files remain (expected 250+)!"
+          echo "Reverting docs/ to previous state..."
+          git checkout -- docs/
+          echo "safeguard_triggered=true" >> $GITHUB_OUTPUT
+          echo "::error::Workflow failed. Manual investigation required."
+          exit 1
+        fi
+
+        echo "✅ Safeguard validation passed: $AFTER files present"
+
     - name: Check for changes
       id: verify-changed-files
       run: |

.gitignore

@@ -90,6 +90,9 @@ claude-docs-helper.sh
 # Upstream reference repository (local development only)
 upstream/
 
+# Archive folder (legacy/orphaned scripts)
+archive/
+
 # ============================================================================
 # Documentation & Tracking Files (DO NOT COMMIT)
 # ============================================================================

CHANGELOG.md

@@ -5,6 +5,50 @@ All notable changes to the enhanced edition of claude-code-docs will be document
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.0] - 2025-12-06
+
+### Breaking Changes
+- **Filename Convention Changed**: All Claude Code CLI docs renamed to use domain-based prefix
+  - Old: `docs__en__<topic>.md` (e.g., `docs__en__hooks.md`)
+  - New: `claude-code__<topic>.md` (e.g., `claude-code__hooks.md`)
+  - Platform docs use: `docs__en__<path>.md`
+- **Scripts Restructured**: Monolithic scripts replaced with modular packages
+  - Added: `scripts/fetcher/` (8 modules for documentation fetching)
+  - Added: `scripts/lookup/` (7 modules for search and validation)
+  - Removed: `main.py`, `update_sitemap.py`, `extract_paths.py`, `clean_manifest.py`
+
+### Added
+- **2x Documentation Coverage**: 571 files (up from ~270)
+- **573 Tracked Paths**: Comprehensive coverage across 6 categories
+  - API Reference: 377 paths (65.8%)
+  - Core Documentation: 82 paths (14.3%)
+  - Prompt Library: 65 paths (11.3%)
+  - Claude Code: 46 paths (8.0%)
+  - Release Notes: 2 paths
+  - Resources: 1 path
+- **Safety Thresholds**: Prevent catastrophic deletion during automated sync
+  - `MIN_DISCOVERY_THRESHOLD`: 200 paths minimum from sitemaps
+  - `MAX_DELETION_PERCENT`: 10% maximum deletion per sync
+  - `MIN_EXPECTED_FILES`: 250 minimum files required
+- **Modular Architecture**: Better code organization and testability
+  - `fetcher/` package: config, manifest, paths, sitemap, content, safeguards, cli
+  - `lookup/` package: config, manifest, search, validation, formatting, cli
+- **Domain-Based Naming**: Clear source identification in filenames
+  - `claude-code__*.md` from code.claude.com
+  - `docs__en__*.md` from platform.claude.com
+- **Version-Aware Upgrades**: Installer detects existing version and shows upgrade info
+
+### Changed
+- **Manifest Structure**: `paths_manifest.json` now tracks 573 paths in 6 categories
+- **Search Index**: Updated to cover all 571 documentation files
+- **Python Packages**: Thin wrappers (`fetch_claude_docs.py`, `lookup_paths.py`) for backward compatibility
+
+### Upgrade Notes
+- **Seamless Upgrade**: Run `install.sh` again to upgrade from any v0.4.x version
+- **No Data Loss**: All user configs remain in `~/.claude/`
+- **Atomic Operation**: Installation uses temp directory, moves atomically
+- The installer will show before/after comparison during upgrade
+
 ## [0.4.2] - 2025-11-25
 
 ### Fixed

CLAUDE.md

@@ -1,18 +1,30 @@
 # Claude Code Documentation Mirror - Enhanced Edition
 
-This repository contains local copies of Claude Code documentation from https://docs.anthropic.com/en/docs/claude-code/
-
-The docs are periodically updated via GitHub Actions.
+> **⛔ CRITICAL: UPSTREAM ISOLATION ⛔**
+>
+> **This repository is COMPLETELY INDEPENDENT. Do NOT:**
+> - Push to, pull from, or sync with the upstream repo (ericbuess/claude-code-docs)
+> - Create PRs to the upstream repo
+> - Add upstream as a remote
+> - Reference upstream in any git operations
+>
+> **All git operations must target `origin` (costiash/claude-code-docs) ONLY.**
+
+This repository contains local copies of Claude documentation from multiple Anthropic sources:
+- **Platform docs**: https://platform.claude.com (API, guides, Agent SDK, etc.)
+- **Claude Code docs**: https://code.claude.com/docs (CLI-specific documentation)
+
+The docs are periodically updated via GitHub Actions with safeguards to prevent mass deletion.
 
 ## Architecture: Single Installation with Optional Python Features
 
 This repository uses a **graceful degradation** approach:
 
 **Installation** (always the same):
-- 273 documentation paths tracked in manifest
-- ~266-270 files downloaded (varies based on fetch success)
+- 573 documentation paths tracked in manifest (6 categories)
+- 571 files downloaded
 - Python scripts for enhanced features
-- Full test suite and GitHub workflows
+- Full test suite (294 tests) and GitHub workflows
 
 **Runtime Features** (Python-dependent):
 - **Without Python 3.9+**: Basic documentation reading via shell scripts
@@ -383,36 +395,53 @@ When Python 3.9+ is installed, these additional capabilities are available:
 - **Full-text search**: `--search "keyword"` searches across all documentation content
 - **Category filtering**: `--category api` lists paths in specific categories
 - **Path validation**: `--validate` checks documentation integrity
-- **Active documentation**: Access to 273 active paths across 7 categories:
-  - Core Documentation (80 paths, 29.3%)
-  - API Reference (79 paths, 28.9%)
-  - Prompt Library (65 paths, 23.8%)
-  - Claude Code (45 paths, 16.5%)
+- **Active documentation**: Access to 573 paths across 6 categories:
+  - API Reference (377 paths, 65.8%) - Includes multi-language SDK docs (Python, TypeScript, Go, Java, Kotlin, Ruby)
+  - Core Documentation (82 paths, 14.3%)
+  - Prompt Library (65 paths, 11.3%)
+  - Claude Code (46 paths, 8.0%)
   - Release Notes (2 paths)
   - Resources (1 path)
-  - Uncategorized (1 path)
 
 See `enhancements/` directory for comprehensive feature documentation and examples.
 
 ## Repository Structure
 
 ```
 /
-├── docs/                   # ~266-270 documentation files (.md format)
+├── docs/                   # 571 documentation files (.md format)
 │   ├── docs_manifest.json  # File tracking manifest
 │   └── .search_index.json  # Full-text search index (Python-generated)
 ├── scripts/
 │   ├── claude-docs-helper.sh       # Main helper (feature detection)
-│   ├── fetch_claude_docs.py        # Documentation fetcher with auto-regeneration
-│   ├── lookup_paths.py             # Search & validation (Python)
-│   └── build_search_index.py       # Index builder (Python)
-├── paths_manifest.json     # Active paths manifest (273 paths tracked)
+│   ├── fetch_claude_docs.py        # Thin wrapper for fetcher package
+│   ├── lookup_paths.py             # Thin wrapper for lookup package
+│   ├── build_search_index.py       # Index builder (Python)
+│   ├── fetcher/                    # Documentation fetching package
+│   │   ├── __init__.py             # Package exports
+│   │   ├── config.py               # Constants and safety thresholds
+│   │   ├── manifest.py             # Manifest file operations
+│   │   ├── paths.py                # Path conversion and categorization
+│   │   ├── sitemap.py              # Sitemap discovery and parsing
+│   │   ├── content.py              # Content fetching and validation
+│   │   ├── safeguards.py           # Safety checks (deletion prevention)
+│   │   └── cli.py                  # Main entry point
+│   └── lookup/                     # Search and validation package
+│       ├── __init__.py             # Package exports
+│       ├── config.py               # Configuration constants
+│       ├── manifest.py             # Manifest loading utilities
+│       ├── search.py               # Search functionality
+│       ├── validation.py           # Path validation
+│       ├── formatting.py           # Output formatting
+│       └── cli.py                  # Main entry point
+├── paths_manifest.json     # Active paths manifest (573 paths, 6 categories)
+├── archive/               # Archived/deprecated scripts (git-ignored)
 ├── enhancements/          # Feature documentation
 │   ├── README.md          # Overview
 │   ├── FEATURES.md        # Technical specs
 │   ├── CAPABILITIES.md    # Detailed capabilities
 │   └── EXAMPLES.md        # Usage examples
-├── tests/                 # Test suite (620 tests, 618 passing)
+├── tests/                 # Test suite (294 tests, 294 passing)
 ├── install.sh            # Installation script
 └── CLAUDE.md             # This file (AI context)
 
@@ -430,15 +459,48 @@ When working on this repository:
 @uninstall.sh - Clean removal
 
 ### Python Features
-@scripts/fetch_claude_docs.py - Documentation fetcher with auto-regeneration
-@scripts/lookup_paths.py - Search & validation
+@scripts/fetch_claude_docs.py - Thin wrapper for fetcher package (backwards compatible)
+@scripts/lookup_paths.py - Thin wrapper for lookup package (backwards compatible)
+@scripts/fetcher/ - Documentation fetching package (8 modules)
+@scripts/lookup/ - Search & validation package (7 modules)
 @scripts/build_search_index.py - Full-text search indexing
-@paths_manifest.json - Active paths manifest (273 paths tracked)
-@tests/ - Test suite (620 tests)
+@paths_manifest.json - Active paths manifest (573 paths, 6 categories)
+@tests/ - Test suite (294 tests)
 
 ### Automation
 @.github/workflows/ - Auto-update workflows (runs every 3 hours)
 
+## Documentation Deletion Safeguards
+
+The automated sync system includes multiple safeguards to prevent catastrophic documentation loss. These were implemented after a critical bug where 80%+ of documentation was deleted due to broken sitemap URLs.
+
+### Safety Thresholds (in `scripts/fetcher/config.py`)
+
+| Constant | Value | Purpose |
+|----------|-------|---------|
+| `MIN_DISCOVERY_THRESHOLD` | 200 | Minimum paths that must be discovered from sitemaps |
+| `MAX_DELETION_PERCENT` | 10 | Maximum percentage of files that can be deleted in one sync |
+| `MIN_EXPECTED_FILES` | 250 | Minimum files that must remain after sync |
+
+### How Safeguards Work
+
+1. **Discovery Validation**: Before fetching, validates that sitemap discovery found enough paths
+2. **Deletion Limiting**: `cleanup_old_files()` refuses to delete more than 10% of existing files
+3. **File Count Validation**: Refuses to proceed if result would have fewer than 250 files
+4. **Workflow Validation**: GitHub Actions validates sync success before committing
+
+### Sitemap Sources
+
+Documentation is discovered from two sitemaps:
+- `https://platform.claude.com/sitemap.xml` - Platform documentation (API, guides, etc.)
+- `https://code.claude.com/docs/sitemap.xml` - Claude Code CLI documentation
+
+### Filename Conventions
+
+Files are named based on their source:
+- Platform docs: `en__docs__section__page.md` (double underscore for path separators)
+- Claude Code docs: `docs__en__page.md` (prefixed with `docs__`)
+
 ## Working on This Repository
 
 **Critical Rule**: Changes must maintain graceful degradation - work with AND without Python.
@@ -463,7 +525,7 @@ python3 scripts/lookup_paths.py --search "mcp"
 pytest tests/ -v
 
 # Run full test suite
-pytest tests/ -q  # Should see: 598 passed, 2 skipped
+pytest tests/ -q  # Should see: 294 passed, 2 skipped
 ```
 
 ## Upstream Compatibility

CONTRIBUTING.md

@@ -7,16 +7,16 @@ Thank you for contributing to the Enhanced Claude Code Documentation Mirror!
 This project extends [ericbuess/claude-code-docs](https://github.com/ericbuess/claude-code-docs) with optional Python features:
 
 **Core Principle: Graceful Degradation**
-- Single installation (273 paths tracked + Python scripts)
+- Single installation (573 paths tracked across 6 categories + Python scripts)
 - Python features activate only when Python 3.9+ is available
 - Everything works without Python (basic `/docs` command)
 - No separate "modes" - just feature detection at runtime
 
 **Design Goals:**
-1. **Honesty**: Accurate claims about what we deliver (273 paths tracked, ~266-270 files downloaded)
+1. **Honesty**: Accurate claims about what we deliver (573 paths tracked, 571 files downloaded)
 2. **Simplicity**: One installation, automatic feature detection
 3. **Compatibility**: Works with upstream, same `/docs` interface
-4. **Testing**: High test coverage (78%), all changes tested
+4. **Testing**: All changes tested (294 tests)
 
 ## Repository URL Strategy
 
@@ -125,8 +125,7 @@ pip install -e ".[dev]"
 ~/.claude-code-docs/claude-docs-helper.sh --validate
 
 # Run tests (REQUIRED before submitting PR)
-pytest tests/ -v  # Should see: 627 passed, 2 skipped
-pytest --cov=scripts --cov-report=term  # Should see: ~78.7%
+pytest tests/ -v  # Should see: 294 passed, 2 skipped
 
 # Test specific changes
 python scripts/lookup_paths.py "your test query"
@@ -331,10 +330,9 @@ def test_search_paths_with_limit():
 ```
 
 **Current test status:**
-- Total: 629 tests
-- Passing: 627 (99.7%)
+- Total: 296 tests
+- Passing: 294 (99.3%)
 - Skipped: 2 (intentional)
-- Coverage: 78.70%
 
 ## Pull Request Guidelines
 
@@ -417,17 +415,14 @@ git push origin v0.x.x
 
 **When to release:**
 - New Python features complete
-- All tests passing (629/629 or 627/629)
+- All tests passing (294/296 or 296/296)
 - Documentation updated
 
 **Process:**
 ```bash
 # Ensure tests pass
 pytest
 
-# Check coverage
-pytest --cov=scripts --cov-report=term  # Should be ~78.7%
-
 # Update versions
 # Edit install.sh, README.md