Sync TF Mirror #18
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync TF Mirror | |
| on: | |
| schedule: | |
| - cron: '0 4 * * 0' # 每周日运行 | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| jobs: | |
| native-mirror: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Source | |
| uses: actions/checkout@v4 | |
| # 拉取现有的 gh-pages 分支数据作为本地缓存，实现增量更新 | |
| - name: Checkout existing mirror data (Incremental Sync) | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| path: public | |
| continue-on-error: true # 第一次运行可能没有 gh-pages 分支，允许报错跳过 | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v3 | |
| with: | |
| terraform_version: "latest" | |
| - name: Install jq | |
| run: sudo apt-get install -y jq | |
| - name: Fetch and Mirror All Versions | |
| run: | | |
| mkdir -p public/terraform | |
| # 定义需要同步的插件列表 | |
| PROVIDERS=( | |
| "hashicorp/random" | |
| "hashicorp/null" | |
| "hashicorp/time" | |
| "hashicorp/local" | |
| "hashicorp/tls" | |
| "hashicorp/http" | |
| "hashicorp/archive" | |
| "hashicorp/external" | |
| ) | |
| for PROVIDER in "${PROVIDERS[@]}"; do | |
| IFS='/' read -r NAMESPACE NAME <<< "$PROVIDER" | |
| echo "========================================" | |
| echo "🔍 Fetching versions for $PROVIDER..." | |
| # 调用官方 API，提取最新的 20 个版本 (若需全部同步，去掉 | tail -n 20) | |
| VERSIONS=$(curl -s "https://registry.terraform.io/v1/providers/${NAMESPACE}/${NAME}" | jq -r '.versions[]' | tail -n 5) | |
| for VERSION in $VERSIONS; do | |
| # 检查该版本是否已经存在于本地缓存中。如果有 .zip 文件，直接跳过下载！ | |
| TARGET_DIR="../public/terraform/registry.terraform.io/${NAMESPACE}/${NAME}/${VERSION}" | |
| if [ -d "$TARGET_DIR" ] && ls "$TARGET_DIR"/*.zip >/dev/null 2>&1; then | |
| echo "⏭️ Skipped: $PROVIDER v$VERSION already exists. Moving to next." | |
| continue | |
| fi | |
| echo "⬇️ Syncing $PROVIDER v$VERSION..." | |
| mkdir -p run_tmp && cd run_tmp | |
| cat > main.tf <<EOF | |
| terraform { | |
| required_providers { | |
| $NAME = { | |
| source = "$PROVIDER" | |
| version = "= $VERSION" | |
| } | |
| } | |
| } | |
| EOF | |
| if ! terraform init -no-color > /dev/null 2>&1; then | |
| echo " ⚠️ Skipped: v$VERSION is too old/incompatible." | |
| cd .. | |
| rm -rf run_tmp | |
| continue | |
| fi | |
| terraform providers mirror \ | |
| -platform=linux_amd64 \ | |
| -platform=linux_arm64 \ | |
| -platform=windows_amd64 \ | |
| -platform=darwin_amd64 \ | |
| -platform=darwin_arm64 \ | |
| ../public/terraform > /dev/null 2>&1 || true | |
| cd .. | |
| rm -rf run_tmp | |
| done | |
| done | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| # 生成 HTML 索引页面，并重建 index.json (此部分代码保持你的原样逻辑，非常完善) | |
| - name: Rebuild Index JSON & Generate HTML | |
| shell: python | |
| run: | | |
| import os | |
| import json | |
| import datetime | |
| import textwrap | |
| from pathlib import Path | |
| MIRROR_ROOT = Path('public/terraform') | |
| MIRROR_URL = 'https://mirror.wgpsec.org/terraform/' | |
| print('📝 Rebuilding index.json and generating index.html...') | |
| providers = [] | |
| base_registry = MIRROR_ROOT / 'registry.terraform.io' | |
| if base_registry.exists(): | |
| for namespace in base_registry.iterdir(): | |
| if not namespace.is_dir(): continue | |
| for name in namespace.iterdir(): | |
| if not name.is_dir(): continue | |
| actual_versions = set() | |
| for zip_file in name.glob('*.zip'): | |
| parts = zip_file.name.split('_') | |
| if len(parts) >= 2: | |
| actual_versions.add(parts[1]) | |
| if not actual_versions: | |
| continue | |
| def version_key(v): | |
| parts = v.replace('-', '.').split('.') | |
| return [int(p) if p.isdigit() else p for p in parts] | |
| versions_sorted = sorted(list(actual_versions), key=version_key, reverse=True) | |
| latest_ver = versions_sorted[0] | |
| index_data = {"versions": {v: {} for v in versions_sorted}} | |
| with open(name / 'index.json', 'w', encoding='utf-8') as f: | |
| json.dump(index_data, f) | |
| providers.append({ | |
| 'namespace': namespace.name, | |
| 'name': name.name, | |
| 'latest': latest_ver, | |
| 'all_versions': versions_sorted, | |
| 'count': len(versions_sorted) | |
| }) | |
| timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC') | |
| sorted_providers = sorted(providers, key=lambda x: (x['namespace'], x['name'])) | |
| config_lines = [f'"registry.terraform.io/{p["namespace"]}/{p["name"]}"' for p in sorted_providers] | |
| config_str = ",\n ".join(config_lines) | |
| cards_html = "" | |
| for p in sorted_providers: | |
| display_limit = 15 | |
| version_tags = "".join([f'<span class="ver-tag">v{v}</span>' for v in p['all_versions'][:display_limit]]) | |
| if p['count'] > display_limit: | |
| version_tags += f'<span class="ver-tag more">+{p["count"] - display_limit} more</span>' | |
| cards_html += f""" | |
| <div class="card"> | |
| <div class="name">{p['namespace']}/{p['name']}</div> | |
| <div class="latest">Latest: v{p['latest']}</div> | |
| <div class="version-list">{version_tags}</div> | |
| </div> | |
| """ | |
| html_template = textwrap.dedent(""" | |
| <!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>Terraform Provider Mirror</title> | |
| <style> | |
| body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; max-width: 1000px; margin: 0 auto; padding: 2rem; color: #333; line-height: 1.6; } | |
| h1 { border-bottom: 2px solid #eee; padding-bottom: 0.5rem; } | |
| .status { background: #f6f8fa; padding: 1rem; border-radius: 6px; margin-bottom: 2rem; border: 1px solid #e1e4e8; } | |
| code { background: #eee; padding: 0.2rem 0.4rem; border-radius: 3px; font-family: monospace; font-size: 0.9em; } | |
| pre { background: #2d333b; color: #adbac7; padding: 1.5rem; border-radius: 6px; overflow-x: auto; font-family: monospace; } | |
| .grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); gap: 1rem; margin-top: 1rem; } | |
| .card { border: 1px solid #e1e4e8; border-radius: 6px; padding: 1.2rem; transition: transform 0.2s; background: #fff; } | |
| .card:hover { transform: translateY(-2px); box-shadow: 0 4px 12px rgba(0,0,0,0.1); } | |
| .name { font-weight: bold; color: #0366d6; font-size: 1.1em; } | |
| .latest { font-weight: bold; color: #28a745; font-size: 0.85em; margin: 6px 0 10px 0; } | |
| .version-list { display: flex; flex-wrap: wrap; gap: 5px; margin-top: 8px; } | |
| .ver-tag { font-size: 0.75em; background: #f1f8ff; color: #586069; padding: 2px 6px; border-radius: 4px; border: 1px solid #e1e4e8;} | |
| .ver-tag.more { background: #fffbe6; color: #d48806; border-color: #ffe58f; } | |
| footer { margin-top: 3rem; text-align: center; color: #666; font-size: 0.85rem; border-top: 1px solid #eee; padding-top: 1rem; } | |
| </style> | |
| </head> | |
| <body> | |
| <h1>Terraform Provider Mirror</h1> | |
| <div class="status"> | |
| <p><strong>Method:</strong> Native CLI (Incremental Sync)</p> | |
| <p><strong>Last Updated:</strong> {{TIMESTAMP}}</p> | |
| <p><strong>Mirror URL:</strong> <code>{{MIRROR_URL}}</code></p> | |
| </div> | |
| <h2>Usage Configuration</h2> | |
| <p>Add the following to your <code>~/.terraformrc</code> (Linux/Mac) or <code>%APPDATA%\\terraform.rc</code> (Windows):</p> | |
| <pre> | |
| provider_installation { | |
| network_mirror { | |
| url = "{{MIRROR_URL}}" | |
| include = [ | |
| {{CONFIG_STR}} | |
| ] | |
| } | |
| direct { | |
| exclude = [ | |
| {{CONFIG_STR}} | |
| ] | |
| } | |
| }</pre> | |
| <h2>Synced Providers ({{COUNT}})</h2> | |
| <div class="grid"> | |
| {{CARDS_HTML}} | |
| </div> | |
| <footer> | |
| wgpsec terraform mirror | <a href="https://github.com/wgpsec/redc-template">Source Repository</a> | |
| </footer> | |
| </body> | |
| </html> | |
| """) | |
| final_html = html_template.replace("{{TIMESTAMP}}", timestamp) \ | |
| .replace("{{MIRROR_URL}}", MIRROR_URL) \ | |
| .replace("{{CONFIG_STR}}", config_str) \ | |
| .replace("{{COUNT}}", str(len(providers))) \ | |
| .replace("{{CARDS_HTML}}", cards_html) | |
| with open(MIRROR_ROOT / 'index.html', 'w', encoding='utf-8') as f: | |
| f.write(final_html) | |
| - name: Deploy to gh-pages | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ./public | |
| publish_branch: gh-pages | |
| keep_files: true | |
| commit_message: "Mirror: Sync latest versions & rebuild index" |