diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..802832d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: CI + +on: + push: + branches: [main, develop] + pull_request: + branches: [main, develop] + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + - run: npm ci + - run: npm run lint + + test: + name: Test + runs-on: ubuntu-latest + strategy: + matrix: + node-version: ['20', '22'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + - run: npm ci + - run: npm test + + format-check: + name: Format Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + - run: npm ci + - run: npm run format:check diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100644 index 0000000..a84cc5a --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1,2 @@ +npm run lint +npm test diff --git a/CHANGELOG.md b/CHANGELOG.md index 99d25c4..55f976d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added + - Rolling window mode for automatic data window maintenance - `clear` command to truncate table without deleting schema - Automatic backfill on service start @@ -26,12 +27,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Progress tracking for all operations ### Changed + - `start` command now auto-backfills by default (rolling window mode) - Documentation reorganized into logical sections - Test files moved to examples/ directory - Improved error messages and user feedback ### Fixed + - Configuration loading from config.yaml - BigQuery credential handling - Service account key path resolution @@ -39,6 +42,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.0.0] - 2024-XX-XX ### Added + - Initial release of BigQuery Plugin for HarperDB - Modulo-based partitioning for distributed ingestion - Adaptive batch sizing based on sync lag @@ -51,6 +55,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Configurable sync phases (initial, catchup, steady) ### Plugin Features + - Horizontal scalability with linear throughput - No coordination overhead between nodes - Deterministic partition assignments @@ -60,6 +65,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Generic storage for any BigQuery schema ### Documentation + - Comprehensive design document - Blog post explaining architecture evolution - System overview showing component interaction @@ -86,6 +92,7 @@ This project follows [Semantic Versioning](https://semver.org/): ### Deprecation Policy Features will be deprecated with: + 1. Warning in release notes 2. Deprecation notice in code 3. Minimum 2 minor versions before removal diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c1e2849..aa9f8b9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,6 +71,7 @@ git commit -m "chore: update dependencies" ``` **Commit Types**: + - `feat`: New feature - `fix`: Bug fix - `docs`: Documentation changes @@ -86,6 +87,7 @@ git push origin your-branch-name ``` Then create a Pull Request on GitHub with: + - Clear title and description - Reference any related issues - Screenshots/examples if applicable @@ -101,6 +103,7 @@ Then create a Pull Request on GitHub with: - Keep functions focused and small **Example**: + ```javascript /** * Generate vessel position data @@ -190,6 +193,7 @@ The BigQuery plugin integrates with HarperDB. When modifying plugin code: 4. Check data consistency **Key Files**: + - `src/sync-engine.js` - Main sync engine logic - `src/validation.js` - Data validation - `schema/harper-bigquery-sync.graphql` - GraphQL schema @@ -204,6 +208,7 @@ The maritime data synthesizer generates test data. When modifying: 4. Check rolling window behavior **Key Files**: + - `src/generator.js` - Data generation - `src/service.js` - Orchestration - `src/bigquery.js` - BigQuery client @@ -233,6 +238,7 @@ By contributing, you agree that your contributions will be licensed under the Ap ## Recognition Contributors will be: + - Listed in package.json contributors - Acknowledged in release notes - Credited in documentation for major features diff --git a/Readme.md b/Readme.md index 65f3b4f..6983850 100644 --- a/Readme.md +++ b/Readme.md @@ -37,11 +37,13 @@ See [System Overview](docs/SYSTEM-OVERVIEW.md) for how they work together, or ju **Quick Start**: `npx maritime-data-synthesizer start` (auto-backfills and maintains rolling window) **Key Commands:** + - `start` - Auto-backfill and continuous generation (rolling window) - `clear` - Clear all data (keeps schema) - perfect for quick resets - `reset N` - Delete and reload with N days of data **Documentation:** + - **[5-Minute Quick Start](docs/QUICKSTART.md)** - Get generating data immediately - **[System Overview](docs/SYSTEM-OVERVIEW.md)** - How plugin + synthesizer work together - **[Full Guide](docs/maritime-data-synthesizer.md)** - Comprehensive synthesizer documentation @@ -50,6 +52,7 @@ See [System Overview](docs/SYSTEM-OVERVIEW.md) for how they work together, or ju ## Architecture Each node: + 1. Discovers cluster topology via Harper's clustering API 2. Calculates its node ID from ordered peer list 3. Pulls only records where `hash(timestamp) % clusterSize == nodeId` @@ -59,6 +62,7 @@ Each node: ## Installation ### Option 1: Deploy on Fabric (Recommended) + 1. Sign up at [fabric.harper.fast](https://fabric.harper.fast) 2. Create a new application 3. Upload this component @@ -66,9 +70,11 @@ Each node: 5. Component auto-deploys across your cluster ### Option 2: Self-Hosted + 1. Deploy Harper cluster (3+ nodes recommended) - [Quick start guide](https://docs.harperdb.io/docs/getting-started/quickstart) 2. Configure clustering between nodes - [Clustering docs](https://docs.harperdb.io/docs/developers/replication) 3. Copy this component to each node: + ```bash harper deploy bigquery-sync /path/to/component ``` @@ -90,27 +96,28 @@ BigQuery records are stored as-is at the top level: ```graphql type BigQueryData @table { - id: ID! @primaryKey - # All BigQuery fields stored directly at top level - _syncedAt: String @createdTime + id: ID! @primaryKey + # All BigQuery fields stored directly at top level + _syncedAt: String @createdTime } ``` Example stored record: + ```json { - "id": "a1b2c3d4e5f6g7h8", - "_syncedAt": "2025-11-04T16:00:00Z", - "timestamp": "2025-11-04T15:59:00Z", - "mmsi": "367123456", - "imo": "IMO9876543", - "vessel_name": "MARITIME VOYAGER", - "vessel_type": "Container Ship", - "latitude": 37.7749, - "longitude": -122.4194, - "speed_knots": 12.5, - "heading": 275, - "status": "Under way using engine" + "id": "a1b2c3d4e5f6g7h8", + "_syncedAt": "2025-11-04T16:00:00Z", + "timestamp": "2025-11-04T15:59:00Z", + "mmsi": "367123456", + "imo": "IMO9876543", + "vessel_name": "MARITIME VOYAGER", + "vessel_type": "Container Ship", + "latitude": 37.7749, + "longitude": -122.4194, + "speed_knots": 12.5, + "heading": 275, + "status": "Under way using engine" } ``` @@ -119,6 +126,7 @@ This provides maximum flexibility - all BigQuery fields are directly queryable w ### BigQuery Setup Ensure service account has: + - `bigquery.jobs.create` permission - `bigquery.tables.getData` permission on target table @@ -138,6 +146,7 @@ Additional considerations for production deployments: ### Batch Size Tuning Adjust based on: + - Record size - Network bandwidth - IOPS capacity @@ -175,12 +184,14 @@ LIMIT 10; ## Monitoring ### Check Sync Status + ```javascript // Query checkpoint table SELECT * FROM SyncCheckpoint ORDER BY nodeId; ``` ### View Recent Audits + ```javascript // Check validation results SELECT * FROM SyncAudit @@ -189,6 +200,7 @@ ORDER BY timestamp DESC; ``` ### Monitor Lag + ```javascript // Calculate current lag SELECT @@ -202,6 +214,7 @@ FROM SyncCheckpoint; ## API Endpoints ### Get Status + ```bash GET /SyncControl ``` @@ -209,6 +222,7 @@ GET /SyncControl Returns current sync status for the node. ### Control Sync + ```bash POST /SyncControl { @@ -220,16 +234,19 @@ POST /SyncControl ## Troubleshooting ### Node Not Ingesting + - Check BigQuery credentials - Verify node can reach BigQuery API - Check checkpoint table for errors ### Data Drift Detected + - Check for partition key collisions - Verify all nodes are running - Review checkpoint timestamps across nodes ### High Lag + - Increase batch sizes - Add more nodes - Check IOPS capacity @@ -239,6 +256,7 @@ POST /SyncControl ## Performance Tuning ### IOPS Calculation + ``` Indexes: 1 primary + 1 timestamp = 2 indexes IOPS per record: ~4 IOPS @@ -249,6 +267,7 @@ Required IOPS: 20,000 per node Learn more about [Harper's storage architecture](https://docs.harperdb.io/docs/reference/storage-algorithm) ### Scaling Guidelines + - 3 nodes: ~15K records/sec total - 6 nodes: ~30K records/sec total - 12 nodes: ~60K records/sec total @@ -264,18 +283,21 @@ Learn more about [Harper's storage architecture](https://docs.harperdb.io/docs/r ## Roadmap ### 🐛 Crawl (Current - v1.0) + **Status:** 🔨 In Progress Single-threaded ingestion (one worker per Harper instance): + - ✅ Modulo-based partitioning for distributed workload - ✅ One BigQuery table ingestion - ✅ Adaptive batch sizing (phase-based: initial/catchup/steady) - ✅ Checkpoint-based recovery per thread (`hostname-workerIndex`) - ✅ Durable thread identity (survives restarts) - ✅ Basic monitoring via GraphQL API (`/SyncControl`) -- ⚠️ **Validation subsystem** (not yet complete - see src/validation.js) +- ⚠️ **Validation subsystem** (not yet complete - see src/validation.js) **Current Limitations:** + - Single worker thread per instance (supports multi-instance clusters) - Manual cluster scaling coordination - Validation endpoint disabled (commented out in src/resources.js) @@ -283,9 +305,11 @@ Single-threaded ingestion (one worker per Harper instance): **Note:** The code already supports multiple worker threads per instance via `server.workerIndex`. Each thread gets a durable identity (`hostname-workerIndex`) that persists across restarts, enabling checkpoint-based recovery. ### 🚶 Walk (Planned - v2.0) + **Status:** 🔨 In Development Multi-threaded, multi-instance Harper cluster support: + - [ ] **Multi-threaded ingestion** - Multiple worker threads per node - [ ] **Full cluster distribution** - Automatic workload distribution across all Harper nodes - [ ] **Dynamic rebalancing** - Handle node additions/removals without manual intervention @@ -293,14 +317,17 @@ Multi-threaded, multi-instance Harper cluster support: - [ ] **Thread-level checkpointing** - Fine-grained recovery per worker thread **Benefits:** + - Linear scaling across cluster nodes - Better resource utilization per node - Automatic failover and rebalancing ### 🏃 Run (Future - v3.0) + **Status:** 📋 Planned Multi-table ingestion with column selection: + - [ ] **Multiple BigQuery tables** - Ingest from multiple tables simultaneously - [ ] **Column selection** - Choose specific columns per table (reduce data transfer) - [ ] **Per-table configuration** - Different batch sizes, intervals, and strategies per table @@ -308,11 +335,13 @@ Multi-table ingestion with column selection: - [ ] **Unified monitoring** - Single dashboard for all table ingestions **Use Cases:** + - Ingest multiple related datasets (e.g., vessels, ports, weather) - Reduce costs by selecting only needed columns - Different sync strategies per data type (real-time vs batch) **Example Configuration (Future):** + ```yaml bigquery: projectId: your-project @@ -339,4 +368,4 @@ bigquery: **Get Started:** Deploy on [Harper Fabric](https://fabric.harper.fast) - free tier available, no credit card required. -**Learn More:** [Harper Documentation](https://docs.harperdb.io) | [harperdb.io](https://harperdb.io) \ No newline at end of file +**Learn More:** [Harper Documentation](https://docs.harperdb.io) | [harperdb.io](https://harperdb.io) diff --git a/bin/cli.js b/bin/cli.js index f207e2a..cf456c8 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -8,221 +8,223 @@ import { MaritimeDataSynthesizer } from '../src/maritime-synthesizer.js'; import { getSynthesizerConfig } from '../src/config-loader.js'; const COMMANDS = { - initialize: 'Initialize BigQuery resources and load historical data', - start: 'Start continuous data generation', - stats: 'Display table statistics', - clear: 'Clear all data from table (keeps schema)', - clean: 'Delete table and all data', - reset: 'Delete table and reinitialize with historical data', - help: 'Show this help message' + initialize: 'Initialize BigQuery resources and load historical data', + start: 'Start continuous data generation', + stats: 'Display table statistics', + clear: 'Clear all data from table (keeps schema)', + clean: 'Delete table and all data', + reset: 'Delete table and reinitialize with historical data', + help: 'Show this help message', }; function showHelp() { - console.log('\nMaritime Vessel Data Synthesizer CLI\n'); - console.log('Usage: maritime-data-synthesizer [options]\n'); - console.log('Commands:'); - - for (const [cmd, desc] of Object.entries(COMMANDS)) { - console.log(` ${cmd.padEnd(15)} ${desc}`); - } - - console.log('\nExamples:'); - console.log(' maritime-data-synthesizer initialize 30 # Load 30 days of historical data'); - console.log(' maritime-data-synthesizer start # Start with auto-backfill (rolling window)'); - console.log(' maritime-data-synthesizer start --no-backfill # Start without backfill'); - console.log(' maritime-data-synthesizer stats # View statistics'); - console.log(' maritime-data-synthesizer clear # Clear all data (keeps table)'); - console.log(' maritime-data-synthesizer reset 60 # Reset with 60 days of data'); - console.log('\nConfiguration:'); - console.log(' All settings are loaded from config.yaml'); - console.log(' - Uses same BigQuery connection as the plugin (bigquery section)'); - console.log(' - Synthesizer settings in synthesizer section'); - console.log(' - See config.yaml for all available options'); - console.log(''); + console.log('\nMaritime Vessel Data Synthesizer CLI\n'); + console.log('Usage: maritime-data-synthesizer [options]\n'); + console.log('Commands:'); + + for (const [cmd, desc] of Object.entries(COMMANDS)) { + console.log(` ${cmd.padEnd(15)} ${desc}`); + } + + console.log('\nExamples:'); + console.log(' maritime-data-synthesizer initialize 30 # Load 30 days of historical data'); + console.log(' maritime-data-synthesizer start # Start with auto-backfill (rolling window)'); + console.log(' maritime-data-synthesizer start --no-backfill # Start without backfill'); + console.log(' maritime-data-synthesizer stats # View statistics'); + console.log(' maritime-data-synthesizer clear # Clear all data (keeps table)'); + console.log(' maritime-data-synthesizer reset 60 # Reset with 60 days of data'); + console.log('\nConfiguration:'); + console.log(' All settings are loaded from config.yaml'); + console.log(' - Uses same BigQuery connection as the plugin (bigquery section)'); + console.log(' - Synthesizer settings in synthesizer section'); + console.log(' - See config.yaml for all available options'); + console.log(''); } async function main() { - const command = process.argv[2]; - const arg = process.argv[3]; - - if (!command || command === 'help') { - showHelp(); - process.exit(0); - } - - if (!COMMANDS[command]) { - console.error(`Unknown command: ${command}`); - showHelp(); - process.exit(1); - } - - try { - // Load configuration from config.yaml - const config = getSynthesizerConfig(); - console.log(`Configuration loaded from config.yaml`); - console.log(` Project: ${config.projectId}`); - console.log(` Dataset: ${config.datasetId}`); - console.log(` Table: ${config.tableId}`); - console.log(''); - - const synthesizer = new MaritimeDataSynthesizer(config); - - switch (command) { - case 'initialize': { - const days = parseInt(arg || '30', 10); - if (days < 1 || days > 365) { - console.error('Days must be between 1 and 365'); - process.exit(1); - } - - console.log(`Initializing with ${days} days of historical data...`); - await synthesizer.initialize(days); - console.log('Initialization complete!'); - break; - } - - case 'start': { - // Check for optional flags - const maintainWindow = !process.argv.includes('--no-backfill'); - const targetDays = config.retentionDays; - - console.log('Starting Maritime Data Synthesizer...\n'); - - if (maintainWindow) { - console.log(`Rolling window mode: Will maintain ${targetDays}-day data window`); - console.log(' - Automatically backfills if data is missing'); - console.log(' - Continuously generates new data'); - console.log(' - Automatically cleans up old data\n'); - } else { - console.log('Generation-only mode: Will only generate new data (no backfill)\n'); - } - - // Set up event listeners - synthesizer.on('batch:inserted', (data) => { - // Already logged by the service - }); - - synthesizer.on('batch:error', (data) => { - console.error('Batch error:', data.error.message); - }); - - synthesizer.on('cleanup:completed', (data) => { - console.log(`Cleanup: deleted ${data.deletedRows} rows older than ${data.cutoffDate}`); - }); - - synthesizer.on('backfill:starting', (data) => { - console.log(`\nBackfill starting: ${data.days} days before ${data.beforeTimestamp.toISOString()}`); - }); - - synthesizer.on('backfill:completed', (data) => { - console.log(`Backfill completed: ${data.recordsInserted.toLocaleString()} records in ${data.totalTime} minutes\n`); - }); - - // Handle shutdown gracefully - process.on('SIGINT', async () => { - console.log('\nShutting down...'); - await synthesizer.stop(); - console.log('Service stopped'); - process.exit(0); - }); - - process.on('SIGTERM', async () => { - console.log('\nShutting down...'); - await synthesizer.stop(); - console.log('Service stopped'); - process.exit(0); - }); - - await synthesizer.start({ - maintainWindow, - targetDays - }); - - // Keep the process running - console.log('\nPress Ctrl+C to stop\n'); - break; - } - - case 'stats': { - console.log('Fetching statistics...\n'); - - const stats = await synthesizer.getBigQueryStats(); - - console.log('Table Metadata:'); - console.log(` Size: ${(parseInt(stats.tableMetadata.numBytes) / 1024 / 1024).toFixed(2)} MB`); - console.log(` Rows: ${parseInt(stats.tableMetadata.numRows).toLocaleString()}`); - console.log(` Created: ${new Date(parseInt(stats.tableMetadata.creationTime)).toLocaleString()}`); - console.log(` Modified: ${new Date(parseInt(stats.tableMetadata.lastModifiedTime)).toLocaleString()}`); - console.log(''); - - console.log('Data Statistics:'); - console.log(` Total Records: ${parseInt(stats.statistics.total_records).toLocaleString()}`); - console.log(` Unique Vessels: ${parseInt(stats.statistics.unique_vessels).toLocaleString()}`); - console.log(` Vessel Types: ${stats.statistics.vessel_types}`); - console.log(` Unique Positions: ${parseInt(stats.statistics.unique_positions).toLocaleString()}`); - console.log(` Oldest Record: ${stats.statistics.oldest_record?.value || 'N/A'}`); - console.log(` Newest Record: ${stats.statistics.newest_record?.value || 'N/A'}`); - console.log(''); - - break; - } - - case 'clear': { - console.log('This will clear all data from the table (schema will be preserved).'); - console.log('Are you sure? (Ctrl+C to cancel)'); - await new Promise(resolve => setTimeout(resolve, 3000)); - - console.log('Clearing data...'); - await synthesizer.clear(); - console.log('Clear complete! Table is empty but schema remains.'); - break; - } - - case 'clean': { - console.log('This will delete all data and the table. Are you sure? (Ctrl+C to cancel)'); - await new Promise(resolve => setTimeout(resolve, 3000)); - - console.log('Cleaning...'); - await synthesizer.clean(); - console.log('Clean complete!'); - break; - } - - case 'reset': { - const days = parseInt(arg || '30', 10); - if (days < 1 || days > 365) { - console.error('Days must be between 1 and 365'); - process.exit(1); - } - - console.log(`This will delete all data and reinitialize with ${days} days. Are you sure? (Ctrl+C to cancel)`); - await new Promise(resolve => setTimeout(resolve, 3000)); - - console.log('Resetting...'); - await synthesizer.reset(days); - console.log('Reset complete!'); - break; - } - - default: - console.error(`Command not implemented: ${command}`); - process.exit(1); - } - - // Exit for non-start commands - if (command !== 'start') { - process.exit(0); - } - } catch (error) { - console.error('Error:', error.message); - if (error.message.includes('config.yaml')) { - console.error('\nMake sure config.yaml exists and has valid bigquery and synthesizer sections'); - } - if (error.code === 'ENOENT' && error.message.includes('service-account-key')) { - console.error('\nMake sure the credentials file specified in config.yaml exists'); - } - process.exit(1); - } + const command = process.argv[2]; + const arg = process.argv[3]; + + if (!command || command === 'help') { + showHelp(); + process.exit(0); + } + + if (!COMMANDS[command]) { + console.error(`Unknown command: ${command}`); + showHelp(); + process.exit(1); + } + + try { + // Load configuration from config.yaml + const config = getSynthesizerConfig(); + console.log(`Configuration loaded from config.yaml`); + console.log(` Project: ${config.projectId}`); + console.log(` Dataset: ${config.datasetId}`); + console.log(` Table: ${config.tableId}`); + console.log(''); + + const synthesizer = new MaritimeDataSynthesizer(config); + + switch (command) { + case 'initialize': { + const days = parseInt(arg || '30', 10); + if (days < 1 || days > 365) { + console.error('Days must be between 1 and 365'); + process.exit(1); + } + + console.log(`Initializing with ${days} days of historical data...`); + await synthesizer.initialize(days); + console.log('Initialization complete!'); + break; + } + + case 'start': { + // Check for optional flags + const maintainWindow = !process.argv.includes('--no-backfill'); + const targetDays = config.retentionDays; + + console.log('Starting Maritime Data Synthesizer...\n'); + + if (maintainWindow) { + console.log(`Rolling window mode: Will maintain ${targetDays}-day data window`); + console.log(' - Automatically backfills if data is missing'); + console.log(' - Continuously generates new data'); + console.log(' - Automatically cleans up old data\n'); + } else { + console.log('Generation-only mode: Will only generate new data (no backfill)\n'); + } + + // Set up event listeners + synthesizer.on('batch:inserted', () => { + // Already logged by the service + }); + + synthesizer.on('batch:error', (data) => { + console.error('Batch error:', data.error.message); + }); + + synthesizer.on('cleanup:completed', (data) => { + console.log(`Cleanup: deleted ${data.deletedRows} rows older than ${data.cutoffDate}`); + }); + + synthesizer.on('backfill:starting', (data) => { + console.log(`\nBackfill starting: ${data.days} days before ${data.beforeTimestamp.toISOString()}`); + }); + + synthesizer.on('backfill:completed', (data) => { + console.log( + `Backfill completed: ${data.recordsInserted.toLocaleString()} records in ${data.totalTime} minutes\n` + ); + }); + + // Handle shutdown gracefully + process.on('SIGINT', async () => { + console.log('\nShutting down...'); + await synthesizer.stop(); + console.log('Service stopped'); + process.exit(0); + }); + + process.on('SIGTERM', async () => { + console.log('\nShutting down...'); + await synthesizer.stop(); + console.log('Service stopped'); + process.exit(0); + }); + + await synthesizer.start({ + maintainWindow, + targetDays, + }); + + // Keep the process running + console.log('\nPress Ctrl+C to stop\n'); + break; + } + + case 'stats': { + console.log('Fetching statistics...\n'); + + const stats = await synthesizer.getBigQueryStats(); + + console.log('Table Metadata:'); + console.log(` Size: ${(parseInt(stats.tableMetadata.numBytes) / 1024 / 1024).toFixed(2)} MB`); + console.log(` Rows: ${parseInt(stats.tableMetadata.numRows).toLocaleString()}`); + console.log(` Created: ${new Date(parseInt(stats.tableMetadata.creationTime)).toLocaleString()}`); + console.log(` Modified: ${new Date(parseInt(stats.tableMetadata.lastModifiedTime)).toLocaleString()}`); + console.log(''); + + console.log('Data Statistics:'); + console.log(` Total Records: ${parseInt(stats.statistics.total_records).toLocaleString()}`); + console.log(` Unique Vessels: ${parseInt(stats.statistics.unique_vessels).toLocaleString()}`); + console.log(` Vessel Types: ${stats.statistics.vessel_types}`); + console.log(` Unique Positions: ${parseInt(stats.statistics.unique_positions).toLocaleString()}`); + console.log(` Oldest Record: ${stats.statistics.oldest_record?.value || 'N/A'}`); + console.log(` Newest Record: ${stats.statistics.newest_record?.value || 'N/A'}`); + console.log(''); + + break; + } + + case 'clear': { + console.log('This will clear all data from the table (schema will be preserved).'); + console.log('Are you sure? (Ctrl+C to cancel)'); + await new Promise((resolve) => setTimeout(resolve, 3000)); + + console.log('Clearing data...'); + await synthesizer.clear(); + console.log('Clear complete! Table is empty but schema remains.'); + break; + } + + case 'clean': { + console.log('This will delete all data and the table. Are you sure? (Ctrl+C to cancel)'); + await new Promise((resolve) => setTimeout(resolve, 3000)); + + console.log('Cleaning...'); + await synthesizer.clean(); + console.log('Clean complete!'); + break; + } + + case 'reset': { + const days = parseInt(arg || '30', 10); + if (days < 1 || days > 365) { + console.error('Days must be between 1 and 365'); + process.exit(1); + } + + console.log(`This will delete all data and reinitialize with ${days} days. Are you sure? (Ctrl+C to cancel)`); + await new Promise((resolve) => setTimeout(resolve, 3000)); + + console.log('Resetting...'); + await synthesizer.reset(days); + console.log('Reset complete!'); + break; + } + + default: + console.error(`Command not implemented: ${command}`); + process.exit(1); + } + + // Exit for non-start commands + if (command !== 'start') { + process.exit(0); + } + } catch (error) { + console.error('Error:', error.message); + if (error.message.includes('config.yaml')) { + console.error('\nMake sure config.yaml exists and has valid bigquery and synthesizer sections'); + } + if (error.code === 'ENOENT' && error.message.includes('service-account-key')) { + console.error('\nMake sure the credentials file specified in config.yaml exists'); + } + process.exit(1); + } } main(); diff --git a/config.yaml b/config.yaml index a1e5294..957235c 100644 --- a/config.yaml +++ b/config.yaml @@ -17,47 +17,47 @@ bigquery: table: vessel_positions timestampColumn: timestamp credentials: service-account-key.json - location: US # BigQuery dataset location (US, EU, or specific region like us-central1) + location: US # BigQuery dataset location (US, EU, or specific region like us-central1) # Maritime vessel data synthesizer configuration (OPTIONAL) # Optional: Override target dataset/table (defaults to bigquery.dataset and bigquery.table) # By default, synthesizer uses the bigquery: section above (same dataset/table as plugin) # Uncomment settings below to override defaults for the synthesizer # synthesizer: - # dataset: maritime_tracking - # table: vessel_positions +# dataset: maritime_tracking +# table: vessel_positions - # Data generation settings (optional, these are the defaults) - # totalVessels: 100000 # Total vessel pool size - # batchSize: 100 # Vessel positions per batch - # generationIntervalMs: 60000 # Time between batches (ms) - 60 seconds +# Data generation settings (optional, these are the defaults) +# totalVessels: 100000 # Total vessel pool size +# batchSize: 100 # Vessel positions per batch +# generationIntervalMs: 60000 # Time between batches (ms) - 60 seconds - # Data retention (optional, these are the defaults) - # retentionDays: 30 # How many days to keep data - # cleanupIntervalHours: 24 # How often to run cleanup +# Data retention (optional, these are the defaults) +# retentionDays: 30 # How many days to keep data +# cleanupIntervalHours: 24 # How often to run cleanup # Default settings sync: # Start timestamp (ISO format) - omit to start from beginning # startTimestamp: "2024-01-01T00:00:00Z" - + # Batch sizes for different phases initialBatchSize: 10000 catchupBatchSize: 1000 steadyBatchSize: 500 - + # Lag thresholds (seconds) - catchupThreshold: 3600 # 1 hour - steadyThreshold: 300 # 5 minutes - + catchupThreshold: 3600 # 1 hour + steadyThreshold: 300 # 5 minutes + # Polling interval in steady state (milliseconds) - pollInterval: 30000 # 30 seconds + pollInterval: 30000 # 30 seconds validation: enabled: true - interval: 300 # Run every 5 minutes + interval: 300 # Run every 5 minutes retry: maxAttempts: 5 backoffMultiplier: 2 - initialDelay: 1000 # milliseconds + initialDelay: 1000 # milliseconds diff --git a/docs/MARITIME-SYNTHESIZER-README.md b/docs/MARITIME-SYNTHESIZER-README.md index 506632a..377256d 100644 --- a/docs/MARITIME-SYNTHESIZER-README.md +++ b/docs/MARITIME-SYNTHESIZER-README.md @@ -5,6 +5,7 @@ A production-ready synthetic data generator that creates realistic vessel tracki ## What It Does Generates millions of realistic vessel position records with: + - **100,000+ vessels** in the global fleet - **6 vessel types** (container ships, bulk carriers, tankers, cargo, passenger, fishing) - **29 major ports** worldwide with realistic traffic patterns @@ -20,8 +21,8 @@ Edit `config.yaml`: ```yaml bigquery: projectId: your-gcp-project-id - dataset: your_dataset # Plugin reads from here - table: your_table # Plugin reads from here + dataset: your_dataset # Plugin reads from here + table: your_table # Plugin reads from here credentials: service-account-key.json location: US @@ -30,8 +31,8 @@ bigquery: synthesizer: # dataset: maritime_tracking # Optional: Override to use different dataset # table: vessel_positions # Optional: Override to use different table - batchSize: 100 # Optional: Defaults to 100 - generationIntervalMs: 60000 # Optional: Defaults to 60000 + batchSize: 100 # Optional: Defaults to 100 + generationIntervalMs: 60000 # Optional: Defaults to 60000 ``` **That's it!** The synthesizer uses the same BigQuery connection and target as your plugin by default. @@ -51,6 +52,7 @@ npx maritime-data-synthesizer start ``` **Automatic rolling window mode:** + - Checks current data and backfills if needed - Generates 100 vessel positions every 60 seconds (144K records/day) - Maintains exactly N days of data automatically @@ -61,16 +63,19 @@ npx maritime-data-synthesizer start ## Why Use This? ### For Testing the BigQuery Plugin + - Generate realistic test data without accessing real vessel tracking systems - Test data pipeline performance at scale - Validate data transformation and aggregation logic ### For Development + - Local development without production data access - Reproducible test datasets - Privacy-compliant synthetic data ### For Analytics & ML + - Train predictive models for vessel arrival times - Develop anomaly detection algorithms - Build maritime traffic visualization dashboards @@ -79,23 +84,27 @@ npx maritime-data-synthesizer start ## Key Features ### Realistic Data + - Vessels move between actual ports (Singapore, Rotterdam, Los Angeles, etc.) - Proper speeds for vessel types (8-30 knots) - Port operations (anchoring, mooring) and ocean transit - Journey tracking with destinations and ETAs ### Optimized for BigQuery + - Uses load jobs (free tier compatible) - Partitioned by timestamp - Clustered by vessel_type, mmsi, report_date - Automatic cleanup of old data ### Simple Configuration + - **Shares config with the plugin** - no duplicate setup! - Same project, credentials, location - Just configure the target dataset/table ### Production-Ready + - Event-driven architecture - Comprehensive error handling - Progress tracking and statistics @@ -129,6 +138,7 @@ report_date STRING Date in YYYYMMDD format ## Example Queries ### Active Vessels by Type + ```sql SELECT vessel_type, @@ -141,6 +151,7 @@ ORDER BY vessel_count DESC ``` ### Vessels in a Region + ```sql SELECT * FROM `your-project.maritime_tracking.vessel_positions` @@ -151,6 +162,7 @@ ORDER BY timestamp DESC ``` ### Port Activity + ```sql SELECT destination as port, @@ -198,8 +210,8 @@ In `config.yaml`: ```yaml bigquery: projectId: your-gcp-project-id - dataset: your_dataset # Default target for synthesizer - table: your_table # Default target for synthesizer + dataset: your_dataset # Default target for synthesizer + table: your_table # Default target for synthesizer credentials: service-account-key.json synthesizer: @@ -220,6 +232,7 @@ synthesizer: **Records per day** = `(86,400,000 / generationIntervalMs) × batchSize` Examples: + - Default (100 × 60s): **144,000 records/day** - High volume (1000 × 60s): **1,440,000 records/day** - Low volume (10 × 600s): **1,440 records/day** diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 3eb96b0..f345c70 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -24,11 +24,11 @@ Edit `config.yaml`: ```yaml bigquery: - projectId: your-gcp-project-id # Your GCP project - dataset: maritime_tracking # Plugin reads from here / Synthesizer writes to here (default) + projectId: your-gcp-project-id # Your GCP project + dataset: maritime_tracking # Plugin reads from here / Synthesizer writes to here (default) table: vessel_positions timestampColumn: timestamp - credentials: service-account-key.json # Path to your service account key + credentials: service-account-key.json # Path to your service account key location: US # Optional: Override synthesizer settings (defaults shown below) @@ -45,6 +45,7 @@ synthesizer: ``` **Key Points:** + - By default, synthesizer writes to the same dataset/table as the plugin reads from - The `bigquery` section is shared (same project, credentials, and target) - The `synthesizer` section is optional - only needed to override defaults or adjust generation settings @@ -71,12 +72,14 @@ npx maritime-data-synthesizer initialize 30 ``` This will: + - Create the `maritime_tracking` dataset (if needed) - Create the `vessel_positions` table with proper schema - Load ~4.3 million historical vessel position records - Show progress updates every 10 batches **Expected output:** + ``` Configuration loaded from config.yaml Project: your-project @@ -103,6 +106,7 @@ npx maritime-data-synthesizer start ``` **Rolling Window Mode** (default): + - Checks current data range - Automatically backfills if you have less than the target window (e.g., 30 days) - Generates new vessel positions continuously (100 every 60 seconds by default) @@ -111,7 +115,8 @@ npx maritime-data-synthesizer start **Example scenarios:** -*Fresh start (no data):* +_Fresh start (no data):_ + ``` $ npx maritime-data-synthesizer start Checking data range (target: 30 days)... @@ -120,7 +125,8 @@ Loading 30 days... (takes ~30-60 min) Starting continuous generation... ``` -*Partial data (only 7 days):* +_Partial data (only 7 days):_ + ``` $ npx maritime-data-synthesizer start Checking data range (target: 30 days)... @@ -129,7 +135,8 @@ Backfilling 23 days to reach 30-day window... Starting continuous generation... ``` -*Sufficient data (30+ days):* +_Sufficient data (30+ days):_ + ``` $ npx maritime-data-synthesizer start Checking data range (target: 30 days)... @@ -139,6 +146,7 @@ Starting continuous generation... ``` **Skip backfill (generation only):** + ```bash npx maritime-data-synthesizer start --no-backfill ``` @@ -146,6 +154,7 @@ npx maritime-data-synthesizer start --no-backfill This will only generate new data going forward without checking or backfilling historical data. **Expected output:** + ``` Configuration loaded from config.yaml Project: your-project @@ -175,6 +184,7 @@ npx maritime-data-synthesizer stats ``` **Example output:** + ``` Configuration loaded from config.yaml Project: your-project @@ -201,17 +211,21 @@ Data Statistics: ### Clear or Reset Data **Clear data (keeps schema):** + ```bash npx maritime-data-synthesizer clear ``` + - Removes all data from table - Preserves table schema and structure - Useful for quick data refresh **Reset everything (deletes table):** + ```bash npx maritime-data-synthesizer reset 30 ``` + - Stops the service if running - Deletes the entire table - Reinitializes with 30 days of data @@ -221,6 +235,7 @@ npx maritime-data-synthesizer reset 30 ### For Different Data Volumes **High Volume** (1.44M records/day): + ```yaml synthesizer: batchSize: 1000 @@ -228,22 +243,25 @@ synthesizer: ``` **Low Volume** (14.4K records/day): + ```yaml synthesizer: batchSize: 100 - generationIntervalMs: 600000 # 10 minutes + generationIntervalMs: 600000 # 10 minutes ``` **Test/Development** (1.44K records/day): + ```yaml synthesizer: batchSize: 10 - generationIntervalMs: 600000 # 10 minutes + generationIntervalMs: 600000 # 10 minutes ``` ### For Longer/Shorter Retention **90-day retention:** + ```yaml synthesizer: retentionDays: 90 @@ -251,10 +269,11 @@ synthesizer: ``` **7-day retention (for testing):** + ```yaml synthesizer: retentionDays: 7 - cleanupIntervalHours: 6 # Clean up more frequently + cleanupIntervalHours: 6 # Clean up more frequently ``` ## Verify It's Working @@ -297,6 +316,7 @@ LIMIT 10 ### "Could not load the default credentials" Make sure: + 1. Your service account key file exists 2. The path in config.yaml matches the filename 3. The service account has BigQuery permissions @@ -304,17 +324,20 @@ Make sure: ### "Dataset already exists" but table creation fails The dataset might exist from a previous run. Either: + - Delete it in BigQuery console and retry - Or just run with the existing dataset (table will be created) ### Slow historical data loading This is normal! Loading 30 days takes ~1 hour because: + - 4.3 million records need to be generated - Each batch is rate-limited to avoid overwhelming BigQuery - Free tier has load job limits (1,500/day) To speed up (for testing): + ```bash npx maritime-data-synthesizer initialize 7 # Just 7 days ``` @@ -329,6 +352,7 @@ npx maritime-data-synthesizer initialize 7 # Just 7 days ## Stopping the Service Press `Ctrl+C` in the terminal running the synthesizer. It will: + - Stop generating new data - Clean up gracefully - Show final statistics diff --git a/docs/ROLLING-WINDOW.md b/docs/ROLLING-WINDOW.md index 58eea14..16a994f 100644 --- a/docs/ROLLING-WINDOW.md +++ b/docs/ROLLING-WINDOW.md @@ -124,8 +124,8 @@ Rolling window behavior is controlled by `retentionDays` in `config.yaml`: ```yaml synthesizer: - retentionDays: 30 # Target window size - cleanupIntervalHours: 24 # How often to clean up old data + retentionDays: 30 # Target window size + cleanupIntervalHours: 24 # How often to clean up old data ``` ## Benefits @@ -135,12 +135,14 @@ synthesizer: No need to run `initialize` before `start`. Just start the service and it handles everything. **Old workflow:** + ```bash npx maritime-data-synthesizer initialize 30 # Manual step npx maritime-data-synthesizer start # Then start ``` **New workflow:** + ```bash npx maritime-data-synthesizer start # That's it! ``` @@ -148,11 +150,13 @@ npx maritime-data-synthesizer start # That's it! ### 2. Graceful Recovery Service can be stopped and restarted at any time. It will: + - Check current state - Backfill if data is missing - Resume generating new data **Example**: Stop service for 5 days, then restart: + ``` Checking data range (target: 30 days)... Found 3,600,000 records covering 25 days @@ -163,6 +167,7 @@ Starting continuous generation... ### 3. Consistent State Always maintains exactly N days of data: + - New data continuously added at the front - Old data automatically removed from the back - Window size remains constant @@ -170,6 +175,7 @@ Always maintains exactly N days of data: ### 4. Production-Ready Perfect for long-running deployments: + - No manual maintenance needed - Self-healing on restart - Predictable resource usage @@ -236,6 +242,7 @@ FROM `project.dataset.table` ``` Calculates: + - `daysCovered = (newest - oldest) / 86400000` - `daysNeeded = targetDays - daysCovered` @@ -255,19 +262,19 @@ The service emits events for monitoring: ```javascript // Backfill events synthesizer.on('backfill:starting', (data) => { - // { days, beforeTimestamp } + // { days, beforeTimestamp } }); synthesizer.on('backfill:progress', (data) => { - // { batchNum, totalBatches, recordsInserted, totalRecords, progress } + // { batchNum, totalBatches, recordsInserted, totalRecords, progress } }); synthesizer.on('backfill:completed', (data) => { - // { recordsInserted, totalTime } + // { recordsInserted, totalTime } }); synthesizer.on('backfill:error', (data) => { - // { error } + // { error } }); ``` @@ -310,22 +317,24 @@ Still supported if you prefer explicit control. Backfill time scales linearly with days: -| Days to Backfill | Records | Estimated Time | -|------------------|---------|----------------| -| 1 day | 144,000 | ~2 minutes | -| 7 days | 1,008,000 | ~14 minutes | -| 23 days | 3,312,000 | ~46 minutes | -| 30 days | 4,320,000 | ~60 minutes | +| Days to Backfill | Records | Estimated Time | +| ---------------- | --------- | -------------- | +| 1 day | 144,000 | ~2 minutes | +| 7 days | 1,008,000 | ~14 minutes | +| 23 days | 3,312,000 | ~46 minutes | +| 30 days | 4,320,000 | ~60 minutes | ### Resource Usage During backfill: + - **Network**: Moderate (1-2 KB per record) - **CPU**: Low (<5%) - **Memory**: ~150 MB baseline - **BigQuery**: 1 load job per batch (1 per second) During steady state: + - **Network**: Minimal (100 records/min) - **CPU**: <1% - **Memory**: ~150 MB @@ -344,6 +353,7 @@ During steady state: ### Q: What if I don't want backfill? **A**: Use `--no-backfill` flag: + ```bash npx maritime-data-synthesizer start --no-backfill ``` @@ -365,6 +375,7 @@ npx maritime-data-synthesizer start --no-backfill If you're currently using manual initialization: **Old workflow:** + ```bash # Step 1: Initialize once npx maritime-data-synthesizer initialize 30 @@ -376,6 +387,7 @@ npx maritime-data-synthesizer start ``` **New workflow:** + ```bash # Just start - everything automatic npx maritime-data-synthesizer start diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 6afa4b1..ea12ae0 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -18,6 +18,7 @@ We release patches for security vulnerabilities for the following versions: Send security vulnerabilities to: **security@harperdb.io** Include: + - Description of the vulnerability - Steps to reproduce - Potential impact @@ -66,12 +67,14 @@ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/key.json For the BigQuery plugin/synthesizer, service accounts need: **Minimum Required**: + - `bigquery.jobs.create` - `bigquery.tables.getData` - `bigquery.tables.create` - `bigquery.tables.updateData` **Not Required**: + - `bigquery.datasets.delete` - `bigquery.tables.delete` (unless using clean/reset) - Admin permissions @@ -79,6 +82,7 @@ For the BigQuery plugin/synthesizer, service accounts need: ### Configuration Security **config.yaml**: + ```yaml # Safe - relative path to credential file bigquery: @@ -90,6 +94,7 @@ bigquery: ``` **.env file**: + ```bash # Always add to .gitignore echo ".env" >> .gitignore @@ -120,6 +125,7 @@ echo ".env.*.local" >> .gitignore **Issue**: config.yaml references credential files **Mitigation**: + - Credential files must be in .gitignore - Use environment variables where possible - Rotate keys regularly @@ -129,6 +135,7 @@ echo ".env.*.local" >> .gitignore **Issue**: Synthetic data may resemble production patterns **Mitigation**: + - Use synthesizer only for testing - Don't use production data characteristics - Sanitize any borrowed patterns @@ -138,6 +145,7 @@ echo ".env.*.local" >> .gitignore **Issue**: Malicious or buggy code could incur costs **Mitigation**: + - Set up billing alerts - Use BigQuery quotas - Monitor query patterns @@ -148,6 +156,7 @@ echo ".env.*.local" >> .gitignore **Issue**: Distributed system requires node trust **Mitigation**: + - Use HarperDB authentication - TLS for inter-node communication - Network isolation where possible @@ -193,6 +202,7 @@ npm audit fix --force # Use with caution ### Automated Scanning We use: + - GitHub Dependabot - npm audit in CI/CD - Snyk (optional) @@ -219,6 +229,7 @@ Security patches are released as soon as possible: - **Low**: Included in next release cycle Users are notified via: + - GitHub Security Advisories - Release notes - Email (for critical issues) @@ -226,6 +237,7 @@ Users are notified via: ## Compliance This project aims to follow: + - OWASP Top 10 - CIS Benchmarks - NIST Cybersecurity Framework (where applicable) @@ -240,6 +252,7 @@ This project aims to follow: ## Contact For security concerns: + - Email: security@harperdb.io - Expect response within 48 hours - PGP key available on request diff --git a/docs/SYSTEM-OVERVIEW.md b/docs/SYSTEM-OVERVIEW.md index 541452a..f833140 100644 --- a/docs/SYSTEM-OVERVIEW.md +++ b/docs/SYSTEM-OVERVIEW.md @@ -7,6 +7,7 @@ This project contains two complementary components that work together: **Purpose**: Syncs data FROM BigQuery INTO HarperDB **What it does**: + - Connects to BigQuery and monitors a source table - Fetches new/updated records based on timestamp - Ingests data into HarperDB with validation @@ -14,10 +15,11 @@ This project contains two complementary components that work together: - Provides GraphQL API for querying synced data **Configuration** (in `config.yaml`): + ```yaml bigquery: projectId: your-gcp-project-id - dataset: maritime_tracking # Reads from here + dataset: maritime_tracking # Reads from here table: vessel_positions timestampColumn: timestamp credentials: service-account-key.json @@ -29,6 +31,7 @@ bigquery: **Purpose**: Generates synthetic data and writes it TO BigQuery **What it does**: + - Creates realistic vessel tracking data at global scale - Generates 100,000+ vessel positions with movement patterns - Writes to BigQuery with proper schema and partitioning @@ -36,9 +39,10 @@ bigquery: - Can be used to test the plugin or for other purposes **Configuration** (in `config.yaml`): + ```yaml synthesizer: - dataset: maritime_tracking # Writes to here + dataset: maritime_tracking # Writes to here table: vessel_positions totalVessels: 100000 batchSize: 100 @@ -52,6 +56,7 @@ synthesizer: ### Shared Configuration Both components use the **same** BigQuery connection: + - Same GCP project - Same credentials (`service-account-key.json`) - Same location (e.g., `US`) @@ -121,9 +126,9 @@ Update `config.yaml`: ```yaml bigquery: projectId: irjudson-demo - dataset: maritime_tracking # Point to synthetic data + dataset: maritime_tracking # Point to synthetic data table: vessel_positions - timestampColumn: timestamp # Use 'timestamp' field + timestampColumn: timestamp # Use 'timestamp' field credentials: service-account-key.json location: US ``` @@ -131,6 +136,7 @@ bigquery: ### 3. Run Plugin Start HarperDB with the plugin, and it will: + - Sync vessel positions from BigQuery - Make them queryable via GraphQL - Keep data up-to-date as synthesizer generates new records @@ -140,6 +146,7 @@ Start HarperDB with the plugin, and it will: For production, keep them separate: ### Plugin Configuration (production data) + ```yaml bigquery: dataset: production_data @@ -148,6 +155,7 @@ bigquery: ``` ### Synthesizer Configuration (test data) + ```yaml synthesizer: dataset: test_data @@ -159,21 +167,25 @@ Both use the same credentials, but read/write different datasets. ## Key Benefits ### Unified Configuration + - Single `config.yaml` for both components - Shared BigQuery connection settings - No duplicate credential management ### Flexible Usage + - Use synthesizer independently for data generation - Use plugin independently for any BigQuery table - Combine them for end-to-end testing ### Realistic Test Data + - Synthesizer creates production-like workloads - Millions of records with realistic patterns - Perfect for load testing and validation ### Cost Optimization + - Both use BigQuery free tier efficiently - Load jobs instead of streaming inserts - Automatic cleanup of old data @@ -181,17 +193,19 @@ Both use the same credentials, but read/write different datasets. ## Configuration Reference ### Required (shared by both) + ```yaml bigquery: - projectId: your-project-id # GCP project - credentials: key.json # Service account key - location: US # BigQuery region + projectId: your-project-id # GCP project + credentials: key.json # Service account key + location: US # BigQuery region ``` ### Plugin-specific + ```yaml bigquery: - dataset: source_dataset # Where to read from + dataset: source_dataset # Where to read from table: source_table timestampColumn: timestamp_field @@ -203,9 +217,10 @@ sync: ``` ### Synthesizer-specific + ```yaml synthesizer: - dataset: target_dataset # Where to write to + dataset: target_dataset # Where to write to table: target_table totalVessels: 100000 batchSize: 100 @@ -246,6 +261,7 @@ harper-bigquery-sync/ ## Quick Commands ### Synthesizer + ```bash # Generate test data npx maritime-data-synthesizer initialize 30 @@ -257,6 +273,7 @@ npx maritime-data-synthesizer reset 30 ``` ### Plugin + ```bash # Runs as HarperDB plugin # See HarperDB documentation for setup diff --git a/docs/blog-post.md b/docs/blog-post.md index 7c0de80..b8b9412 100644 --- a/docs/blog-post.md +++ b/docs/blog-post.md @@ -29,6 +29,7 @@ END **Why it failed:** The bottleneck was pulling FROM BigQuery, not writing TO Harper. **Key issues:** + - One process = one API client = hard rate limits - No parallelism in BigQuery queries - Single point of failure @@ -59,6 +60,7 @@ END **Why it failed:** Still no parallelism. Worse—operational complexity exploded. **The hidden costs of "simple" locks:** + - Which node gets the lock? (consensus problem) - Crashed holder vs. slow holder? (timeout tuning nightmare) - Network partition = split-brain scenarios @@ -81,7 +83,7 @@ END Hash each record's timestamp, modulo by cluster size: ```javascript -nodeId = hash(record.timestamp) % clusterSize +nodeId = hash(record.timestamp) % clusterSize; ``` **In practice:** Node 0 in a 3-node cluster pulls only `hash(timestamp) % 3 == 0` records. @@ -106,7 +108,7 @@ Harper's native clustering API ([docs](https://docs.harperdb.io/docs/developers/ ```javascript const nodes = await harperCluster.getNodes(); const sorted = nodes.sort((a, b) => a.id.localeCompare(b.id)); -const myNodeId = sorted.findIndex(n => n.id === harperCluster.currentNode.id); +const myNodeId = sorted.findIndex((n) => n.id === harperCluster.currentNode.id); ``` Deterministic sorting = consistent partition assignments across all nodes. @@ -131,9 +133,9 @@ const query = ` ```javascript await harperTable.putBatch(records); await checkpointTable.put({ - nodeId: myNodeId, - lastTimestamp: records[records.length - 1].timestamp, - recordsIngested: totalIngested + nodeId: myNodeId, + lastTimestamp: records[records.length - 1].timestamp, + recordsIngested: totalIngested, }); ``` @@ -147,9 +149,9 @@ Batch size adjusts to lag: ```javascript function calculateBatchSize(lag) { - if (lag > 3600) return 10000; // Hours behind - if (lag > 300) return 1000; // Minutes behind - return 500; // Near real-time + if (lag > 3600) return 10000; // Hours behind + if (lag > 300) return 1000; // Minutes behind + return 500; // Near real-time } ``` @@ -178,6 +180,7 @@ graph TB ``` **Each node independently:** + - Discovers cluster topology - Calculates its partition - Polls BigQuery for its partition only @@ -189,21 +192,25 @@ graph TB ## Why This Works **No coordination overhead** + - Zero inter-node communication for ingestion - No locks, leader election, or consensus - Just independent, parallel work **Linear scalability** + - 3 nodes = 3x throughput - 6 nodes = 6x throughput - Each handles 1/n of data **Independent failure recovery** + - Node crashes? Others keep running - Crashed node restarts from last checkpoint - Zero cluster-wide impact **Predictable performance** + - No variable coordination latency - Performance = f(partition size, BigQuery response time) @@ -221,7 +228,7 @@ graph TB ```yaml clustering: - nodeId: node-001 # Never changes + nodeId: node-001 # Never changes peers: [node-001, node-002, node-003] ``` @@ -265,6 +272,7 @@ Quarterly capacity planning replaces frequent topology changes. **Initial approach (V1):** Compare BigQuery counts with Harper counts every 5 minutes. **Why it failed:** + - **Eventual consistency:** Harper is eventually consistent—counts don't reflect cluster state immediately - **Estimate variance:** `count()` returns performance-optimized estimates with large, inconsistent ranges - **False positives:** Alerts fired during normal operation, not actual data loss @@ -282,6 +290,7 @@ Quarterly capacity planning replaces frequent topology changes. **3. Spot checks** — Randomly verify 5-10 records exist in both BigQuery and Harper **Key metrics:** + - **Lag:** Seconds behind BigQuery - **Throughput:** Records/sec per node - **Phase:** Initial | Catchup | Steady @@ -297,28 +306,28 @@ Quarterly capacity planning replaces frequent topology changes. ```graphql type BigQueryData @table { - id: ID! @primaryKey - timestamp: String! @indexed - deviceId: String @indexed - data: Any - _syncedAt: String @createdTime + id: ID! @primaryKey + timestamp: String! @indexed + deviceId: String @indexed + data: Any + _syncedAt: String @createdTime } type SyncCheckpoint @table { - nodeId: Int! @primaryKey - lastTimestamp: String! - recordsIngested: Long! - phase: String! # initial | catchup | steady + nodeId: Int! @primaryKey + lastTimestamp: String! + recordsIngested: Long! + phase: String! # initial | catchup | steady } type SyncAudit @table { - id: ID! @primaryKey - timestamp: String! @indexed - nodeId: Int! - bigQueryCount: Long! - harperCount: Long! - delta: Long! - status: String! + id: ID! @primaryKey + timestamp: String! @indexed + nodeId: Int! + bigQueryCount: Long! + harperCount: Long! + delta: Long! + status: String! } ``` @@ -326,17 +335,17 @@ type SyncAudit @table { ```javascript async function ingestBatch(records) { - try { - await harperTable.putBatch(records); - await updateCheckpoint(records[records.length - 1].timestamp); - } catch (error) { - if (isRetriable(error)) { - await sleep(exponentialBackoff()); - return ingestBatch(records); - } - logger.error('Unrecoverable', { error, records }); - // Skip, continue - } + try { + await harperTable.putBatch(records); + await updateCheckpoint(records[records.length - 1].timestamp); + } catch (error) { + if (isRetriable(error)) { + await sleep(exponentialBackoff()); + return ingestBatch(records); + } + logger.error('Unrecoverable', { error, records }); + // Skip, continue + } } ``` @@ -344,11 +353,11 @@ async function ingestBatch(records) { ```javascript for (const record of records) { - if (!record.timestamp) { - await audit.logSkipped(record, 'missing_timestamp'); - continue; - } - // Process + if (!record.timestamp) { + await audit.logSkipped(record, 'missing_timestamp'); + continue; + } + // Process } ``` @@ -357,15 +366,18 @@ for (const record of records) { ## Performance Results **Throughput** (3 nodes, modest hardware): + - Steady: 15K records/sec total (5K per node) - Catchup: 30K records/sec (10K per node) **Latency:** + - Steady-state lag: <30 sec - Initial sync: 6 hours for 100M records - Catchup from 1hr lag: 10 min **IOPS** (2 indexes): + - 4 IOPS per record - 5K records/sec/node = 20K IOPS - SSD handles comfortably @@ -389,12 +401,14 @@ for (const record of records) { ## When to Use This Pattern **✅ Good fit:** + - Large, continuously updating datasets - Horizontal scalability requirements - Stable cluster topology - Source supports partitioned queries **❌ Poor fit:** + - Minute-by-minute autoscaling needs - Immediate strong consistency requirements - Source lacks efficient partition support @@ -418,18 +432,21 @@ This pattern applies beyond BigQuery and Harper—anywhere you need large-scale To validate this architecture with realistic workloads, we built the **Maritime Vessel Data Synthesizer**: **What it does:** + - Generates 100,000+ vessel positions with realistic movement patterns - Simulates global maritime traffic across 29 major ports - Produces 144,000+ records/day to BigQuery - Uses the same `config.yaml` configuration as the plugin **Why it matters:** + - Test the full ingestion pipeline without production data - Validate partition distribution across nodes - Load test with millions of records - Privacy-compliant development and testing **Quick start:** + ```bash npx maritime-data-synthesizer initialize 30 # Load 30 days npx maritime-data-synthesizer start # Continuous generation @@ -446,6 +463,7 @@ See [docs/QUICKSTART.md](docs/QUICKSTART.md) for the 5-minute setup guide or [MA **Need help?** The team at Harper is ready to discuss data ingestion challenges. **Resources:** + - [Design Document](#) — Full technical details - [GitHub Repository](#) — Complete implementation -- [HarperDB Docs](https://docs.harperdb.io) — Platform documentation \ No newline at end of file +- [HarperDB Docs](https://docs.harperdb.io) — Platform documentation diff --git a/docs/design-document.md b/docs/design-document.md index 270df36..c31b9e1 100644 --- a/docs/design-document.md +++ b/docs/design-document.md @@ -1,6 +1,6 @@ # Scaling BigQuery to Harper Ingestion: Why Simple Solutions Don't Work and What Does -*How we evolved from a single-process bottleneck to a distributed architecture that scales linearly* +_How we evolved from a single-process bottleneck to a distributed architecture that scales linearly_ **About Harper:** Harper is a distributed application platform that unifies database, cache, and application server into a single system. [Learn more at harperdb.io](https://harperdb.io) @@ -33,6 +33,7 @@ To validate this ingestion system at scale without production data, we developed ### Why We Built This Testing distributed ingestion requires: + - **Realistic data volume**: Millions of records to validate performance - **Continuous generation**: Real-time ingestion patterns - **Predictable patterns**: Known data for validation @@ -43,12 +44,14 @@ Testing distributed ingestion requires: The synthesizer generates realistic maritime vessel tracking data: **Scale:** + - 100,000+ vessels with persistent identities (MMSI, IMO numbers) - 144,000+ position updates per day (default configuration) - Global coverage across 29 major ports - Configurable from 1,440 to 1.44M records/day **Realism:** + - 6 vessel types (container ships, tankers, bulk carriers, cargo, passenger, fishing) - Physics-based movement using Haversine distance calculations - Port-to-port journeys with realistic speeds (10-25 knots depending on type) @@ -56,18 +59,19 @@ The synthesizer generates realistic maritime vessel tracking data: - Weighted port traffic (Singapore, Rotterdam, Shanghai busier than smaller ports) **Data Structure** (actual generated record): + ```json { - "timestamp": "2025-11-07T16:30:00.000Z", - "mmsi": "367123456", - "imo": "IMO9876543", - "vessel_name": "MARITIME VOYAGER", - "vessel_type": "Container Ship", - "latitude": 37.7749, - "longitude": -122.4194, - "speed_knots": 12.5, - "heading": 275, - "status": "Under way using engine" + "timestamp": "2025-11-07T16:30:00.000Z", + "mmsi": "367123456", + "imo": "IMO9876543", + "vessel_name": "MARITIME VOYAGER", + "vessel_type": "Container Ship", + "latitude": 37.7749, + "longitude": -122.4194, + "speed_knots": 12.5, + "heading": 275, + "status": "Under way using engine" } ``` @@ -78,16 +82,16 @@ The synthesizer and plugin share configuration via `config.yaml`: ```yaml bigquery: projectId: your-project-id - dataset: maritime_tracking # Same dataset/table by default + dataset: maritime_tracking # Same dataset/table by default table: vessel_positions credentials: service-account-key.json # Optional: override target for synthesizer synthesizer: # dataset: test_data # Uncomment to use different dataset - batchSize: 100 # Positions per batch - generationIntervalMs: 60000 # 60 seconds between batches - retentionDays: 30 # Auto-cleanup after 30 days + batchSize: 100 # Positions per batch + generationIntervalMs: 60000 # 60 seconds between batches + retentionDays: 30 # Auto-cleanup after 30 days ``` **By default**, the synthesizer writes to the same BigQuery table the plugin reads from—perfect for testing end-to-end. @@ -95,11 +99,13 @@ synthesizer: ### Key Features **Rolling Window Mode:** + - `npx maritime-data-synthesizer start` - Automatically maintains N-day data window - Self-healing: Restarts detect gaps and backfill automatically - No manual management after initial setup **Data Management:** + - `initialize N` - Load N days of historical data - `clear` - Truncate data (keeps table schema) - `reset N` - Delete and reload with N days @@ -130,12 +136,14 @@ END LOOP ``` **What we liked:** + - Dead simple to implement - Easy to reason about - Single checkpoint to track - DNS load balancing distributes writes across Harper nodes **What broke:** + - **The bottleneck is pulling FROM BigQuery, not writing TO Harper.** DNS round-robin solved the wrong problem - One process = one BigQuery API client = one set of rate limits - Limited to a single machine's CPU, memory, and network for the pull operation @@ -172,12 +180,14 @@ END LOOP We could implement the lock in a Harper table, Redis, or ZooKeeper. **What we liked:** + - Multiple nodes meant automatic failover - If the active node dies, another picks up - Solves the single process bottleneck from v1 - **Prevents wasted work:** Without the lock, all nodes would pull the same data from BigQuery (wasting query costs) and write duplicates to Harper (wasting IOPS, even though primary keys dedupe) **What we hated:** + - **Only one node pulling at a time—no actual parallelism** - **The "simple" lock is deceptively complex:** - Which node gets the lock? (requires consensus) @@ -261,6 +271,7 @@ async discoverCluster() { ``` **Key points:** + - Node ID is `hostname-workerIndex` (e.g., `node1-0`, `node2-0`) - Deterministic lexicographic sorting ensures all nodes agree on partition assignments - Falls back to single-node mode if `server.nodes` is empty @@ -301,6 +312,7 @@ async pullPartition({ nodeId, clusterSize, lastTimestamp, batchSize }) { ``` **Why `UNIX_MICROS` instead of `FARM_FINGERPRINT`?** + - Deterministic: Same timestamp always maps to same partition - Fast: Integer modulo is cheaper than string hashing - Chronological distribution: Adjacent timestamps spread across nodes (better for time-series data) @@ -367,6 +379,7 @@ async updateCheckpoint(records) { ``` **Key differences from naive approach:** + - Uses Harper's `transaction()` for atomic batch writes - Harper auto-generates primary key `id` - we don't manually create it - All BigQuery columns stored at top level (no nested `data` field) @@ -426,13 +439,14 @@ async updatePhase() { ``` **Configuration** (`config.yaml`): + ```yaml sync: - initialBatchSize: 10000 # Fast catch-up when hours behind - catchupBatchSize: 1000 # Moderate pace when minutes behind - steadyBatchSize: 500 # Small frequent polls when near real-time - catchupThreshold: 3600 # 1 hour (seconds) - steadyThreshold: 300 # 5 minutes (seconds) + initialBatchSize: 10000 # Fast catch-up when hours behind + catchupBatchSize: 1000 # Moderate pace when minutes behind + steadyBatchSize: 500 # Small frequent polls when near real-time + catchupThreshold: 3600 # 1 hour (seconds) + steadyThreshold: 300 # 5 minutes (seconds) ``` This means initial sync is fast (large batches), and steady-state is efficient (small, frequent polls). @@ -469,6 +483,7 @@ flowchart TB ``` **Each node independently:** + - Discovers cluster topology - Calculates which partition it owns - Polls BigQuery for its partition only @@ -493,6 +508,7 @@ Need more throughput? Add nodes. Each node handles 1/n of the data: ### Independent Failure Recovery If a node crashes: + - Other nodes keep running - Crashed node restarts from its last checkpoint - No cluster-wide impact @@ -508,6 +524,7 @@ No variable coordination latency. Each node's performance is deterministic based The one downside: **cluster topology must be relatively stable.** If you add or remove nodes, the modulo changes: + - Old: `hash(ts) % 3` - New: `hash(ts) % 4` @@ -529,11 +546,13 @@ The system gets resilience from Harper's built-in durable node identity: **Node ID formula:** `${hostname}-${workerIndex}` (e.g., `node-001-0`, `node-001-1`, `node-002-0`) **Why this works:** + - If `node-001` thread 0 crashes and restarts, it comes back as `node-001-0` again - Same node ID = same partition assignment = continues from checkpoint - No manual configuration needed - Harper's runtime provides this automatically This means each ingestion thread has a stable, persistent identity that survives: + - Process crashes - Rolling updates - Planned maintenance @@ -553,6 +572,7 @@ Plan capacity quarterly, make changes during maintenance windows. **3. Implement Comprehensive Monitoring** Essential alerts: + - **Drift alert:** `|bigquery_count - harper_count| > 1000` for 5+ minutes - **Lag alert:** Behind BigQuery by >30 minutes - **Dead node alert:** No checkpoint update in 5 minutes @@ -567,6 +587,7 @@ Essential alerts: **5. Test Recovery Regularly** Monthly chaos engineering: + - Kill random nodes, verify recovery from checkpoint - Force restarts to validate checkpoint integrity - Inject data corruption to test drift detection @@ -576,6 +597,7 @@ Monthly chaos engineering: ### Future Enhancement: Automatic Rebalancing We could implement a rebalancing protocol: + 1. Detect topology change 2. Pause all nodes briefly 3. Recalculate partitions @@ -614,6 +636,7 @@ Instead of comparing estimates, validate what matters: is ingestion progressing Three complementary approaches: **1. Checkpoint Progress Monitoring** + ```javascript // Monitor that ingestion is progressing const checkpoint = await tables.SyncCheckpoint.get(nodeId); @@ -622,7 +645,7 @@ const lagSeconds = (Date.now() - new Date(checkpoint.lastTimestamp)) / 1000; // Alert if stalled for 10+ minutes if (timeSinceLastSync > 600000) { - alert('Ingestion stalled'); + alert('Ingestion stalled'); } ``` @@ -665,19 +688,23 @@ async smokeTest() { ⚠️ **Note**: Full validation is not yet implemented. The validation subsystem exists in `src/validation.js` but is currently commented out in `src/resources.js` and `src/index.js`. This will be completed in v1.0. Planned validation approach: + ```javascript // Verify Harper records exist in BigQuery const harperSample = await tables.BigQueryData.search({ limit: 5 }); for (const record of harperSample) { - const exists = await bigquery.query(` + const exists = await bigquery.query( + ` SELECT 1 FROM \`dataset.table\` WHERE timestamp = @ts AND id = @id LIMIT 1 - `, { ts: record.timestamp, id: record.id }); + `, + { ts: record.timestamp, id: record.id } + ); - if (!exists) { - // Phantom record: exists in Harper but not in BigQuery - } + if (!exists) { + // Phantom record: exists in Harper but not in BigQuery + } } ``` @@ -703,37 +730,38 @@ for (const record of harperSample) { # BigQuery records are stored as-is with metadata fields # Harper auto-generates 'id' field if not provided type BigQueryData @table { - id: ID @primaryKey - # All BigQuery fields stored directly at top level - # Metadata fields: - # _syncedAt: Date @createdTime + id: ID @primaryKey + # All BigQuery fields stored directly at top level + # Metadata fields: + # _syncedAt: Date @createdTime } # Per-node checkpoint type SyncCheckpoint @table { - nodeId: Int! @primaryKey - lastTimestamp: Date! - recordsIngested: Long! - lastSyncTime: Date! - phase: String! - batchSize: Int! + nodeId: Int! @primaryKey + lastTimestamp: Date! + recordsIngested: Long! + lastSyncTime: Date! + phase: String! + batchSize: Int! } # Audit log for validation/errors type SyncAudit @table { - id: ID! @primaryKey - timestamp: Date! @indexed @createdTime - nodeId: Int - bigQueryCount: Long - harperCount: Long - delta: Long - status: String! - reason: String - recordSample: String + id: ID! @primaryKey + timestamp: Date! @indexed @createdTime + nodeId: Int + bigQueryCount: Long + harperCount: Long + delta: Long + status: String! + reason: String + recordSample: String } ``` **Key differences from naive schema:** + - `BigQueryData` is generic - all BigQuery columns stored at top level (no predefined fields) - Checkpoint includes `lastSyncTime`, `phase`, and `batchSize` for monitoring - Audit table flexible - supports both validation and error logging @@ -779,10 +807,11 @@ async runSyncCycle() { ``` **Error handling strategy:** + - Errors logged but don't crash the process - Failed cycle retries on next scheduled poll (interval-based retry) - Checkpoint preserved from last successful batch -- ⚠️ **TODO**: Add exponential backoff for transient BigQuery errors +- ⚠️ **TODO**: Add exponential backoff for transient BigQuery errors ### Missing Data Handling @@ -790,19 +819,20 @@ async runSyncCycle() { ```javascript for (const record of records) { - // Validate timestamp exists - if (!convertedRecord[timestampColumn]) { - logger.warn(`Missing timestamp column '${timestampColumn}', skipping record`); - await this.logSkippedRecord(convertedRecord, `missing_${timestampColumn}`); - continue; - } - - // Process valid record - validRecords.push(mappedRecord); + // Validate timestamp exists + if (!convertedRecord[timestampColumn]) { + logger.warn(`Missing timestamp column '${timestampColumn}', skipping record`); + await this.logSkippedRecord(convertedRecord, `missing_${timestampColumn}`); + continue; + } + + // Process valid record + validRecords.push(mappedRecord); } ``` **Skipped record logging** (`src/sync-engine.js:450-468`): + ```javascript async logSkippedRecord(record, reason) { const auditEntry = { @@ -825,6 +855,7 @@ Skipped records are logged to `SyncAudit` table for monitoring and debugging. ### Throughput (Projected) With 3 nodes on modest hardware: + - **Steady-state:** ~15,000 records/second total (5,000 per node) - **Catch-up:** ~30,000 records/second (10,000 per node) @@ -837,11 +868,13 @@ With 3 nodes on modest hardware: ### IOPS Usage (Estimated) With 2 indexes per table: + - ~4 IOPS per record - 5,000 records/sec per node = ~20,000 IOPS - SSD handles this comfortably ⚠️ **Note**: These are projected performance numbers based on architecture design. Actual performance will vary based on: + - BigQuery query response times - Network latency between BigQuery and Harper - Record size and structure @@ -877,12 +910,14 @@ Node stability vs. flexibility is acceptable. Most systems don't need dynamic sc ## When to Use This Pattern **Good fit:** + - Large datasets with continuous updates - Need for horizontal scalability - Relatively stable cluster topology - Source system supports partitioned queries **Not ideal:** + - Extremely dynamic cluster sizes (Note: Harper doesn't support autoscaling—nodes must be added/removed manually with rebalancing consideration) - Need for strong consistency across nodes immediately - Source doesn't support efficient partitioned queries @@ -903,4 +938,4 @@ Sometimes the best distributed algorithm is no coordination at all. - [HarperDB Documentation](https://docs.harperdb.io) - [Harper Clustering Documentation](https://docs.harperdb.io/docs/developers/replication) -- [Harper Fabric](https://fabric.harper.fast) \ No newline at end of file +- [Harper Fabric](https://fabric.harper.fast) diff --git a/docs/maritime-data-synthesizer.md b/docs/maritime-data-synthesizer.md index 37e9fff..fc39410 100644 --- a/docs/maritime-data-synthesizer.md +++ b/docs/maritime-data-synthesizer.md @@ -28,26 +28,26 @@ The Maritime Vessel Data Synthesizer generates realistic synthetic tracking data Each vessel position record includes: -| Field | Type | Description | -|-------|------|-------------| -| `mmsi` | STRING | 9-digit Maritime Mobile Service Identity | -| `imo` | STRING | 7-digit International Maritime Organization number | -| `vessel_name` | STRING | Vessel name (e.g., "MV OCEAN FORTUNE 42") | -| `vessel_type` | STRING | CONTAINER, BULK_CARRIER, TANKER, CARGO, PASSENGER, FISHING | -| `flag` | STRING | Two-letter country code | -| `length` | INTEGER | Vessel length in meters | -| `beam` | INTEGER | Vessel width in meters | -| `draft` | FLOAT | Vessel draft (depth) in meters | -| `latitude` | FLOAT | Current latitude (-90 to 90) | -| `longitude` | FLOAT | Current longitude (-180 to 180) | -| `speed_knots` | FLOAT | Current speed in knots | -| `course` | INTEGER | Direction of travel (0-360 degrees) | -| `heading` | INTEGER | Vessel heading (0-360 degrees) | -| `status` | STRING | Vessel operational status | -| `destination` | STRING | Destination port name | -| `eta` | TIMESTAMP | Estimated time of arrival | -| `timestamp` | TIMESTAMP | Record timestamp | -| `report_date` | STRING | Date in YYYYMMDD format | +| Field | Type | Description | +| ------------- | --------- | ---------------------------------------------------------- | +| `mmsi` | STRING | 9-digit Maritime Mobile Service Identity | +| `imo` | STRING | 7-digit International Maritime Organization number | +| `vessel_name` | STRING | Vessel name (e.g., "MV OCEAN FORTUNE 42") | +| `vessel_type` | STRING | CONTAINER, BULK_CARRIER, TANKER, CARGO, PASSENGER, FISHING | +| `flag` | STRING | Two-letter country code | +| `length` | INTEGER | Vessel length in meters | +| `beam` | INTEGER | Vessel width in meters | +| `draft` | FLOAT | Vessel draft (depth) in meters | +| `latitude` | FLOAT | Current latitude (-90 to 90) | +| `longitude` | FLOAT | Current longitude (-180 to 180) | +| `speed_knots` | FLOAT | Current speed in knots | +| `course` | INTEGER | Direction of travel (0-360 degrees) | +| `heading` | INTEGER | Vessel heading (0-360 degrees) | +| `status` | STRING | Vessel operational status | +| `destination` | STRING | Destination port name | +| `eta` | TIMESTAMP | Estimated time of arrival | +| `timestamp` | TIMESTAMP | Record timestamp | +| `report_date` | STRING | Date in YYYYMMDD format | ## Installation @@ -84,17 +84,20 @@ CLEANUP_INTERVAL_HOURS=24 # Clean up old data daily ### Configuration Guide **GENERATION_INTERVAL_MS**: Time between batches + - Lower = more frequent updates, more BigQuery load jobs - Default: 60000 (1 minute) - Range: 10000-300000 (10 seconds to 5 minutes) **BATCH_SIZE**: Records per batch + - Higher = fewer BigQuery jobs, more records per insert - Default: 100 - Range: 50-1000 - Free tier limit: ~1,500 load jobs per day **Records per day** = `(86,400,000 / GENERATION_INTERVAL_MS) × BATCH_SIZE` + - Default: `(86400000 / 60000) × 100 = 144,000 records/day` - At 1000 batch size: 1.44M records/day @@ -132,17 +135,21 @@ npx maritime-data-synthesizer help ### Typical Workflow 1. **Initialize with historical data**: + ```bash npx maritime-data-synthesizer initialize 30 ``` + This creates the BigQuery table and loads 30 days of historical vessel positions. - Time: ~30-60 minutes for 30 days - Data: ~4.3M records (144K/day × 30 days) 2. **Start continuous generation**: + ```bash npx maritime-data-synthesizer start ``` + This starts generating new vessel positions every minute. - Press Ctrl+C to stop @@ -158,25 +165,25 @@ const { MaritimeDataSynthesizer } = require('./src'); // Create synthesizer instance const synthesizer = new MaritimeDataSynthesizer({ - totalVessels: 100000, - batchSize: 100, - generationIntervalMs: 60000, - retentionDays: 30 + totalVessels: 100000, + batchSize: 100, + generationIntervalMs: 60000, + retentionDays: 30, }); // Set up event listeners synthesizer.on('batch:inserted', (data) => { - console.log(`Inserted ${data.records} records`); + console.log(`Inserted ${data.records} records`); }); synthesizer.on('batch:error', (data) => { - console.error('Error:', data.error); + console.error('Error:', data.error); }); // Initialize and start async function run() { - await synthesizer.initialize(30); // 30 days of historical data - await synthesizer.start(); + await synthesizer.initialize(30); // 30 days of historical data + await synthesizer.start(); } run(); @@ -207,23 +214,27 @@ run(); ### Data Generation Strategy **Vessel Pool**: + - Pre-generates 10,000 vessels with persistent identifiers - Each vessel has fixed attributes (MMSI, IMO, type, dimensions) - Vessels are reused across batches for consistency **Journey Simulation**: + - Each vessel maintains a journey state (origin → destination) - 30% of vessels are in port at any time (anchored or moored) - 70% are at sea, moving toward their destination - When a vessel reaches its destination, it enters port and eventually starts a new journey **Movement Calculation**: + - Uses Haversine formula for great circle distances - Calculates bearing between current position and destination - Moves vessel based on speed and course - Accounts for different speeds by vessel type and status **Geographic Distribution**: + - Major ports weighted by actual traffic volume - Asia-Pacific: 50% (Singapore, Shanghai, Hong Kong, etc.) - Europe: 20% (Rotterdam, Antwerp, Hamburg, etc.) @@ -236,6 +247,7 @@ run(); ### Throughput With default settings: + - **144,000 records/day** (100 records × 1,440 batches) - **1.44M records/day** at 1,000 batch size - **4.3M records** for 30 days of historical data @@ -243,6 +255,7 @@ With default settings: ### BigQuery Costs (Free Tier) The synthesizer is optimized for BigQuery free tier: + - **Storage**: 10 GB free (30 days ≈ 2-3 GB) - **Queries**: 1 TB/month free (plenty for monitoring) - **Load Jobs**: 1,500/table/day limit (default config uses ~1,440/day) @@ -327,24 +340,28 @@ LIMIT 100 ## Use Cases ### Maritime Analytics + - Track vessel movements and patterns - Analyze port activity and congestion - Study shipping routes and trade flows - Monitor vessel speeds and efficiency ### Machine Learning + - Train models for vessel trajectory prediction - Anomaly detection for unusual vessel behavior - Port arrival time estimation - Route optimization algorithms ### Visualization & Dashboards + - Real-time vessel tracking maps - Port activity heatmaps - Trade flow visualization - Fleet management dashboards ### Testing & Development + - Test maritime tracking applications - Develop AIS (Automatic Identification System) tools - Validate geospatial queries and analytics @@ -355,6 +372,7 @@ LIMIT 100 The synthesizer emits comprehensive events for monitoring: ### Service Events + - `service:starting` - Service initialization begun - `service:started` - Service running - `service:stopping` - Shutdown initiated @@ -362,6 +380,7 @@ The synthesizer emits comprehensive events for monitoring: - `service:error` - Fatal error occurred ### Initialization Events + - `init:starting` - Historical data load beginning - `init:bigquery-ready` - Schema created - `init:data-generation-starting` - Batch generation starting @@ -370,6 +389,7 @@ The synthesizer emits comprehensive events for monitoring: - `init:error` - Initialization failed ### Batch Events + - `batch:generating` - Record generation started - `batch:generated` - Records ready for insert - `batch:inserting` - Insert job submitted to BigQuery @@ -377,6 +397,7 @@ The synthesizer emits comprehensive events for monitoring: - `batch:error` - Insert failed ### Cleanup Events + - `cleanup:starting` - Retention cleanup started - `cleanup:completed` - Old data deleted - `cleanup:error` - Cleanup failed @@ -384,19 +405,23 @@ The synthesizer emits comprehensive events for monitoring: ## Troubleshooting ### "GCP_PROJECT_ID must be set" + - Ensure `.env` file exists with `GCP_PROJECT_ID=your-project-id` - Or set environment variable: `export GCP_PROJECT_ID=your-project-id` ### "Load job completed with errors" + - Check BigQuery quota limits (1,500 load jobs per table per day) - Verify table schema matches data format - Review BigQuery logs in GCP Console ### High Memory Usage + - Reduce `TOTAL_VESSELS` (default 100,000) - Decrease `BATCH_SIZE` to process smaller batches ### Slow Historical Data Loading + - Increase `BATCH_SIZE` to insert more records per job - Reduce number of days to load - Consider loading in stages @@ -404,17 +429,20 @@ The synthesizer emits comprehensive events for monitoring: ## Technical Details ### Coordinate System + - **Latitude**: -90° (South Pole) to +90° (North Pole) - **Longitude**: -180° (Date Line West) to +180° (Date Line East) - **Precision**: 6 decimal places (~0.1 meters) ### Navigation Calculations + - **Distance**: Haversine formula (great circle) - **Bearing**: Forward azimuth calculation - **New Position**: Given distance and bearing - **Speed**: Nautical miles per hour (knots) ### BigQuery Optimization + - **Partitioning**: By timestamp (DAY) - **Clustering**: By vessel_type, mmsi, report_date - **Load Jobs**: NDJSON format via temp files diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000..0225963 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,23 @@ +import harperConfig from '@harperdb/code-guidelines/eslint'; + +export default [ + ...harperConfig, + // Custom configuration for BigQuery sync plugin + { + ignores: ['dist/', 'node_modules/', 'coverage/', 'ext/maritime-data-synthesizer/**', 'examples/**'], + }, + { + rules: { + // Allow unused vars that start with underscore (intentional unused) + 'no-unused-vars': [ + 'error', + { + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + }, + ], + // Allow unused function parameters (common in callbacks) + '@typescript-eslint/no-unused-vars': 'off', + }, + }, +]; diff --git a/examples/README.md b/examples/README.md index 29385b1..9d4fcb0 100644 --- a/examples/README.md +++ b/examples/README.md @@ -5,6 +5,7 @@ This directory contains example and demonstration scripts for the maritime vesse ## Demo Scripts ### test-config.js + Demonstrates loading and parsing configuration from `config.yaml`. Shows how the synthesizer and plugin share configuration. ```bash @@ -12,6 +13,7 @@ node examples/test-config.js ``` ### test-generator.js + Demonstrates the vessel data generator creating realistic vessel positions. ```bash @@ -19,6 +21,7 @@ node examples/test-generator.js ``` ### test-rolling-window.js + Demonstrates the rolling window feature with different scenarios (empty table, partial data, sufficient data). ```bash @@ -26,6 +29,7 @@ node examples/test-rolling-window.js ``` ### test-clear-commands.js + Compares the `clear`, `clean`, and `reset` commands with use cases and workflows. ```bash @@ -33,6 +37,7 @@ node examples/test-clear-commands.js ``` ### test-bigquery-config.js + Tests BigQuery configuration and connection (requires valid credentials). ```bash diff --git a/package-lock.json b/package-lock.json index 78fd6f8..212b08f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,2113 +1,2130 @@ { - "name": "@harperdb/harper-bigquery-sync", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@harperdb/harper-bigquery-sync", - "version": "1.0.0", - "license": "Apache-2.0", - "dependencies": { - "@google-cloud/bigquery": "^7.0.0", - "yaml": "^2.8.1" - }, - "bin": { - "maritime-data-synthesizer": "bin/cli.js" - }, - "devDependencies": { - "@harperdb/code-guidelines": "^0.0.5", - "eslint": "^9.35.0", - "prettier": "^3.6.2" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@eslint-community/eslint-utils": { - "version": "4.9.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.0.tgz", - "integrity": "sha512-ayVFHdtZ+hsq1t2Dy24wCmGXGe4q9Gu3smhLYALJrr473ZH27MsnSL+LKUlimp4BWJqMDMLmPpx/Q9R3OAlL4g==", - "dev": true, - "license": "MIT", - "dependencies": { - "eslint-visitor-keys": "^3.4.3" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - }, - "peerDependencies": { - "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" - } - }, - "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@eslint-community/regexpp": { - "version": "4.12.2", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", - "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.0.0 || ^14.0.0 || >=16.0.0" - } - }, - "node_modules/@eslint/config-array": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz", - "integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@eslint/object-schema": "^2.1.7", - "debug": "^4.3.1", - "minimatch": "^3.1.2" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/config-helpers": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", - "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@eslint/core": "^0.17.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/core": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", - "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@types/json-schema": "^7.0.15" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/eslintrc": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.1.tgz", - "integrity": "sha512-gtF186CXhIl1p4pJNGZw8Yc6RlshoePRvE0X91oPGb3vZ8pM3qOS9W9NGPat9LziaBV7XrJWGylNQXkGcnM3IQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "^6.12.4", - "debug": "^4.3.2", - "espree": "^10.0.1", - "globals": "^14.0.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", - "minimatch": "^3.1.2", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@eslint/js": { - "version": "9.39.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.0.tgz", - "integrity": "sha512-BIhe0sW91JGPiaF1mOuPy5v8NflqfjIcDNpC+LbW9f609WVRX1rArrhi6Z2ymvrAry9jw+5POTj4t2t62o8Bmw==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://eslint.org/donate" - } - }, - "node_modules/@eslint/object-schema": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", - "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/plugin-kit": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", - "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@eslint/core": "^0.17.0", - "levn": "^0.4.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@google-cloud/bigquery": { - "version": "7.9.4", - "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.9.4.tgz", - "integrity": "sha512-C7jeI+9lnCDYK3cRDujcBsPgiwshWKn/f0BiaJmClplfyosCLfWE83iGQ0eKH113UZzjR9c9q7aZQg0nU388sw==", - "license": "Apache-2.0", - "dependencies": { - "@google-cloud/common": "^5.0.0", - "@google-cloud/paginator": "^5.0.2", - "@google-cloud/precise-date": "^4.0.0", - "@google-cloud/promisify": "4.0.0", - "arrify": "^2.0.1", - "big.js": "^6.0.0", - "duplexify": "^4.0.0", - "extend": "^3.0.2", - "is": "^3.3.0", - "stream-events": "^1.0.5", - "uuid": "^9.0.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/common": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-5.0.2.tgz", - "integrity": "sha512-V7bmBKYQyu0eVG2BFejuUjlBt+zrya6vtsKdY+JxMM/dNntPF41vZ9+LhOshEUH01zOHEqBSvI7Dad7ZS6aUeA==", - "license": "Apache-2.0", - "dependencies": { - "@google-cloud/projectify": "^4.0.0", - "@google-cloud/promisify": "^4.0.0", - "arrify": "^2.0.1", - "duplexify": "^4.1.1", - "extend": "^3.0.2", - "google-auth-library": "^9.0.0", - "html-entities": "^2.5.2", - "retry-request": "^7.0.0", - "teeny-request": "^9.0.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/paginator": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", - "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", - "license": "Apache-2.0", - "dependencies": { - "arrify": "^2.0.0", - "extend": "^3.0.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/precise-date": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-4.0.0.tgz", - "integrity": "sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==", - "license": "Apache-2.0", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/projectify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", - "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", - "license": "Apache-2.0", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/promisify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", - "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@harperdb/code-guidelines": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/@harperdb/code-guidelines/-/code-guidelines-0.0.5.tgz", - "integrity": "sha512-+RvvkTe7APFeaoOYxLQMsv4qtOhER+LAUpnA+lCkV56Q/Uv6RAPtRd/k08u5WnNDe27VCmXi/tUxiGElPF27WQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@tsconfig/node-ts": "23.6.1", - "@tsconfig/node20": "20.1.6", - "eslint-config-prettier": "10.1.8", - "eslint-plugin-prettier": "5.5.4" - }, - "peerDependencies": { - "@types/node": ">= 20", - "eslint": ">= 9", - "prettier": ">= 3", - "typescript": ">= 5.9" - } - }, - "node_modules/@humanfs/core": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", - "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.18.0" - } - }, - "node_modules/@humanfs/node": { - "version": "0.16.7", - "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", - "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@humanfs/core": "^0.19.1", - "@humanwhocodes/retry": "^0.4.0" - }, - "engines": { - "node": ">=18.18.0" - } - }, - "node_modules/@humanwhocodes/module-importer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", - "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=12.22" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, - "node_modules/@humanwhocodes/retry": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", - "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, - "node_modules/@pkgr/core": { - "version": "0.2.9", - "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.9.tgz", - "integrity": "sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/pkgr" - } - }, - "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tsconfig/node-ts": { - "version": "23.6.1", - "resolved": "https://registry.npmjs.org/@tsconfig/node-ts/-/node-ts-23.6.1.tgz", - "integrity": "sha512-1E5cUp+S65pLKKI9VrGMQPWDHxOEq3dAGM2onG3fLeSRwWbylYFwhIjnzJikjSN7w2nCgwxmv8ifvUKDFkK38Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node20": { - "version": "20.1.6", - "resolved": "https://registry.npmjs.org/@tsconfig/node20/-/node20-20.1.6.tgz", - "integrity": "sha512-sz+Hqx9zwZDpZIV871WSbUzSqNIsXzghZydypnfgzPKLltVJfkINfUeTct31n/tTSa9ZE1ZOfKdRre1uHHquYQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/caseless": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", - "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", - "license": "MIT" - }, - "node_modules/@types/estree": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", - "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/node": { - "version": "24.9.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.9.2.tgz", - "integrity": "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA==", - "license": "MIT", - "peer": true, - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/request": { - "version": "2.48.13", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.13.tgz", - "integrity": "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==", - "license": "MIT", - "dependencies": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.5" - } - }, - "node_modules/@types/tough-cookie": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", - "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", - "license": "MIT" - }, - "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", - "dev": true, - "license": "MIT", - "peer": true, - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-jsx": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", - "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" - } - }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, - "node_modules/arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/big.js": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz", - "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==", - "license": "MIT", - "engines": { - "node": "*" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/bigjs" - } - }, - "node_modules/bignumber.js": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", - "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", - "license": "BSD-3-Clause" - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, - "license": "MIT" - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT" - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/duplexify": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", - "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", - "license": "MIT", - "dependencies": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.2" - } - }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/end-of-stream": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", - "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "license": "MIT", - "dependencies": { - "once": "^1.4.0" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/eslint": { - "version": "9.39.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.0.tgz", - "integrity": "sha512-iy2GE3MHrYTL5lrCtMZ0X1KLEKKUjmK0kzwcnefhR66txcEmXZD2YWgR5GNdcEwkNx3a0siYkSvl0vIC+Svjmg==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "@eslint-community/eslint-utils": "^4.8.0", - "@eslint-community/regexpp": "^4.12.1", - "@eslint/config-array": "^0.21.1", - "@eslint/config-helpers": "^0.4.2", - "@eslint/core": "^0.17.0", - "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.39.0", - "@eslint/plugin-kit": "^0.4.1", - "@humanfs/node": "^0.16.6", - "@humanwhocodes/module-importer": "^1.0.1", - "@humanwhocodes/retry": "^0.4.2", - "@types/estree": "^1.0.6", - "ajv": "^6.12.4", - "chalk": "^4.0.0", - "cross-spawn": "^7.0.6", - "debug": "^4.3.2", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.4.0", - "eslint-visitor-keys": "^4.2.1", - "espree": "^10.4.0", - "esquery": "^1.5.0", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^8.0.0", - "find-up": "^5.0.0", - "glob-parent": "^6.0.2", - "ignore": "^5.2.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.2", - "natural-compare": "^1.4.0", - "optionator": "^0.9.3" - }, - "bin": { - "eslint": "bin/eslint.js" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://eslint.org/donate" - }, - "peerDependencies": { - "jiti": "*" - }, - "peerDependenciesMeta": { - "jiti": { - "optional": true - } - } - }, - "node_modules/eslint-config-prettier": { - "version": "10.1.8", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.8.tgz", - "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", - "dev": true, - "license": "MIT", - "peer": true, - "bin": { - "eslint-config-prettier": "bin/cli.js" - }, - "funding": { - "url": "https://opencollective.com/eslint-config-prettier" - }, - "peerDependencies": { - "eslint": ">=7.0.0" - } - }, - "node_modules/eslint-plugin-prettier": { - "version": "5.5.4", - "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.5.4.tgz", - "integrity": "sha512-swNtI95SToIz05YINMA6Ox5R057IMAmWZ26GqPxusAp1TZzj+IdY9tXNWWD3vkF/wEqydCONcwjTFpxybBqZsg==", - "dev": true, - "license": "MIT", - "dependencies": { - "prettier-linter-helpers": "^1.0.0", - "synckit": "^0.11.7" - }, - "engines": { - "node": "^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint-plugin-prettier" - }, - "peerDependencies": { - "@types/eslint": ">=8.0.0", - "eslint": ">=8.0.0", - "eslint-config-prettier": ">= 7.0.0 <10.0.0 || >=10.1.0", - "prettier": ">=3.0.0" - }, - "peerDependenciesMeta": { - "@types/eslint": { - "optional": true - }, - "eslint-config-prettier": { - "optional": true - } - } - }, - "node_modules/eslint-scope": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", - "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/espree": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", - "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "acorn": "^8.15.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^4.2.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/esquery": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", - "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "estraverse": "^5.1.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "estraverse": "^5.2.0" - }, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-diff": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", - "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", - "dev": true, - "license": "Apache-2.0" - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true, - "license": "MIT" - }, - "node_modules/file-entry-cache": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", - "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "flat-cache": "^4.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/find-up": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", - "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^6.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/flat-cache": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", - "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", - "dev": true, - "license": "MIT", - "dependencies": { - "flatted": "^3.2.9", - "keyv": "^4.5.4" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/flatted": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", - "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", - "dev": true, - "license": "ISC" - }, - "node_modules/form-data": { - "version": "2.5.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.5.tgz", - "integrity": "sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.35", - "safe-buffer": "^5.2.1" - }, - "engines": { - "node": ">= 0.12" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gaxios": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", - "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gcp-metadata": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", - "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", - "license": "Apache-2.0", - "dependencies": { - "gaxios": "^6.1.1", - "google-logging-utils": "^0.0.2", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.3" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/globals": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", - "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/google-auth-library": { - "version": "9.15.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", - "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", - "license": "Apache-2.0", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^6.1.1", - "gcp-metadata": "^6.1.0", - "gtoken": "^7.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/google-logging-utils": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", - "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gtoken": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", - "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", - "license": "MIT", - "dependencies": { - "gaxios": "^6.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/html-entities": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz", - "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/mdevils" - }, - { - "type": "patreon", - "url": "https://patreon.com/mdevils" - } - ], - "license": "MIT" - }, - "node_modules/http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "license": "MIT", - "dependencies": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/http-proxy-agent/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/ignore": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", - "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/import-fresh": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", - "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/is": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/is/-/is-3.3.2.tgz", - "integrity": "sha512-a2xr4E3s1PjDS8ORcGgXpWx6V+liNs+O3JRD2mb9aeugD7rtkkZ0zgLdYgw0tWsKhsdiezGYptSiMlVazCBTuQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, - "license": "ISC" - }, - "node_modules/js-yaml": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", - "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, - "node_modules/json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true, - "license": "MIT" - }, - "node_modules/json-stable-stringify-without-jsonify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/jwa": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", - "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", - "license": "MIT", - "dependencies": { - "buffer-equal-constant-time": "^1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "license": "MIT", - "dependencies": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "dev": true, - "license": "MIT", - "dependencies": { - "json-buffer": "3.0.1" - } - }, - "node_modules/levn": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", - "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "prelude-ls": "^1.2.1", - "type-check": "~0.4.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/locate-path": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", - "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^5.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/optionator": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", - "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", - "dev": true, - "license": "MIT", - "dependencies": { - "deep-is": "^0.1.3", - "fast-levenshtein": "^2.0.6", - "levn": "^0.4.1", - "prelude-ls": "^1.2.1", - "type-check": "^0.4.0", - "word-wrap": "^1.2.5" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", - "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^3.0.2" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "callsites": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/prelude-ls": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", - "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/prettier": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", - "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", - "dev": true, - "license": "MIT", - "peer": true, - "bin": { - "prettier": "bin/prettier.cjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" - } - }, - "node_modules/prettier-linter-helpers": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", - "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-diff": "^1.1.2" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/retry-request": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", - "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", - "license": "MIT", - "dependencies": { - "@types/request": "^2.48.8", - "extend": "^3.0.2", - "teeny-request": "^9.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/stream-events": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", - "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", - "license": "MIT", - "dependencies": { - "stubs": "^3.0.0" - } - }, - "node_modules/stream-shift": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", - "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", - "license": "MIT" - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/stubs": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", - "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", - "license": "MIT" - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/synckit": { - "version": "0.11.11", - "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.11.tgz", - "integrity": "sha512-MeQTA1r0litLUf0Rp/iisCaL8761lKAZHaimlbGK4j0HysC4PLfqygQj9srcs0m2RdtDYnF8UuYyKpbjHYp7Jw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@pkgr/core": "^0.2.9" - }, - "engines": { - "node": "^14.18.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/synckit" - } - }, - "node_modules/teeny-request": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", - "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", - "license": "Apache-2.0", - "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^2.6.9", - "stream-events": "^1.0.5", - "uuid": "^9.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/teeny-request/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "license": "MIT", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "license": "MIT", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, - "node_modules/type-check": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", - "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", - "dev": true, - "license": "MIT", - "dependencies": { - "prelude-ls": "^1.2.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "peer": true, - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "license": "MIT" - }, - "node_modules/uri-js": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "license": "MIT" - }, - "node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/word-wrap": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", - "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC" - }, - "node_modules/yaml": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz", - "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", - "license": "ISC", - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14.6" - } - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - } - } + "name": "@harperdb/harper-bigquery-sync", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@harperdb/harper-bigquery-sync", + "version": "1.0.0", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/bigquery": "^7.0.0", + "yaml": "^2.8.1" + }, + "bin": { + "maritime-data-synthesizer": "bin/cli.js" + }, + "devDependencies": { + "@harperdb/code-guidelines": "^0.0.5", + "eslint": "^9.35.0", + "husky": "^9.1.7", + "prettier": "^3.6.2" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.0.tgz", + "integrity": "sha512-ayVFHdtZ+hsq1t2Dy24wCmGXGe4q9Gu3smhLYALJrr473ZH27MsnSL+LKUlimp4BWJqMDMLmPpx/Q9R3OAlL4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz", + "integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.7", + "debug": "^4.3.1", + "minimatch": "^3.1.2" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.1.tgz", + "integrity": "sha512-gtF186CXhIl1p4pJNGZw8Yc6RlshoePRvE0X91oPGb3vZ8pM3qOS9W9NGPat9LziaBV7XrJWGylNQXkGcnM3IQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "9.39.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.0.tgz", + "integrity": "sha512-BIhe0sW91JGPiaF1mOuPy5v8NflqfjIcDNpC+LbW9f609WVRX1rArrhi6Z2ymvrAry9jw+5POTj4t2t62o8Bmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", + "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", + "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0", + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@google-cloud/bigquery": { + "version": "7.9.4", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.9.4.tgz", + "integrity": "sha512-C7jeI+9lnCDYK3cRDujcBsPgiwshWKn/f0BiaJmClplfyosCLfWE83iGQ0eKH113UZzjR9c9q7aZQg0nU388sw==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/common": "^5.0.0", + "@google-cloud/paginator": "^5.0.2", + "@google-cloud/precise-date": "^4.0.0", + "@google-cloud/promisify": "4.0.0", + "arrify": "^2.0.1", + "big.js": "^6.0.0", + "duplexify": "^4.0.0", + "extend": "^3.0.2", + "is": "^3.3.0", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/common": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-5.0.2.tgz", + "integrity": "sha512-V7bmBKYQyu0eVG2BFejuUjlBt+zrya6vtsKdY+JxMM/dNntPF41vZ9+LhOshEUH01zOHEqBSvI7Dad7ZS6aUeA==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "extend": "^3.0.2", + "google-auth-library": "^9.0.0", + "html-entities": "^2.5.2", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/paginator": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", + "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", + "license": "Apache-2.0", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/precise-date": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-4.0.0.tgz", + "integrity": "sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@harperdb/code-guidelines": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@harperdb/code-guidelines/-/code-guidelines-0.0.5.tgz", + "integrity": "sha512-+RvvkTe7APFeaoOYxLQMsv4qtOhER+LAUpnA+lCkV56Q/Uv6RAPtRd/k08u5WnNDe27VCmXi/tUxiGElPF27WQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@tsconfig/node-ts": "23.6.1", + "@tsconfig/node20": "20.1.6", + "eslint-config-prettier": "10.1.8", + "eslint-plugin-prettier": "5.5.4" + }, + "peerDependencies": { + "@types/node": ">= 20", + "eslint": ">= 9", + "prettier": ">= 3", + "typescript": ">= 5.9" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", + "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.7", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", + "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.1", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@pkgr/core": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.9.tgz", + "integrity": "sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/pkgr" + } + }, + "node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tsconfig/node-ts": { + "version": "23.6.1", + "resolved": "https://registry.npmjs.org/@tsconfig/node-ts/-/node-ts-23.6.1.tgz", + "integrity": "sha512-1E5cUp+S65pLKKI9VrGMQPWDHxOEq3dAGM2onG3fLeSRwWbylYFwhIjnzJikjSN7w2nCgwxmv8ifvUKDFkK38Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node20": { + "version": "20.1.6", + "resolved": "https://registry.npmjs.org/@tsconfig/node20/-/node20-20.1.6.tgz", + "integrity": "sha512-sz+Hqx9zwZDpZIV871WSbUzSqNIsXzghZydypnfgzPKLltVJfkINfUeTct31n/tTSa9ZE1ZOfKdRre1uHHquYQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.9.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.9.2.tgz", + "integrity": "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA==", + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/request": { + "version": "2.48.13", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.13.tgz", + "integrity": "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==", + "license": "MIT", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.5" + } + }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "license": "MIT" + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/big.js": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.2.tgz", + "integrity": "sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==", + "license": "MIT", + "engines": { + "node": "*" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/bigjs" + } + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "9.39.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.0.tgz", + "integrity": "sha512-iy2GE3MHrYTL5lrCtMZ0X1KLEKKUjmK0kzwcnefhR66txcEmXZD2YWgR5GNdcEwkNx3a0siYkSvl0vIC+Svjmg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.1", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", + "@eslint/eslintrc": "^3.3.1", + "@eslint/js": "9.39.0", + "@eslint/plugin-kit": "^0.4.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-config-prettier": { + "version": "10.1.8", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.8.tgz", + "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "eslint-config-prettier": "bin/cli.js" + }, + "funding": { + "url": "https://opencollective.com/eslint-config-prettier" + }, + "peerDependencies": { + "eslint": ">=7.0.0" + } + }, + "node_modules/eslint-plugin-prettier": { + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.5.4.tgz", + "integrity": "sha512-swNtI95SToIz05YINMA6Ox5R057IMAmWZ26GqPxusAp1TZzj+IdY9tXNWWD3vkF/wEqydCONcwjTFpxybBqZsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "prettier-linter-helpers": "^1.0.0", + "synckit": "^0.11.7" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint-plugin-prettier" + }, + "peerDependencies": { + "@types/eslint": ">=8.0.0", + "eslint": ">=8.0.0", + "eslint-config-prettier": ">= 7.0.0 <10.0.0 || >=10.1.0", + "prettier": ">=3.0.0" + }, + "peerDependenciesMeta": { + "@types/eslint": { + "optional": true + }, + "eslint-config-prettier": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", + "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-diff": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", + "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", + "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "dev": true, + "license": "ISC" + }, + "node_modules/form-data": { + "version": "2.5.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.5.tgz", + "integrity": "sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.35", + "safe-buffer": "^5.2.1" + }, + "engines": { + "node": ">= 0.12" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", + "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^6.1.1", + "google-logging-utils": "^0.0.2", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/google-auth-library": { + "version": "9.15.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", + "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-logging-utils": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", + "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "license": "MIT", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/html-entities": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz", + "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, + "node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "license": "MIT", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/husky": { + "version": "9.1.7", + "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz", + "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==", + "dev": true, + "license": "MIT", + "bin": { + "husky": "bin.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/typicode" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/is": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/is/-/is-3.3.2.tgz", + "integrity": "sha512-a2xr4E3s1PjDS8ORcGgXpWx6V+liNs+O3JRD2mb9aeugD7rtkkZ0zgLdYgw0tWsKhsdiezGYptSiMlVazCBTuQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/prettier": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", + "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-diff": "^1.1.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "license": "MIT", + "dependencies": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "license": "MIT", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "license": "MIT" + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", + "license": "MIT" + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/synckit": { + "version": "0.11.11", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.11.tgz", + "integrity": "sha512-MeQTA1r0litLUf0Rp/iisCaL8761lKAZHaimlbGK4j0HysC4PLfqygQj9srcs0m2RdtDYnF8UuYyKpbjHYp7Jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@pkgr/core": "^0.2.9" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/synckit" + } + }, + "node_modules/teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "license": "Apache-2.0", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/teeny-request/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/teeny-request/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "license": "MIT" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/yaml": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz", + "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } } diff --git a/package.json b/package.json index 7eab7c0..ab7a88a 100644 --- a/package.json +++ b/package.json @@ -1,67 +1,69 @@ { - "name": "@harperdb/harper-bigquery-sync", - "version": "1.0.0", - "description": "BigQuery sync plugin for Harper", - "license": "Apache-2.0", - "author": { - "name": "HarperDB, Inc.", - "email": "opensource@harperdb.io", - "url": "https://harper.fast/" - }, - "contributors": [], - "homepage": "https://harper.fast/", - "keywords": [ - "harperdb", - "harper", - "plugin", - "bigquery" - ], - "type": "module", - "main": "dist/index.js", - "exports": { - ".": "./src/index.js", - "./config": "./config.yaml" - }, - "bin": { - "maritime-data-synthesizer": "./bin/cli.js" - }, - "files": [ - "dist/", - "schema/", - "config.yaml", - "LICENSE", - "README.md" - ], - "scripts": { - "test": "node --test test/**/*.test.js", - "test:coverage": "node --test --experimental-test-coverage test/**/*.test.js", - "test:config": "node examples/test-bigquery-config.js", - "lint": "eslint .", - "format": "prettier .", - "format:check": "npm run format -- --check", - "format:write": "npm run format -- --write" - }, - "dependencies": { - "@google-cloud/bigquery": "^7.0.0", - "yaml": "^2.8.1" - }, - "devDependencies": { - "@harperdb/code-guidelines": "^0.0.5", - "eslint": "^9.35.0", - "prettier": "^3.6.2" - }, - "engines": { - "node": ">=20" - }, - "devEngines": { - "runtime": { - "name": "node", - "version": ">=20", - "onFail": "error" - }, - "packageManager": { - "name": "npm", - "onFail": "error" - } - } + "name": "@harperdb/harper-bigquery-sync", + "version": "1.0.0", + "description": "BigQuery sync plugin for Harper", + "license": "Apache-2.0", + "author": { + "name": "HarperDB, Inc.", + "email": "opensource@harperdb.io", + "url": "https://harper.fast/" + }, + "contributors": [], + "homepage": "https://harper.fast/", + "keywords": [ + "harperdb", + "harper", + "plugin", + "bigquery" + ], + "type": "module", + "main": "dist/index.js", + "exports": { + ".": "./src/index.js", + "./config": "./config.yaml" + }, + "bin": { + "maritime-data-synthesizer": "./bin/cli.js" + }, + "files": [ + "dist/", + "schema/", + "config.yaml", + "LICENSE", + "README.md" + ], + "scripts": { + "test": "node --test test/config-loader.test.js test/sync-engine.test.js", + "test:coverage": "node --test --experimental-test-coverage test/config-loader.test.js test/sync-engine.test.js", + "test:config": "node examples/test-bigquery-config.js", + "lint": "eslint .", + "format": "prettier .", + "format:check": "npm run format -- --check", + "format:write": "npm run format -- --write", + "prepare": "husky" + }, + "dependencies": { + "@google-cloud/bigquery": "^7.0.0", + "yaml": "^2.8.1" + }, + "devDependencies": { + "@harperdb/code-guidelines": "^0.0.5", + "eslint": "^9.35.0", + "husky": "^9.1.7", + "prettier": "^3.6.2" + }, + "engines": { + "node": ">=20" + }, + "devEngines": { + "runtime": { + "name": "node", + "version": ">=20", + "onFail": "error" + }, + "packageManager": { + "name": "npm", + "onFail": "error" + } + } } diff --git a/prettier.config.js b/prettier.config.js new file mode 100644 index 0000000..c79127d --- /dev/null +++ b/prettier.config.js @@ -0,0 +1,3 @@ +import harperConfig from '@harperdb/code-guidelines/prettier'; + +export default harperConfig; diff --git a/schema/harper-bigquery-sync.graphql b/schema/harper-bigquery-sync.graphql index f5db88e..ad50767 100644 --- a/schema/harper-bigquery-sync.graphql +++ b/schema/harper-bigquery-sync.graphql @@ -5,31 +5,31 @@ # BigQuery records are stored as-is with metadata fields # Harper will auto-generate 'id' field if not provided type BigQueryData @table { - id: ID @primaryKey - # All BigQuery fields stored directly at top level - # Metadata fields: - # _syncedAt: Date @createdTime + id: ID @primaryKey + # All BigQuery fields stored directly at top level + # Metadata fields: + # _syncedAt: Date @createdTime } # Checkpoint table for ingestion type SyncCheckpoint @table { - nodeId: Int! @primaryKey - lastTimestamp: Date! - recordsIngested: Long! - lastSyncTime: Date! - phase: String! - batchSize: Int! + nodeId: Int! @primaryKey + lastTimestamp: Date! + recordsIngested: Long! + lastSyncTime: Date! + phase: String! + batchSize: Int! } # Audit table for ingestion type SyncAudit @table { - id: ID! @primaryKey - timestamp: Date! @indexed @createdTime - nodeId: Int - bigQueryCount: Long - harperCount: Long - delta: Long - status: String! - reason: String - recordSample: String -} \ No newline at end of file + id: ID! @primaryKey + timestamp: Date! @indexed @createdTime + nodeId: Int + bigQueryCount: Long + harperCount: Long + delta: Long + status: String! + reason: String + recordSample: String +} diff --git a/src/bigquery-client.js b/src/bigquery-client.js index 0ab3b2a..81b4161 100644 --- a/src/bigquery-client.js +++ b/src/bigquery-client.js @@ -2,41 +2,42 @@ // File: bigquery-client.js // BigQuery API client with partition-aware queries -/* global config */ - import { BigQuery } from '@google-cloud/bigquery'; export class BigQueryClient { - constructor(config) { - logger.info('[BigQueryClient] Constructor called - initializing BigQuery client'); - logger.debug(`[BigQueryClient] Config - projectId: ${config.bigquery.projectId}, dataset: ${config.bigquery.dataset}, table: ${config.bigquery.table}, location: ${config.bigquery.location}`); - this.config = config; - this.client = new BigQuery({ - projectId: config.bigquery.projectId, - keyFilename: config.bigquery.credentials, - location: config.bigquery.location - }); - - this.dataset = config.bigquery.dataset; - this.table = config.bigquery.table; - this.timestampColumn = config.bigquery.timestampColumn; - logger.info('[BigQueryClient] Client initialized successfully'); - } - - async resolveParams(params) { - const entries = Object.entries(params); - const resolvedEntries = await Promise.all( - entries.map(async ([key, value]) => [key, await value]) - ); - return Object.fromEntries(resolvedEntries); - } - - async pullPartition({ nodeId, clusterSize, lastTimestamp, batchSize }) { - - logger.info(`[BigQueryClient.pullPartition] Pulling partition - nodeId: ${nodeId}, clusterSize: ${clusterSize}, batchSize: ${batchSize}`); - logger.debug(`[BigQueryClient.pullPartition] Query parameters - lastTimestamp: ${lastTimestamp} type: ${typeof(lastTimestamp)}, timestampColumn: ${this.timestampColumn}`); - - const query = ` + constructor(config) { + logger.info('[BigQueryClient] Constructor called - initializing BigQuery client'); + logger.debug( + `[BigQueryClient] Config - projectId: ${config.bigquery.projectId}, dataset: ${config.bigquery.dataset}, table: ${config.bigquery.table}, location: ${config.bigquery.location}` + ); + this.config = config; + this.client = new BigQuery({ + projectId: config.bigquery.projectId, + keyFilename: config.bigquery.credentials, + location: config.bigquery.location, + }); + + this.dataset = config.bigquery.dataset; + this.table = config.bigquery.table; + this.timestampColumn = config.bigquery.timestampColumn; + logger.info('[BigQueryClient] Client initialized successfully'); + } + + async resolveParams(params) { + const entries = Object.entries(params); + const resolvedEntries = await Promise.all(entries.map(async ([key, value]) => [key, await value])); + return Object.fromEntries(resolvedEntries); + } + + async pullPartition({ nodeId, clusterSize, lastTimestamp, batchSize }) { + logger.info( + `[BigQueryClient.pullPartition] Pulling partition - nodeId: ${nodeId}, clusterSize: ${clusterSize}, batchSize: ${batchSize}` + ); + logger.debug( + `[BigQueryClient.pullPartition] Query parameters - lastTimestamp: ${lastTimestamp} type: ${typeof lastTimestamp}, timestampColumn: ${this.timestampColumn}` + ); + + const query = ` SELECT * FROM \`${this.dataset}.${this.table}\` WHERE @@ -51,72 +52,75 @@ export class BigQueryClient { LIMIT CAST(@batchSize AS INT64) `; - // Assume these might return Promises: - const params = await this.resolveParams({ - nodeId, - clusterSize, - lastTimestamp, - batchSize, - }); - - const options = { - query, - params: params - }; - - logger.trace(`[BigQueryClient.pullPartition] Generated SQL query: ${query}`); - - try { - logger.debug('[BigQueryClient.pullPartition] Executing BigQuery query...'); - const startTime = Date.now(); - const [rows] = await this.client.query(options); - const duration = Date.now() - startTime; - logger.info(`[BigQueryClient.pullPartition] Query complete - returned ${rows.length} rows in ${duration}ms`); - logger.debug(`[BigQueryClient.pullPartition] First row timestamp: ${rows.length > 0 ? Date(rows[0][this.timestampColumn]) : 'N/A'}`); - return rows; - } catch (error) { - // Always log full error detail - logger.error('[BigQueryClient.pullPartition] BigQuery query failed'); - logger.error(`Error name: ${error.name}`); - logger.error(`Error message: ${error.message}`); - logger.error(`Error stack: ${error.stack}`); - - // BigQuery often includes structured info - if (error.errors) { - for (const e of error.errors) { - logger.error(`BigQuery error reason: ${e.reason}`); - logger.error(`BigQuery error location: ${e.location}`); - logger.error(`BigQuery error message: ${e.message}`); - } - } - } - } - - async normalizeToIso(ts) { - if (ts == null) return null; - - if (ts instanceof Date) return ts.toISOString(); - - if (typeof ts === 'number') return new Date(ts).toISOString(); - - if (typeof ts === 'string') { - // If someone passed "Wed Nov 05 2025 16:11:45 GMT-0700 (Mountain ...)" - // normalize it to ISO; reject if not parseable. - const d = new Date(ts); - if (!Number.isNaN(d.getTime())) return d.toISOString(); - throw new Error(`Unparseable timestamp string: ${ts}`); - } - - if (typeof ts.toISOString === 'function') return ts.toISOString(); - - throw new Error(`Unsupported lastTimestamp type: ${typeof ts}`); - } - - - async countPartition({ nodeId, clusterSize }) { - logger.info(`[BigQueryClient.countPartition] Counting partition records - nodeId: ${nodeId}, clusterSize: ${clusterSize}`); - - const query = ` + // Assume these might return Promises: + const params = await this.resolveParams({ + nodeId, + clusterSize, + lastTimestamp, + batchSize, + }); + + const options = { + query, + params: params, + }; + + logger.trace(`[BigQueryClient.pullPartition] Generated SQL query: ${query}`); + + try { + logger.debug('[BigQueryClient.pullPartition] Executing BigQuery query...'); + const startTime = Date.now(); + const [rows] = await this.client.query(options); + const duration = Date.now() - startTime; + logger.info(`[BigQueryClient.pullPartition] Query complete - returned ${rows.length} rows in ${duration}ms`); + logger.debug( + `[BigQueryClient.pullPartition] First row timestamp: ${rows.length > 0 ? Date(rows[0][this.timestampColumn]) : 'N/A'}` + ); + return rows; + } catch (error) { + // Always log full error detail + logger.error('[BigQueryClient.pullPartition] BigQuery query failed'); + logger.error(`Error name: ${error.name}`); + logger.error(`Error message: ${error.message}`); + logger.error(`Error stack: ${error.stack}`); + + // BigQuery often includes structured info + if (error.errors) { + for (const e of error.errors) { + logger.error(`BigQuery error reason: ${e.reason}`); + logger.error(`BigQuery error location: ${e.location}`); + logger.error(`BigQuery error message: ${e.message}`); + } + } + } + } + + async normalizeToIso(ts) { + if (ts === null || ts === undefined) return null; + + if (ts instanceof Date) return ts.toISOString(); + + if (typeof ts === 'number') return new Date(ts).toISOString(); + + if (typeof ts === 'string') { + // If someone passed "Wed Nov 05 2025 16:11:45 GMT-0700 (Mountain ...)" + // normalize it to ISO; reject if not parseable. + const d = new Date(ts); + if (!Number.isNaN(d.getTime())) return d.toISOString(); + throw new Error(`Unparseable timestamp string: ${ts}`); + } + + if (typeof ts.toISOString === 'function') return ts.toISOString(); + + throw new Error(`Unsupported lastTimestamp type: ${typeof ts}`); + } + + async countPartition({ nodeId, clusterSize }) { + logger.info( + `[BigQueryClient.countPartition] Counting partition records - nodeId: ${nodeId}, clusterSize: ${clusterSize}` + ); + + const query = ` SELECT COUNT(*) as count FROM \`${this.dataset}.${this.table}\` WHERE MOD( @@ -125,32 +129,34 @@ export class BigQueryClient { ) = @nodeId `; - logger.trace(`[BigQueryClient.countPartition] Count query: ${query}`); - - const options = { - query, - params: { clusterSize, nodeId } - }; - - try { - logger.debug('[BigQueryClient.countPartition] Executing count query...'); - const startTime = Date.now(); - const [rows] = await this.client.query(options); - const duration = Date.now() - startTime; - const count = rows[0].count; - logger.info(`[BigQueryClient.countPartition] Count complete - ${count} records in partition (took ${duration}ms)`); - return count; - } catch (error) { - logger.error(`[BigQueryClient.countPartition] Count query error: ${error.message}`, error); - throw error; - } - } - - async verifyRecord(record) { - logger.debug(`[BigQueryClient.verifyRecord] Verifying record - timestamp: ${record.timestamp}`); - // Verify a specific record exists in BigQuery by timestamp and unique identifier - // Note: This assumes a unique identifier field exists - adapt to your schema - const query = ` + logger.trace(`[BigQueryClient.countPartition] Count query: ${query}`); + + const options = { + query, + params: { clusterSize, nodeId }, + }; + + try { + logger.debug('[BigQueryClient.countPartition] Executing count query...'); + const startTime = Date.now(); + const [rows] = await this.client.query(options); + const duration = Date.now() - startTime; + const count = rows[0].count; + logger.info( + `[BigQueryClient.countPartition] Count complete - ${count} records in partition (took ${duration}ms)` + ); + return count; + } catch (error) { + logger.error(`[BigQueryClient.countPartition] Count query error: ${error.message}`, error); + throw error; + } + } + + async verifyRecord(record) { + logger.debug(`[BigQueryClient.verifyRecord] Verifying record - timestamp: ${record.timestamp}`); + // Verify a specific record exists in BigQuery by timestamp and unique identifier + // Note: This assumes a unique identifier field exists - adapt to your schema + const query = ` SELECT 1 FROM \`${this.dataset}.${this.table}\` WHERE ${this.timestampColumn} = @timestamp @@ -158,25 +164,25 @@ export class BigQueryClient { LIMIT 1 `; - logger.trace(`[BigQueryClient.verifyRecord] Verification query: ${query}`); - - const options = { - query, - params: { - timestamp: record.timestamp, - recordId: record.id - } - }; - - try { - logger.debug('[BigQueryClient.verifyRecord] Executing verification query...'); - const [rows] = await this.client.query(options); - const exists = rows.length > 0; - logger.debug(`[BigQueryClient.verifyRecord] Record ${exists ? 'EXISTS' : 'NOT FOUND'} in BigQuery`); - return exists; - } catch (error) { - logger.error(`[BigQueryClient.verifyRecord] Verification error: ${error.message}`, error); - return false; - } - } -} \ No newline at end of file + logger.trace(`[BigQueryClient.verifyRecord] Verification query: ${query}`); + + const options = { + query, + params: { + timestamp: record.timestamp, + recordId: record.id, + }, + }; + + try { + logger.debug('[BigQueryClient.verifyRecord] Executing verification query...'); + const [rows] = await this.client.query(options); + const exists = rows.length > 0; + logger.debug(`[BigQueryClient.verifyRecord] Record ${exists ? 'EXISTS' : 'NOT FOUND'} in BigQuery`); + return exists; + } catch (error) { + logger.error(`[BigQueryClient.verifyRecord] Verification error: ${error.message}`, error); + return false; + } + } +} diff --git a/src/bigquery.js b/src/bigquery.js index 8cc2557..76d0bf1 100644 --- a/src/bigquery.js +++ b/src/bigquery.js @@ -9,193 +9,193 @@ import os from 'os'; import path from 'path'; class MaritimeBigQueryClient { - constructor(config = {}) { - this.projectId = config.projectId || process.env.GCP_PROJECT_ID; - this.datasetId = config.datasetId || process.env.BIGQUERY_DATASET || 'maritime_tracking'; - this.tableId = config.tableId || process.env.BIGQUERY_TABLE || 'vessel_positions'; - this.retentionDays = config.retentionDays || parseInt(process.env.RETENTION_DAYS || '30', 10); - this.location = config.location || process.env.BIGQUERY_LOCATION || 'US'; - - if (!this.projectId) { - throw new Error('projectId must be set in config or GCP_PROJECT_ID environment variable'); - } - - // Configure BigQuery client - const bqConfig = { - projectId: this.projectId - }; - - // Add credentials if provided (path to service account key file) - if (config.credentials) { - bqConfig.keyFilename = config.credentials; - } - - this.bigquery = new BigQuery(bqConfig); - - this.dataset = this.bigquery.dataset(this.datasetId); - this.table = this.dataset.table(this.tableId); - } - - /** - * Define the BigQuery schema for vessel position data - */ - getSchema() { - return [ - { name: 'mmsi', type: 'STRING', mode: 'REQUIRED' }, - { name: 'imo', type: 'STRING', mode: 'REQUIRED' }, - { name: 'vessel_name', type: 'STRING', mode: 'REQUIRED' }, - { name: 'vessel_type', type: 'STRING', mode: 'REQUIRED' }, - { name: 'flag', type: 'STRING', mode: 'REQUIRED' }, - { name: 'length', type: 'INTEGER', mode: 'REQUIRED' }, - { name: 'beam', type: 'INTEGER', mode: 'REQUIRED' }, - { name: 'draft', type: 'FLOAT', mode: 'REQUIRED' }, - { name: 'latitude', type: 'FLOAT', mode: 'REQUIRED' }, - { name: 'longitude', type: 'FLOAT', mode: 'REQUIRED' }, - { name: 'speed_knots', type: 'FLOAT', mode: 'REQUIRED' }, - { name: 'course', type: 'INTEGER', mode: 'REQUIRED' }, - { name: 'heading', type: 'INTEGER', mode: 'REQUIRED' }, - { name: 'status', type: 'STRING', mode: 'REQUIRED' }, - { name: 'destination', type: 'STRING', mode: 'NULLABLE' }, - { name: 'eta', type: 'TIMESTAMP', mode: 'NULLABLE' }, - { name: 'timestamp', type: 'TIMESTAMP', mode: 'REQUIRED' }, - { name: 'report_date', type: 'STRING', mode: 'REQUIRED' } - ]; - } - - /** - * Initialize BigQuery resources (dataset and table) - */ - async initialize() { - try { - // Create dataset if it doesn't exist - const [datasetExists] = await this.dataset.exists(); - if (!datasetExists) { - console.log(`Creating dataset: ${this.datasetId}`); - await this.bigquery.createDataset(this.datasetId, { - location: this.location - }); - console.log(`Dataset ${this.datasetId} created`); - } else { - console.log(`Dataset ${this.datasetId} already exists`); - } - - // Create table if it doesn't exist - const [tableExists] = await this.table.exists(); - if (!tableExists) { - console.log(`Creating table: ${this.tableId}`); - const options = { - schema: this.getSchema(), - location: this.location, - timePartitioning: { - type: 'DAY', - field: 'timestamp' - }, - clustering: { - fields: ['vessel_type', 'mmsi', 'report_date'] - } - }; - - await this.dataset.createTable(this.tableId, options); - console.log(`Table ${this.tableId} created with schema and partitioning`); - } else { - console.log(`Table ${this.tableId} already exists`); - } - - return true; - } catch (error) { - console.error('Error initializing BigQuery resources:', error); - throw error; - } - } - - /** - * Insert batch of records into BigQuery using Load Job (free tier compatible) - * Includes retry logic for transient network errors - */ - async insertBatch(records, maxRetries = 5) { - if (!records || records.length === 0) { - throw new Error('No records to insert'); - } - - const tmpFile = path.join(os.tmpdir(), `maritime-vessels-${Date.now()}.ndjson`); - - try { - // Write records to temporary NDJSON file - const ndjson = records.map(record => JSON.stringify(record)).join('\n'); - fs.writeFileSync(tmpFile, ndjson, 'utf8'); - - // Retry loop for transient network errors - let lastError; - for (let attempt = 1; attempt <= maxRetries; attempt++) { - try { - // Load file into BigQuery - this waits for job completion - await this.table.load(tmpFile, { - sourceFormat: 'NEWLINE_DELIMITED_JSON', - writeDisposition: 'WRITE_APPEND', - autodetect: false, - schema: { fields: this.getSchema() } - }); - - // Success - clean up and return - fs.unlinkSync(tmpFile); - - if (attempt > 1) { - console.log(`Batch inserted successfully after ${attempt} attempts`); - } - - return { - success: true, - recordCount: records.length - }; - } catch (loadError) { - lastError = loadError; - - // Check if this is a retryable error (network timeout, rate limit, etc.) - const isRetryable = - loadError.code === 'ETIMEDOUT' || - loadError.code === 'ECONNRESET' || - loadError.code === 'ENOTFOUND' || - (loadError.code === 429) || // Rate limit - (loadError.code >= 500 && loadError.code < 600); // Server errors - - if (!isRetryable || attempt === maxRetries) { - // Non-retryable error or final attempt - clean up and throw - if (fs.existsSync(tmpFile)) { - fs.unlinkSync(tmpFile); - } - throw loadError; - } - - // Exponential backoff: 2^attempt seconds (2s, 4s, 8s, 16s, 32s) - const backoffMs = Math.pow(2, attempt) * 1000; - console.log(`Upload failed (attempt ${attempt}/${maxRetries}): ${loadError.message}`); - console.log(`Retrying in ${backoffMs / 1000}s...`); - - await new Promise(resolve => setTimeout(resolve, backoffMs)); - } - } - - // Should never reach here, but just in case - throw lastError; - } catch (error) { - // Clean up temp file on any error - if (fs.existsSync(tmpFile)) { - fs.unlinkSync(tmpFile); - } - console.error('Error inserting batch:', error.message); - throw error; - } - } - - /** - * Get table statistics - */ - async getStats() { - try { - const [metadata] = await this.table.getMetadata(); - - // Query for additional statistics - const query = ` + constructor(config = {}) { + this.projectId = config.projectId || process.env.GCP_PROJECT_ID; + this.datasetId = config.datasetId || process.env.BIGQUERY_DATASET || 'maritime_tracking'; + this.tableId = config.tableId || process.env.BIGQUERY_TABLE || 'vessel_positions'; + this.retentionDays = config.retentionDays || parseInt(process.env.RETENTION_DAYS || '30', 10); + this.location = config.location || process.env.BIGQUERY_LOCATION || 'US'; + + if (!this.projectId) { + throw new Error('projectId must be set in config or GCP_PROJECT_ID environment variable'); + } + + // Configure BigQuery client + const bqConfig = { + projectId: this.projectId, + }; + + // Add credentials if provided (path to service account key file) + if (config.credentials) { + bqConfig.keyFilename = config.credentials; + } + + this.bigquery = new BigQuery(bqConfig); + + this.dataset = this.bigquery.dataset(this.datasetId); + this.table = this.dataset.table(this.tableId); + } + + /** + * Define the BigQuery schema for vessel position data + */ + getSchema() { + return [ + { name: 'mmsi', type: 'STRING', mode: 'REQUIRED' }, + { name: 'imo', type: 'STRING', mode: 'REQUIRED' }, + { name: 'vessel_name', type: 'STRING', mode: 'REQUIRED' }, + { name: 'vessel_type', type: 'STRING', mode: 'REQUIRED' }, + { name: 'flag', type: 'STRING', mode: 'REQUIRED' }, + { name: 'length', type: 'INTEGER', mode: 'REQUIRED' }, + { name: 'beam', type: 'INTEGER', mode: 'REQUIRED' }, + { name: 'draft', type: 'FLOAT', mode: 'REQUIRED' }, + { name: 'latitude', type: 'FLOAT', mode: 'REQUIRED' }, + { name: 'longitude', type: 'FLOAT', mode: 'REQUIRED' }, + { name: 'speed_knots', type: 'FLOAT', mode: 'REQUIRED' }, + { name: 'course', type: 'INTEGER', mode: 'REQUIRED' }, + { name: 'heading', type: 'INTEGER', mode: 'REQUIRED' }, + { name: 'status', type: 'STRING', mode: 'REQUIRED' }, + { name: 'destination', type: 'STRING', mode: 'NULLABLE' }, + { name: 'eta', type: 'TIMESTAMP', mode: 'NULLABLE' }, + { name: 'timestamp', type: 'TIMESTAMP', mode: 'REQUIRED' }, + { name: 'report_date', type: 'STRING', mode: 'REQUIRED' }, + ]; + } + + /** + * Initialize BigQuery resources (dataset and table) + */ + async initialize() { + try { + // Create dataset if it doesn't exist + const [datasetExists] = await this.dataset.exists(); + if (!datasetExists) { + console.log(`Creating dataset: ${this.datasetId}`); + await this.bigquery.createDataset(this.datasetId, { + location: this.location, + }); + console.log(`Dataset ${this.datasetId} created`); + } else { + console.log(`Dataset ${this.datasetId} already exists`); + } + + // Create table if it doesn't exist + const [tableExists] = await this.table.exists(); + if (!tableExists) { + console.log(`Creating table: ${this.tableId}`); + const options = { + schema: this.getSchema(), + location: this.location, + timePartitioning: { + type: 'DAY', + field: 'timestamp', + }, + clustering: { + fields: ['vessel_type', 'mmsi', 'report_date'], + }, + }; + + await this.dataset.createTable(this.tableId, options); + console.log(`Table ${this.tableId} created with schema and partitioning`); + } else { + console.log(`Table ${this.tableId} already exists`); + } + + return true; + } catch (error) { + console.error('Error initializing BigQuery resources:', error); + throw error; + } + } + + /** + * Insert batch of records into BigQuery using Load Job (free tier compatible) + * Includes retry logic for transient network errors + */ + async insertBatch(records, maxRetries = 5) { + if (!records || records.length === 0) { + throw new Error('No records to insert'); + } + + const tmpFile = path.join(os.tmpdir(), `maritime-vessels-${Date.now()}.ndjson`); + + try { + // Write records to temporary NDJSON file + const ndjson = records.map((record) => JSON.stringify(record)).join('\n'); + fs.writeFileSync(tmpFile, ndjson, 'utf8'); + + // Retry loop for transient network errors + let lastError; + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + // Load file into BigQuery - this waits for job completion + await this.table.load(tmpFile, { + sourceFormat: 'NEWLINE_DELIMITED_JSON', + writeDisposition: 'WRITE_APPEND', + autodetect: false, + schema: { fields: this.getSchema() }, + }); + + // Success - clean up and return + fs.unlinkSync(tmpFile); + + if (attempt > 1) { + console.log(`Batch inserted successfully after ${attempt} attempts`); + } + + return { + success: true, + recordCount: records.length, + }; + } catch (loadError) { + lastError = loadError; + + // Check if this is a retryable error (network timeout, rate limit, etc.) + const isRetryable = + loadError.code === 'ETIMEDOUT' || + loadError.code === 'ECONNRESET' || + loadError.code === 'ENOTFOUND' || + loadError.code === 429 || // Rate limit + (loadError.code >= 500 && loadError.code < 600); // Server errors + + if (!isRetryable || attempt === maxRetries) { + // Non-retryable error or final attempt - clean up and throw + if (fs.existsSync(tmpFile)) { + fs.unlinkSync(tmpFile); + } + throw loadError; + } + + // Exponential backoff: 2^attempt seconds (2s, 4s, 8s, 16s, 32s) + const backoffMs = Math.pow(2, attempt) * 1000; + console.log(`Upload failed (attempt ${attempt}/${maxRetries}): ${loadError.message}`); + console.log(`Retrying in ${backoffMs / 1000}s...`); + + await new Promise((resolve) => setTimeout(resolve, backoffMs)); + } + } + + // Should never reach here, but just in case + throw lastError; + } catch (error) { + // Clean up temp file on any error + if (fs.existsSync(tmpFile)) { + fs.unlinkSync(tmpFile); + } + console.error('Error inserting batch:', error.message); + throw error; + } + } + + /** + * Get table statistics + */ + async getStats() { + try { + const [metadata] = await this.table.getMetadata(); + + // Query for additional statistics + const query = ` SELECT COUNT(*) as total_records, COUNT(DISTINCT mmsi) as unique_vessels, @@ -206,112 +206,112 @@ class MaritimeBigQueryClient { FROM \`${this.projectId}.${this.datasetId}.${this.tableId}\` `; - const [rows] = await this.bigquery.query({ query }); - - return { - tableMetadata: { - numBytes: metadata.numBytes, - numRows: metadata.numRows, - creationTime: metadata.creationTime, - lastModifiedTime: metadata.lastModifiedTime - }, - statistics: rows[0] - }; - } catch (error) { - console.error('Error getting statistics:', error); - throw error; - } - } - - /** - * Clean up old data based on retention policy - */ - async cleanupOldData() { - try { - const cutoffDate = new Date(); - cutoffDate.setDate(cutoffDate.getDate() - this.retentionDays); - - const query = ` + const [rows] = await this.bigquery.query({ query }); + + return { + tableMetadata: { + numBytes: metadata.numBytes, + numRows: metadata.numRows, + creationTime: metadata.creationTime, + lastModifiedTime: metadata.lastModifiedTime, + }, + statistics: rows[0], + }; + } catch (error) { + console.error('Error getting statistics:', error); + throw error; + } + } + + /** + * Clean up old data based on retention policy + */ + async cleanupOldData() { + try { + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - this.retentionDays); + + const query = ` DELETE FROM \`${this.projectId}.${this.datasetId}.${this.tableId}\` WHERE TIMESTAMP(timestamp) < TIMESTAMP('${cutoffDate.toISOString()}') `; - console.log(`Cleaning up data older than ${cutoffDate.toISOString()}`); - - const [job] = await this.bigquery.createQueryJob({ - query, - location: this.location - }); - - const [response] = await job.getQueryResults(); - - console.log(`Cleanup completed. Deleted rows: ${response.length}`); - - return { - success: true, - cutoffDate: cutoffDate.toISOString(), - deletedRows: response.length - }; - } catch (error) { - console.error('Error cleaning up old data:', error); - throw error; - } - } - - /** - * Clear all data from table (truncate) without deleting the table - */ - async clearData() { - try { - const query = ` + console.log(`Cleaning up data older than ${cutoffDate.toISOString()}`); + + const [job] = await this.bigquery.createQueryJob({ + query, + location: this.location, + }); + + const [response] = await job.getQueryResults(); + + console.log(`Cleanup completed. Deleted rows: ${response.length}`); + + return { + success: true, + cutoffDate: cutoffDate.toISOString(), + deletedRows: response.length, + }; + } catch (error) { + console.error('Error cleaning up old data:', error); + throw error; + } + } + + /** + * Clear all data from table (truncate) without deleting the table + */ + async clearData() { + try { + const query = ` DELETE FROM \`${this.projectId}.${this.datasetId}.${this.tableId}\` WHERE TRUE `; - console.log(`Clearing all data from ${this.tableId}...`); - - const [job] = await this.bigquery.createQueryJob({ - query, - location: this.location - }); - - await job.getQueryResults(); - - console.log('All data cleared from table (schema preserved)'); - - return { success: true }; - } catch (error) { - console.error('Error clearing data:', error); - throw error; - } - } - - /** - * Delete all data and table - */ - async deleteTable() { - try { - const [exists] = await this.table.exists(); - if (exists) { - await this.table.delete(); - console.log(`Table ${this.tableId} deleted`); - return { success: true }; - } else { - console.log(`Table ${this.tableId} does not exist`); - return { success: true, message: 'Table does not exist' }; - } - } catch (error) { - console.error('Error deleting table:', error); - throw error; - } - } - - /** - * Query vessels by type - */ - async getVesselsByType(vesselType, limit = 100) { - try { - const query = ` + console.log(`Clearing all data from ${this.tableId}...`); + + const [job] = await this.bigquery.createQueryJob({ + query, + location: this.location, + }); + + await job.getQueryResults(); + + console.log('All data cleared from table (schema preserved)'); + + return { success: true }; + } catch (error) { + console.error('Error clearing data:', error); + throw error; + } + } + + /** + * Delete all data and table + */ + async deleteTable() { + try { + const [exists] = await this.table.exists(); + if (exists) { + await this.table.delete(); + console.log(`Table ${this.tableId} deleted`); + return { success: true }; + } else { + console.log(`Table ${this.tableId} does not exist`); + return { success: true, message: 'Table does not exist' }; + } + } catch (error) { + console.error('Error deleting table:', error); + throw error; + } + } + + /** + * Query vessels by type + */ + async getVesselsByType(vesselType, limit = 100) { + try { + const query = ` SELECT * FROM \`${this.projectId}.${this.datasetId}.${this.tableId}\` WHERE vessel_type = @vesselType @@ -319,25 +319,25 @@ class MaritimeBigQueryClient { LIMIT @limit `; - const options = { - query, - params: { vesselType, limit } - }; - - const [rows] = await this.bigquery.query(options); - return rows; - } catch (error) { - console.error('Error querying vessels by type:', error); - throw error; - } - } - - /** - * Query vessels in a geographic bounding box - */ - async getVesselsInBoundingBox(minLat, maxLat, minLon, maxLon, limit = 1000) { - try { - const query = ` + const options = { + query, + params: { vesselType, limit }, + }; + + const [rows] = await this.bigquery.query(options); + return rows; + } catch (error) { + console.error('Error querying vessels by type:', error); + throw error; + } + } + + /** + * Query vessels in a geographic bounding box + */ + async getVesselsInBoundingBox(minLat, maxLat, minLon, maxLon, limit = 1000) { + try { + const query = ` SELECT * FROM \`${this.projectId}.${this.datasetId}.${this.tableId}\` WHERE latitude BETWEEN @minLat AND @maxLat @@ -347,18 +347,18 @@ class MaritimeBigQueryClient { LIMIT @limit `; - const options = { - query, - params: { minLat, maxLat, minLon, maxLon, limit } - }; - - const [rows] = await this.bigquery.query(options); - return rows; - } catch (error) { - console.error('Error querying vessels in bounding box:', error); - throw error; - } - } + const options = { + query, + params: { minLat, maxLat, minLon, maxLon, limit }, + }; + + const [rows] = await this.bigquery.query(options); + return rows; + } catch (error) { + console.error('Error querying vessels in bounding box:', error); + throw error; + } + } } export default MaritimeBigQueryClient; diff --git a/src/config-loader.js b/src/config-loader.js index bb97b02..43edf05 100644 --- a/src/config-loader.js +++ b/src/config-loader.js @@ -15,20 +15,20 @@ const __dirname = dirname(__filename); * Load configuration from config.yaml */ export function loadConfig(configPath = null) { - try { - // Default to config.yaml in project root - const path = configPath || join(__dirname, '..', 'config.yaml'); - const fileContent = readFileSync(path, 'utf8'); - const config = parse(fileContent); + try { + // Default to config.yaml in project root + const path = configPath || join(__dirname, '..', 'config.yaml'); + const fileContent = readFileSync(path, 'utf8'); + const config = parse(fileContent); - if (!config) { - throw new Error('Failed to parse config.yaml'); - } + if (!config) { + throw new Error('Failed to parse config.yaml'); + } - return config; - } catch (error) { - throw new Error(`Failed to load configuration: ${error.message}`); - } + return config; + } catch (error) { + throw new Error(`Failed to load configuration: ${error.message}`); + } } /** @@ -36,56 +36,56 @@ export function loadConfig(configPath = null) { * Uses bigquery section as primary config, with optional synthesizer overrides */ export function getSynthesizerConfig(config = null) { - const fullConfig = config || loadConfig(); + const fullConfig = config || loadConfig(); - if (!fullConfig.bigquery) { - throw new Error('bigquery section missing in config.yaml'); - } + if (!fullConfig.bigquery) { + throw new Error('bigquery section missing in config.yaml'); + } - // Use bigquery settings as defaults, with optional synthesizer overrides - return { - // BigQuery connection (from bigquery section) - projectId: fullConfig.bigquery.projectId, - credentials: fullConfig.bigquery.credentials, - location: fullConfig.bigquery.location || 'US', + // Use bigquery settings as defaults, with optional synthesizer overrides + return { + // BigQuery connection (from bigquery section) + projectId: fullConfig.bigquery.projectId, + credentials: fullConfig.bigquery.credentials, + location: fullConfig.bigquery.location || 'US', - // Target dataset/table: Use bigquery settings by default, synthesizer overrides if present - datasetId: fullConfig.synthesizer?.dataset || fullConfig.bigquery.dataset, - tableId: fullConfig.synthesizer?.table || fullConfig.bigquery.table, + // Target dataset/table: Use bigquery settings by default, synthesizer overrides if present + datasetId: fullConfig.synthesizer?.dataset || fullConfig.bigquery.dataset, + tableId: fullConfig.synthesizer?.table || fullConfig.bigquery.table, - // Data generation settings (from synthesizer section with defaults) - totalVessels: fullConfig.synthesizer?.totalVessels || 100000, - batchSize: fullConfig.synthesizer?.batchSize || 100, - generationIntervalMs: fullConfig.synthesizer?.generationIntervalMs || 60000, + // Data generation settings (from synthesizer section with defaults) + totalVessels: fullConfig.synthesizer?.totalVessels || 100000, + batchSize: fullConfig.synthesizer?.batchSize || 100, + generationIntervalMs: fullConfig.synthesizer?.generationIntervalMs || 60000, - // Data retention (from synthesizer section with defaults) - retentionDays: fullConfig.synthesizer?.retentionDays || 30, - cleanupIntervalHours: fullConfig.synthesizer?.cleanupIntervalHours || 24 - }; + // Data retention (from synthesizer section with defaults) + retentionDays: fullConfig.synthesizer?.retentionDays || 30, + cleanupIntervalHours: fullConfig.synthesizer?.cleanupIntervalHours || 24, + }; } /** * Get BigQuery configuration for the plugin (for reference) */ export function getPluginConfig(config = null) { - const fullConfig = config || loadConfig(); + const fullConfig = config || loadConfig(); - if (!fullConfig.bigquery) { - throw new Error('bigquery section missing in config.yaml'); - } + if (!fullConfig.bigquery) { + throw new Error('bigquery section missing in config.yaml'); + } - return { - projectId: fullConfig.bigquery.projectId, - dataset: fullConfig.bigquery.dataset, - table: fullConfig.bigquery.table, - timestampColumn: fullConfig.bigquery.timestampColumn, - credentials: fullConfig.bigquery.credentials, - location: fullConfig.bigquery.location || 'US' - }; + return { + projectId: fullConfig.bigquery.projectId, + dataset: fullConfig.bigquery.dataset, + table: fullConfig.bigquery.table, + timestampColumn: fullConfig.bigquery.timestampColumn, + credentials: fullConfig.bigquery.credentials, + location: fullConfig.bigquery.location || 'US', + }; } export default { - loadConfig, - getSynthesizerConfig, - getPluginConfig + loadConfig, + getSynthesizerConfig, + getPluginConfig, }; diff --git a/src/generator.js b/src/generator.js index 144f651..f22b81f 100644 --- a/src/generator.js +++ b/src/generator.js @@ -5,443 +5,473 @@ // Major ports around the world with coordinates and traffic weight const MAJOR_PORTS = [ - // Asia-Pacific (50% of global maritime traffic) - { name: 'Singapore', lat: 1.2644, lon: 103.8223, weight: 10, region: 'Asia' }, - { name: 'Shanghai', lat: 31.2304, lon: 121.4737, weight: 9, region: 'Asia' }, - { name: 'Hong Kong', lat: 22.3193, lon: 114.1694, weight: 7, region: 'Asia' }, - { name: 'Busan', lat: 35.1796, lon: 129.0756, weight: 6, region: 'Asia' }, - { name: 'Guangzhou', lat: 23.1291, lon: 113.2644, weight: 5, region: 'Asia' }, - { name: 'Qingdao', lat: 36.0671, lon: 120.3826, weight: 5, region: 'Asia' }, - { name: 'Tokyo', lat: 35.6528, lon: 139.8394, weight: 4, region: 'Asia' }, - { name: 'Port Klang', lat: 3.0041, lon: 101.3653, weight: 4, region: 'Asia' }, - { name: 'Kaohsiung', lat: 22.6163, lon: 120.2997, weight: 3, region: 'Asia' }, - { name: 'Tianjin', lat: 39.0842, lon: 117.2010, weight: 4, region: 'Asia' }, - - // Europe (20% of global maritime traffic) - { name: 'Rotterdam', lat: 51.9225, lon: 4.4792, weight: 7, region: 'Europe' }, - { name: 'Antwerp', lat: 51.2194, lon: 4.4025, weight: 5, region: 'Europe' }, - { name: 'Hamburg', lat: 53.5511, lon: 9.9937, weight: 4, region: 'Europe' }, - { name: 'Valencia', lat: 39.4699, lon: -0.3763, weight: 3, region: 'Europe' }, - { name: 'Piraeus', lat: 37.9472, lon: 23.6472, weight: 3, region: 'Europe' }, - { name: 'Felixstowe', lat: 51.9542, lon: 1.3511, weight: 2, region: 'Europe' }, - - // Middle East (10% of global maritime traffic) - { name: 'Dubai', lat: 25.2769, lon: 55.2963, weight: 6, region: 'Middle East' }, - { name: 'Jeddah', lat: 21.5433, lon: 39.1728, weight: 3, region: 'Middle East' }, - - // Americas (15% of global maritime traffic) - { name: 'Los Angeles', lat: 33.7405, lon: -118.2720, weight: 6, region: 'Americas' }, - { name: 'Long Beach', lat: 33.7683, lon: -118.1956, weight: 5, region: 'Americas' }, - { name: 'New York/New Jersey', lat: 40.6700, lon: -74.0400, weight: 5, region: 'Americas' }, - { name: 'Savannah', lat: 32.0809, lon: -81.0912, weight: 3, region: 'Americas' }, - { name: 'Houston', lat: 29.7604, lon: -95.3698, weight: 4, region: 'Americas' }, - { name: 'Vancouver', lat: 49.2827, lon: -123.1207, weight: 3, region: 'Americas' }, - { name: 'Santos', lat: -23.9618, lon: -46.3322, weight: 3, region: 'Americas' }, - { name: 'Manzanillo', lat: 19.0544, lon: -104.3188, weight: 2, region: 'Americas' }, - - // Africa (5% of global maritime traffic) - { name: 'Cape Town', lat: -33.9249, lon: 18.4241, weight: 2, region: 'Africa' }, - { name: 'Durban', lat: -29.8587, lon: 31.0218, weight: 2, region: 'Africa' }, - { name: 'Lagos', lat: 6.4526, lon: 3.3958, weight: 2, region: 'Africa' } + // Asia-Pacific (50% of global maritime traffic) + { name: 'Singapore', lat: 1.2644, lon: 103.8223, weight: 10, region: 'Asia' }, + { name: 'Shanghai', lat: 31.2304, lon: 121.4737, weight: 9, region: 'Asia' }, + { name: 'Hong Kong', lat: 22.3193, lon: 114.1694, weight: 7, region: 'Asia' }, + { name: 'Busan', lat: 35.1796, lon: 129.0756, weight: 6, region: 'Asia' }, + { name: 'Guangzhou', lat: 23.1291, lon: 113.2644, weight: 5, region: 'Asia' }, + { name: 'Qingdao', lat: 36.0671, lon: 120.3826, weight: 5, region: 'Asia' }, + { name: 'Tokyo', lat: 35.6528, lon: 139.8394, weight: 4, region: 'Asia' }, + { name: 'Port Klang', lat: 3.0041, lon: 101.3653, weight: 4, region: 'Asia' }, + { name: 'Kaohsiung', lat: 22.6163, lon: 120.2997, weight: 3, region: 'Asia' }, + { name: 'Tianjin', lat: 39.0842, lon: 117.201, weight: 4, region: 'Asia' }, + + // Europe (20% of global maritime traffic) + { name: 'Rotterdam', lat: 51.9225, lon: 4.4792, weight: 7, region: 'Europe' }, + { name: 'Antwerp', lat: 51.2194, lon: 4.4025, weight: 5, region: 'Europe' }, + { name: 'Hamburg', lat: 53.5511, lon: 9.9937, weight: 4, region: 'Europe' }, + { name: 'Valencia', lat: 39.4699, lon: -0.3763, weight: 3, region: 'Europe' }, + { name: 'Piraeus', lat: 37.9472, lon: 23.6472, weight: 3, region: 'Europe' }, + { name: 'Felixstowe', lat: 51.9542, lon: 1.3511, weight: 2, region: 'Europe' }, + + // Middle East (10% of global maritime traffic) + { name: 'Dubai', lat: 25.2769, lon: 55.2963, weight: 6, region: 'Middle East' }, + { name: 'Jeddah', lat: 21.5433, lon: 39.1728, weight: 3, region: 'Middle East' }, + + // Americas (15% of global maritime traffic) + { name: 'Los Angeles', lat: 33.7405, lon: -118.272, weight: 6, region: 'Americas' }, + { name: 'Long Beach', lat: 33.7683, lon: -118.1956, weight: 5, region: 'Americas' }, + { name: 'New York/New Jersey', lat: 40.67, lon: -74.04, weight: 5, region: 'Americas' }, + { name: 'Savannah', lat: 32.0809, lon: -81.0912, weight: 3, region: 'Americas' }, + { name: 'Houston', lat: 29.7604, lon: -95.3698, weight: 4, region: 'Americas' }, + { name: 'Vancouver', lat: 49.2827, lon: -123.1207, weight: 3, region: 'Americas' }, + { name: 'Santos', lat: -23.9618, lon: -46.3322, weight: 3, region: 'Americas' }, + { name: 'Manzanillo', lat: 19.0544, lon: -104.3188, weight: 2, region: 'Americas' }, + + // Africa (5% of global maritime traffic) + { name: 'Cape Town', lat: -33.9249, lon: 18.4241, weight: 2, region: 'Africa' }, + { name: 'Durban', lat: -29.8587, lon: 31.0218, weight: 2, region: 'Africa' }, + { name: 'Lagos', lat: 6.4526, lon: 3.3958, weight: 2, region: 'Africa' }, ]; // Vessel types with characteristics const VESSEL_TYPES = [ - { - type: 'CONTAINER', - speedRange: [18, 25], // knots - lengthRange: [200, 400], // meters - draftRange: [10, 16], // meters - distribution: 0.35 // 35% of vessels - }, - { - type: 'BULK_CARRIER', - speedRange: [12, 16], - lengthRange: [150, 300], - draftRange: [8, 14], - distribution: 0.25 - }, - { - type: 'TANKER', - speedRange: [13, 17], - lengthRange: [180, 330], - draftRange: [10, 18], - distribution: 0.20 - }, - { - type: 'CARGO', - speedRange: [14, 19], - lengthRange: [120, 250], - draftRange: [7, 12], - distribution: 0.10 - }, - { - type: 'PASSENGER', - speedRange: [20, 30], - lengthRange: [200, 360], - draftRange: [7, 9], - distribution: 0.05 - }, - { - type: 'FISHING', - speedRange: [8, 14], - lengthRange: [30, 100], - draftRange: [3, 6], - distribution: 0.05 - } + { + type: 'CONTAINER', + speedRange: [18, 25], // knots + lengthRange: [200, 400], // meters + draftRange: [10, 16], // meters + distribution: 0.35, // 35% of vessels + }, + { + type: 'BULK_CARRIER', + speedRange: [12, 16], + lengthRange: [150, 300], + draftRange: [8, 14], + distribution: 0.25, + }, + { + type: 'TANKER', + speedRange: [13, 17], + lengthRange: [180, 330], + draftRange: [10, 18], + distribution: 0.2, + }, + { + type: 'CARGO', + speedRange: [14, 19], + lengthRange: [120, 250], + draftRange: [7, 12], + distribution: 0.1, + }, + { + type: 'PASSENGER', + speedRange: [20, 30], + lengthRange: [200, 360], + draftRange: [7, 9], + distribution: 0.05, + }, + { + type: 'FISHING', + speedRange: [8, 14], + lengthRange: [30, 100], + draftRange: [3, 6], + distribution: 0.05, + }, ]; // Vessel status options -const VESSEL_STATUS = [ - 'UNDERWAY_USING_ENGINE', - 'AT_ANCHOR', - 'MOORED', - 'UNDERWAY_SAILING', - 'NOT_UNDER_COMMAND' -]; +const _VESSEL_STATUS = ['UNDERWAY_USING_ENGINE', 'AT_ANCHOR', 'MOORED', 'UNDERWAY_SAILING', 'NOT_UNDER_COMMAND']; // Flag states (top maritime nations) const FLAG_STATES = [ - 'PA', 'LR', 'MH', 'HK', 'SG', 'MT', 'BS', 'CY', 'IM', 'GR', - 'CN', 'KR', 'JP', 'US', 'GB', 'NO', 'DE', 'IT', 'NL', 'DK' + 'PA', + 'LR', + 'MH', + 'HK', + 'SG', + 'MT', + 'BS', + 'CY', + 'IM', + 'GR', + 'CN', + 'KR', + 'JP', + 'US', + 'GB', + 'NO', + 'DE', + 'IT', + 'NL', + 'DK', ]; class MaritimeVesselGenerator { - constructor(config = {}) { - this.totalVessels = config.totalVessels || 100000; // Global fleet - this.vesselsPerBatch = config.vesselsPerBatch || 100; - this.vesselPool = []; - this.journeys = new Map(); // Track ongoing journeys - - // Initialize vessel pool - this.initializeVesselPool(); - console.log(`Initialized vessel pool with ${this.vesselPool.length} vessels`); - } - - /** - * Initialize a pool of vessels with persistent identifiers - */ - initializeVesselPool() { - const cacheSize = Math.min(10000, this.totalVessels); - - for (let i = 0; i < cacheSize; i++) { - const vesselType = this.selectVesselType(); - const vessel = { - mmsi: this.generateMMSI(), - imo: this.generateIMO(), - name: this.generateVesselName(vesselType.type, i), - type: vesselType.type, - flag: FLAG_STATES[Math.floor(Math.random() * FLAG_STATES.length)], - length: Math.floor(Math.random() * (vesselType.lengthRange[1] - vesselType.lengthRange[0]) + vesselType.lengthRange[0]), - beam: 0, // will calculate from length - draft: parseFloat((Math.random() * (vesselType.draftRange[1] - vesselType.draftRange[0]) + vesselType.draftRange[0]).toFixed(1)), - maxSpeed: vesselType.speedRange[1], - cruiseSpeed: Math.floor(Math.random() * (vesselType.speedRange[1] - vesselType.speedRange[0]) + vesselType.speedRange[0]) - }; - - // Beam typically 1/8 to 1/6 of length - vessel.beam = Math.floor(vessel.length / 7); - - this.vesselPool.push(vessel); - } - } - - /** - * Generate a 9-digit MMSI (Maritime Mobile Service Identity) - */ - generateMMSI() { - // Format: MIDxxxxxx where MID is Maritime Identification Digits (country code) - const mids = [ - '201', '211', '219', '227', '232', '236', '240', '244', '247', // European countries - '303', '310', '338', // North American countries - '412', '413', '414', '416', '419', // Asian countries - '563', '564', '565', '566', '567' // Southeast Asian countries - ]; - - const mid = mids[Math.floor(Math.random() * mids.length)]; - const remaining = String(Math.floor(Math.random() * 1000000)).padStart(6, '0'); - - return mid + remaining; - } - - /** - * Generate a 7-digit IMO number - */ - generateIMO() { - const base = String(Math.floor(Math.random() * 1000000) + 1000000); - return base.substring(0, 7); - } - - /** - * Generate vessel name - */ - generateVesselName(type, index) { - const prefixes = ['MV', 'MT', 'MSC', 'COSCO', 'MAERSK', 'EVERGREEN', 'CMA CGM']; - const names = ['FORTUNE', 'VOYAGER', 'EXPLORER', 'STAR', 'OCEAN', 'SPIRIT', 'PIONEER', 'LIBERTY']; - - const prefix = prefixes[Math.floor(Math.random() * prefixes.length)]; - const name = names[Math.floor(Math.random() * names.length)]; - - return `${prefix} ${name} ${index % 100}`; - } - - /** - * Select vessel type based on distribution - */ - selectVesselType() { - const rand = Math.random(); - let cumulative = 0; - - for (const vesselType of VESSEL_TYPES) { - cumulative += vesselType.distribution; - if (rand <= cumulative) { - return vesselType; - } - } - - return VESSEL_TYPES[0]; - } - - /** - * Select a random port weighted by traffic volume - */ - selectWeightedPort() { - const totalWeight = MAJOR_PORTS.reduce((sum, port) => sum + port.weight, 0); - const rand = Math.random() * totalWeight; - let cumulative = 0; - - for (const port of MAJOR_PORTS) { - cumulative += port.weight; - if (rand <= cumulative) { - return port; - } - } - - return MAJOR_PORTS[0]; - } - - /** - * Calculate distance between two points (Haversine formula) - */ - calculateDistance(lat1, lon1, lat2, lon2) { - const R = 6371; // Earth radius in km - const dLat = (lat2 - lat1) * Math.PI / 180; - const dLon = (lon2 - lon1) * Math.PI / 180; - - const a = Math.sin(dLat/2) * Math.sin(dLat/2) + - Math.cos(lat1 * Math.PI / 180) * Math.cos(lat2 * Math.PI / 180) * - Math.sin(dLon/2) * Math.sin(dLon/2); - - const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1-a)); - const distance = R * c; // Distance in km - - return distance * 0.539957; // Convert to nautical miles - } - - /** - * Calculate bearing between two points - */ - calculateBearing(lat1, lon1, lat2, lon2) { - const dLon = (lon2 - lon1) * Math.PI / 180; - const y = Math.sin(dLon) * Math.cos(lat2 * Math.PI / 180); - const x = Math.cos(lat1 * Math.PI / 180) * Math.sin(lat2 * Math.PI / 180) - - Math.sin(lat1 * Math.PI / 180) * Math.cos(lat2 * Math.PI / 180) * Math.cos(dLon); - - const bearing = Math.atan2(y, x) * 180 / Math.PI; - return (bearing + 360) % 360; - } - - /** - * Calculate new position given start position, bearing, and distance - */ - calculateNewPosition(lat, lon, bearing, distance) { - const R = 6371; // Earth radius in km - const distanceKm = distance * 1.852; // Convert nautical miles to km - - const bearingRad = bearing * Math.PI / 180; - const lat1 = lat * Math.PI / 180; - const lon1 = lon * Math.PI / 180; - - const lat2 = Math.asin( - Math.sin(lat1) * Math.cos(distanceKm/R) + - Math.cos(lat1) * Math.sin(distanceKm/R) * Math.cos(bearingRad) - ); - - const lon2 = lon1 + Math.atan2( - Math.sin(bearingRad) * Math.sin(distanceKm/R) * Math.cos(lat1), - Math.cos(distanceKm/R) - Math.sin(lat1) * Math.sin(lat2) - ); - - return { - lat: lat2 * 180 / Math.PI, - lon: lon2 * 180 / Math.PI - }; - } - - /** - * Generate vessel position based on journey status - */ - generateVesselPosition(vessel, timestamp) { - const journeyId = vessel.mmsi; - let journey = this.journeys.get(journeyId); - - // 30% chance to be in port, 70% at sea - const inPort = Math.random() < 0.3; - - if (!journey || journey.completed) { - // Start new journey - const origin = this.selectWeightedPort(); - let destination = this.selectWeightedPort(); - - // Ensure different origin and destination - let attempts = 0; - while (destination.name === origin.name && attempts < 10) { - destination = this.selectWeightedPort(); - attempts++; - } - - journey = { - origin, - destination, - startTime: timestamp, - currentLat: origin.lat + (Math.random() - 0.5) * 0.05, // Small offset within port - currentLon: origin.lon + (Math.random() - 0.5) * 0.05, - completed: false, - inPort: true - }; - - this.journeys.set(journeyId, journey); - } - - let status, speed, course; - - if (inPort || journey.inPort) { - // Vessel in port - status = Math.random() < 0.5 ? 'AT_ANCHOR' : 'MOORED'; - speed = parseFloat((Math.random() * 0.5).toFixed(1)); // Very slow or stationary - course = Math.floor(Math.random() * 360); - - // Small random movement within port area - journey.currentLat += (Math.random() - 0.5) * 0.01; - journey.currentLon += (Math.random() - 0.5) * 0.01; - - // 20% chance to leave port - if (Math.random() < 0.2) { - journey.inPort = false; - } - } else { - // Vessel at sea, moving toward destination - status = 'UNDERWAY_USING_ENGINE'; - - const distanceToDestination = this.calculateDistance( - journey.currentLat, - journey.currentLon, - journey.destination.lat, - journey.destination.lon - ); - - // Speed variation: 80-100% of cruise speed - speed = parseFloat((vessel.cruiseSpeed * (0.8 + Math.random() * 0.2)).toFixed(1)); - course = Math.floor(this.calculateBearing( - journey.currentLat, - journey.currentLon, - journey.destination.lat, - journey.destination.lon - )); - - // Move vessel (assume 1 hour between reports) - const distanceTraveled = speed; // nautical miles in 1 hour - - if (distanceToDestination <= distanceTraveled * 2) { - // Arriving at destination - journey.currentLat = journey.destination.lat + (Math.random() - 0.5) * 0.05; - journey.currentLon = journey.destination.lon + (Math.random() - 0.5) * 0.05; - journey.inPort = true; - speed = 0.5; - status = 'AT_ANCHOR'; - } else { - // Continue journey - const newPos = this.calculateNewPosition( - journey.currentLat, - journey.currentLon, - course, - distanceTraveled - ); - - journey.currentLat = newPos.lat; - journey.currentLon = newPos.lon; - } - } - - return { - latitude: parseFloat(journey.currentLat.toFixed(6)), - longitude: parseFloat(journey.currentLon.toFixed(6)), - speed, - course, - status, - destination: journey.destination.name - }; - } - - /** - * Generate a batch of vessel position records - */ - generateBatch(count = this.vesselsPerBatch, timestampOffset = 0) { - const records = []; - const now = new Date(Date.now() - timestampOffset); - - for (let i = 0; i < count; i++) { - const vessel = this.vesselPool[Math.floor(Math.random() * this.vesselPool.length)]; - - // Add some time variation within the batch (spread over last hour) - const recordTime = new Date(now.getTime() - Math.random() * 3600000); - - const position = this.generateVesselPosition(vessel, recordTime); - - const record = { - mmsi: vessel.mmsi, - imo: vessel.imo, - vessel_name: vessel.name, - vessel_type: vessel.type, - flag: vessel.flag, - length: vessel.length, - beam: vessel.beam, - draft: vessel.draft, - latitude: position.latitude, - longitude: position.longitude, - speed_knots: position.speed, - course: position.course, - heading: position.course, // Simplified: heading = course - status: position.status, - destination: position.destination, - eta: new Date(recordTime.getTime() + Math.random() * 7 * 24 * 3600000).toISOString(), // Random ETA within 7 days - timestamp: recordTime.toISOString(), - report_date: recordTime.toISOString().split('T')[0].replace(/-/g, '') // YYYYMMDD - }; - - records.push(record); - } - - return records; - } - - /** - * Get statistics about the vessel pool - */ - getStats() { - const typeDistribution = {}; - - for (const vessel of this.vesselPool) { - typeDistribution[vessel.type] = (typeDistribution[vessel.type] || 0) + 1; - } - - return { - totalVessels: this.vesselPool.length, - typeDistribution, - portsCount: MAJOR_PORTS.length, - activeJourneys: this.journeys.size - }; - } + constructor(config = {}) { + this.totalVessels = config.totalVessels || 100000; // Global fleet + this.vesselsPerBatch = config.vesselsPerBatch || 100; + this.vesselPool = []; + this.journeys = new Map(); // Track ongoing journeys + + // Initialize vessel pool + this.initializeVesselPool(); + console.log(`Initialized vessel pool with ${this.vesselPool.length} vessels`); + } + + /** + * Initialize a pool of vessels with persistent identifiers + */ + initializeVesselPool() { + const cacheSize = Math.min(10000, this.totalVessels); + + for (let i = 0; i < cacheSize; i++) { + const vesselType = this.selectVesselType(); + const vessel = { + mmsi: this.generateMMSI(), + imo: this.generateIMO(), + name: this.generateVesselName(vesselType.type, i), + type: vesselType.type, + flag: FLAG_STATES[Math.floor(Math.random() * FLAG_STATES.length)], + length: Math.floor( + Math.random() * (vesselType.lengthRange[1] - vesselType.lengthRange[0]) + vesselType.lengthRange[0] + ), + beam: 0, // will calculate from length + draft: parseFloat( + (Math.random() * (vesselType.draftRange[1] - vesselType.draftRange[0]) + vesselType.draftRange[0]).toFixed(1) + ), + maxSpeed: vesselType.speedRange[1], + cruiseSpeed: Math.floor( + Math.random() * (vesselType.speedRange[1] - vesselType.speedRange[0]) + vesselType.speedRange[0] + ), + }; + + // Beam typically 1/8 to 1/6 of length + vessel.beam = Math.floor(vessel.length / 7); + + this.vesselPool.push(vessel); + } + } + + /** + * Generate a 9-digit MMSI (Maritime Mobile Service Identity) + */ + generateMMSI() { + // Format: MIDxxxxxx where MID is Maritime Identification Digits (country code) + const mids = [ + '201', + '211', + '219', + '227', + '232', + '236', + '240', + '244', + '247', // European countries + '303', + '310', + '338', // North American countries + '412', + '413', + '414', + '416', + '419', // Asian countries + '563', + '564', + '565', + '566', + '567', // Southeast Asian countries + ]; + + const mid = mids[Math.floor(Math.random() * mids.length)]; + const remaining = String(Math.floor(Math.random() * 1000000)).padStart(6, '0'); + + return mid + remaining; + } + + /** + * Generate a 7-digit IMO number + */ + generateIMO() { + const base = String(Math.floor(Math.random() * 1000000) + 1000000); + return base.substring(0, 7); + } + + /** + * Generate vessel name + */ + generateVesselName(type, index) { + const prefixes = ['MV', 'MT', 'MSC', 'COSCO', 'MAERSK', 'EVERGREEN', 'CMA CGM']; + const names = ['FORTUNE', 'VOYAGER', 'EXPLORER', 'STAR', 'OCEAN', 'SPIRIT', 'PIONEER', 'LIBERTY']; + + const prefix = prefixes[Math.floor(Math.random() * prefixes.length)]; + const name = names[Math.floor(Math.random() * names.length)]; + + return `${prefix} ${name} ${index % 100}`; + } + + /** + * Select vessel type based on distribution + */ + selectVesselType() { + const rand = Math.random(); + let cumulative = 0; + + for (const vesselType of VESSEL_TYPES) { + cumulative += vesselType.distribution; + if (rand <= cumulative) { + return vesselType; + } + } + + return VESSEL_TYPES[0]; + } + + /** + * Select a random port weighted by traffic volume + */ + selectWeightedPort() { + const totalWeight = MAJOR_PORTS.reduce((sum, port) => sum + port.weight, 0); + const rand = Math.random() * totalWeight; + let cumulative = 0; + + for (const port of MAJOR_PORTS) { + cumulative += port.weight; + if (rand <= cumulative) { + return port; + } + } + + return MAJOR_PORTS[0]; + } + + /** + * Calculate distance between two points (Haversine formula) + */ + calculateDistance(lat1, lon1, lat2, lon2) { + const R = 6371; // Earth radius in km + const dLat = ((lat2 - lat1) * Math.PI) / 180; + const dLon = ((lon2 - lon1) * Math.PI) / 180; + + const a = + Math.sin(dLat / 2) * Math.sin(dLat / 2) + + Math.cos((lat1 * Math.PI) / 180) * Math.cos((lat2 * Math.PI) / 180) * Math.sin(dLon / 2) * Math.sin(dLon / 2); + + const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); + const distance = R * c; // Distance in km + + return distance * 0.539957; // Convert to nautical miles + } + + /** + * Calculate bearing between two points + */ + calculateBearing(lat1, lon1, lat2, lon2) { + const dLon = ((lon2 - lon1) * Math.PI) / 180; + const y = Math.sin(dLon) * Math.cos((lat2 * Math.PI) / 180); + const x = + Math.cos((lat1 * Math.PI) / 180) * Math.sin((lat2 * Math.PI) / 180) - + Math.sin((lat1 * Math.PI) / 180) * Math.cos((lat2 * Math.PI) / 180) * Math.cos(dLon); + + const bearing = (Math.atan2(y, x) * 180) / Math.PI; + return (bearing + 360) % 360; + } + + /** + * Calculate new position given start position, bearing, and distance + */ + calculateNewPosition(lat, lon, bearing, distance) { + const R = 6371; // Earth radius in km + const distanceKm = distance * 1.852; // Convert nautical miles to km + + const bearingRad = (bearing * Math.PI) / 180; + const lat1 = (lat * Math.PI) / 180; + const lon1 = (lon * Math.PI) / 180; + + const lat2 = Math.asin( + Math.sin(lat1) * Math.cos(distanceKm / R) + Math.cos(lat1) * Math.sin(distanceKm / R) * Math.cos(bearingRad) + ); + + const lon2 = + lon1 + + Math.atan2( + Math.sin(bearingRad) * Math.sin(distanceKm / R) * Math.cos(lat1), + Math.cos(distanceKm / R) - Math.sin(lat1) * Math.sin(lat2) + ); + + return { + lat: (lat2 * 180) / Math.PI, + lon: (lon2 * 180) / Math.PI, + }; + } + + /** + * Generate vessel position based on journey status + */ + generateVesselPosition(vessel, timestamp) { + const journeyId = vessel.mmsi; + let journey = this.journeys.get(journeyId); + + // 30% chance to be in port, 70% at sea + const inPort = Math.random() < 0.3; + + if (!journey || journey.completed) { + // Start new journey + const origin = this.selectWeightedPort(); + let destination = this.selectWeightedPort(); + + // Ensure different origin and destination + let attempts = 0; + while (destination.name === origin.name && attempts < 10) { + destination = this.selectWeightedPort(); + attempts++; + } + + journey = { + origin, + destination, + startTime: timestamp, + currentLat: origin.lat + (Math.random() - 0.5) * 0.05, // Small offset within port + currentLon: origin.lon + (Math.random() - 0.5) * 0.05, + completed: false, + inPort: true, + }; + + this.journeys.set(journeyId, journey); + } + + let status, speed, course; + + if (inPort || journey.inPort) { + // Vessel in port + status = Math.random() < 0.5 ? 'AT_ANCHOR' : 'MOORED'; + speed = parseFloat((Math.random() * 0.5).toFixed(1)); // Very slow or stationary + course = Math.floor(Math.random() * 360); + + // Small random movement within port area + journey.currentLat += (Math.random() - 0.5) * 0.01; + journey.currentLon += (Math.random() - 0.5) * 0.01; + + // 20% chance to leave port + if (Math.random() < 0.2) { + journey.inPort = false; + } + } else { + // Vessel at sea, moving toward destination + status = 'UNDERWAY_USING_ENGINE'; + + const distanceToDestination = this.calculateDistance( + journey.currentLat, + journey.currentLon, + journey.destination.lat, + journey.destination.lon + ); + + // Speed variation: 80-100% of cruise speed + speed = parseFloat((vessel.cruiseSpeed * (0.8 + Math.random() * 0.2)).toFixed(1)); + course = Math.floor( + this.calculateBearing(journey.currentLat, journey.currentLon, journey.destination.lat, journey.destination.lon) + ); + + // Move vessel (assume 1 hour between reports) + const distanceTraveled = speed; // nautical miles in 1 hour + + if (distanceToDestination <= distanceTraveled * 2) { + // Arriving at destination + journey.currentLat = journey.destination.lat + (Math.random() - 0.5) * 0.05; + journey.currentLon = journey.destination.lon + (Math.random() - 0.5) * 0.05; + journey.inPort = true; + speed = 0.5; + status = 'AT_ANCHOR'; + } else { + // Continue journey + const newPos = this.calculateNewPosition(journey.currentLat, journey.currentLon, course, distanceTraveled); + + journey.currentLat = newPos.lat; + journey.currentLon = newPos.lon; + } + } + + return { + latitude: parseFloat(journey.currentLat.toFixed(6)), + longitude: parseFloat(journey.currentLon.toFixed(6)), + speed, + course, + status, + destination: journey.destination.name, + }; + } + + /** + * Generate a batch of vessel position records + */ + generateBatch(count = this.vesselsPerBatch, timestampOffset = 0) { + const records = []; + const now = new Date(Date.now() - timestampOffset); + + for (let i = 0; i < count; i++) { + const vessel = this.vesselPool[Math.floor(Math.random() * this.vesselPool.length)]; + + // Add some time variation within the batch (spread over last hour) + const recordTime = new Date(now.getTime() - Math.random() * 3600000); + + const position = this.generateVesselPosition(vessel, recordTime); + + const record = { + mmsi: vessel.mmsi, + imo: vessel.imo, + vessel_name: vessel.name, + vessel_type: vessel.type, + flag: vessel.flag, + length: vessel.length, + beam: vessel.beam, + draft: vessel.draft, + latitude: position.latitude, + longitude: position.longitude, + speed_knots: position.speed, + course: position.course, + heading: position.course, // Simplified: heading = course + status: position.status, + destination: position.destination, + eta: new Date(recordTime.getTime() + Math.random() * 7 * 24 * 3600000).toISOString(), // Random ETA within 7 days + timestamp: recordTime.toISOString(), + report_date: recordTime.toISOString().split('T')[0].replace(/-/g, ''), // YYYYMMDD + }; + + records.push(record); + } + + return records; + } + + /** + * Get statistics about the vessel pool + */ + getStats() { + const typeDistribution = {}; + + for (const vessel of this.vesselPool) { + typeDistribution[vessel.type] = (typeDistribution[vessel.type] || 0) + 1; + } + + return { + totalVessels: this.vesselPool.length, + typeDistribution, + portsCount: MAJOR_PORTS.length, + activeJourneys: this.journeys.size, + }; + } } export default MaritimeVesselGenerator; diff --git a/src/globals.js b/src/globals.js index fe22a10..2cd748b 100644 --- a/src/globals.js +++ b/src/globals.js @@ -1,21 +1,21 @@ class Globals { - constructor() { - if(Globals.instance) { - return Globals.instance; - } - this.data = {}; - Globals.instance = this; - } - set(key, value) { - this.data[key] = value; - } - get(key) { - return this.data[key]; - } + constructor() { + if (Globals.instance) { + return Globals.instance; + } + this.data = {}; + Globals.instance = this; + } + set(key, value) { + this.data[key] = value; + } + get(key) { + return this.data[key]; + } } const globals = new Globals(); export { globals, Globals }; -export default Globals; \ No newline at end of file +export default Globals; diff --git a/src/index.js b/src/index.js index 36922ae..b3cc572 100644 --- a/src/index.js +++ b/src/index.js @@ -4,14 +4,13 @@ import { globals } from './globals.js'; import { SyncEngine } from './sync-engine.js'; // TODO: Validation not yet implemented - requires additional testing // import { ValidationService } from './validation.js'; -import { logger } from '@google-cloud/bigquery/build/src/logger.js'; export async function handleApplication(scope) { - const logger = scope.logger; - const options = scope.options.getAll(); - const syncEngine = new SyncEngine(options); - syncEngine.initialize(); - globals.set('syncEngine', syncEngine); - // TODO: Validation not yet implemented - requires additional testing - // globals.set('validator', new ValidationService(options)); -} \ No newline at end of file + const _logger = scope.logger; + const options = scope.options.getAll(); + const syncEngine = new SyncEngine(options); + syncEngine.initialize(); + globals.set('syncEngine', syncEngine); + // TODO: Validation not yet implemented - requires additional testing + // globals.set('validator', new ValidationService(options)); +} diff --git a/src/maritime-synthesizer.js b/src/maritime-synthesizer.js index ea3f39d..efc9a29 100644 --- a/src/maritime-synthesizer.js +++ b/src/maritime-synthesizer.js @@ -7,10 +7,6 @@ import MaritimeDataSynthesizer from './service.js'; import MaritimeVesselGenerator from './generator.js'; import MaritimeBigQueryClient from './bigquery.js'; -export { - MaritimeDataSynthesizer, - MaritimeVesselGenerator, - MaritimeBigQueryClient -}; +export { MaritimeDataSynthesizer, MaritimeVesselGenerator, MaritimeBigQueryClient }; export default MaritimeDataSynthesizer; diff --git a/src/resources.js b/src/resources.js index 665bf7c..d0b1634 100644 --- a/src/resources.js +++ b/src/resources.js @@ -9,96 +9,95 @@ /* global tables, Resource */ import { globals } from './globals.js'; - + // Main data table resource export class BigQueryData extends tables.BigQueryData { - async get(id) { - logger.debug(`[BigQueryData.get] Fetching record with id: ${id}`); - const result = await super.get(id); - logger.debug(`[BigQueryData.get] Record ${result ? 'found' : 'not found'}`); - return result; - } + async get(id) { + logger.debug(`[BigQueryData.get] Fetching record with id: ${id}`); + const result = await super.get(id); + logger.debug(`[BigQueryData.get] Record ${result ? 'found' : 'not found'}`); + return result; + } - async search(params) { - // This allows us to search on dynamic attributes. - params.allowConditionsOnDynamicAttributes = true; - logger.debug(`[BigQueryData.search] Searching with params: ${JSON.stringify(params).substring(0, 200)}`); - const results = await super.search(params); - logger.info(`[BigQueryData.search] Search returned ${results.length} records`); - return results; - } + async search(params) { + // This allows us to search on dynamic attributes. + params.allowConditionsOnDynamicAttributes = true; + logger.debug(`[BigQueryData.search] Searching with params: ${JSON.stringify(params).substring(0, 200)}`); + const results = await super.search(params); + logger.info(`[BigQueryData.search] Search returned ${results.length} records`); + return results; + } } // Checkpoint resource export class SyncCheckpoint extends tables.SyncCheckpoint { - async getForNode(nodeId) { - logger.debug(`[SyncCheckpoint.getForNode] Fetching checkpoint for nodeId: ${nodeId}`); - const checkpoint = await super.get(nodeId); - logger.debug(`[SyncCheckpoint.getForNode] Checkpoint ${checkpoint ? 'found' : 'not found'}`); - return checkpoint; - } + async getForNode(nodeId) { + logger.debug(`[SyncCheckpoint.getForNode] Fetching checkpoint for nodeId: ${nodeId}`); + const checkpoint = await super.get(nodeId); + logger.debug(`[SyncCheckpoint.getForNode] Checkpoint ${checkpoint ? 'found' : 'not found'}`); + return checkpoint; + } - async updateCheckpoint(nodeId, data) { - logger.info(`[SyncCheckpoint.updateCheckpoint] Updating checkpoint for nodeId: ${nodeId}`); - logger.debug(`[SyncCheckpoint.updateCheckpoint] Data: ${JSON.stringify(data).substring(0, 200)}`); - const result = await super.put({ nodeId, ...data }); - logger.info(`[SyncCheckpoint.updateCheckpoint] Checkpoint updated successfully`); - return result; - } + async updateCheckpoint(nodeId, data) { + logger.info(`[SyncCheckpoint.updateCheckpoint] Updating checkpoint for nodeId: ${nodeId}`); + logger.debug(`[SyncCheckpoint.updateCheckpoint] Data: ${JSON.stringify(data).substring(0, 200)}`); + const result = await super.put({ nodeId, ...data }); + logger.info(`[SyncCheckpoint.updateCheckpoint] Checkpoint updated successfully`); + return result; + } } // Audit resource export class SyncAudit extends tables.SyncAudit { - async getRecent(hours = 24) { - logger.debug(`[SyncAudit.getRecent] Fetching audit records from last ${hours} hours`); - const cutoff = new Date(Date.now() - hours * 3600000).toISOString(); - logger.debug(`[SyncAudit.getRecent] Cutoff timestamp: ${cutoff}`); - const results = await super.search({ - conditions: [{ timestamp: { $gt: cutoff } }], - orderBy: 'timestamp DESC' - }); - logger.info(`[SyncAudit.getRecent] Retrieved ${results.length} audit records`); - return results; - } + async getRecent(hours = 24) { + logger.debug(`[SyncAudit.getRecent] Fetching audit records from last ${hours} hours`); + const cutoff = new Date(Date.now() - hours * 3600000).toISOString(); + logger.debug(`[SyncAudit.getRecent] Cutoff timestamp: ${cutoff}`); + const results = await super.search({ + conditions: [{ timestamp: { $gt: cutoff } }], + orderBy: 'timestamp DESC', + }); + logger.info(`[SyncAudit.getRecent] Retrieved ${results.length} audit records`); + return results; + } } // Control endpoint export class SyncControl extends Resource { - async get() { - logger.debug('[SyncControl.get] Status request received'); - const status = await globals.get('syncEngine').getStatus(); - const response = { - status, - uptime: process.uptime(), - version: '1.0.0' - }; - logger.info(`[SyncControl.get] Returning status - running: ${status.running}, phase: ${status.phase}`); - return response; - } + async get() { + logger.debug('[SyncControl.get] Status request received'); + const status = await globals.get('syncEngine').getStatus(); + const response = { + status, + uptime: process.uptime(), + version: '1.0.0', + }; + logger.info(`[SyncControl.get] Returning status - running: ${status.running}, phase: ${status.phase}`); + return response; + } - async post({ action }) { - logger.info(`[SyncControl.post] Control action received: ${action}`); - switch(action) { - case 'start': - logger.info('[SyncControl.post] Starting sync engine'); - await globals.get('syncEngine').start(); - logger.info('[SyncControl.post] Sync engine started successfully'); - return { message: 'Sync started' }; - case 'stop': - logger.info('[SyncControl.post] Stopping sync engine'); - await globals.get('syncEngine').stop(); - logger.info('[SyncControl.post] Sync engine stopped successfully'); - return { message: 'Sync stopped' }; - // TODO: Validation not yet implemented - requires additional testing - // case 'validate': - // logger.info('[SyncControl.post] Triggering validation'); - // await globals.get('validator').runValidation(); - // logger.info('[SyncControl.post] Validation completed'); - // return { message: 'Validation triggered' }; - default: - logger.warn(`[SyncControl.post] Unknown action requested: ${action}`); - throw new Error(`Unknown action: ${action}`); - } - } + async post({ action }) { + logger.info(`[SyncControl.post] Control action received: ${action}`); + switch (action) { + case 'start': + logger.info('[SyncControl.post] Starting sync engine'); + await globals.get('syncEngine').start(); + logger.info('[SyncControl.post] Sync engine started successfully'); + return { message: 'Sync started' }; + case 'stop': + logger.info('[SyncControl.post] Stopping sync engine'); + await globals.get('syncEngine').stop(); + logger.info('[SyncControl.post] Sync engine stopped successfully'); + return { message: 'Sync stopped' }; + // TODO: Validation not yet implemented - requires additional testing + // case 'validate': + // logger.info('[SyncControl.post] Triggering validation'); + // await globals.get('validator').runValidation(); + // logger.info('[SyncControl.post] Validation completed'); + // return { message: 'Validation triggered' }; + default: + logger.warn(`[SyncControl.post] Unknown action requested: ${action}`); + throw new Error(`Unknown action: ${action}`); + } + } } - diff --git a/src/service.js b/src/service.js index dc860e8..371ea2b 100644 --- a/src/service.js +++ b/src/service.js @@ -8,149 +8,150 @@ import MaritimeVesselGenerator from './generator.js'; import MaritimeBigQueryClient from './bigquery.js'; class MaritimeDataSynthesizer extends EventEmitter { - constructor(config = {}) { - super(); - - this.config = { - totalVessels: parseInt(config.totalVessels || process.env.TOTAL_VESSELS || '100000', 10), - batchSize: parseInt(config.batchSize || process.env.BATCH_SIZE || '100', 10), - generationIntervalMs: parseInt(config.generationIntervalMs || process.env.GENERATION_INTERVAL_MS || '60000', 10), - retentionDays: parseInt(config.retentionDays || process.env.RETENTION_DAYS || '30', 10), - cleanupIntervalHours: parseInt(config.cleanupIntervalHours || process.env.CLEANUP_INTERVAL_HOURS || '24', 10), - ...config - }; - - this.generator = new MaritimeVesselGenerator({ - totalVessels: this.config.totalVessels, - vesselsPerBatch: this.config.batchSize - }); - - this.bigquery = new MaritimeBigQueryClient({ - projectId: config.projectId, - datasetId: config.datasetId, - tableId: config.tableId, - credentials: config.credentials, - location: config.location, - retentionDays: this.config.retentionDays - }); - - this.isRunning = false; - this.generationTimer = null; - this.cleanupTimer = null; - this.stats = { - totalBatchesGenerated: 0, - totalRecordsInserted: 0, - errors: 0, - startTime: null - }; - } - - /** - * Initialize BigQuery resources and optionally load historical data - */ - async initialize(daysOfHistoricalData = 0) { - try { - this.emit('init:starting', { days: daysOfHistoricalData }); - - // Initialize BigQuery schema - await this.bigquery.initialize(); - this.emit('init:bigquery-ready'); - - if (daysOfHistoricalData > 0) { - await this.loadHistoricalData(daysOfHistoricalData); - } - - this.emit('init:completed', { days: daysOfHistoricalData }); - return true; - } catch (error) { - this.emit('init:error', { error }); - throw error; - } - } - - /** - * Load historical data for specified number of days - */ - async loadHistoricalData(days) { - try { - this.emit('init:data-generation-starting', { days }); - - const recordsPerDay = Math.floor((24 * 60 * 60 * 1000) / this.config.generationIntervalMs) * this.config.batchSize; - const totalRecords = recordsPerDay * days; - const totalBatches = Math.ceil(totalRecords / this.config.batchSize); - - console.log(`Loading ${days} days of historical data...`); - console.log(` Records per day: ${recordsPerDay.toLocaleString()}`); - console.log(` Total records: ${totalRecords.toLocaleString()}`); - console.log(` Total batches: ${totalBatches.toLocaleString()}`); - console.log(` Estimated time: ${Math.ceil(totalBatches * 2 / 60)} minutes`); - - let recordsInserted = 0; - const startTime = Date.now(); - - for (let batchNum = 0; batchNum < totalBatches; batchNum++) { - // Calculate time offset for this batch (spread evenly across the days) - const timeOffsetMs = (batchNum / totalBatches) * days * 24 * 60 * 60 * 1000; - - // Generate batch with timestamp offset - const records = this.generator.generateBatch(this.config.batchSize, timeOffsetMs); - - // Insert into BigQuery - await this.bigquery.insertBatch(records); - - recordsInserted += records.length; - - // Emit progress - const progress = ((batchNum + 1) / totalBatches * 100).toFixed(1); - this.emit('init:progress', { - batchNum: batchNum + 1, - totalBatches, - recordsInserted, - totalRecords, - progress: parseFloat(progress) - }); - - // Log progress periodically - if ((batchNum + 1) % 10 === 0 || batchNum === totalBatches - 1) { - const elapsed = (Date.now() - startTime) / 1000; - const rate = recordsInserted / elapsed; - const remaining = (totalRecords - recordsInserted) / rate; - - console.log( - `Progress: ${progress}% | ` + - `Batch ${batchNum + 1}/${totalBatches} | ` + - `Records: ${recordsInserted.toLocaleString()}/${totalRecords.toLocaleString()} | ` + - `Rate: ${Math.floor(rate)} records/sec | ` + - `ETA: ${Math.ceil(remaining / 60)} min` - ); - } - - // Rate limiting: wait 1 second between batches to avoid overwhelming BigQuery - if (batchNum < totalBatches - 1) { - await new Promise(resolve => setTimeout(resolve, 1000)); - } - } - - const totalTime = ((Date.now() - startTime) / 1000 / 60).toFixed(1); - console.log(`Historical data loaded: ${recordsInserted.toLocaleString()} records in ${totalTime} minutes`); - - this.emit('init:data-generation-completed', { - recordsInserted, - totalTime: parseFloat(totalTime) - }); - } catch (error) { - console.error('Error loading historical data:', error); - this.emit('init:data-generation-error', { error }); - throw error; - } - } - - /** - * Check current data range and determine if backfill is needed - */ - async checkDataRange() { - try { - const query = ` + constructor(config = {}) { + super(); + + this.config = { + totalVessels: parseInt(config.totalVessels || process.env.TOTAL_VESSELS || '100000', 10), + batchSize: parseInt(config.batchSize || process.env.BATCH_SIZE || '100', 10), + generationIntervalMs: parseInt(config.generationIntervalMs || process.env.GENERATION_INTERVAL_MS || '60000', 10), + retentionDays: parseInt(config.retentionDays || process.env.RETENTION_DAYS || '30', 10), + cleanupIntervalHours: parseInt(config.cleanupIntervalHours || process.env.CLEANUP_INTERVAL_HOURS || '24', 10), + ...config, + }; + + this.generator = new MaritimeVesselGenerator({ + totalVessels: this.config.totalVessels, + vesselsPerBatch: this.config.batchSize, + }); + + this.bigquery = new MaritimeBigQueryClient({ + projectId: config.projectId, + datasetId: config.datasetId, + tableId: config.tableId, + credentials: config.credentials, + location: config.location, + retentionDays: this.config.retentionDays, + }); + + this.isRunning = false; + this.generationTimer = null; + this.cleanupTimer = null; + this.stats = { + totalBatchesGenerated: 0, + totalRecordsInserted: 0, + errors: 0, + startTime: null, + }; + } + + /** + * Initialize BigQuery resources and optionally load historical data + */ + async initialize(daysOfHistoricalData = 0) { + try { + this.emit('init:starting', { days: daysOfHistoricalData }); + + // Initialize BigQuery schema + await this.bigquery.initialize(); + this.emit('init:bigquery-ready'); + + if (daysOfHistoricalData > 0) { + await this.loadHistoricalData(daysOfHistoricalData); + } + + this.emit('init:completed', { days: daysOfHistoricalData }); + return true; + } catch (error) { + this.emit('init:error', { error }); + throw error; + } + } + + /** + * Load historical data for specified number of days + */ + async loadHistoricalData(days) { + try { + this.emit('init:data-generation-starting', { days }); + + const recordsPerDay = + Math.floor((24 * 60 * 60 * 1000) / this.config.generationIntervalMs) * this.config.batchSize; + const totalRecords = recordsPerDay * days; + const totalBatches = Math.ceil(totalRecords / this.config.batchSize); + + console.log(`Loading ${days} days of historical data...`); + console.log(` Records per day: ${recordsPerDay.toLocaleString()}`); + console.log(` Total records: ${totalRecords.toLocaleString()}`); + console.log(` Total batches: ${totalBatches.toLocaleString()}`); + console.log(` Estimated time: ${Math.ceil((totalBatches * 2) / 60)} minutes`); + + let recordsInserted = 0; + const startTime = Date.now(); + + for (let batchNum = 0; batchNum < totalBatches; batchNum++) { + // Calculate time offset for this batch (spread evenly across the days) + const timeOffsetMs = (batchNum / totalBatches) * days * 24 * 60 * 60 * 1000; + + // Generate batch with timestamp offset + const records = this.generator.generateBatch(this.config.batchSize, timeOffsetMs); + + // Insert into BigQuery + await this.bigquery.insertBatch(records); + + recordsInserted += records.length; + + // Emit progress + const progress = (((batchNum + 1) / totalBatches) * 100).toFixed(1); + this.emit('init:progress', { + batchNum: batchNum + 1, + totalBatches, + recordsInserted, + totalRecords, + progress: parseFloat(progress), + }); + + // Log progress periodically + if ((batchNum + 1) % 10 === 0 || batchNum === totalBatches - 1) { + const elapsed = (Date.now() - startTime) / 1000; + const rate = recordsInserted / elapsed; + const remaining = (totalRecords - recordsInserted) / rate; + + console.log( + `Progress: ${progress}% | ` + + `Batch ${batchNum + 1}/${totalBatches} | ` + + `Records: ${recordsInserted.toLocaleString()}/${totalRecords.toLocaleString()} | ` + + `Rate: ${Math.floor(rate)} records/sec | ` + + `ETA: ${Math.ceil(remaining / 60)} min` + ); + } + + // Rate limiting: wait 1 second between batches to avoid overwhelming BigQuery + if (batchNum < totalBatches - 1) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } + + const totalTime = ((Date.now() - startTime) / 1000 / 60).toFixed(1); + console.log(`Historical data loaded: ${recordsInserted.toLocaleString()} records in ${totalTime} minutes`); + + this.emit('init:data-generation-completed', { + recordsInserted, + totalTime: parseFloat(totalTime), + }); + } catch (error) { + console.error('Error loading historical data:', error); + this.emit('init:data-generation-error', { error }); + throw error; + } + } + + /** + * Check current data range and determine if backfill is needed + */ + async checkDataRange() { + try { + const query = ` SELECT MIN(timestamp) as oldest, MAX(timestamp) as newest, @@ -158,375 +159,372 @@ class MaritimeDataSynthesizer extends EventEmitter { FROM \`${this.bigquery.projectId}.${this.bigquery.datasetId}.${this.bigquery.tableId}\` `; - const [rows] = await this.bigquery.bigquery.query({ query }); - - if (rows.length === 0 || rows[0].total_records === '0') { - return { - hasData: false, - oldestTimestamp: null, - newestTimestamp: null, - totalRecords: 0, - daysCovered: 0 - }; - } - - const oldest = new Date(rows[0].oldest.value); - const newest = new Date(rows[0].newest.value); - const daysCovered = (newest - oldest) / (24 * 60 * 60 * 1000); - - return { - hasData: true, - oldestTimestamp: oldest, - newestTimestamp: newest, - totalRecords: parseInt(rows[0].total_records), - daysCovered: Math.floor(daysCovered) - }; - } catch (error) { - // Table might not exist yet - if (error.message.includes('Not found')) { - return { - hasData: false, - oldestTimestamp: null, - newestTimestamp: null, - totalRecords: 0, - daysCovered: 0 - }; - } - throw error; - } - } - - /** - * Start the data generation service with rolling window support - */ - async start(options = {}) { - if (this.isRunning) { - console.log('Service is already running'); - return; - } - - const maintainWindow = options.maintainWindow !== false; // Default true - const targetDays = options.targetDays || this.config.retentionDays; - - try { - this.emit('service:starting'); - - this.isRunning = true; - this.stats.startTime = new Date(); - - // Check if we need to backfill - if (maintainWindow) { - console.log(`Checking data range (target: ${targetDays} days)...`); - const dataRange = await this.checkDataRange(); - - if (!dataRange.hasData) { - console.log('No existing data found. Initializing with historical data...'); - await this.initialize(targetDays); - } else { - console.log(`Found ${dataRange.totalRecords.toLocaleString()} records covering ${dataRange.daysCovered} days`); - console.log(` Oldest: ${dataRange.oldestTimestamp.toISOString()}`); - console.log(` Newest: ${dataRange.newestTimestamp.toISOString()}`); - - const daysNeeded = targetDays - dataRange.daysCovered; - if (daysNeeded > 1) { - console.log(`Backfilling ${Math.floor(daysNeeded)} days to reach ${targetDays}-day window...`); - await this.backfillHistoricalData(Math.floor(daysNeeded), dataRange.oldestTimestamp); - } else { - console.log(`Data window is sufficient (${dataRange.daysCovered}/${targetDays} days)`); - } - } - } - - // Start generation loop - console.log('\nStarting continuous generation...'); - await this.generateAndInsertBatch(); // Run immediately - this.generationTimer = setInterval( - () => this.generateAndInsertBatch(), - this.config.generationIntervalMs - ); - - // Start cleanup loop - setTimeout(() => { - this.cleanupOldData(); - this.cleanupTimer = setInterval( - () => this.cleanupOldData(), - this.config.cleanupIntervalHours * 60 * 60 * 1000 - ); - }, 60000); // Wait 1 minute before first cleanup - - console.log('\nMaritime Data Synthesizer started'); - console.log(` Batch size: ${this.config.batchSize} vessels`); - console.log(` Generation interval: ${this.config.generationIntervalMs / 1000} seconds`); - console.log(` Records per day: ~${Math.floor((24 * 60 * 60 * 1000) / this.config.generationIntervalMs) * this.config.batchSize}`); - console.log(` Rolling window: ${this.config.retentionDays} days`); - console.log(` Cleanup interval: ${this.config.cleanupIntervalHours} hours`); - - this.emit('service:started'); - } catch (error) { - this.isRunning = false; - this.emit('service:error', { error }); - throw error; - } - } - - /** - * Backfill historical data before a specific timestamp - */ - async backfillHistoricalData(days, beforeTimestamp) { - try { - this.emit('backfill:starting', { days, beforeTimestamp }); - - const recordsPerDay = Math.floor((24 * 60 * 60 * 1000) / this.config.generationIntervalMs) * this.config.batchSize; - const totalRecords = recordsPerDay * days; - const totalBatches = Math.ceil(totalRecords / this.config.batchSize); - - console.log(`Backfilling ${days} days of historical data...`); - console.log(` Records per day: ${recordsPerDay.toLocaleString()}`); - console.log(` Total records: ${totalRecords.toLocaleString()}`); - console.log(` Total batches: ${totalBatches.toLocaleString()}`); - - let recordsInserted = 0; - const startTime = Date.now(); - const oldestTimestamp = beforeTimestamp.getTime(); - - for (let batchNum = 0; batchNum < totalBatches; batchNum++) { - // Calculate time offset - going backwards from beforeTimestamp - const timeOffsetMs = (batchNum / totalBatches) * days * 24 * 60 * 60 * 1000; - const batchTimestamp = oldestTimestamp - (days * 24 * 60 * 60 * 1000) + timeOffsetMs; - - // Generate batch with timestamp offset - const records = this.generator.generateBatch(this.config.batchSize, Date.now() - batchTimestamp); - - // Insert into BigQuery - await this.bigquery.insertBatch(records); - - recordsInserted += records.length; - - // Emit progress - const progress = ((batchNum + 1) / totalBatches * 100).toFixed(1); - this.emit('backfill:progress', { - batchNum: batchNum + 1, - totalBatches, - recordsInserted, - totalRecords, - progress: parseFloat(progress) - }); - - // Log progress periodically - if ((batchNum + 1) % 10 === 0 || batchNum === totalBatches - 1) { - const elapsed = (Date.now() - startTime) / 1000; - const rate = recordsInserted / elapsed; - const remaining = (totalRecords - recordsInserted) / rate; - - console.log( - `Backfill: ${progress}% | ` + - `Batch ${batchNum + 1}/${totalBatches} | ` + - `Records: ${recordsInserted.toLocaleString()}/${totalRecords.toLocaleString()} | ` + - `Rate: ${Math.floor(rate)} records/sec | ` + - `ETA: ${Math.ceil(remaining / 60)} min` - ); - } - - // Rate limiting - if (batchNum < totalBatches - 1) { - await new Promise(resolve => setTimeout(resolve, 1000)); - } - } - - const totalTime = ((Date.now() - startTime) / 1000 / 60).toFixed(1); - console.log(`Backfill completed: ${recordsInserted.toLocaleString()} records in ${totalTime} minutes`); - - this.emit('backfill:completed', { - recordsInserted, - totalTime: parseFloat(totalTime) - }); - } catch (error) { - console.error('Error backfilling historical data:', error); - this.emit('backfill:error', { error }); - throw error; - } - } - - /** - * Generate a batch of records and insert into BigQuery - */ - async generateAndInsertBatch() { - try { - this.emit('batch:generating', { size: this.config.batchSize }); - - const records = this.generator.generateBatch(this.config.batchSize); - - this.emit('batch:generated', { - records: records.length, - sample: records[0] - }); - - this.emit('batch:inserting', { records: records.length }); - - const result = await this.bigquery.insertBatch(records); - - this.stats.totalBatchesGenerated++; - this.stats.totalRecordsInserted += records.length; - - this.emit('batch:inserted', { - records: records.length, - totalBatches: this.stats.totalBatchesGenerated, - totalRecords: this.stats.totalRecordsInserted, - jobId: result.jobId - }); - - console.log( - `Batch inserted: ${records.length} records | ` + - `Total: ${this.stats.totalRecordsInserted.toLocaleString()} records | ` + - `Batches: ${this.stats.totalBatchesGenerated}` - ); - } catch (error) { - this.stats.errors++; - this.emit('batch:error', { error, errorCount: this.stats.errors }); - console.error('Error generating/inserting batch:', error.message); - } - } - - /** - * Clean up old data based on retention policy - */ - async cleanupOldData() { - try { - this.emit('cleanup:starting', { retentionDays: this.config.retentionDays }); - - const result = await this.bigquery.cleanupOldData(); - - this.emit('cleanup:completed', { - cutoffDate: result.cutoffDate, - deletedRows: result.deletedRows - }); - - console.log(`Cleanup completed: deleted ${result.deletedRows} rows older than ${result.cutoffDate}`); - } catch (error) { - this.emit('cleanup:error', { error }); - console.error('Error during cleanup:', error.message); - } - } - - /** - * Stop the service - */ - async stop() { - if (!this.isRunning) { - console.log('Service is not running'); - return; - } - - try { - this.emit('service:stopping'); - - this.isRunning = false; - - if (this.generationTimer) { - clearInterval(this.generationTimer); - this.generationTimer = null; - } - - if (this.cleanupTimer) { - clearInterval(this.cleanupTimer); - this.cleanupTimer = null; - } - - console.log('Maritime Data Synthesizer stopped'); - this.emit('service:stopped', { stats: this.getStats() }); - } catch (error) { - this.emit('service:error', { error }); - throw error; - } - } - - /** - * Get service statistics - */ - getStats() { - const uptime = this.stats.startTime - ? (Date.now() - this.stats.startTime.getTime()) / 1000 - : 0; - - return { - ...this.stats, - uptime: Math.floor(uptime), - isRunning: this.isRunning, - generatorStats: this.generator.getStats(), - config: this.config - }; - } - - /** - * Get BigQuery table statistics - */ - async getBigQueryStats() { - return await this.bigquery.getStats(); - } - - /** - * Clear all data from table (truncate) without deleting the table - */ - async clear() { - try { - this.emit('clear:starting'); - - await this.bigquery.clearData(); - - this.emit('clear:completed'); - console.log('All data cleared from table'); - - return true; - } catch (error) { - this.emit('clear:error', { error }); - throw error; - } - } - - /** - * Delete all data and table - */ - async clean() { - try { - this.emit('clean:starting'); - - await this.bigquery.deleteTable(); - - this.emit('clean:completed'); - console.log('All data and table deleted'); - - return true; - } catch (error) { - this.emit('clean:error', { error }); - throw error; - } - } - - /** - * Reset: delete and reinitialize with historical data - */ - async reset(daysOfHistoricalData = 30) { - try { - console.log('Resetting maritime data synthesizer...'); - - // Stop if running - if (this.isRunning) { - await this.stop(); - } - - // Delete table - await this.clean(); - - // Reinitialize - await this.initialize(daysOfHistoricalData); - - console.log('Reset completed'); - return true; - } catch (error) { - console.error('Error during reset:', error); - throw error; - } - } + const [rows] = await this.bigquery.bigquery.query({ query }); + + if (rows.length === 0 || rows[0].total_records === '0') { + return { + hasData: false, + oldestTimestamp: null, + newestTimestamp: null, + totalRecords: 0, + daysCovered: 0, + }; + } + + const oldest = new Date(rows[0].oldest.value); + const newest = new Date(rows[0].newest.value); + const daysCovered = (newest - oldest) / (24 * 60 * 60 * 1000); + + return { + hasData: true, + oldestTimestamp: oldest, + newestTimestamp: newest, + totalRecords: parseInt(rows[0].total_records), + daysCovered: Math.floor(daysCovered), + }; + } catch (error) { + // Table might not exist yet + if (error.message.includes('Not found')) { + return { + hasData: false, + oldestTimestamp: null, + newestTimestamp: null, + totalRecords: 0, + daysCovered: 0, + }; + } + throw error; + } + } + + /** + * Start the data generation service with rolling window support + */ + async start(options = {}) { + if (this.isRunning) { + console.log('Service is already running'); + return; + } + + const maintainWindow = options.maintainWindow !== false; // Default true + const targetDays = options.targetDays || this.config.retentionDays; + + try { + this.emit('service:starting'); + + this.isRunning = true; + this.stats.startTime = new Date(); + + // Check if we need to backfill + if (maintainWindow) { + console.log(`Checking data range (target: ${targetDays} days)...`); + const dataRange = await this.checkDataRange(); + + if (!dataRange.hasData) { + console.log('No existing data found. Initializing with historical data...'); + await this.initialize(targetDays); + } else { + console.log( + `Found ${dataRange.totalRecords.toLocaleString()} records covering ${dataRange.daysCovered} days` + ); + console.log(` Oldest: ${dataRange.oldestTimestamp.toISOString()}`); + console.log(` Newest: ${dataRange.newestTimestamp.toISOString()}`); + + const daysNeeded = targetDays - dataRange.daysCovered; + if (daysNeeded > 1) { + console.log(`Backfilling ${Math.floor(daysNeeded)} days to reach ${targetDays}-day window...`); + await this.backfillHistoricalData(Math.floor(daysNeeded), dataRange.oldestTimestamp); + } else { + console.log(`Data window is sufficient (${dataRange.daysCovered}/${targetDays} days)`); + } + } + } + + // Start generation loop + console.log('\nStarting continuous generation...'); + await this.generateAndInsertBatch(); // Run immediately + this.generationTimer = setInterval(() => this.generateAndInsertBatch(), this.config.generationIntervalMs); + + // Start cleanup loop + setTimeout(() => { + this.cleanupOldData(); + this.cleanupTimer = setInterval(() => this.cleanupOldData(), this.config.cleanupIntervalHours * 60 * 60 * 1000); + }, 60000); // Wait 1 minute before first cleanup + + console.log('\nMaritime Data Synthesizer started'); + console.log(` Batch size: ${this.config.batchSize} vessels`); + console.log(` Generation interval: ${this.config.generationIntervalMs / 1000} seconds`); + console.log( + ` Records per day: ~${Math.floor((24 * 60 * 60 * 1000) / this.config.generationIntervalMs) * this.config.batchSize}` + ); + console.log(` Rolling window: ${this.config.retentionDays} days`); + console.log(` Cleanup interval: ${this.config.cleanupIntervalHours} hours`); + + this.emit('service:started'); + } catch (error) { + this.isRunning = false; + this.emit('service:error', { error }); + throw error; + } + } + + /** + * Backfill historical data before a specific timestamp + */ + async backfillHistoricalData(days, beforeTimestamp) { + try { + this.emit('backfill:starting', { days, beforeTimestamp }); + + const recordsPerDay = + Math.floor((24 * 60 * 60 * 1000) / this.config.generationIntervalMs) * this.config.batchSize; + const totalRecords = recordsPerDay * days; + const totalBatches = Math.ceil(totalRecords / this.config.batchSize); + + console.log(`Backfilling ${days} days of historical data...`); + console.log(` Records per day: ${recordsPerDay.toLocaleString()}`); + console.log(` Total records: ${totalRecords.toLocaleString()}`); + console.log(` Total batches: ${totalBatches.toLocaleString()}`); + + let recordsInserted = 0; + const startTime = Date.now(); + const oldestTimestamp = beforeTimestamp.getTime(); + + for (let batchNum = 0; batchNum < totalBatches; batchNum++) { + // Calculate time offset - going backwards from beforeTimestamp + const timeOffsetMs = (batchNum / totalBatches) * days * 24 * 60 * 60 * 1000; + const batchTimestamp = oldestTimestamp - days * 24 * 60 * 60 * 1000 + timeOffsetMs; + + // Generate batch with timestamp offset + const records = this.generator.generateBatch(this.config.batchSize, Date.now() - batchTimestamp); + + // Insert into BigQuery + await this.bigquery.insertBatch(records); + + recordsInserted += records.length; + + // Emit progress + const progress = (((batchNum + 1) / totalBatches) * 100).toFixed(1); + this.emit('backfill:progress', { + batchNum: batchNum + 1, + totalBatches, + recordsInserted, + totalRecords, + progress: parseFloat(progress), + }); + + // Log progress periodically + if ((batchNum + 1) % 10 === 0 || batchNum === totalBatches - 1) { + const elapsed = (Date.now() - startTime) / 1000; + const rate = recordsInserted / elapsed; + const remaining = (totalRecords - recordsInserted) / rate; + + console.log( + `Backfill: ${progress}% | ` + + `Batch ${batchNum + 1}/${totalBatches} | ` + + `Records: ${recordsInserted.toLocaleString()}/${totalRecords.toLocaleString()} | ` + + `Rate: ${Math.floor(rate)} records/sec | ` + + `ETA: ${Math.ceil(remaining / 60)} min` + ); + } + + // Rate limiting + if (batchNum < totalBatches - 1) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } + + const totalTime = ((Date.now() - startTime) / 1000 / 60).toFixed(1); + console.log(`Backfill completed: ${recordsInserted.toLocaleString()} records in ${totalTime} minutes`); + + this.emit('backfill:completed', { + recordsInserted, + totalTime: parseFloat(totalTime), + }); + } catch (error) { + console.error('Error backfilling historical data:', error); + this.emit('backfill:error', { error }); + throw error; + } + } + + /** + * Generate a batch of records and insert into BigQuery + */ + async generateAndInsertBatch() { + try { + this.emit('batch:generating', { size: this.config.batchSize }); + + const records = this.generator.generateBatch(this.config.batchSize); + + this.emit('batch:generated', { + records: records.length, + sample: records[0], + }); + + this.emit('batch:inserting', { records: records.length }); + + const result = await this.bigquery.insertBatch(records); + + this.stats.totalBatchesGenerated++; + this.stats.totalRecordsInserted += records.length; + + this.emit('batch:inserted', { + records: records.length, + totalBatches: this.stats.totalBatchesGenerated, + totalRecords: this.stats.totalRecordsInserted, + jobId: result.jobId, + }); + + console.log( + `Batch inserted: ${records.length} records | ` + + `Total: ${this.stats.totalRecordsInserted.toLocaleString()} records | ` + + `Batches: ${this.stats.totalBatchesGenerated}` + ); + } catch (error) { + this.stats.errors++; + this.emit('batch:error', { error, errorCount: this.stats.errors }); + console.error('Error generating/inserting batch:', error.message); + } + } + + /** + * Clean up old data based on retention policy + */ + async cleanupOldData() { + try { + this.emit('cleanup:starting', { retentionDays: this.config.retentionDays }); + + const result = await this.bigquery.cleanupOldData(); + + this.emit('cleanup:completed', { + cutoffDate: result.cutoffDate, + deletedRows: result.deletedRows, + }); + + console.log(`Cleanup completed: deleted ${result.deletedRows} rows older than ${result.cutoffDate}`); + } catch (error) { + this.emit('cleanup:error', { error }); + console.error('Error during cleanup:', error.message); + } + } + + /** + * Stop the service + */ + async stop() { + if (!this.isRunning) { + console.log('Service is not running'); + return; + } + + try { + this.emit('service:stopping'); + + this.isRunning = false; + + if (this.generationTimer) { + clearInterval(this.generationTimer); + this.generationTimer = null; + } + + if (this.cleanupTimer) { + clearInterval(this.cleanupTimer); + this.cleanupTimer = null; + } + + console.log('Maritime Data Synthesizer stopped'); + this.emit('service:stopped', { stats: this.getStats() }); + } catch (error) { + this.emit('service:error', { error }); + throw error; + } + } + + /** + * Get service statistics + */ + getStats() { + const uptime = this.stats.startTime ? (Date.now() - this.stats.startTime.getTime()) / 1000 : 0; + + return { + ...this.stats, + uptime: Math.floor(uptime), + isRunning: this.isRunning, + generatorStats: this.generator.getStats(), + config: this.config, + }; + } + + /** + * Get BigQuery table statistics + */ + async getBigQueryStats() { + return await this.bigquery.getStats(); + } + + /** + * Clear all data from table (truncate) without deleting the table + */ + async clear() { + try { + this.emit('clear:starting'); + + await this.bigquery.clearData(); + + this.emit('clear:completed'); + console.log('All data cleared from table'); + + return true; + } catch (error) { + this.emit('clear:error', { error }); + throw error; + } + } + + /** + * Delete all data and table + */ + async clean() { + try { + this.emit('clean:starting'); + + await this.bigquery.deleteTable(); + + this.emit('clean:completed'); + console.log('All data and table deleted'); + + return true; + } catch (error) { + this.emit('clean:error', { error }); + throw error; + } + } + + /** + * Reset: delete and reinitialize with historical data + */ + async reset(daysOfHistoricalData = 30) { + try { + console.log('Resetting maritime data synthesizer...'); + + // Stop if running + if (this.isRunning) { + await this.stop(); + } + + // Delete table + await this.clean(); + + // Reinitialize + await this.initialize(daysOfHistoricalData); + + console.log('Reset completed'); + return true; + } catch (error) { + console.error('Error during reset:', error); + throw error; + } + } } export default MaritimeDataSynthesizer; diff --git a/src/sync-engine.js b/src/sync-engine.js index cd7805f..e3e3034 100644 --- a/src/sync-engine.js +++ b/src/sync-engine.js @@ -2,476 +2,499 @@ // File: sync-engine.js // Core synchronization engine with modulo-based partitioning -/* global config, harperCluster, tables */ +/* global tables */ import { BigQueryClient } from './bigquery-client.js'; -import { globals } from './globals.js'; export class SyncEngine { - constructor(config) { - logger.info('[SyncEngine] Constructor called - initializing sync engine'); - - logger.info("Hostname: " + server.hostname); - logger.info("Worker Id: " + server.workerIndex); - logger.info("Nodes: " + server.nodes); - - this.initialized = false; - this.config = config; - this.client = new BigQueryClient(this.config); - this.running = false; - this.nodeId = null; - this.clusterSize = null; - this.currentPhase = 'initial'; - this.lastCheckpoint = null; - this.pollTimer = null; - logger.debug('[SyncEngine] Constructor complete - initial state set'); - } - - async initialize() { - if (! this.initialized) { - logger.info('[SyncEngine.initialize] Starting initialization process'); - - // Discover cluster topology using Harper's native clustering - logger.debug('[SyncEngine.initialize] Discovering cluster topology'); - const clusterInfo = await this.discoverCluster(); - logger.debug(clusterInfo); - this.nodeId = clusterInfo.nodeId; - this.clusterSize = clusterInfo.clusterSize; - - logger.info(`[SyncEngine.initialize] Node initialized: ID=${this.nodeId}, ClusterSize=${this.clusterSize}`); - - // Load last checkpoint - logger.debug('[SyncEngine.initialize] Loading checkpoint from database'); - this.lastCheckpoint = await this.loadCheckpoint(); - - if (this.lastCheckpoint) { - this.currentPhase = this.lastCheckpoint.phase || 'initial'; - logger.info(`[SyncEngine.initialize] Resuming from checkpoint: ${this.lastCheckpoint.lastTimestamp}, phase: ${this.currentPhase}`); - } else { - logger.info('[SyncEngine.initialize] No checkpoint found - starting fresh'); - // First run - start from beginning or configurable start time - this.lastCheckpoint = { - nodeId: this.nodeId, - lastTimestamp: this.config.sync.startTimestamp || '1970-01-01T00:00:00Z', - recordsIngested: 0, - phase: 'initial' - }; - logger.debug(`[SyncEngine.initialize] Created new checkpoint starting at ${this.lastCheckpoint.lastTimestamp}`); - } - logger.info('[SyncEngine.initialize] Initialization complete'); - this.initialized = true; - } else { - logger.info('[SyncEngine.initialize] Object already initialized'); - } - } - - async discoverCluster() { - logger.debug('[SyncEngine.discoverCluster] Querying Harper cluster API'); - const currentNodeId = [server.hostname, server.workerIndex].join('-'); - logger.info(`[SyncEngine.discoverCluster] Current node ID: ${currentNodeId}`); - - // Get cluster nodes from server.nodes if available - let nodes; - if (server.nodes && Array.isArray(server.nodes) && server.nodes.length > 0) { - nodes = server.nodes.map(node => `${node.hostname}-${node.workerIndex || 0}`); - logger.info(`[SyncEngine.discoverCluster] Found ${nodes.length} nodes from server.nodes`); - } else { - logger.info('[SyncEngine.discoverCluster] No cluster nodes found, running in single-node mode'); - nodes = [currentNodeId]; - } - - // Sort deterministically (lexicographic) - nodes.sort(); - logger.info(`[SyncEngine.discoverCluster] Sorted nodes: ${nodes.join(', ')}`); - - // Find our position - const nodeIndex = nodes.findIndex(n => n === currentNodeId); - - if (nodeIndex === -1) { - logger.error(`[SyncEngine.discoverCluster] Current node '${currentNodeId}' not found in cluster nodes: ${nodes.join(', ')}`); - throw new Error(`Current node ${currentNodeId} not found in cluster`); - } - - logger.info(`[SyncEngine.discoverCluster] Node position determined: index=${nodeIndex}, clusterSize=${nodes.length}`); - return { - nodeId: nodeIndex, - clusterSize: nodes.length, - nodes: nodes - }; - } - - async loadCheckpoint() { - logger.debug(`[SyncEngine.loadCheckpoint] Attempting to load checkpoint for nodeId=${this.nodeId}`); - try { - const checkpoint = await tables.SyncCheckpoint.get(this.nodeId); - logger.debug(`[SyncEngine.loadCheckpoint] Checkpoint found: ${JSON.stringify(checkpoint)}`); - return checkpoint; - } catch (error) { - // If checkpoint not found, return null; otherwise log and rethrow so callers can handle it. - if (error && (error.code === 'NOT_FOUND' || /not\s*found/i.test(error.message || ''))) { - logger.debug('[SyncEngine.loadCheckpoint] No checkpoint found in database (first run)'); - return null; - } - logger.error(`[SyncEngine.loadCheckpoint] Error loading checkpoint: ${error.message}`, error); - throw error; - } - } - - async start() { - logger.info('[SyncEngine.start] Start method called'); - if (this.running) { - logger.warn('[SyncEngine.start] Sync already running - ignoring duplicate start request'); - return; - } - - this.running = true; - logger.info('[SyncEngine.start] Starting sync loop'); - this.schedulePoll(); - logger.debug('[SyncEngine.start] First poll scheduled'); - } - - async stop() { - logger.info('[SyncEngine.stop] Stop method called'); - this.running = false; - if (this.pollTimer) { - logger.debug('[SyncEngine.stop] Clearing poll timer'); - clearTimeout(this.pollTimer); - this.pollTimer = null; - } - logger.info('[SyncEngine.stop] Sync stopped'); - } - - schedulePoll() { - if (!this.running) { - logger.debug('[SyncEngine.schedulePoll] Not running - skipping poll schedule'); - return; - } - - const interval = this.calculatePollInterval(); - logger.debug(`[SyncEngine.schedulePoll] Scheduling next poll in ${interval}ms (phase: ${this.currentPhase})`); - this.pollTimer = setTimeout(() => this.runSyncCycle(), interval); - } - - calculatePollInterval() { - logger.debug(`[SyncEngine.calculatePollInterval] Calculating interval for phase: ${this.currentPhase}`); - // Adaptive polling based on phase - let interval; - switch(this.currentPhase) { - case 'initial': - interval = 1000; // 1 second - aggressive catch-up - break; - case 'catchup': - interval = 5000; // 5 seconds - break; - case 'steady': - interval = this.config.sync.pollInterval || 30000; // 30 seconds - break; - default: - interval = 10000; - } - logger.debug(`[SyncEngine.calculatePollInterval] Interval calculated: ${interval}ms`); - return interval; - } - - async runSyncCycle() { - logger.info(`[SyncEngine.runSyncCycle] Starting sync cycle for node ${this.nodeId}, phase: ${this.currentPhase}`); - try { - const batchSize = this.calculateBatchSize(); - logger.debug(`[SyncEngine.runSyncCycle] Batch size: ${batchSize}`); - - // Pull records for this node's partition - logger.debug(`[SyncEngine.runSyncCycle] Pulling partition data from BigQuery - nodeId: ${this.nodeId}, clusterSize: ${this.clusterSize}, lastTimestamp: ${this.lastCheckpoint.lastTimestamp}`); - const records = await this.client.pullPartition({ - nodeId: this.nodeId, - clusterSize: this.clusterSize, - lastTimestamp: this.lastCheckpoint.lastTimestamp, - batchSize - }); - logger.info(`[SyncEngine.runSyncCycle] Received ${records.length} records from BigQuery`); - - if (records.length === 0) { - logger.info(`[SyncEngine.runSyncCycle] No new records found - transitioning to steady state`); - this.currentPhase = 'steady'; - } else { - - // Write to Harper - logger.debug(`[SyncEngine.runSyncCycle] Ingesting ${records.length} records into Harper`); - await this.ingestRecords(records); - logger.debug('[SyncEngine.runSyncCycle] Ingest complete'); - - // Update checkpoint - logger.debug('[SyncEngine.runSyncCycle] Updating checkpoint'); - await this.updateCheckpoint(records); - logger.debug('[SyncEngine.runSyncCycle] Checkpoint updated'); - - // Update phase based on lag - logger.debug('[SyncEngine.runSyncCycle] Updating phase based on lag'); - await this.updatePhase(); - logger.debug(`[SyncEngine.runSyncCycle] Phase after update: ${this.currentPhase}`); - } - - logger.info(`[SyncEngine.runSyncCycle] Sync cycle complete`); - } catch (error) { - logger.error(`[SyncEngine.runSyncCycle] Sync cycle error: ${error.message}`, error); - // Continue despite errors - don't crash the component - } finally { - // Schedule next poll - logger.debug('[SyncEngine.runSyncCycle] Scheduling next poll'); - this.schedulePoll(); - } - } - - calculateBatchSize() { - logger.debug(`[SyncEngine.calculateBatchSize] Calculating batch size for phase: ${this.currentPhase}`); - let batchSize; - switch(this.currentPhase) { - case 'initial': - batchSize = this.config.sync.initialBatchSize || 10000; - break; - case 'catchup': - batchSize = this.config.sync.catchupBatchSize || 1000; - break; - case 'steady': - batchSize = this.config.sync.steadyBatchSize || 500; - break; - default: - batchSize = 1000; - } - logger.debug(`[SyncEngine.calculateBatchSize] Batch size: ${batchSize}`); - return batchSize; - } - - convertBigQueryTypes(record) { - // Convert BigQuery types to JavaScript primitives - // All timestamp/datetime types are converted to Date objects for Harper's timestamp type - const converted = {}; - for (const [key, value] of Object.entries(record)) { - if (value === null || value === undefined) { - converted[key] = value; - } else if (typeof value === 'bigint') { - // Convert BigInt to number or string depending on size - converted[key] = value <= Number.MAX_SAFE_INTEGER ? Number(value) : value.toString(); - } else if (value && typeof value === 'object') { - // Handle various BigQuery object types - const constructorName = value.constructor?.name; - - // BigQuery Timestamp/DateTime objects - if (constructorName === 'BigQueryTimestamp' || constructorName === 'BigQueryDatetime' || constructorName === 'BigQueryDate') { - // Convert to Date object - Harper's timestamp type expects Date objects - if (value.value) { - // value.value contains the ISO string - const dateObj = new Date(value.value); - logger.trace(`[SyncEngine.convertBigQueryTypes] Converted ${constructorName} '${key}': ${value.value} -> Date(${dateObj.toISOString()})`); - converted[key] = dateObj; - } else if (typeof value.toJSON === 'function') { - const jsonValue = value.toJSON(); - const dateObj = new Date(jsonValue); - logger.trace(`[SyncEngine.convertBigQueryTypes] Converted ${constructorName} '${key}' via toJSON: ${jsonValue} -> Date(${dateObj.toISOString()})`); - converted[key] = dateObj; - } else { - logger.warn(`[SyncEngine.convertBigQueryTypes] Unable to convert ${constructorName} for key ${key}`); - converted[key] = value; - } - } else if (typeof value.toISOString === 'function') { - // Already a Date object - keep as-is - converted[key] = value; - } else if (typeof value.toJSON === 'function') { - // Object with toJSON method - convert - const jsonValue = value.toJSON(); - // If it looks like an ISO date string, convert to Date - if (typeof jsonValue === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(jsonValue)) { - const dateObj = new Date(jsonValue); - logger.trace(`[SyncEngine.convertBigQueryTypes] Converted generic timestamp '${key}': ${jsonValue} -> Date(${dateObj.toISOString()})`); - converted[key] = dateObj; - } else { - converted[key] = jsonValue; - } - } else { - converted[key] = value; - } - } else { - converted[key] = value; - } - } - return converted; - } - - async ingestRecords(records) { - - logger.trace(`[SyncEngine.ingestRecords] Processing records: ${JSON.stringify(records)} records for ingestion`); - logger.debug(`[SyncEngine.ingestRecords] Processing ${records.length} records for ingestion`); - const validRecords = []; - const timestampColumn = this.config.bigquery.timestampColumn; - - for (const record of records) { - try { - // Convert BigQuery types to JavaScript primitives - const convertedRecord = this.convertBigQueryTypes(record); - logger.trace(`[SyncEngine.ingestRecords] Converted record: ${JSON.stringify(convertedRecord)}`); - - // Validate timestamp exists - if (!convertedRecord[timestampColumn]) { - logger.warn(`[SyncEngine.ingestRecords] Missing timestamp column '${timestampColumn}', skipping record: ${JSON.stringify(convertedRecord).substring(0, 100)}`); - await this.logSkippedRecord(convertedRecord, `missing_${timestampColumn}`); - continue; - } - - // Remove 'id' field from BigQuery data if it exists (not needed since transaction_date is the primary key) - const { id: _unusedId, ...cleanedRecord } = convertedRecord; - - // Store BigQuery record as-is with metadata - // transaction_date is the primary key (defined in schema) - const mappedRecord = { - ...cleanedRecord, // All BigQuery fields at top level (timestamps converted to Date objects) - _syncedAt: new Date() // Add sync timestamp as Date object - }; - - validRecords.push(mappedRecord); - } catch (error) { - logger.error(`[SyncEngine.ingestRecords] Error processing record: ${error.message}`, error); - logger.error(`[SyncEngine.ingestRecords] Error stack: ${error.stack}`); - await this.logSkippedRecord(record, `processing_error: ${error.message}`); - } - } - - logger.info(`[SyncEngine.ingestRecords] Validated ${validRecords.length}/${records.length} records`); - // logger.debug(`[SyncEngine.ingestRecords] Cleaned Records: ` + validRecords); - - // Batch write to Harper using internal API - if (validRecords.length > 0) { - logger.info(`[SyncEngine.ingestRecords] Writing ${validRecords.length} records to Harper`); - - // Debug: Log first record to see exact structure - const firstRecord = validRecords[0]; - logger.info(`[SyncEngine.ingestRecords] First record keys: ${Object.keys(firstRecord).join(', ')}`); - logger.info(`[SyncEngine.ingestRecords] First record sample: ${JSON.stringify(firstRecord).substring(0, 500)}`); - - // Check for undefined values - for (const [key, value] of Object.entries(firstRecord)) { - if (value === undefined) { - logger.error(`[SyncEngine.ingestRecords] Field '${key}' is undefined!`); - } - } - - let lastResult; - transaction((txn) => { - logger.info(`[SyncEngine.ingestRecords] Cleaned Records[0]: ${JSON.stringify(validRecords[0]).substring(0, 500)}`); - try { - // logger.error(`[SyncEngine.ingestRecords] Records to create ${JSON.stringify(validRecords, null, 2)}`); - for (const rec of validRecords) { - lastResult = tables.BigQueryData.create(rec); - } - } catch (error) { - // Always log full error detail - logger.error('[SyncEngine.ingestRecords] Harper create failed'); - logger.error(`Error name: ${error.name}`); - logger.error(`Error message: ${error.message}`); - logger.error(`Error stack: ${error.stack}`); - - // BigQuery often includes structured info - if (error.errors) { - for (const e of error.errors) { - logger.error(`BigQuery error reason: ${e.reason}`); - logger.error(`BigQuery error location: ${e.location}`); - logger.error(`BigQuery error message: ${e.message}`); - } - } - } - }); - logger.info('[SyncEngine.ingestRecords] Created validRecords in database/table, result:' + lastResult); - - logger.info(`[SyncEngine.ingestRecords] Successfully wrote ${validRecords.length} records to BigQueryData table`); - } else { - logger.warn('[SyncEngine.ingestRecords] No valid records to write'); - } - } - - async updateCheckpoint(records) { - logger.debug(`[SyncEngine.updateCheckpoint] Updating checkpoint with ${records.length} records`); - const lastRecord = records.at(-1); - const timestampColumn = this.config.bigquery.timestampColumn; - const lastTimestamp = lastRecord[timestampColumn]; - - if (!lastTimestamp) { - logger.error(`[SyncEngine.updateCheckpoint] Last record missing timestamp column '${timestampColumn}'`); - throw new Error(`Missing timestamp column in last record: ${timestampColumn}`); - } - - // Convert Date object to ISO string for storage in checkpoint - const lastTimestampString = lastTimestamp instanceof Date ? lastTimestamp.toISOString() : String(lastTimestamp); - logger.debug(`[SyncEngine.updateCheckpoint] Last record timestamp: ${lastTimestampString}`); - - this.lastCheckpoint = { - nodeId: this.nodeId, - lastTimestamp: lastTimestampString, - recordsIngested: this.lastCheckpoint.recordsIngested + records.length, - lastSyncTime: new Date().toISOString(), - phase: this.currentPhase, - batchSize: this.calculateBatchSize() - }; - - logger.debug(`[SyncEngine.updateCheckpoint] New checkpoint: ${JSON.stringify(this.lastCheckpoint)}`); - await tables.SyncCheckpoint.put(this.lastCheckpoint); - logger.info(`[SyncEngine.updateCheckpoint] Checkpoint saved - total records ingested: ${this.lastCheckpoint.recordsIngested}`); - } - - async updatePhase() { - logger.debug('[SyncEngine.updatePhase] Calculating sync lag and updating phase'); - // Calculate lag in seconds - const now = Date.now(); - const lastTimestamp = new Date(this.lastCheckpoint.lastTimestamp).getTime(); - const lagSeconds = (now - lastTimestamp) / 1000; - - logger.debug(`[SyncEngine.updatePhase] Current lag: ${lagSeconds.toFixed(2)} seconds`); - const oldPhase = this.currentPhase; - - // Update phase based on lag thresholds - if (lagSeconds > (this.config.sync.catchupThreshold || 3600)) { - this.currentPhase = 'initial'; - } else if (lagSeconds > (this.config.sync.steadyThreshold || 300)) { - this.currentPhase = 'catchup'; - } else { - this.currentPhase = 'steady'; - } - - if (oldPhase !== this.currentPhase) { - logger.info(`[SyncEngine.updatePhase] Phase transition: ${oldPhase} -> ${this.currentPhase} (lag: ${lagSeconds.toFixed(2)}s)`); - } else { - logger.debug(`[SyncEngine.updatePhase] Phase unchanged: ${this.currentPhase}`); - } - } - - async logSkippedRecord(record, reason) { - logger.warn(`[SyncEngine.logSkippedRecord] Logging skipped record - reason: ${reason}`); - // Log to audit table for monitoring - const auditEntry = { - id: `skip-${Date.now()}-${Math.random()}`, - timestamp: new Date().toISOString(), - nodeId: this.nodeId, - status: 'skipped', - reason, - recordSample: JSON.stringify(record).substring(0, 500) - }; - logger.debug(`[SyncEngine.logSkippedRecord] Audit entry: ${JSON.stringify(auditEntry)}`); - await tables.SyncAudit.put(auditEntry); - logger.debug('[SyncEngine.logSkippedRecord] Skipped record logged to audit table'); - } - - async getStatus() { - logger.debug('[SyncEngine.getStatus] Status requested'); - const status = { - nodeId: this.nodeId, - clusterSize: this.clusterSize, - running: this.running, - phase: this.currentPhase, - lastCheckpoint: this.lastCheckpoint - }; - logger.debug(`[SyncEngine.getStatus] Returning status: ${JSON.stringify(status)}`); - return status; - } + constructor(config) { + logger.info('[SyncEngine] Constructor called - initializing sync engine'); + + logger.info('Hostname: ' + server.hostname); + logger.info('Worker Id: ' + server.workerIndex); + logger.info('Nodes: ' + server.nodes); + + this.initialized = false; + this.config = config; + this.client = new BigQueryClient(this.config); + this.running = false; + this.nodeId = null; + this.clusterSize = null; + this.currentPhase = 'initial'; + this.lastCheckpoint = null; + this.pollTimer = null; + logger.debug('[SyncEngine] Constructor complete - initial state set'); + } + + async initialize() { + if (!this.initialized) { + logger.info('[SyncEngine.initialize] Starting initialization process'); + + // Discover cluster topology using Harper's native clustering + logger.debug('[SyncEngine.initialize] Discovering cluster topology'); + const clusterInfo = await this.discoverCluster(); + logger.debug(clusterInfo); + this.nodeId = clusterInfo.nodeId; + this.clusterSize = clusterInfo.clusterSize; + + logger.info(`[SyncEngine.initialize] Node initialized: ID=${this.nodeId}, ClusterSize=${this.clusterSize}`); + + // Load last checkpoint + logger.debug('[SyncEngine.initialize] Loading checkpoint from database'); + this.lastCheckpoint = await this.loadCheckpoint(); + + if (this.lastCheckpoint) { + this.currentPhase = this.lastCheckpoint.phase || 'initial'; + logger.info( + `[SyncEngine.initialize] Resuming from checkpoint: ${this.lastCheckpoint.lastTimestamp}, phase: ${this.currentPhase}` + ); + } else { + logger.info('[SyncEngine.initialize] No checkpoint found - starting fresh'); + // First run - start from beginning or configurable start time + this.lastCheckpoint = { + nodeId: this.nodeId, + lastTimestamp: this.config.sync.startTimestamp || '1970-01-01T00:00:00Z', + recordsIngested: 0, + phase: 'initial', + }; + logger.debug(`[SyncEngine.initialize] Created new checkpoint starting at ${this.lastCheckpoint.lastTimestamp}`); + } + logger.info('[SyncEngine.initialize] Initialization complete'); + this.initialized = true; + } else { + logger.info('[SyncEngine.initialize] Object already initialized'); + } + } + + async discoverCluster() { + logger.debug('[SyncEngine.discoverCluster] Querying Harper cluster API'); + const currentNodeId = [server.hostname, server.workerIndex].join('-'); + logger.info(`[SyncEngine.discoverCluster] Current node ID: ${currentNodeId}`); + + // Get cluster nodes from server.nodes if available + let nodes; + if (server.nodes && Array.isArray(server.nodes) && server.nodes.length > 0) { + nodes = server.nodes.map((node) => `${node.hostname}-${node.workerIndex || 0}`); + logger.info(`[SyncEngine.discoverCluster] Found ${nodes.length} nodes from server.nodes`); + } else { + logger.info('[SyncEngine.discoverCluster] No cluster nodes found, running in single-node mode'); + nodes = [currentNodeId]; + } + + // Sort deterministically (lexicographic) + nodes.sort(); + logger.info(`[SyncEngine.discoverCluster] Sorted nodes: ${nodes.join(', ')}`); + + // Find our position + const nodeIndex = nodes.findIndex((n) => n === currentNodeId); + + if (nodeIndex === -1) { + logger.error( + `[SyncEngine.discoverCluster] Current node '${currentNodeId}' not found in cluster nodes: ${nodes.join(', ')}` + ); + throw new Error(`Current node ${currentNodeId} not found in cluster`); + } + + logger.info( + `[SyncEngine.discoverCluster] Node position determined: index=${nodeIndex}, clusterSize=${nodes.length}` + ); + return { + nodeId: nodeIndex, + clusterSize: nodes.length, + nodes: nodes, + }; + } + + async loadCheckpoint() { + logger.debug(`[SyncEngine.loadCheckpoint] Attempting to load checkpoint for nodeId=${this.nodeId}`); + try { + const checkpoint = await tables.SyncCheckpoint.get(this.nodeId); + logger.debug(`[SyncEngine.loadCheckpoint] Checkpoint found: ${JSON.stringify(checkpoint)}`); + return checkpoint; + } catch (error) { + // If checkpoint not found, return null; otherwise log and rethrow so callers can handle it. + if (error && (error.code === 'NOT_FOUND' || /not\s*found/i.test(error.message || ''))) { + logger.debug('[SyncEngine.loadCheckpoint] No checkpoint found in database (first run)'); + return null; + } + logger.error(`[SyncEngine.loadCheckpoint] Error loading checkpoint: ${error.message}`, error); + throw error; + } + } + + async start() { + logger.info('[SyncEngine.start] Start method called'); + if (this.running) { + logger.warn('[SyncEngine.start] Sync already running - ignoring duplicate start request'); + return; + } + + this.running = true; + logger.info('[SyncEngine.start] Starting sync loop'); + this.schedulePoll(); + logger.debug('[SyncEngine.start] First poll scheduled'); + } + + async stop() { + logger.info('[SyncEngine.stop] Stop method called'); + this.running = false; + if (this.pollTimer) { + logger.debug('[SyncEngine.stop] Clearing poll timer'); + clearTimeout(this.pollTimer); + this.pollTimer = null; + } + logger.info('[SyncEngine.stop] Sync stopped'); + } + + schedulePoll() { + if (!this.running) { + logger.debug('[SyncEngine.schedulePoll] Not running - skipping poll schedule'); + return; + } + + const interval = this.calculatePollInterval(); + logger.debug(`[SyncEngine.schedulePoll] Scheduling next poll in ${interval}ms (phase: ${this.currentPhase})`); + this.pollTimer = setTimeout(() => this.runSyncCycle(), interval); + } + + calculatePollInterval() { + logger.debug(`[SyncEngine.calculatePollInterval] Calculating interval for phase: ${this.currentPhase}`); + // Adaptive polling based on phase + let interval; + switch (this.currentPhase) { + case 'initial': + interval = 1000; // 1 second - aggressive catch-up + break; + case 'catchup': + interval = 5000; // 5 seconds + break; + case 'steady': + interval = this.config.sync.pollInterval || 30000; // 30 seconds + break; + default: + interval = 10000; + } + logger.debug(`[SyncEngine.calculatePollInterval] Interval calculated: ${interval}ms`); + return interval; + } + + async runSyncCycle() { + logger.info(`[SyncEngine.runSyncCycle] Starting sync cycle for node ${this.nodeId}, phase: ${this.currentPhase}`); + try { + const batchSize = this.calculateBatchSize(); + logger.debug(`[SyncEngine.runSyncCycle] Batch size: ${batchSize}`); + + // Pull records for this node's partition + logger.debug( + `[SyncEngine.runSyncCycle] Pulling partition data from BigQuery - nodeId: ${this.nodeId}, clusterSize: ${this.clusterSize}, lastTimestamp: ${this.lastCheckpoint.lastTimestamp}` + ); + const records = await this.client.pullPartition({ + nodeId: this.nodeId, + clusterSize: this.clusterSize, + lastTimestamp: this.lastCheckpoint.lastTimestamp, + batchSize, + }); + logger.info(`[SyncEngine.runSyncCycle] Received ${records.length} records from BigQuery`); + + if (records.length === 0) { + logger.info(`[SyncEngine.runSyncCycle] No new records found - transitioning to steady state`); + this.currentPhase = 'steady'; + } else { + // Write to Harper + logger.debug(`[SyncEngine.runSyncCycle] Ingesting ${records.length} records into Harper`); + await this.ingestRecords(records); + logger.debug('[SyncEngine.runSyncCycle] Ingest complete'); + + // Update checkpoint + logger.debug('[SyncEngine.runSyncCycle] Updating checkpoint'); + await this.updateCheckpoint(records); + logger.debug('[SyncEngine.runSyncCycle] Checkpoint updated'); + + // Update phase based on lag + logger.debug('[SyncEngine.runSyncCycle] Updating phase based on lag'); + await this.updatePhase(); + logger.debug(`[SyncEngine.runSyncCycle] Phase after update: ${this.currentPhase}`); + } + + logger.info(`[SyncEngine.runSyncCycle] Sync cycle complete`); + } catch (error) { + logger.error(`[SyncEngine.runSyncCycle] Sync cycle error: ${error.message}`, error); + // Continue despite errors - don't crash the component + } finally { + // Schedule next poll + logger.debug('[SyncEngine.runSyncCycle] Scheduling next poll'); + this.schedulePoll(); + } + } + + calculateBatchSize() { + logger.debug(`[SyncEngine.calculateBatchSize] Calculating batch size for phase: ${this.currentPhase}`); + let batchSize; + switch (this.currentPhase) { + case 'initial': + batchSize = this.config.sync.initialBatchSize || 10000; + break; + case 'catchup': + batchSize = this.config.sync.catchupBatchSize || 1000; + break; + case 'steady': + batchSize = this.config.sync.steadyBatchSize || 500; + break; + default: + batchSize = 1000; + } + logger.debug(`[SyncEngine.calculateBatchSize] Batch size: ${batchSize}`); + return batchSize; + } + + convertBigQueryTypes(record) { + // Convert BigQuery types to JavaScript primitives + // All timestamp/datetime types are converted to Date objects for Harper's timestamp type + const converted = {}; + for (const [key, value] of Object.entries(record)) { + if (value === null || value === undefined) { + converted[key] = value; + } else if (typeof value === 'bigint') { + // Convert BigInt to number or string depending on size + converted[key] = value <= Number.MAX_SAFE_INTEGER ? Number(value) : value.toString(); + } else if (value && typeof value === 'object') { + // Handle various BigQuery object types + const constructorName = value.constructor?.name; + + // BigQuery Timestamp/DateTime objects + if ( + constructorName === 'BigQueryTimestamp' || + constructorName === 'BigQueryDatetime' || + constructorName === 'BigQueryDate' + ) { + // Convert to Date object - Harper's timestamp type expects Date objects + if (value.value) { + // value.value contains the ISO string + const dateObj = new Date(value.value); + logger.trace( + `[SyncEngine.convertBigQueryTypes] Converted ${constructorName} '${key}': ${value.value} -> Date(${dateObj.toISOString()})` + ); + converted[key] = dateObj; + } else if (typeof value.toJSON === 'function') { + const jsonValue = value.toJSON(); + const dateObj = new Date(jsonValue); + logger.trace( + `[SyncEngine.convertBigQueryTypes] Converted ${constructorName} '${key}' via toJSON: ${jsonValue} -> Date(${dateObj.toISOString()})` + ); + converted[key] = dateObj; + } else { + logger.warn(`[SyncEngine.convertBigQueryTypes] Unable to convert ${constructorName} for key ${key}`); + converted[key] = value; + } + } else if (typeof value.toISOString === 'function') { + // Already a Date object - keep as-is + converted[key] = value; + } else if (typeof value.toJSON === 'function') { + // Object with toJSON method - convert + const jsonValue = value.toJSON(); + // If it looks like an ISO date string, convert to Date + if (typeof jsonValue === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(jsonValue)) { + const dateObj = new Date(jsonValue); + logger.trace( + `[SyncEngine.convertBigQueryTypes] Converted generic timestamp '${key}': ${jsonValue} -> Date(${dateObj.toISOString()})` + ); + converted[key] = dateObj; + } else { + converted[key] = jsonValue; + } + } else { + converted[key] = value; + } + } else { + converted[key] = value; + } + } + return converted; + } + + async ingestRecords(records) { + logger.trace(`[SyncEngine.ingestRecords] Processing records: ${JSON.stringify(records)} records for ingestion`); + logger.debug(`[SyncEngine.ingestRecords] Processing ${records.length} records for ingestion`); + const validRecords = []; + const timestampColumn = this.config.bigquery.timestampColumn; + + for (const record of records) { + try { + // Convert BigQuery types to JavaScript primitives + const convertedRecord = this.convertBigQueryTypes(record); + logger.trace(`[SyncEngine.ingestRecords] Converted record: ${JSON.stringify(convertedRecord)}`); + + // Validate timestamp exists + if (!convertedRecord[timestampColumn]) { + logger.warn( + `[SyncEngine.ingestRecords] Missing timestamp column '${timestampColumn}', skipping record: ${JSON.stringify(convertedRecord).substring(0, 100)}` + ); + await this.logSkippedRecord(convertedRecord, `missing_${timestampColumn}`); + continue; + } + + // Remove 'id' field from BigQuery data if it exists (not needed since transaction_date is the primary key) + const { id: _unusedId, ...cleanedRecord } = convertedRecord; + + // Store BigQuery record as-is with metadata + // transaction_date is the primary key (defined in schema) + const mappedRecord = { + ...cleanedRecord, // All BigQuery fields at top level (timestamps converted to Date objects) + _syncedAt: new Date(), // Add sync timestamp as Date object + }; + + validRecords.push(mappedRecord); + } catch (error) { + logger.error(`[SyncEngine.ingestRecords] Error processing record: ${error.message}`, error); + logger.error(`[SyncEngine.ingestRecords] Error stack: ${error.stack}`); + await this.logSkippedRecord(record, `processing_error: ${error.message}`); + } + } + + logger.info(`[SyncEngine.ingestRecords] Validated ${validRecords.length}/${records.length} records`); + // logger.debug(`[SyncEngine.ingestRecords] Cleaned Records: ` + validRecords); + + // Batch write to Harper using internal API + if (validRecords.length > 0) { + logger.info(`[SyncEngine.ingestRecords] Writing ${validRecords.length} records to Harper`); + + // Debug: Log first record to see exact structure + const firstRecord = validRecords[0]; + logger.info(`[SyncEngine.ingestRecords] First record keys: ${Object.keys(firstRecord).join(', ')}`); + logger.info(`[SyncEngine.ingestRecords] First record sample: ${JSON.stringify(firstRecord).substring(0, 500)}`); + + // Check for undefined values + for (const [key, value] of Object.entries(firstRecord)) { + if (value === undefined) { + logger.error(`[SyncEngine.ingestRecords] Field '${key}' is undefined!`); + } + } + + let lastResult; + transaction((_txn) => { + logger.info( + `[SyncEngine.ingestRecords] Cleaned Records[0]: ${JSON.stringify(validRecords[0]).substring(0, 500)}` + ); + try { + // logger.error(`[SyncEngine.ingestRecords] Records to create ${JSON.stringify(validRecords, null, 2)}`); + for (const rec of validRecords) { + lastResult = tables.BigQueryData.create(rec); + } + } catch (error) { + // Always log full error detail + logger.error('[SyncEngine.ingestRecords] Harper create failed'); + logger.error(`Error name: ${error.name}`); + logger.error(`Error message: ${error.message}`); + logger.error(`Error stack: ${error.stack}`); + + // BigQuery often includes structured info + if (error.errors) { + for (const e of error.errors) { + logger.error(`BigQuery error reason: ${e.reason}`); + logger.error(`BigQuery error location: ${e.location}`); + logger.error(`BigQuery error message: ${e.message}`); + } + } + } + }); + logger.info('[SyncEngine.ingestRecords] Created validRecords in database/table, result:' + lastResult); + + logger.info(`[SyncEngine.ingestRecords] Successfully wrote ${validRecords.length} records to BigQueryData table`); + } else { + logger.warn('[SyncEngine.ingestRecords] No valid records to write'); + } + } + + async updateCheckpoint(records) { + logger.debug(`[SyncEngine.updateCheckpoint] Updating checkpoint with ${records.length} records`); + const lastRecord = records.at(-1); + const timestampColumn = this.config.bigquery.timestampColumn; + const lastTimestamp = lastRecord[timestampColumn]; + + if (!lastTimestamp) { + logger.error(`[SyncEngine.updateCheckpoint] Last record missing timestamp column '${timestampColumn}'`); + throw new Error(`Missing timestamp column in last record: ${timestampColumn}`); + } + + // Convert Date object to ISO string for storage in checkpoint + const lastTimestampString = lastTimestamp instanceof Date ? lastTimestamp.toISOString() : String(lastTimestamp); + logger.debug(`[SyncEngine.updateCheckpoint] Last record timestamp: ${lastTimestampString}`); + + this.lastCheckpoint = { + nodeId: this.nodeId, + lastTimestamp: lastTimestampString, + recordsIngested: this.lastCheckpoint.recordsIngested + records.length, + lastSyncTime: new Date().toISOString(), + phase: this.currentPhase, + batchSize: this.calculateBatchSize(), + }; + + logger.debug(`[SyncEngine.updateCheckpoint] New checkpoint: ${JSON.stringify(this.lastCheckpoint)}`); + await tables.SyncCheckpoint.put(this.lastCheckpoint); + logger.info( + `[SyncEngine.updateCheckpoint] Checkpoint saved - total records ingested: ${this.lastCheckpoint.recordsIngested}` + ); + } + + async updatePhase() { + logger.debug('[SyncEngine.updatePhase] Calculating sync lag and updating phase'); + // Calculate lag in seconds + const now = Date.now(); + const lastTimestamp = new Date(this.lastCheckpoint.lastTimestamp).getTime(); + const lagSeconds = (now - lastTimestamp) / 1000; + + logger.debug(`[SyncEngine.updatePhase] Current lag: ${lagSeconds.toFixed(2)} seconds`); + const oldPhase = this.currentPhase; + + // Update phase based on lag thresholds + if (lagSeconds > (this.config.sync.catchupThreshold || 3600)) { + this.currentPhase = 'initial'; + } else if (lagSeconds > (this.config.sync.steadyThreshold || 300)) { + this.currentPhase = 'catchup'; + } else { + this.currentPhase = 'steady'; + } + + if (oldPhase !== this.currentPhase) { + logger.info( + `[SyncEngine.updatePhase] Phase transition: ${oldPhase} -> ${this.currentPhase} (lag: ${lagSeconds.toFixed(2)}s)` + ); + } else { + logger.debug(`[SyncEngine.updatePhase] Phase unchanged: ${this.currentPhase}`); + } + } + + async logSkippedRecord(record, reason) { + logger.warn(`[SyncEngine.logSkippedRecord] Logging skipped record - reason: ${reason}`); + // Log to audit table for monitoring + const auditEntry = { + id: `skip-${Date.now()}-${Math.random()}`, + timestamp: new Date().toISOString(), + nodeId: this.nodeId, + status: 'skipped', + reason, + recordSample: JSON.stringify(record).substring(0, 500), + }; + logger.debug(`[SyncEngine.logSkippedRecord] Audit entry: ${JSON.stringify(auditEntry)}`); + await tables.SyncAudit.put(auditEntry); + logger.debug('[SyncEngine.logSkippedRecord] Skipped record logged to audit table'); + } + + async getStatus() { + logger.debug('[SyncEngine.getStatus] Status requested'); + const status = { + nodeId: this.nodeId, + clusterSize: this.clusterSize, + running: this.running, + phase: this.currentPhase, + lastCheckpoint: this.lastCheckpoint, + }; + logger.debug(`[SyncEngine.getStatus] Returning status: ${JSON.stringify(status)}`); + return status; + } } // Export additional classes for use in resources.js // TODO: Validation not yet implemented - requires additional testing // export { ValidationService } from './validation.js'; -export { BigQueryClient } from './bigquery-client.js'; \ No newline at end of file +export { BigQueryClient } from './bigquery-client.js'; diff --git a/src/validation.js b/src/validation.js index 5f7d554..b20b59b 100644 --- a/src/validation.js +++ b/src/validation.js @@ -15,285 +15,307 @@ import { BigQueryClient } from './bigquery-client.js'; import { createHash } from 'node:crypto'; export class ValidationService { - constructor(config) { - this.config = config; - logger.info('[ValidationService] Constructor called - initializing validation service'); - this.bigqueryClient = new BigQueryClient(config); - logger.debug('[ValidationService] BigQuery client initialized for validation'); - } - - async runValidation() { - logger.info('[ValidationService.runValidation] Starting validation suite'); - - const results = { - timestamp: new Date().toISOString(), - checks: {} - }; - - try { - // 1. Checkpoint progress monitoring - logger.debug('[ValidationService.runValidation] Running checkpoint progress validation'); - results.checks.progress = await this.validateProgress(); - logger.info(`[ValidationService.runValidation] Progress check complete: ${results.checks.progress.status}`); - - // 2. Smoke test - can we query recent data? - logger.debug('[ValidationService.runValidation] Running smoke test'); - results.checks.smokeTest = await this.smokeTest(); - logger.info(`[ValidationService.runValidation] Smoke test complete: ${results.checks.smokeTest.status}`); - - // 3. Spot check random records - logger.debug('[ValidationService.runValidation] Running spot check'); - results.checks.spotCheck = await this.spotCheckRecords(); - logger.info(`[ValidationService.runValidation] Spot check complete: ${results.checks.spotCheck.status}`); - - // Determine overall status - const allHealthy = Object.values(results.checks).every( - check => check.status === 'healthy' || check.status === 'ok' - ); - - results.overallStatus = allHealthy ? 'healthy' : 'issues_detected'; - logger.info(`[ValidationService.runValidation] Overall validation status: ${results.overallStatus}`); - - // Log to audit table - logger.debug('[ValidationService.runValidation] Logging validation results to audit table'); - await this.logAudit(results); - - return results; - - } catch (error) { - logger.error(`[ValidationService.runValidation] Validation failed: ${error.message}`, error); - results.overallStatus = 'error'; - results.error = error.message; - await this.logAudit(results); - throw error; - } - } - - async validateProgress() { - logger.debug('[ValidationService.validateProgress] Validating checkpoint progress'); - const clusterInfo = await this.discoverCluster(); - logger.debug(`[ValidationService.validateProgress] Cluster info - nodeId: ${clusterInfo.nodeId}, clusterSize: ${clusterInfo.clusterSize}`); - - const checkpoint = await tables.SyncCheckpoint.get(clusterInfo.nodeId); - - if (!checkpoint) { - logger.warn('[ValidationService.validateProgress] No checkpoint found - node may not have started'); - return { - status: 'no_checkpoint', - message: 'No checkpoint found - node may not have started' - }; - } - - logger.debug(`[ValidationService.validateProgress] Checkpoint found - lastTimestamp: ${checkpoint.lastTimestamp}, recordsIngested: ${checkpoint.recordsIngested}`); - - const timeSinceLastSync = Date.now() - new Date(checkpoint.lastSyncTime).getTime(); - const lagSeconds = (Date.now() - new Date(checkpoint.lastTimestamp).getTime()) / 1000; - - logger.debug(`[ValidationService.validateProgress] Time since last sync: ${timeSinceLastSync}ms, lag: ${lagSeconds.toFixed(2)}s`); - - // Alert if no progress in 10 minutes - if (timeSinceLastSync > 600000) { - logger.warn(`[ValidationService.validateProgress] Sync appears STALLED - no progress in ${(timeSinceLastSync / 1000 / 60).toFixed(2)} minutes`); - return { - status: 'stalled', - message: 'No ingestion progress in 10+ minutes', - timeSinceLastSync, - lastTimestamp: checkpoint.lastTimestamp - }; - } - - // Check lag - let lagStatus = 'healthy'; - if (lagSeconds > 3600) lagStatus = 'severely_lagging'; - else if (lagSeconds > 300) lagStatus = 'lagging'; - - logger.info(`[ValidationService.validateProgress] Progress validation complete - status: ${lagStatus}, lag: ${lagSeconds.toFixed(2)}s`); - - return { - status: lagStatus, - lagSeconds, - recordsIngested: checkpoint.recordsIngested, - phase: checkpoint.phase, - lastTimestamp: checkpoint.lastTimestamp - }; - } - - async smokeTest() { - logger.debug('[ValidationService.smokeTest] Running smoke test - checking for recent data'); - const fiveMinutesAgo = new Date(Date.now() - 300000).toISOString(); - logger.debug(`[ValidationService.smokeTest] Looking for records after ${fiveMinutesAgo}`); - - try { - // Can we query recent data? - logger.debug('[ValidationService.smokeTest] Querying BigQueryData table for recent records'); - const recentRecords = await tables.BigQueryData.search({ - conditions: [{ timestamp: { $gt: fiveMinutesAgo } }], - limit: 1, - orderBy: 'timestamp DESC' - }); - - logger.debug(`[ValidationService.smokeTest] Query returned ${recentRecords.length} records`); - - if (recentRecords.length === 0) { - logger.warn('[ValidationService.smokeTest] No recent data found in last 5 minutes'); - return { - status: 'no_recent_data', - message: 'No records found in last 5 minutes' - }; - } - - const latestRecord = recentRecords[0]; - const recordLagSeconds = (Date.now() - new Date(latestRecord.timestamp).getTime()) / 1000; - - logger.info(`[ValidationService.smokeTest] Smoke test passed - latest record is ${Math.round(recordLagSeconds)}s old (timestamp: ${latestRecord.timestamp})`); - - return { - status: 'healthy', - latestTimestamp: latestRecord.timestamp, - lagSeconds: recordLagSeconds, - message: `Latest record is ${Math.round(recordLagSeconds)}s old` - }; - - } catch (error) { - logger.error(`[ValidationService.smokeTest] Query failed: ${error.message}`, error); - return { - status: 'query_failed', - message: 'Failed to query Harper', - error: error.message - }; - } - } - - async spotCheckRecords() { - logger.debug('[ValidationService.spotCheckRecords] Starting spot check validation'); - const clusterInfo = await this.discoverCluster(); - logger.debug(`[ValidationService.spotCheckRecords] Using nodeId: ${clusterInfo.nodeId}, clusterSize: ${clusterInfo.clusterSize}`); - const issues = []; - - try { - // Get 5 recent records from Harper - logger.debug('[ValidationService.spotCheckRecords] Fetching 5 recent records from Harper'); - const harperSample = await tables.BigQueryData.search({ - limit: 5, - orderBy: 'timestamp DESC' - }); - - logger.debug(`[ValidationService.spotCheckRecords] Retrieved ${harperSample.length} records from Harper`); - - if (harperSample.length === 0) { - logger.warn('[ValidationService.spotCheckRecords] No records found in Harper for validation'); - return { - status: 'no_data', - message: 'No records in Harper to validate' - }; - } - - // Verify each exists in BigQuery - logger.debug(`[ValidationService.spotCheckRecords] Verifying ${harperSample.length} Harper records exist in BigQuery`); - for (const record of harperSample) { - logger.trace(`[ValidationService.spotCheckRecords] Verifying Harper record: id=${record.id}, timestamp=${record.timestamp}`); - const exists = await this.bigqueryClient.verifyRecord(record); - if (!exists) { - logger.warn(`[ValidationService.spotCheckRecords] Phantom record found - exists in Harper but not BigQuery: ${record.id}`); - issues.push({ - type: 'phantom_record', - timestamp: record.timestamp, - id: record.id, - message: 'Record exists in Harper but not in BigQuery' - }); - } - } - - // Reverse check: verify recent BigQuery records exist in Harper - const oneHourAgo = new Date(Date.now() - 3600000).toISOString(); - logger.debug(`[ValidationService.spotCheckRecords] Fetching recent BigQuery records (after ${oneHourAgo})`); - const bqSample = await this.bigqueryClient.pullPartition({ - nodeId: clusterInfo.nodeId, - clusterSize: clusterInfo.clusterSize, - lastTimestamp: oneHourAgo, // Last hour - batchSize: 5 - }); - - logger.debug(`[ValidationService.spotCheckRecords] Retrieved ${bqSample.length} records from BigQuery for reverse check`); - - for (const record of bqSample) { - const id = this.generateRecordId(record); - logger.trace(`[ValidationService.spotCheckRecords] Checking if BigQuery record exists in Harper: id=${id}`); - const exists = await tables.BigQueryData.get(id); - if (!exists) { - logger.warn(`[ValidationService.spotCheckRecords] Missing record - exists in BigQuery but not Harper: ${id}`); - issues.push({ - type: 'missing_record', - timestamp: record.timestamp, - id, - message: 'Record exists in BigQuery but not in Harper' - }); - } - } - - const totalChecked = harperSample.length + bqSample.length; - const status = issues.length === 0 ? 'healthy' : 'issues_found'; - logger.info(`[ValidationService.spotCheckRecords] Spot check complete - status: ${status}, checked: ${totalChecked} records, issues: ${issues.length}`); - - return { - status, - samplesChecked: totalChecked, - issues, - message: issues.length === 0 - ? `Checked ${totalChecked} records, all match` - : `Found ${issues.length} mismatches` - }; - - } catch (error) { - logger.error(`[ValidationService.spotCheckRecords] Spot check failed: ${error.message}`, error); - return { - status: 'check_failed', - message: 'Spot check failed', - error: error.message - }; - } - } - - generateRecordId(record) { - logger.trace(`[ValidationService.generateRecordId] Generating ID for validation - timestamp: ${record.timestamp}`); - // Match the ID generation in sync-engine.js - // Note: Adapt this to match your record's unique identifier strategy - const hash = createHash('sha256') - .update(`${record.timestamp}-${record.id || ''}`) - .digest('hex'); - const id = hash.substring(0, 16); - logger.trace(`[ValidationService.generateRecordId] Generated ID: ${id}`); - return id; - } - - async logAudit(results) { - logger.debug('[ValidationService.logAudit] Logging validation audit results'); - const auditEntry = { - id: `validation-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, - timestamp: results.timestamp, - nodeId: (await this.discoverCluster()).nodeId, - status: results.overallStatus, - checkResults: JSON.stringify(results.checks), - message: results.error || 'Validation completed' - }; - logger.debug(`[ValidationService.logAudit] Audit entry: ${JSON.stringify(auditEntry).substring(0, 200)}...`); - await tables.SyncAudit.put(auditEntry); - logger.info('[ValidationService.logAudit] Validation audit logged to SyncAudit table'); - } - - async discoverCluster() { - logger.trace('[ValidationService.discoverCluster] Discovering cluster topology for validation'); - const nodes = await harperCluster.getNodes(); - logger.trace(`[ValidationService.discoverCluster] Found ${nodes.length} nodes`); - const sortedNodes = nodes.sort((a, b) => a.id.localeCompare(b.id)); - const currentNodeId = harperCluster.currentNode.id; - const nodeIndex = sortedNodes.findIndex(n => n.id === currentNodeId); - - logger.trace(`[ValidationService.discoverCluster] Current node: ${currentNodeId}, index: ${nodeIndex}, clusterSize: ${sortedNodes.length}`); - - return { - nodeId: nodeIndex, - clusterSize: sortedNodes.length - }; - } -} \ No newline at end of file + constructor(config) { + this.config = config; + logger.info('[ValidationService] Constructor called - initializing validation service'); + this.bigqueryClient = new BigQueryClient(config); + logger.debug('[ValidationService] BigQuery client initialized for validation'); + } + + async runValidation() { + logger.info('[ValidationService.runValidation] Starting validation suite'); + + const results = { + timestamp: new Date().toISOString(), + checks: {}, + }; + + try { + // 1. Checkpoint progress monitoring + logger.debug('[ValidationService.runValidation] Running checkpoint progress validation'); + results.checks.progress = await this.validateProgress(); + logger.info(`[ValidationService.runValidation] Progress check complete: ${results.checks.progress.status}`); + + // 2. Smoke test - can we query recent data? + logger.debug('[ValidationService.runValidation] Running smoke test'); + results.checks.smokeTest = await this.smokeTest(); + logger.info(`[ValidationService.runValidation] Smoke test complete: ${results.checks.smokeTest.status}`); + + // 3. Spot check random records + logger.debug('[ValidationService.runValidation] Running spot check'); + results.checks.spotCheck = await this.spotCheckRecords(); + logger.info(`[ValidationService.runValidation] Spot check complete: ${results.checks.spotCheck.status}`); + + // Determine overall status + const allHealthy = Object.values(results.checks).every( + (check) => check.status === 'healthy' || check.status === 'ok' + ); + + results.overallStatus = allHealthy ? 'healthy' : 'issues_detected'; + logger.info(`[ValidationService.runValidation] Overall validation status: ${results.overallStatus}`); + + // Log to audit table + logger.debug('[ValidationService.runValidation] Logging validation results to audit table'); + await this.logAudit(results); + + return results; + } catch (error) { + logger.error(`[ValidationService.runValidation] Validation failed: ${error.message}`, error); + results.overallStatus = 'error'; + results.error = error.message; + await this.logAudit(results); + throw error; + } + } + + async validateProgress() { + logger.debug('[ValidationService.validateProgress] Validating checkpoint progress'); + const clusterInfo = await this.discoverCluster(); + logger.debug( + `[ValidationService.validateProgress] Cluster info - nodeId: ${clusterInfo.nodeId}, clusterSize: ${clusterInfo.clusterSize}` + ); + + const checkpoint = await tables.SyncCheckpoint.get(clusterInfo.nodeId); + + if (!checkpoint) { + logger.warn('[ValidationService.validateProgress] No checkpoint found - node may not have started'); + return { + status: 'no_checkpoint', + message: 'No checkpoint found - node may not have started', + }; + } + + logger.debug( + `[ValidationService.validateProgress] Checkpoint found - lastTimestamp: ${checkpoint.lastTimestamp}, recordsIngested: ${checkpoint.recordsIngested}` + ); + + const timeSinceLastSync = Date.now() - new Date(checkpoint.lastSyncTime).getTime(); + const lagSeconds = (Date.now() - new Date(checkpoint.lastTimestamp).getTime()) / 1000; + + logger.debug( + `[ValidationService.validateProgress] Time since last sync: ${timeSinceLastSync}ms, lag: ${lagSeconds.toFixed(2)}s` + ); + + // Alert if no progress in 10 minutes + if (timeSinceLastSync > 600000) { + logger.warn( + `[ValidationService.validateProgress] Sync appears STALLED - no progress in ${(timeSinceLastSync / 1000 / 60).toFixed(2)} minutes` + ); + return { + status: 'stalled', + message: 'No ingestion progress in 10+ minutes', + timeSinceLastSync, + lastTimestamp: checkpoint.lastTimestamp, + }; + } + + // Check lag + let lagStatus = 'healthy'; + if (lagSeconds > 3600) lagStatus = 'severely_lagging'; + else if (lagSeconds > 300) lagStatus = 'lagging'; + + logger.info( + `[ValidationService.validateProgress] Progress validation complete - status: ${lagStatus}, lag: ${lagSeconds.toFixed(2)}s` + ); + + return { + status: lagStatus, + lagSeconds, + recordsIngested: checkpoint.recordsIngested, + phase: checkpoint.phase, + lastTimestamp: checkpoint.lastTimestamp, + }; + } + + async smokeTest() { + logger.debug('[ValidationService.smokeTest] Running smoke test - checking for recent data'); + const fiveMinutesAgo = new Date(Date.now() - 300000).toISOString(); + logger.debug(`[ValidationService.smokeTest] Looking for records after ${fiveMinutesAgo}`); + + try { + // Can we query recent data? + logger.debug('[ValidationService.smokeTest] Querying BigQueryData table for recent records'); + const recentRecords = await tables.BigQueryData.search({ + conditions: [{ timestamp: { $gt: fiveMinutesAgo } }], + limit: 1, + orderBy: 'timestamp DESC', + }); + + logger.debug(`[ValidationService.smokeTest] Query returned ${recentRecords.length} records`); + + if (recentRecords.length === 0) { + logger.warn('[ValidationService.smokeTest] No recent data found in last 5 minutes'); + return { + status: 'no_recent_data', + message: 'No records found in last 5 minutes', + }; + } + + const latestRecord = recentRecords[0]; + const recordLagSeconds = (Date.now() - new Date(latestRecord.timestamp).getTime()) / 1000; + + logger.info( + `[ValidationService.smokeTest] Smoke test passed - latest record is ${Math.round(recordLagSeconds)}s old (timestamp: ${latestRecord.timestamp})` + ); + + return { + status: 'healthy', + latestTimestamp: latestRecord.timestamp, + lagSeconds: recordLagSeconds, + message: `Latest record is ${Math.round(recordLagSeconds)}s old`, + }; + } catch (error) { + logger.error(`[ValidationService.smokeTest] Query failed: ${error.message}`, error); + return { + status: 'query_failed', + message: 'Failed to query Harper', + error: error.message, + }; + } + } + + async spotCheckRecords() { + logger.debug('[ValidationService.spotCheckRecords] Starting spot check validation'); + const clusterInfo = await this.discoverCluster(); + logger.debug( + `[ValidationService.spotCheckRecords] Using nodeId: ${clusterInfo.nodeId}, clusterSize: ${clusterInfo.clusterSize}` + ); + const issues = []; + + try { + // Get 5 recent records from Harper + logger.debug('[ValidationService.spotCheckRecords] Fetching 5 recent records from Harper'); + const harperSample = await tables.BigQueryData.search({ + limit: 5, + orderBy: 'timestamp DESC', + }); + + logger.debug(`[ValidationService.spotCheckRecords] Retrieved ${harperSample.length} records from Harper`); + + if (harperSample.length === 0) { + logger.warn('[ValidationService.spotCheckRecords] No records found in Harper for validation'); + return { + status: 'no_data', + message: 'No records in Harper to validate', + }; + } + + // Verify each exists in BigQuery + logger.debug( + `[ValidationService.spotCheckRecords] Verifying ${harperSample.length} Harper records exist in BigQuery` + ); + for (const record of harperSample) { + logger.trace( + `[ValidationService.spotCheckRecords] Verifying Harper record: id=${record.id}, timestamp=${record.timestamp}` + ); + const exists = await this.bigqueryClient.verifyRecord(record); + if (!exists) { + logger.warn( + `[ValidationService.spotCheckRecords] Phantom record found - exists in Harper but not BigQuery: ${record.id}` + ); + issues.push({ + type: 'phantom_record', + timestamp: record.timestamp, + id: record.id, + message: 'Record exists in Harper but not in BigQuery', + }); + } + } + + // Reverse check: verify recent BigQuery records exist in Harper + const oneHourAgo = new Date(Date.now() - 3600000).toISOString(); + logger.debug(`[ValidationService.spotCheckRecords] Fetching recent BigQuery records (after ${oneHourAgo})`); + const bqSample = await this.bigqueryClient.pullPartition({ + nodeId: clusterInfo.nodeId, + clusterSize: clusterInfo.clusterSize, + lastTimestamp: oneHourAgo, // Last hour + batchSize: 5, + }); + + logger.debug( + `[ValidationService.spotCheckRecords] Retrieved ${bqSample.length} records from BigQuery for reverse check` + ); + + for (const record of bqSample) { + const id = this.generateRecordId(record); + logger.trace(`[ValidationService.spotCheckRecords] Checking if BigQuery record exists in Harper: id=${id}`); + const exists = await tables.BigQueryData.get(id); + if (!exists) { + logger.warn(`[ValidationService.spotCheckRecords] Missing record - exists in BigQuery but not Harper: ${id}`); + issues.push({ + type: 'missing_record', + timestamp: record.timestamp, + id, + message: 'Record exists in BigQuery but not in Harper', + }); + } + } + + const totalChecked = harperSample.length + bqSample.length; + const status = issues.length === 0 ? 'healthy' : 'issues_found'; + logger.info( + `[ValidationService.spotCheckRecords] Spot check complete - status: ${status}, checked: ${totalChecked} records, issues: ${issues.length}` + ); + + return { + status, + samplesChecked: totalChecked, + issues, + message: + issues.length === 0 ? `Checked ${totalChecked} records, all match` : `Found ${issues.length} mismatches`, + }; + } catch (error) { + logger.error(`[ValidationService.spotCheckRecords] Spot check failed: ${error.message}`, error); + return { + status: 'check_failed', + message: 'Spot check failed', + error: error.message, + }; + } + } + + generateRecordId(record) { + logger.trace(`[ValidationService.generateRecordId] Generating ID for validation - timestamp: ${record.timestamp}`); + // Match the ID generation in sync-engine.js + // Note: Adapt this to match your record's unique identifier strategy + const hash = createHash('sha256') + .update(`${record.timestamp}-${record.id || ''}`) + .digest('hex'); + const id = hash.substring(0, 16); + logger.trace(`[ValidationService.generateRecordId] Generated ID: ${id}`); + return id; + } + + async logAudit(results) { + logger.debug('[ValidationService.logAudit] Logging validation audit results'); + const auditEntry = { + id: `validation-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, + timestamp: results.timestamp, + nodeId: (await this.discoverCluster()).nodeId, + status: results.overallStatus, + checkResults: JSON.stringify(results.checks), + message: results.error || 'Validation completed', + }; + logger.debug(`[ValidationService.logAudit] Audit entry: ${JSON.stringify(auditEntry).substring(0, 200)}...`); + await tables.SyncAudit.put(auditEntry); + logger.info('[ValidationService.logAudit] Validation audit logged to SyncAudit table'); + } + + async discoverCluster() { + logger.trace('[ValidationService.discoverCluster] Discovering cluster topology for validation'); + const nodes = await harperCluster.getNodes(); + logger.trace(`[ValidationService.discoverCluster] Found ${nodes.length} nodes`); + const sortedNodes = nodes.sort((a, b) => a.id.localeCompare(b.id)); + const currentNodeId = harperCluster.currentNode.id; + const nodeIndex = sortedNodes.findIndex((n) => n.id === currentNodeId); + + logger.trace( + `[ValidationService.discoverCluster] Current node: ${currentNodeId}, index: ${nodeIndex}, clusterSize: ${sortedNodes.length}` + ); + + return { + nodeId: nodeIndex, + clusterSize: sortedNodes.length, + }; + } +} diff --git a/test/README.md b/test/README.md index 01f99d1..3d5f665 100644 --- a/test/README.md +++ b/test/README.md @@ -9,14 +9,18 @@ npm test ## Test Suite ### ✅ Config Loader Tests (`config-loader.test.js`) + Tests for configuration loading and merging: + - BigQuery config defaults - Synthesizer overrides - Default values - Custom settings ### ✅ Sync Engine Tests (`sync-engine.test.js`) + Tests for core sync engine logic: + - Phase calculation (initial/catchup/steady) - Batch size calculation - Record ID generation @@ -25,6 +29,7 @@ Tests for core sync engine logic: - Timestamp validation ### ⏸️ Generator Tests (`generator.test.js.skip`) + **Currently skipped** due to memory leak in journey tracking. The maritime vessel generator works correctly in production but has a memory issue when repeatedly instantiating generators in tests. The `journeys` Map grows unbounded during test execution. @@ -36,6 +41,7 @@ The maritime vessel generator works correctly in production but has a memory iss Current coverage: **19/19 tests passing** (core functionality) Areas tested: + - Configuration management ✅ - Sync engine logic ✅ - Data partitioning ✅ @@ -43,6 +49,7 @@ Areas tested: - Record validation ✅ Areas not tested: + - Generator (memory leak) ⏸️ - BigQuery integration (requires live instance) ⏭️ - Harper integration (requires live instance) ⏭️ @@ -50,5 +57,6 @@ Areas not tested: ## Integration Testing For integration testing with live BigQuery and Harper instances, see: + - `examples/test-bigquery-config.js` - Tests BigQuery connection - Manual testing with `npx maritime-data-synthesizer start` diff --git a/test/config-loader.test.js b/test/config-loader.test.js index d02203a..8583072 100644 --- a/test/config-loader.test.js +++ b/test/config-loader.test.js @@ -7,91 +7,91 @@ import assert from 'node:assert'; import { getSynthesizerConfig } from '../src/config-loader.js'; describe('Config Loader', () => { - describe('getSynthesizerConfig', () => { - it('should use bigquery config as defaults', () => { - const mockConfig = { - bigquery: { - projectId: 'test-project', - dataset: 'test_dataset', - table: 'test_table', - credentials: 'test-key.json', - location: 'US' - } - }; + describe('getSynthesizerConfig', () => { + it('should use bigquery config as defaults', () => { + const mockConfig = { + bigquery: { + projectId: 'test-project', + dataset: 'test_dataset', + table: 'test_table', + credentials: 'test-key.json', + location: 'US', + }, + }; - const config = getSynthesizerConfig(mockConfig); + const config = getSynthesizerConfig(mockConfig); - assert.strictEqual(config.projectId, 'test-project'); - assert.strictEqual(config.datasetId, 'test_dataset'); - assert.strictEqual(config.tableId, 'test_table'); - assert.strictEqual(config.credentials, 'test-key.json'); - assert.strictEqual(config.location, 'US'); - }); + assert.strictEqual(config.projectId, 'test-project'); + assert.strictEqual(config.datasetId, 'test_dataset'); + assert.strictEqual(config.tableId, 'test_table'); + assert.strictEqual(config.credentials, 'test-key.json'); + assert.strictEqual(config.location, 'US'); + }); - it('should allow synthesizer overrides for dataset/table', () => { - const mockConfig = { - bigquery: { - projectId: 'test-project', - dataset: 'default_dataset', - table: 'default_table', - credentials: 'test-key.json', - location: 'US' - }, - synthesizer: { - dataset: 'override_dataset', - table: 'override_table' - } - }; + it('should allow synthesizer overrides for dataset/table', () => { + const mockConfig = { + bigquery: { + projectId: 'test-project', + dataset: 'default_dataset', + table: 'default_table', + credentials: 'test-key.json', + location: 'US', + }, + synthesizer: { + dataset: 'override_dataset', + table: 'override_table', + }, + }; - const config = getSynthesizerConfig(mockConfig); + const config = getSynthesizerConfig(mockConfig); - assert.strictEqual(config.datasetId, 'override_dataset'); - assert.strictEqual(config.tableId, 'override_table'); - }); + assert.strictEqual(config.datasetId, 'override_dataset'); + assert.strictEqual(config.tableId, 'override_table'); + }); - it('should use default values for synthesizer settings', () => { - const mockConfig = { - bigquery: { - projectId: 'test-project', - dataset: 'test_dataset', - table: 'test_table', - credentials: 'test-key.json' - } - }; + it('should use default values for synthesizer settings', () => { + const mockConfig = { + bigquery: { + projectId: 'test-project', + dataset: 'test_dataset', + table: 'test_table', + credentials: 'test-key.json', + }, + }; - const config = getSynthesizerConfig(mockConfig); + const config = getSynthesizerConfig(mockConfig); - assert.strictEqual(config.totalVessels, 100000); - assert.strictEqual(config.batchSize, 100); - assert.strictEqual(config.generationIntervalMs, 60000); - assert.strictEqual(config.retentionDays, 30); - assert.strictEqual(config.cleanupIntervalHours, 24); - }); + assert.strictEqual(config.totalVessels, 100000); + assert.strictEqual(config.batchSize, 100); + assert.strictEqual(config.generationIntervalMs, 60000); + assert.strictEqual(config.retentionDays, 30); + assert.strictEqual(config.cleanupIntervalHours, 24); + }); - it('should allow custom synthesizer settings', () => { - const mockConfig = { - bigquery: { - projectId: 'test-project', - dataset: 'test_dataset', - table: 'test_table', - credentials: 'test-key.json' - }, - synthesizer: { - totalVessels: 50000, - batchSize: 200, - generationIntervalMs: 30000, - retentionDays: 60, - cleanupIntervalHours: 12 - } - }; + it('should allow custom synthesizer settings', () => { + const mockConfig = { + bigquery: { + projectId: 'test-project', + dataset: 'test_dataset', + table: 'test_table', + credentials: 'test-key.json', + }, + synthesizer: { + totalVessels: 50000, + batchSize: 200, + generationIntervalMs: 30000, + retentionDays: 60, + cleanupIntervalHours: 12, + }, + }; - const config = getSynthesizerConfig(mockConfig); + const config = getSynthesizerConfig(mockConfig); - assert.strictEqual(config.totalVessels, 50000); - assert.strictEqual(config.batchSize, 200); - assert.strictEqual(config.generationIntervalMs, 30000); - assert.strictEqual(config.retentionDays, 60); - assert.strictEqual(config.cleanupIntervalHours, 12); - }); - }); + assert.strictEqual(config.totalVessels, 50000); + assert.strictEqual(config.batchSize, 200); + assert.strictEqual(config.generationIntervalMs, 30000); + assert.strictEqual(config.retentionDays, 60); + assert.strictEqual(config.cleanupIntervalHours, 12); + }); + }); }); diff --git a/test/sync-engine.test.js b/test/sync-engine.test.js index b36a605..f06934f 100644 --- a/test/sync-engine.test.js +++ b/test/sync-engine.test.js @@ -4,318 +4,318 @@ * Note: These are basic unit tests. Integration tests require a running Harper instance. */ -import { describe, it, mock } from 'node:test'; +import { describe, it } from 'node:test'; import assert from 'node:assert'; describe('Sync Engine', () => { - describe('Phase calculation', () => { - it('should determine initial phase when lag is very high', () => { - // Simulating phase logic from calculatePhase() - const now = Date.now(); - const lastTimestamp = now - (7 * 24 * 60 * 60 * 1000); // 7 days ago - const lagSeconds = (now - lastTimestamp) / 1000; - - const catchupThreshold = 3600; // 1 hour - const steadyThreshold = 300; // 5 minutes - - let phase; - if (lagSeconds > catchupThreshold) { - phase = 'initial'; - } else if (lagSeconds > steadyThreshold) { - phase = 'catchup'; - } else { - phase = 'steady'; - } - - assert.strictEqual(phase, 'initial'); - }); - - it('should determine catchup phase when lag is moderate', () => { - const now = Date.now(); - const lastTimestamp = now - (30 * 60 * 1000); // 30 minutes ago - const lagSeconds = (now - lastTimestamp) / 1000; - - const catchupThreshold = 3600; - const steadyThreshold = 300; - - let phase; - if (lagSeconds > catchupThreshold) { - phase = 'initial'; - } else if (lagSeconds > steadyThreshold) { - phase = 'catchup'; - } else { - phase = 'steady'; - } - - assert.strictEqual(phase, 'catchup'); - }); - - it('should determine steady phase when lag is low', () => { - const now = Date.now(); - const lastTimestamp = now - (2 * 60 * 1000); // 2 minutes ago - const lagSeconds = (now - lastTimestamp) / 1000; - - const catchupThreshold = 3600; - const steadyThreshold = 300; - - let phase; - if (lagSeconds > catchupThreshold) { - phase = 'initial'; - } else if (lagSeconds > steadyThreshold) { - phase = 'catchup'; - } else { - phase = 'steady'; - } - - assert.strictEqual(phase, 'steady'); - }); - }); - - describe('Batch size calculation', () => { - it('should use large batch size for initial phase', () => { - const phase = 'initial'; - const config = { - initialBatchSize: 10000, - catchupBatchSize: 1000, - steadyBatchSize: 500 - }; - - let batchSize; - switch (phase) { - case 'initial': - batchSize = config.initialBatchSize; - break; - case 'catchup': - batchSize = config.catchupBatchSize; - break; - case 'steady': - batchSize = config.steadyBatchSize; - break; - default: - batchSize = config.steadyBatchSize; - } - - assert.strictEqual(batchSize, 10000); - }); - - it('should use medium batch size for catchup phase', () => { - const phase = 'catchup'; - const config = { - initialBatchSize: 10000, - catchupBatchSize: 1000, - steadyBatchSize: 500 - }; - - let batchSize; - switch (phase) { - case 'initial': - batchSize = config.initialBatchSize; - break; - case 'catchup': - batchSize = config.catchupBatchSize; - break; - case 'steady': - batchSize = config.steadyBatchSize; - break; - default: - batchSize = config.steadyBatchSize; - } - - assert.strictEqual(batchSize, 1000); - }); - - it('should use small batch size for steady phase', () => { - const phase = 'steady'; - const config = { - initialBatchSize: 10000, - catchupBatchSize: 1000, - steadyBatchSize: 500 - }; - - let batchSize; - switch (phase) { - case 'initial': - batchSize = config.initialBatchSize; - break; - case 'catchup': - batchSize = config.catchupBatchSize; - break; - case 'steady': - batchSize = config.steadyBatchSize; - break; - default: - batchSize = config.steadyBatchSize; - } - - assert.strictEqual(batchSize, 500); - }); - }); - - describe('Record ID generation', () => { - it('should generate consistent IDs from same input', async () => { - const crypto = await import('node:crypto'); - - const record = { - timestamp: '2024-01-01T00:00:00.000Z', - mmsi: '367123456' - }; - - const id1 = crypto.createHash('sha256') - .update(`${record.timestamp}-${record.mmsi}`) - .digest('hex') - .substring(0, 16); - - const id2 = crypto.createHash('sha256') - .update(`${record.timestamp}-${record.mmsi}`) - .digest('hex') - .substring(0, 16); - - assert.strictEqual(id1, id2); - }); - - it('should generate different IDs for different records', async () => { - const crypto = await import('node:crypto'); - - const record1 = { - timestamp: '2024-01-01T00:00:00.000Z', - mmsi: '367123456' - }; - - const record2 = { - timestamp: '2024-01-01T00:01:00.000Z', - mmsi: '367123456' - }; - - const id1 = crypto.createHash('sha256') - .update(`${record1.timestamp}-${record1.mmsi}`) - .digest('hex') - .substring(0, 16); - - const id2 = crypto.createHash('sha256') - .update(`${record2.timestamp}-${record2.mmsi}`) - .digest('hex') - .substring(0, 16); - - assert.notStrictEqual(id1, id2); - }); - }); - - describe('Modulo partitioning', () => { - it('should distribute records evenly across nodes', () => { - const clusterSize = 3; - const records = 1000; - const distribution = [0, 0, 0]; - - for (let i = 0; i < records; i++) { - const timestamp = Date.now() + i * 1000; - const nodeId = timestamp % clusterSize; - distribution[nodeId]++; - } - - // Each node should get approximately 1/3 of records (within 10% tolerance) - const expected = records / clusterSize; - const tolerance = expected * 0.1; - - for (const count of distribution) { - assert.ok(Math.abs(count - expected) <= tolerance); - } - }); - - it('should assign same timestamp to same node consistently', () => { - const clusterSize = 3; - const timestamp = 1704067200000; // Fixed timestamp - - const nodeId1 = timestamp % clusterSize; - const nodeId2 = timestamp % clusterSize; - - assert.strictEqual(nodeId1, nodeId2); - }); - }); - - describe('Poll interval calculation', () => { - it('should use minimal interval for initial phase', () => { - const phase = 'initial'; - const config = { - pollInterval: 30000 // 30 seconds - }; - - let interval; - if (phase === 'initial' || phase === 'catchup') { - interval = 1000; // Poll aggressively - } else { - interval = config.pollInterval; - } - - assert.strictEqual(interval, 1000); - }); - - it('should use minimal interval for catchup phase', () => { - const phase = 'catchup'; - const config = { - pollInterval: 30000 - }; - - let interval; - if (phase === 'initial' || phase === 'catchup') { - interval = 1000; - } else { - interval = config.pollInterval; - } - - assert.strictEqual(interval, 1000); - }); - - it('should use configured interval for steady phase', () => { - const phase = 'steady'; - const config = { - pollInterval: 30000 - }; - - let interval; - if (phase === 'initial' || phase === 'catchup') { - interval = 1000; - } else { - interval = config.pollInterval; - } - - assert.strictEqual(interval, 30000); - }); - }); - - describe('Timestamp validation', () => { - it('should accept valid ISO 8601 timestamps', () => { - const validTimestamps = [ - '2024-01-01T00:00:00Z', - '2024-01-01T00:00:00.000Z', - '2024-12-31T23:59:59.999Z' - ]; - - for (const timestamp of validTimestamps) { - const date = new Date(timestamp); - assert.ok(!isNaN(date.getTime())); - } - }); - - it('should reject invalid timestamps', () => { - const invalidTimestamps = [ - null, - undefined, - '', - 'not-a-date', - '2024-13-01T00:00:00Z', // Invalid month - '2024-01-32T00:00:00Z' // Invalid day - ]; - - for (const timestamp of invalidTimestamps) { - if (!timestamp) { - assert.ok(true); // null/undefined are invalid - } else { - const date = new Date(timestamp); - // Invalid dates should be NaN or have wrong values - const isValid = !isNaN(date.getTime()) && date.toISOString().startsWith(timestamp.substring(0, 10)); - assert.ok(!isValid); - } - } - }); - }); + describe('Phase calculation', () => { + it('should determine initial phase when lag is very high', () => { + // Simulating phase logic from calculatePhase() + const now = Date.now(); + const lastTimestamp = now - 7 * 24 * 60 * 60 * 1000; // 7 days ago + const lagSeconds = (now - lastTimestamp) / 1000; + + const catchupThreshold = 3600; // 1 hour + const steadyThreshold = 300; // 5 minutes + + let phase; + if (lagSeconds > catchupThreshold) { + phase = 'initial'; + } else if (lagSeconds > steadyThreshold) { + phase = 'catchup'; + } else { + phase = 'steady'; + } + + assert.strictEqual(phase, 'initial'); + }); + + it('should determine catchup phase when lag is moderate', () => { + const now = Date.now(); + const lastTimestamp = now - 30 * 60 * 1000; // 30 minutes ago + const lagSeconds = (now - lastTimestamp) / 1000; + + const catchupThreshold = 3600; + const steadyThreshold = 300; + + let phase; + if (lagSeconds > catchupThreshold) { + phase = 'initial'; + } else if (lagSeconds > steadyThreshold) { + phase = 'catchup'; + } else { + phase = 'steady'; + } + + assert.strictEqual(phase, 'catchup'); + }); + + it('should determine steady phase when lag is low', () => { + const now = Date.now(); + const lastTimestamp = now - 2 * 60 * 1000; // 2 minutes ago + const lagSeconds = (now - lastTimestamp) / 1000; + + const catchupThreshold = 3600; + const steadyThreshold = 300; + + let phase; + if (lagSeconds > catchupThreshold) { + phase = 'initial'; + } else if (lagSeconds > steadyThreshold) { + phase = 'catchup'; + } else { + phase = 'steady'; + } + + assert.strictEqual(phase, 'steady'); + }); + }); + + describe('Batch size calculation', () => { + it('should use large batch size for initial phase', () => { + const phase = 'initial'; + const config = { + initialBatchSize: 10000, + catchupBatchSize: 1000, + steadyBatchSize: 500, + }; + + let batchSize; + switch (phase) { + case 'initial': + batchSize = config.initialBatchSize; + break; + case 'catchup': + batchSize = config.catchupBatchSize; + break; + case 'steady': + batchSize = config.steadyBatchSize; + break; + default: + batchSize = config.steadyBatchSize; + } + + assert.strictEqual(batchSize, 10000); + }); + + it('should use medium batch size for catchup phase', () => { + const phase = 'catchup'; + const config = { + initialBatchSize: 10000, + catchupBatchSize: 1000, + steadyBatchSize: 500, + }; + + let batchSize; + switch (phase) { + case 'initial': + batchSize = config.initialBatchSize; + break; + case 'catchup': + batchSize = config.catchupBatchSize; + break; + case 'steady': + batchSize = config.steadyBatchSize; + break; + default: + batchSize = config.steadyBatchSize; + } + + assert.strictEqual(batchSize, 1000); + }); + + it('should use small batch size for steady phase', () => { + const phase = 'steady'; + const config = { + initialBatchSize: 10000, + catchupBatchSize: 1000, + steadyBatchSize: 500, + }; + + let batchSize; + switch (phase) { + case 'initial': + batchSize = config.initialBatchSize; + break; + case 'catchup': + batchSize = config.catchupBatchSize; + break; + case 'steady': + batchSize = config.steadyBatchSize; + break; + default: + batchSize = config.steadyBatchSize; + } + + assert.strictEqual(batchSize, 500); + }); + }); + + describe('Record ID generation', () => { + it('should generate consistent IDs from same input', async () => { + const crypto = await import('node:crypto'); + + const record = { + timestamp: '2024-01-01T00:00:00.000Z', + mmsi: '367123456', + }; + + const id1 = crypto + .createHash('sha256') + .update(`${record.timestamp}-${record.mmsi}`) + .digest('hex') + .substring(0, 16); + + const id2 = crypto + .createHash('sha256') + .update(`${record.timestamp}-${record.mmsi}`) + .digest('hex') + .substring(0, 16); + + assert.strictEqual(id1, id2); + }); + + it('should generate different IDs for different records', async () => { + const crypto = await import('node:crypto'); + + const record1 = { + timestamp: '2024-01-01T00:00:00.000Z', + mmsi: '367123456', + }; + + const record2 = { + timestamp: '2024-01-01T00:01:00.000Z', + mmsi: '367123456', + }; + + const id1 = crypto + .createHash('sha256') + .update(`${record1.timestamp}-${record1.mmsi}`) + .digest('hex') + .substring(0, 16); + + const id2 = crypto + .createHash('sha256') + .update(`${record2.timestamp}-${record2.mmsi}`) + .digest('hex') + .substring(0, 16); + + assert.notStrictEqual(id1, id2); + }); + }); + + describe('Modulo partitioning', () => { + it('should distribute records evenly across nodes', () => { + const clusterSize = 3; + const records = 1000; + const distribution = [0, 0, 0]; + + for (let i = 0; i < records; i++) { + const timestamp = Date.now() + i * 1000; + const nodeId = timestamp % clusterSize; + distribution[nodeId]++; + } + + // Each node should get approximately 1/3 of records (within 10% tolerance) + const expected = records / clusterSize; + const tolerance = expected * 0.1; + + for (const count of distribution) { + assert.ok(Math.abs(count - expected) <= tolerance); + } + }); + + it('should assign same timestamp to same node consistently', () => { + const clusterSize = 3; + const timestamp = 1704067200000; // Fixed timestamp + + const nodeId1 = timestamp % clusterSize; + const nodeId2 = timestamp % clusterSize; + + assert.strictEqual(nodeId1, nodeId2); + }); + }); + + describe('Poll interval calculation', () => { + it('should use minimal interval for initial phase', () => { + const phase = 'initial'; + const config = { + pollInterval: 30000, // 30 seconds + }; + + let interval; + if (phase === 'initial' || phase === 'catchup') { + interval = 1000; // Poll aggressively + } else { + interval = config.pollInterval; + } + + assert.strictEqual(interval, 1000); + }); + + it('should use minimal interval for catchup phase', () => { + const phase = 'catchup'; + const config = { + pollInterval: 30000, + }; + + let interval; + if (phase === 'initial' || phase === 'catchup') { + interval = 1000; + } else { + interval = config.pollInterval; + } + + assert.strictEqual(interval, 1000); + }); + + it('should use configured interval for steady phase', () => { + const phase = 'steady'; + const config = { + pollInterval: 30000, + }; + + let interval; + if (phase === 'initial' || phase === 'catchup') { + interval = 1000; + } else { + interval = config.pollInterval; + } + + assert.strictEqual(interval, 30000); + }); + }); + + describe('Timestamp validation', () => { + it('should accept valid ISO 8601 timestamps', () => { + const validTimestamps = ['2024-01-01T00:00:00Z', '2024-01-01T00:00:00.000Z', '2024-12-31T23:59:59.999Z']; + + for (const timestamp of validTimestamps) { + const date = new Date(timestamp); + assert.ok(!isNaN(date.getTime())); + } + }); + + it('should reject invalid timestamps', () => { + const invalidTimestamps = [ + null, + undefined, + '', + 'not-a-date', + '2024-13-01T00:00:00Z', // Invalid month + '2024-01-32T00:00:00Z', // Invalid day + ]; + + for (const timestamp of invalidTimestamps) { + if (!timestamp) { + assert.ok(true); // null/undefined are invalid + } else { + const date = new Date(timestamp); + // Invalid dates should be NaN or have wrong values + const isValid = !isNaN(date.getTime()) && date.toISOString().startsWith(timestamp.substring(0, 10)); + assert.ok(!isValid); + } + } + }); + }); });