Merge pull request #15 from HarperFast/feature/enhanced-logging

Enhanced logging for Grafana observability (Issue #11)

Author: irjudson

GLOBALS_LOGGING_RESEARCH.md

@@ -21,36 +21,47 @@ const __dirname = dirname(__filename);
 export function loadConfig(configPath = null) {
 	try {
 		let config;
+		let source;
 
 		// Handle different input types
 		if (configPath === null || configPath === undefined) {
 			// Default to config.yaml in project root
 			const path = join(__dirname, '..', 'config.yaml');
+			logger.debug(`[ConfigLoader.loadConfig] Loading config from default path: ${path}`);
 			const fileContent = readFileSync(path, 'utf8');
 			config = parse(fileContent);
+			source = path;
 		} else if (typeof configPath === 'string') {
 			// Path to config file
+			logger.debug(`[ConfigLoader.loadConfig] Loading config from: ${configPath}`);
 			const fileContent = readFileSync(configPath, 'utf8');
 			config = parse(fileContent);
+			source = configPath;
 		} else if (typeof configPath === 'object') {
 			// Config object passed directly (for testing)
+			logger.debug('[ConfigLoader.loadConfig] Using config object passed directly');
 			// Check if it's an options object with 'config' property
 			if (configPath.config) {
 				config = configPath.config;
 			} else {
 				config = configPath;
 			}
+			source = 'object';
 		} else {
 			throw new Error('configPath must be a string, object, or null');
 		}
 
 		if (!config) {
+			logger.error('[ConfigLoader.loadConfig] Failed to parse configuration');
 			throw new Error('Failed to parse configuration');
 		}
 
+		logger.info(`[ConfigLoader.loadConfig] Successfully loaded config from: ${source}`);
+
 		// Normalize to multi-table format if needed
 		return normalizeConfig(config);
 	} catch (error) {
+		logger.error(`[ConfigLoader.loadConfig] Configuration loading failed: ${error.message}`);
 		throw new Error(`Failed to load configuration: ${error.message}`);
 	}
 }
@@ -64,17 +75,22 @@ export function loadConfig(configPath = null) {
  */
 function normalizeConfig(config) {
 	if (!config.bigquery) {
+		logger.error('[ConfigLoader.normalizeConfig] bigquery section missing in configuration');
 		throw new Error('bigquery section missing in configuration');
 	}
 
 	// Check if already in multi-table format
 	if (config.bigquery.tables && Array.isArray(config.bigquery.tables)) {
+		logger.info(
+			`[ConfigLoader.normalizeConfig] Config already in multi-table format with ${config.bigquery.tables.length} tables`
+		);
 		// Validate multi-table configuration
 		validateMultiTableConfig(config);
 		return config;
 	}
 
 	// Legacy single-table format - wrap in tables array
+	logger.info('[ConfigLoader.normalizeConfig] Converting legacy single-table config to multi-table format');
 	const legacyBigQueryConfig = config.bigquery;
 
 	// Extract table-specific config
@@ -92,6 +108,10 @@ function normalizeConfig(config) {
 		},
 	};
 
+	logger.debug(
+		`[ConfigLoader.normalizeConfig] Created table config: ${tableConfig.dataset}.${tableConfig.table} -> ${tableConfig.targetTable}`
+	);
+
 	// Create normalized multi-table config
 	const normalizedConfig = {
 		operations: config.operations, // Preserve operations config if present
@@ -108,6 +128,7 @@ function normalizeConfig(config) {
 		},
 	};
 
+	logger.info('[ConfigLoader.normalizeConfig] Successfully normalized config to multi-table format');
 	return normalizedConfig;
 }
 
@@ -118,11 +139,15 @@ function normalizeConfig(config) {
  * @private
  */
 function validateMultiTableConfig(config) {
+	logger.debug('[ConfigLoader.validateMultiTableConfig] Validating multi-table configuration');
+
 	if (!config.bigquery.tables || !Array.isArray(config.bigquery.tables)) {
+		logger.error('[ConfigLoader.validateMultiTableConfig] bigquery.tables must be an array');
 		throw new Error('bigquery.tables must be an array');
 	}
 
 	if (config.bigquery.tables.length === 0) {
+		logger.error('[ConfigLoader.validateMultiTableConfig] bigquery.tables array cannot be empty');
 		throw new Error('bigquery.tables array cannot be empty');
 	}
 
@@ -132,29 +157,38 @@ function validateMultiTableConfig(config) {
 	for (const table of config.bigquery.tables) {
 		// Check required fields
 		if (!table.id) {
+			logger.error('[ConfigLoader.validateMultiTableConfig] Missing required field: table.id');
 			throw new Error('Missing required field: table.id');
 		}
 		if (!table.dataset) {
+			logger.error(`[ConfigLoader.validateMultiTableConfig] Missing 'dataset' for table: ${table.id}`);
 			throw new Error(`Missing required field 'dataset' for table: ${table.id}`);
 		}
 		if (!table.table) {
+			logger.error(`[ConfigLoader.validateMultiTableConfig] Missing 'table' for table: ${table.id}`);
 			throw new Error(`Missing required field 'table' for table: ${table.id}`);
 		}
 		if (!table.timestampColumn) {
+			logger.error(`[ConfigLoader.validateMultiTableConfig] Missing 'timestampColumn' for table: ${table.id}`);
 			throw new Error(`Missing required field 'timestampColumn' for table: ${table.id}`);
 		}
 		if (!table.targetTable) {
+			logger.error(`[ConfigLoader.validateMultiTableConfig] Missing 'targetTable' for table: ${table.id}`);
 			throw new Error(`Missing required field 'targetTable' for table: ${table.id}`);
 		}
 
 		// Check for duplicate IDs
 		if (tableIds.has(table.id)) {
+			logger.error(`[ConfigLoader.validateMultiTableConfig] Duplicate table ID: ${table.id}`);
 			throw new Error(`Duplicate table ID: ${table.id}`);
 		}
 		tableIds.add(table.id);
 
 		// Check for duplicate target Harper tables
 		if (targetTables.has(table.targetTable)) {
+			logger.error(
+				`[ConfigLoader.validateMultiTableConfig] Duplicate targetTable '${table.targetTable}' for: ${table.id}`
+			);
 			throw new Error(
 				`Duplicate targetTable '${table.targetTable}' for table: ${table.id}. ` +
 					`Each BigQuery table must sync to a DIFFERENT Harper table. ` +
@@ -164,7 +198,15 @@ function validateMultiTableConfig(config) {
 			);
 		}
 		targetTables.add(table.targetTable);
+
+		logger.debug(
+			`[ConfigLoader.validateMultiTableConfig] Validated table: ${table.id} (${table.dataset}.${table.table} -> ${table.targetTable})`
+		);
 	}
+
+	logger.info(
+		`[ConfigLoader.validateMultiTableConfig] Successfully validated ${config.bigquery.tables.length} table configurations`
+	);
 }
 
 /**

LOGGING_ANALYSIS_BY_FILE.md

@@ -7,10 +7,17 @@ class Globals {
 		Globals.instance = this;
 	}
 	set(key, value) {
+		logger.debug(`[Globals.set] Setting '${key}' = ${JSON.stringify(value)}`);
 		this.data[key] = value;
 	}
 	get(key) {
-		return this.data[key];
+		const value = this.data[key];
+		if (value === undefined) {
+			logger.debug(`[Globals.get] Key '${key}' not found`);
+		} else {
+			logger.debug(`[Globals.get] Retrieved '${key}' = ${JSON.stringify(value)}`);
+		}
+		return value;
 	}
 }
 

src/config-loader.js

@@ -13,20 +13,25 @@
  */
 export function formatColumnList(columns) {
 	if (!Array.isArray(columns)) {
+		logger.error('[formatColumnList] Invalid input: columns must be an array');
 		throw new Error('columns must be an array');
 	}
 
 	if (columns.length === 0) {
+		logger.error('[formatColumnList] Invalid input: columns array cannot be empty');
 		throw new Error('columns array cannot be empty');
 	}
 
 	// Special case: ['*'] means SELECT *
 	if (columns.length === 1 && columns[0] === '*') {
+		logger.debug('[formatColumnList] Using wildcard SELECT *');
 		return '*';
 	}
 
 	// Format as comma-separated list with proper spacing
-	return columns.join(', ');
+	const formatted = columns.join(', ');
+	logger.debug(`[formatColumnList] Formatted ${columns.length} columns: ${formatted}`);
+	return formatted;
 }
 
 /**
@@ -41,16 +46,24 @@ export function formatColumnList(columns) {
  */
 export function buildPullPartitionQuery({ dataset, table, timestampColumn, columns }) {
 	if (!dataset || !table || !timestampColumn) {
+		logger.error(
+			'[buildPullPartitionQuery] Missing required parameters: dataset, table, and timestampColumn are required'
+		);
 		throw new Error('dataset, table, and timestampColumn are required');
 	}
 
 	if (!columns || !Array.isArray(columns)) {
+		logger.error('[buildPullPartitionQuery] Invalid columns parameter: must be a non-empty array');
 		throw new Error('columns must be a non-empty array');
 	}
 
+	logger.info(
+		`[buildPullPartitionQuery] Building pull query for ${dataset}.${table} with ${columns.length === 1 && columns[0] === '*' ? 'all columns' : `${columns.length} columns`}`
+	);
+
 	const columnList = formatColumnList(columns);
 
-	return `
+	const query = `
     SELECT ${columnList}
     FROM \`${dataset}.${table}\`
     WHERE
@@ -64,6 +77,9 @@ export function buildPullPartitionQuery({ dataset, table, timestampColumn, colum
     ORDER BY ${timestampColumn} ASC
     LIMIT CAST(@batchSize AS INT64)
   `;
+
+	logger.debug('[buildPullPartitionQuery] Query construction complete');
+	return query;
 }
 
 /**
@@ -126,13 +142,18 @@ export class QueryBuilder {
 	 */
 	constructor({ dataset, table, timestampColumn, columns = ['*'] }) {
 		if (!dataset || !table || !timestampColumn) {
+			logger.error('[QueryBuilder] Missing required parameters: dataset, table, and timestampColumn are required');
 			throw new Error('dataset, table, and timestampColumn are required');
 		}
 
 		this.dataset = dataset;
 		this.table = table;
 		this.timestampColumn = timestampColumn;
 		this.columns = columns;
+
+		logger.info(
+			`[QueryBuilder] Initialized for ${dataset}.${table} with timestamp column '${timestampColumn}' and ${columns.length === 1 && columns[0] === '*' ? 'all columns' : `${columns.length} columns`}`
+		);
 	}
 
 	/**