Skip to content

Enhanced logging for Grafana observability (Issue #11) #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
irjudson merged 2 commits into from
Nov 14, 2025
Merged

Enhanced logging for Grafana observability (Issue #11) #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6e578ca
Add comprehensive logging for Grafana observability (Issue #11)
irjudson Nov 14, 2025
ce3fa36
Fix prettier formatting for markdown documentation
irjudson Nov 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
528 changes: 528 additions & 0 deletions GLOBALS_LOGGING_RESEARCH.md
View file
Open in desktop

Large diffs are not rendered by default.

468 changes: 468 additions & 0 deletions LOGGING_ANALYSIS_BY_FILE.md
View file
Open in desktop

Large diffs are not rendered by default.

42 changes: 42 additions & 0 deletions src/config-loader.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,36 +21,47 @@ const __dirname = dirname(__filename);
export function loadConfig(configPath = null) {
try {
let config;
let source;

// Handle different input types
if (configPath === null || configPath === undefined) {
// Default to config.yaml in project root
const path = join(__dirname, '..', 'config.yaml');
logger.debug(`[ConfigLoader.loadConfig] Loading config from default path: ${path}`);
const fileContent = readFileSync(path, 'utf8');
config = parse(fileContent);
source = path;
} else if (typeof configPath === 'string') {
// Path to config file
logger.debug(`[ConfigLoader.loadConfig] Loading config from: ${configPath}`);
const fileContent = readFileSync(configPath, 'utf8');
config = parse(fileContent);
source = configPath;
} else if (typeof configPath === 'object') {
// Config object passed directly (for testing)
logger.debug('[ConfigLoader.loadConfig] Using config object passed directly');
// Check if it's an options object with 'config' property
if (configPath.config) {
config = configPath.config;
} else {
config = configPath;
}
source = 'object';
} else {
throw new Error('configPath must be a string, object, or null');
}

if (!config) {
logger.error('[ConfigLoader.loadConfig] Failed to parse configuration');
throw new Error('Failed to parse configuration');
}

logger.info(`[ConfigLoader.loadConfig] Successfully loaded config from: ${source}`);

// Normalize to multi-table format if needed
return normalizeConfig(config);
} catch (error) {
logger.error(`[ConfigLoader.loadConfig] Configuration loading failed: ${error.message}`);
throw new Error(`Failed to load configuration: ${error.message}`);
}
}
Expand All @@ -64,17 +75,22 @@ export function loadConfig(configPath = null) {
*/
function normalizeConfig(config) {
if (!config.bigquery) {
logger.error('[ConfigLoader.normalizeConfig] bigquery section missing in configuration');
throw new Error('bigquery section missing in configuration');
}

// Check if already in multi-table format
if (config.bigquery.tables && Array.isArray(config.bigquery.tables)) {
logger.info(
`[ConfigLoader.normalizeConfig] Config already in multi-table format with ${config.bigquery.tables.length} tables`
);
// Validate multi-table configuration
validateMultiTableConfig(config);
return config;
}

// Legacy single-table format - wrap in tables array
logger.info('[ConfigLoader.normalizeConfig] Converting legacy single-table config to multi-table format');
const legacyBigQueryConfig = config.bigquery;

// Extract table-specific config
Expand All @@ -92,6 +108,10 @@ function normalizeConfig(config) {
},
};

logger.debug(
`[ConfigLoader.normalizeConfig] Created table config: ${tableConfig.dataset}.${tableConfig.table} -> ${tableConfig.targetTable}`
);

// Create normalized multi-table config
const normalizedConfig = {
operations: config.operations, // Preserve operations config if present
Expand All @@ -108,6 +128,7 @@ function normalizeConfig(config) {
},
};

logger.info('[ConfigLoader.normalizeConfig] Successfully normalized config to multi-table format');
return normalizedConfig;
}

Expand All @@ -118,11 +139,15 @@ function normalizeConfig(config) {
* @private
*/
function validateMultiTableConfig(config) {
logger.debug('[ConfigLoader.validateMultiTableConfig] Validating multi-table configuration');

if (!config.bigquery.tables || !Array.isArray(config.bigquery.tables)) {
logger.error('[ConfigLoader.validateMultiTableConfig] bigquery.tables must be an array');
throw new Error('bigquery.tables must be an array');
}

if (config.bigquery.tables.length === 0) {
logger.error('[ConfigLoader.validateMultiTableConfig] bigquery.tables array cannot be empty');
throw new Error('bigquery.tables array cannot be empty');
}

Expand All @@ -132,29 +157,38 @@ function validateMultiTableConfig(config) {
for (const table of config.bigquery.tables) {
// Check required fields
if (!table.id) {
logger.error('[ConfigLoader.validateMultiTableConfig] Missing required field: table.id');
throw new Error('Missing required field: table.id');
}
if (!table.dataset) {
logger.error(`[ConfigLoader.validateMultiTableConfig] Missing 'dataset' for table: ${table.id}`);
throw new Error(`Missing required field 'dataset' for table: ${table.id}`);
}
if (!table.table) {
logger.error(`[ConfigLoader.validateMultiTableConfig] Missing 'table' for table: ${table.id}`);
throw new Error(`Missing required field 'table' for table: ${table.id}`);
}
if (!table.timestampColumn) {
logger.error(`[ConfigLoader.validateMultiTableConfig] Missing 'timestampColumn' for table: ${table.id}`);
throw new Error(`Missing required field 'timestampColumn' for table: ${table.id}`);
}
if (!table.targetTable) {
logger.error(`[ConfigLoader.validateMultiTableConfig] Missing 'targetTable' for table: ${table.id}`);
throw new Error(`Missing required field 'targetTable' for table: ${table.id}`);
}

// Check for duplicate IDs
if (tableIds.has(table.id)) {
logger.error(`[ConfigLoader.validateMultiTableConfig] Duplicate table ID: ${table.id}`);
throw new Error(`Duplicate table ID: ${table.id}`);
}
tableIds.add(table.id);

// Check for duplicate target Harper tables
if (targetTables.has(table.targetTable)) {
logger.error(
`[ConfigLoader.validateMultiTableConfig] Duplicate targetTable '${table.targetTable}' for: ${table.id}`
);
throw new Error(
`Duplicate targetTable '${table.targetTable}' for table: ${table.id}. ` +
`Each BigQuery table must sync to a DIFFERENT Harper table. ` +
Expand All @@ -164,7 +198,15 @@ function validateMultiTableConfig(config) {
);
}
targetTables.add(table.targetTable);

logger.debug(
`[ConfigLoader.validateMultiTableConfig] Validated table: ${table.id} (${table.dataset}.${table.table} -> ${table.targetTable})`
);
}

logger.info(
`[ConfigLoader.validateMultiTableConfig] Successfully validated ${config.bigquery.tables.length} table configurations`
);
}

/**
Expand Down
9 changes: 8 additions & 1 deletion src/globals.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,17 @@ class Globals {
Globals.instance = this;
}
set(key, value) {
logger.debug(`[Globals.set] Setting '${key}' = ${JSON.stringify(value)}`);
this.data[key] = value;
}
get(key) {
return this.data[key];
const value = this.data[key];
if (value === undefined) {
logger.debug(`[Globals.get] Key '${key}' not found`);
} else {
logger.debug(`[Globals.get] Retrieved '${key}' = ${JSON.stringify(value)}`);
}
return value;
}
}

Expand Down
25 changes: 23 additions & 2 deletions src/query-builder.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,25 @@
*/
export function formatColumnList(columns) {
if (!Array.isArray(columns)) {
logger.error('[formatColumnList] Invalid input: columns must be an array');
throw new Error('columns must be an array');
}

if (columns.length === 0) {
logger.error('[formatColumnList] Invalid input: columns array cannot be empty');
throw new Error('columns array cannot be empty');
}

// Special case: ['*'] means SELECT *
if (columns.length === 1 && columns[0] === '*') {
logger.debug('[formatColumnList] Using wildcard SELECT *');
return '*';
}

// Format as comma-separated list with proper spacing
return columns.join(', ');
const formatted = columns.join(', ');
logger.debug(`[formatColumnList] Formatted ${columns.length} columns: ${formatted}`);
return formatted;
}

/**
Expand All @@ -41,16 +46,24 @@ export function formatColumnList(columns) {
*/
export function buildPullPartitionQuery({ dataset, table, timestampColumn, columns }) {
if (!dataset || !table || !timestampColumn) {
logger.error(
'[buildPullPartitionQuery] Missing required parameters: dataset, table, and timestampColumn are required'
);
throw new Error('dataset, table, and timestampColumn are required');
}

if (!columns || !Array.isArray(columns)) {
logger.error('[buildPullPartitionQuery] Invalid columns parameter: must be a non-empty array');
throw new Error('columns must be a non-empty array');
}

logger.info(
`[buildPullPartitionQuery] Building pull query for ${dataset}.${table} with ${columns.length === 1 && columns[0] === '*' ? 'all columns' : `${columns.length} columns`}`
);

const columnList = formatColumnList(columns);

return `
const query = `
SELECT ${columnList}
FROM \`${dataset}.${table}\`
WHERE
Expand All @@ -64,6 +77,9 @@ export function buildPullPartitionQuery({ dataset, table, timestampColumn, colum
ORDER BY ${timestampColumn} ASC
LIMIT CAST(@batchSize AS INT64)
`;

logger.debug('[buildPullPartitionQuery] Query construction complete');
return query;
}

/**
Expand Down Expand Up @@ -126,13 +142,18 @@ export class QueryBuilder {
*/
constructor({ dataset, table, timestampColumn, columns = ['*'] }) {
if (!dataset || !table || !timestampColumn) {
logger.error('[QueryBuilder] Missing required parameters: dataset, table, and timestampColumn are required');
throw new Error('dataset, table, and timestampColumn are required');
}

this.dataset = dataset;
this.table = table;
this.timestampColumn = timestampColumn;
this.columns = columns;

logger.info(
`[QueryBuilder] Initialized for ${dataset}.${table} with timestamp column '${timestampColumn}' and ${columns.length === 1 && columns[0] === '*' ? 'all columns' : `${columns.length} columns`}`
);
}

/**
Expand Down
Loading