Debugging
Effective debugging is crucial for building reliable data integrations. This guide shows you advanced debugging techniques, tools, and strategies for troubleshooting Databite connectors, flows, and integrations.Overview
Debugging in Databite involves:- Logging and Monitoring for real-time insights
- Error Tracking and analysis
- Performance Profiling to identify bottlenecks
- Network Debugging for API issues
- State Inspection for flow debugging
- Testing Strategies for reproducible issues
Logging and Monitoring
Structured Logging
Copy
Ask AI
interface LogEntry {
timestamp: string;
level: "debug" | "info" | "warn" | "error";
message: string;
context: any;
error?: Error;
metadata?: any;
}
class Logger {
private logs: LogEntry[] = [];
private level: string = "info";
constructor(level: string = "info") {
this.level = level;
}
debug(message: string, context: any = {}) {
this.log("debug", message, context);
}
info(message: string, context: any = {}) {
this.log("info", message, context);
}
warn(message: string, context: any = {}) {
this.log("warn", message, context);
}
error(message: string, error: Error, context: any = {}) {
this.log("error", message, { ...context, error });
}
private log(level: string, message: string, context: any) {
const entry: LogEntry = {
timestamp: new Date().toISOString(),
level,
message,
context,
metadata: {
userId: context.userId,
sessionId: context.sessionId,
requestId: context.requestId,
},
};
this.logs.push(entry);
// Output to console with colors
const colors = {
debug: "\x1b[36m", // Cyan
info: "\x1b[32m", // Green
warn: "\x1b[33m", // Yellow
error: "\x1b[31m", // Red
};
console.log(
`${
colors[level as keyof typeof colors]
}[${level.toUpperCase()}]\x1b[0m ` + `${entry.timestamp} - ${message}`,
context
);
}
getLogs(level?: string): LogEntry[] {
if (level) {
return this.logs.filter((log) => log.level === level);
}
return this.logs;
}
exportLogs(): string {
return JSON.stringify(this.logs, null, 2);
}
}
const logger = new Logger("debug");
Context-Aware Logging
Copy
Ask AI
class ContextLogger {
private context: any = {};
setContext(context: any) {
this.context = { ...this.context, ...context };
}
clearContext() {
this.context = {};
}
debug(message: string, additionalContext: any = {}) {
logger.debug(message, { ...this.context, ...additionalContext });
}
info(message: string, additionalContext: any = {}) {
logger.info(message, { ...this.context, ...additionalContext });
}
warn(message: string, additionalContext: any = {}) {
logger.warn(message, { ...this.context, ...additionalContext });
}
error(message: string, error: Error, additionalContext: any = {}) {
logger.error(message, error, { ...this.context, ...additionalContext });
}
}
const contextLogger = new ContextLogger();
// Usage in connector
const connector = new ConnectorBuilder()
.identity({
id: "debug-connector",
name: "Debug Connector",
})
.actions([
{
id: "debug_action",
name: "Debug Action",
description: "Action with comprehensive logging",
inputs: {
input: {
type: "string",
label: "Input",
required: true,
},
},
execute: async (inputs, context) => {
contextLogger.setContext({
connectorId: "debug-connector",
actionId: "debug_action",
userId: context.userId,
requestId: context.requestId,
});
contextLogger.info("Action started", { inputs });
try {
const response = await fetch("https://api.service.com/data", {
method: "POST",
body: JSON.stringify(inputs),
});
contextLogger.info("API call completed", {
status: response.status,
statusText: response.statusText,
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const data = await response.json();
contextLogger.info("Action completed successfully", { data });
return data;
} catch (error) {
contextLogger.error("Action failed", error as Error, { inputs });
throw error;
}
},
},
])
.build();
Error Tracking
Error Classification
Copy
Ask AI
enum ErrorType {
NETWORK_ERROR = "NETWORK_ERROR",
AUTHENTICATION_ERROR = "AUTHENTICATION_ERROR",
VALIDATION_ERROR = "VALIDATION_ERROR",
RATE_LIMIT_ERROR = "RATE_LIMIT_ERROR",
SERVER_ERROR = "SERVER_ERROR",
TIMEOUT_ERROR = "TIMEOUT_ERROR",
UNKNOWN_ERROR = "UNKNOWN_ERROR",
}
interface ErrorDetails {
type: ErrorType;
message: string;
stack?: string;
context: any;
timestamp: string;
retryable: boolean;
severity: "low" | "medium" | "high" | "critical";
}
class ErrorTracker {
private errors: ErrorDetails[] = [];
trackError(error: Error, context: any = {}) {
const errorDetails: ErrorDetails = {
type: this.classifyError(error),
message: error.message,
stack: error.stack,
context,
timestamp: new Date().toISOString(),
retryable: this.isRetryable(error),
severity: this.getSeverity(error),
};
this.errors.push(errorDetails);
// Log error
logger.error("Error tracked", error, errorDetails);
// Send to external service if configured
this.sendToExternalService(errorDetails);
}
private classifyError(error: any): ErrorType {
if (error.code === "ENOTFOUND" || error.code === "ECONNREFUSED") {
return ErrorType.NETWORK_ERROR;
}
if (error.status === 401) {
return ErrorType.AUTHENTICATION_ERROR;
}
if (error.status === 429) {
return ErrorType.RATE_LIMIT_ERROR;
}
if (error.status >= 500) {
return ErrorType.SERVER_ERROR;
}
if (error.name === "TimeoutError") {
return ErrorType.TIMEOUT_ERROR;
}
return ErrorType.UNKNOWN_ERROR;
}
private isRetryable(error: any): boolean {
const retryableTypes = [
ErrorType.NETWORK_ERROR,
ErrorType.RATE_LIMIT_ERROR,
ErrorType.SERVER_ERROR,
ErrorType.TIMEOUT_ERROR,
];
return retryableTypes.includes(this.classifyError(error));
}
private getSeverity(error: any): "low" | "medium" | "high" | "critical" {
if (error.status === 401) return "high";
if (error.status >= 500) return "high";
if (error.name === "TimeoutError") return "medium";
return "low";
}
private sendToExternalService(errorDetails: ErrorDetails) {
// Send to external error tracking service
// e.g., Sentry, Bugsnag, etc.
}
getErrors(type?: ErrorType): ErrorDetails[] {
if (type) {
return this.errors.filter((error) => error.type === type);
}
return this.errors;
}
getErrorStats() {
const stats = {
total: this.errors.length,
byType: {} as Record<ErrorType, number>,
bySeverity: {} as Record<string, number>,
retryable: 0,
};
this.errors.forEach((error) => {
stats.byType[error.type] = (stats.byType[error.type] || 0) + 1;
stats.bySeverity[error.severity] =
(stats.bySeverity[error.severity] || 0) + 1;
if (error.retryable) stats.retryable++;
});
return stats;
}
}
const errorTracker = new ErrorTracker();
Error Recovery
Copy
Ask AI
class ErrorRecovery {
private retryAttempts: Map<string, number> = new Map();
private maxRetries = 3;
async executeWithRecovery<T>(
operation: () => Promise<T>,
context: any = {}
): Promise<T> {
const operationId = this.generateOperationId();
let lastError: Error;
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
try {
const result = await operation();
this.retryAttempts.delete(operationId);
return result;
} catch (error) {
lastError = error as Error;
if (attempt === this.maxRetries) {
errorTracker.trackError(lastError, {
...context,
operationId,
attempts: attempt + 1,
});
throw lastError;
}
if (!this.shouldRetry(lastError, attempt)) {
errorTracker.trackError(lastError, {
...context,
operationId,
attempts: attempt + 1,
});
throw lastError;
}
this.retryAttempts.set(operationId, attempt + 1);
const delay = this.calculateDelay(attempt);
await this.sleep(delay);
}
}
throw lastError!;
}
private shouldRetry(error: Error, attempt: number): boolean {
const retryableErrors = [
"NETWORK_ERROR",
"RATE_LIMIT_ERROR",
"SERVER_ERROR",
"TIMEOUT_ERROR",
];
return retryableErrors.includes(error.name) && attempt < this.maxRetries;
}
private calculateDelay(attempt: number): number {
// Exponential backoff with jitter
const baseDelay = 1000;
const delay = baseDelay * Math.pow(2, attempt);
const jitter = Math.random() * 1000;
return delay + jitter;
}
private sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
private generateOperationId(): string {
return Math.random().toString(36).substr(2, 9);
}
}
const errorRecovery = new ErrorRecovery();
Performance Profiling
Performance Monitoring
Copy
Ask AI
class PerformanceMonitor {
private metrics: Map<string, any[]> = new Map();
startTimer(operation: string): () => void {
const startTime = performance.now();
return () => {
const endTime = performance.now();
const duration = endTime - startTime;
this.recordMetric(operation, {
duration,
timestamp: new Date().toISOString(),
});
};
}
recordMetric(operation: string, metric: any) {
if (!this.metrics.has(operation)) {
this.metrics.set(operation, []);
}
this.metrics.get(operation)!.push(metric);
}
getMetrics(operation?: string): any {
if (operation) {
return this.metrics.get(operation) || [];
}
const allMetrics: any = {};
this.metrics.forEach((metrics, op) => {
allMetrics[op] = metrics;
});
return allMetrics;
}
getStats(operation: string): any {
const metrics = this.metrics.get(operation) || [];
if (metrics.length === 0) {
return null;
}
const durations = metrics.map((m) => m.duration);
const sum = durations.reduce((a, b) => a + b, 0);
const avg = sum / durations.length;
const min = Math.min(...durations);
const max = Math.max(...durations);
return {
count: metrics.length,
average: avg,
min,
max,
total: sum,
};
}
}
const performanceMonitor = new PerformanceMonitor();
// Usage in connector
const connector = new ConnectorBuilder()
.actions([
{
id: "profiled_action",
name: "Profiled Action",
description: "Action with performance monitoring",
inputs: {},
execute: async (inputs, context) => {
const endTimer = performanceMonitor.startTimer("profiled_action");
try {
const response = await fetch("https://api.service.com/data");
const data = await response.json();
endTimer();
return data;
} catch (error) {
endTimer();
throw error;
}
},
},
])
.build();
Memory Monitoring
Copy
Ask AI
class MemoryMonitor {
private memorySnapshots: any[] = [];
takeSnapshot(label: string) {
const snapshot = {
label,
timestamp: new Date().toISOString(),
memory: process.memoryUsage(),
heapUsed: process.memoryUsage().heapUsed,
heapTotal: process.memoryUsage().heapTotal,
external: process.memoryUsage().external,
};
this.memorySnapshots.push(snapshot);
logger.debug("Memory snapshot taken", snapshot);
return snapshot;
}
getSnapshots(): any[] {
return this.memorySnapshots;
}
getMemoryStats(): any {
const snapshots = this.memorySnapshots;
if (snapshots.length === 0) {
return null;
}
const heapUsed = snapshots.map((s) => s.heapUsed);
const heapTotal = snapshots.map((s) => s.heapTotal);
return {
count: snapshots.length,
heapUsed: {
min: Math.min(...heapUsed),
max: Math.max(...heapUsed),
avg: heapUsed.reduce((a, b) => a + b, 0) / heapUsed.length,
},
heapTotal: {
min: Math.min(...heapTotal),
max: Math.max(...heapTotal),
avg: heapTotal.reduce((a, b) => a + b, 0) / heapTotal.length,
},
};
}
}
const memoryMonitor = new MemoryMonitor();
Network Debugging
Request/Response Logging
Copy
Ask AI
class NetworkLogger {
private requests: any[] = [];
logRequest(request: any) {
const logEntry = {
timestamp: new Date().toISOString(),
method: request.method,
url: request.url,
headers: request.headers,
body: request.body,
};
this.requests.push(logEntry);
logger.debug("Request logged", logEntry);
}
logResponse(response: any, duration: number) {
const logEntry = {
timestamp: new Date().toISOString(),
status: response.status,
statusText: response.statusText,
headers: response.headers,
body: response.body,
duration,
};
this.requests.push(logEntry);
logger.debug("Response logged", logEntry);
}
getRequests(): any[] {
return this.requests;
}
getRequestStats(): any {
const requests = this.requests;
if (requests.length === 0) {
return null;
}
const durations = requests
.map((r) => r.duration)
.filter((d) => d !== undefined);
const statusCodes = requests.map((r) => r.status);
return {
total: requests.length,
averageDuration: durations.reduce((a, b) => a + b, 0) / durations.length,
statusCodes: statusCodes.reduce((acc, code) => {
acc[code] = (acc[code] || 0) + 1;
return acc;
}, {} as Record<number, number>),
};
}
}
const networkLogger = new NetworkLogger();
// Intercept fetch requests
const originalFetch = global.fetch;
global.fetch = async (input: RequestInfo | URL, init?: RequestInit) => {
const request = {
method: init?.method || "GET",
url: input.toString(),
headers: init?.headers,
body: init?.body,
};
networkLogger.logRequest(request);
const startTime = performance.now();
try {
const response = await originalFetch(input, init);
const duration = performance.now() - startTime;
networkLogger.logResponse(response, duration);
return response;
} catch (error) {
const duration = performance.now() - startTime;
networkLogger.logResponse(
{
status: 0,
statusText: "Network Error",
headers: {},
body: error,
},
duration
);
throw error;
}
};
Flow Debugging
Flow State Inspection
Copy
Ask AI
class FlowDebugger {
private flowStates: Map<string, any> = new Map();
inspectFlow(flowId: string, state: any) {
this.flowStates.set(flowId, {
...state,
timestamp: new Date().toISOString(),
});
logger.debug("Flow state inspected", {
flowId,
state,
timestamp: new Date().toISOString(),
});
}
getFlowState(flowId: string): any {
return this.flowStates.get(flowId);
}
getAllFlowStates(): any {
const states: any = {};
this.flowStates.forEach((state, flowId) => {
states[flowId] = state;
});
return states;
}
clearFlowState(flowId: string) {
this.flowStates.delete(flowId);
}
clearAllFlowStates() {
this.flowStates.clear();
}
}
const flowDebugger = new FlowDebugger();
// Usage in flow execution
const debugFlow = new FlowBuilder()
.generic("start", {
title: "Start",
action: "process",
})
.transform("process", {
result: "{{input}}",
})
.build();
// Wrap flow execution with debugging
const executeFlowWithDebugging = async (flow: any, context: any) => {
const flowId = Math.random().toString(36).substr(2, 9);
try {
const result = await flow.execute(context);
flowDebugger.inspectFlow(flowId, {
success: true,
result,
context,
});
return result;
} catch (error) {
flowDebugger.inspectFlow(flowId, {
success: false,
error: error.message,
context,
});
throw error;
}
};
Flow Step Debugging
Copy
Ask AI
class FlowStepDebugger {
private stepLogs: any[] = [];
logStep(stepId: string, stepType: string, data: any) {
const logEntry = {
stepId,
stepType,
data,
timestamp: new Date().toISOString(),
};
this.stepLogs.push(logEntry);
logger.debug("Flow step logged", logEntry);
}
getStepLogs(stepId?: string): any[] {
if (stepId) {
return this.stepLogs.filter((log) => log.stepId === stepId);
}
return this.stepLogs;
}
getStepStats(): any {
const logs = this.stepLogs;
if (logs.length === 0) {
return null;
}
const stepTypes = logs.map((log) => log.stepType);
const stepCounts = stepTypes.reduce((acc, type) => {
acc[type] = (acc[type] || 0) + 1;
return acc;
}, {} as Record<string, number>);
return {
total: logs.length,
byType: stepCounts,
};
}
}
const flowStepDebugger = new FlowStepDebugger();
Testing Strategies
Debug Test Cases
Copy
Ask AI
describe("Debug Test Cases", () => {
let connector: any;
let logger: Logger;
let errorTracker: ErrorTracker;
beforeEach(() => {
connector = createTestConnector();
logger = new Logger("debug");
errorTracker = new ErrorTracker();
});
it("should log all operations", async () => {
const result = await connector.actions.test_action.execute({
input: "test",
});
expect(result).toBeDefined();
const logs = logger.getLogs();
expect(logs.length).toBeGreaterThan(0);
const debugLogs = logs.filter((log) => log.level === "debug");
expect(debugLogs.length).toBeGreaterThan(0);
});
it("should track errors properly", async () => {
try {
await connector.actions.error_action.execute({});
} catch (error) {
errorTracker.trackError(error as Error, { test: true });
}
const errors = errorTracker.getErrors();
expect(errors.length).toBe(1);
expect(errors[0].context.test).toBe(true);
});
it("should monitor performance", async () => {
const endTimer = performanceMonitor.startTimer("test_operation");
await connector.actions.test_action.execute({
input: "test",
});
endTimer();
const stats = performanceMonitor.getStats("test_operation");
expect(stats).toBeDefined();
expect(stats.count).toBe(1);
expect(stats.average).toBeGreaterThan(0);
});
});
Debug Test Data
Copy
Ask AI
class DebugTestData {
static createConnectorConfig(overrides: any = {}) {
return {
apiUrl: "https://test.api.com",
apiKey: "test-key",
timeout: 30000,
retries: 3,
...overrides,
};
}
static createFlowContext(overrides: any = {}) {
return {
config: this.createConnectorConfig(),
tokens: {},
form: {},
query: {},
...overrides,
};
}
static createErrorScenario(type: string) {
const scenarios = {
network: new Error("Network error"),
auth: new Error("Authentication failed"),
validation: new Error("Validation failed"),
timeout: new Error("Request timeout"),
};
return (
scenarios[type as keyof typeof scenarios] || new Error("Unknown error")
);
}
}
Debug Tools
Debug Dashboard
Copy
Ask AI
import React, { useState, useEffect } from "react";
function DebugDashboard() {
const [logs, setLogs] = useState([]);
const [errors, setErrors] = useState([]);
const [metrics, setMetrics] = useState({});
useEffect(() => {
const interval = setInterval(() => {
setLogs(logger.getLogs());
setErrors(errorTracker.getErrors());
setMetrics(performanceMonitor.getMetrics());
}, 1000);
return () => clearInterval(interval);
}, []);
return (
<div className="debug-dashboard">
<h2>Debug Dashboard</h2>
<div className="debug-section">
<h3>Logs</h3>
<div className="logs">
{logs.map((log, index) => (
<div key={index} className={`log log-${log.level}`}>
<span className="timestamp">{log.timestamp}</span>
<span className="level">{log.level}</span>
<span className="message">{log.message}</span>
</div>
))}
</div>
</div>
<div className="debug-section">
<h3>Errors</h3>
<div className="errors">
{errors.map((error, index) => (
<div key={index} className="error">
<span className="type">{error.type}</span>
<span className="message">{error.message}</span>
<span className="timestamp">{error.timestamp}</span>
</div>
))}
</div>
</div>
<div className="debug-section">
<h3>Performance</h3>
<div className="metrics">
{Object.entries(metrics).map(([operation, stats]) => (
<div key={operation} className="metric">
<span className="operation">{operation}</span>
<span className="stats">{JSON.stringify(stats)}</span>
</div>
))}
</div>
</div>
</div>
);
}
Debug CLI
Copy
Ask AI
class DebugCLI {
private commands: Map<string, Function> = new Map();
constructor() {
this.registerCommands();
}
private registerCommands() {
this.commands.set("logs", this.showLogs.bind(this));
this.commands.set("errors", this.showErrors.bind(this));
this.commands.set("metrics", this.showMetrics.bind(this));
this.commands.set("flows", this.showFlows.bind(this));
this.commands.set("clear", this.clearData.bind(this));
}
private showLogs() {
const logs = logger.getLogs();
console.table(logs);
}
private showErrors() {
const errors = errorTracker.getErrors();
console.table(errors);
}
private showMetrics() {
const metrics = performanceMonitor.getMetrics();
console.table(metrics);
}
private showFlows() {
const flows = flowDebugger.getAllFlowStates();
console.table(flows);
}
private clearData() {
logger.clearLogs();
errorTracker.clearErrors();
flowDebugger.clearAllFlowStates();
console.log("Debug data cleared");
}
execute(command: string) {
const handler = this.commands.get(command);
if (handler) {
handler();
} else {
console.log("Unknown command:", command);
console.log("Available commands:", Array.from(this.commands.keys()));
}
}
}
const debugCLI = new DebugCLI();
Best Practices
Debug Configuration
Use environment variables to control debug settings and enable/disable
debugging features in different environments.
Copy
Ask AI
const debugConfig = {
enabled: process.env.DEBUG_ENABLED === "true",
level: process.env.DEBUG_LEVEL || "info",
logToFile: process.env.DEBUG_LOG_TO_FILE === "true",
logFile: process.env.DEBUG_LOG_FILE || "debug.log",
enablePerformanceMonitoring: process.env.DEBUG_PERFORMANCE === "true",
enableErrorTracking: process.env.DEBUG_ERROR_TRACKING === "true",
};
if (debugConfig.enabled) {
const logger = new Logger(debugConfig.level);
const errorTracker = new ErrorTracker();
const performanceMonitor = new PerformanceMonitor();
}
Debug Data Management
Be careful with debug data in production. Ensure sensitive information is not
logged and debug data is properly cleaned up.
Copy
Ask AI
class DebugDataManager {
private maxLogs = 1000;
private maxErrors = 100;
private maxMetrics = 500;
cleanupLogs() {
const logs = logger.getLogs();
if (logs.length > this.maxLogs) {
const excess = logs.length - this.maxLogs;
logger.clearLogs();
logger.info(`Cleaned up ${excess} old log entries`);
}
}
cleanupErrors() {
const errors = errorTracker.getErrors();
if (errors.length > this.maxErrors) {
const excess = errors.length - this.maxErrors;
errorTracker.clearErrors();
logger.info(`Cleaned up ${excess} old error entries`);
}
}
cleanupMetrics() {
const metrics = performanceMonitor.getMetrics();
Object.keys(metrics).forEach((operation) => {
const operationMetrics = metrics[operation];
if (operationMetrics.length > this.maxMetrics) {
const excess = operationMetrics.length - this.maxMetrics;
operationMetrics.splice(0, excess);
logger.info(`Cleaned up ${excess} old metrics for ${operation}`);
}
});
}
cleanupAll() {
this.cleanupLogs();
this.cleanupErrors();
this.cleanupMetrics();
}
}
const debugDataManager = new DebugDataManager();
// Run cleanup every 5 minutes
setInterval(() => {
debugDataManager.cleanupAll();
}, 5 * 60 * 1000);
Next Steps
Now that you understand debugging techniques, you can:- Implement Monitoring: Set up comprehensive monitoring and alerting
- Add Logging: Implement structured logging throughout your application
- Create Debug Tools: Build custom debug tools and dashboards
- Optimize Performance: Use profiling data to optimize performance