Skip to main content

Debugging

Effective debugging is crucial for building reliable data integrations. This guide shows you advanced debugging techniques, tools, and strategies for troubleshooting Databite connectors, flows, and integrations.

Overview

Debugging in Databite involves:
  • Logging and Monitoring for real-time insights
  • Error Tracking and analysis
  • Performance Profiling to identify bottlenecks
  • Network Debugging for API issues
  • State Inspection for flow debugging
  • Testing Strategies for reproducible issues

Logging and Monitoring

Structured Logging

interface LogEntry {
  timestamp: string;
  level: "debug" | "info" | "warn" | "error";
  message: string;
  context: any;
  error?: Error;
  metadata?: any;
}

class Logger {
  private logs: LogEntry[] = [];
  private level: string = "info";

  constructor(level: string = "info") {
    this.level = level;
  }

  debug(message: string, context: any = {}) {
    this.log("debug", message, context);
  }

  info(message: string, context: any = {}) {
    this.log("info", message, context);
  }

  warn(message: string, context: any = {}) {
    this.log("warn", message, context);
  }

  error(message: string, error: Error, context: any = {}) {
    this.log("error", message, { ...context, error });
  }

  private log(level: string, message: string, context: any) {
    const entry: LogEntry = {
      timestamp: new Date().toISOString(),
      level,
      message,
      context,
      metadata: {
        userId: context.userId,
        sessionId: context.sessionId,
        requestId: context.requestId,
      },
    };

    this.logs.push(entry);

    // Output to console with colors
    const colors = {
      debug: "\x1b[36m", // Cyan
      info: "\x1b[32m", // Green
      warn: "\x1b[33m", // Yellow
      error: "\x1b[31m", // Red
    };

    console.log(
      `${
        colors[level as keyof typeof colors]
      }[${level.toUpperCase()}]\x1b[0m ` + `${entry.timestamp} - ${message}`,
      context
    );
  }

  getLogs(level?: string): LogEntry[] {
    if (level) {
      return this.logs.filter((log) => log.level === level);
    }
    return this.logs;
  }

  exportLogs(): string {
    return JSON.stringify(this.logs, null, 2);
  }
}

const logger = new Logger("debug");

Context-Aware Logging

class ContextLogger {
  private context: any = {};

  setContext(context: any) {
    this.context = { ...this.context, ...context };
  }

  clearContext() {
    this.context = {};
  }

  debug(message: string, additionalContext: any = {}) {
    logger.debug(message, { ...this.context, ...additionalContext });
  }

  info(message: string, additionalContext: any = {}) {
    logger.info(message, { ...this.context, ...additionalContext });
  }

  warn(message: string, additionalContext: any = {}) {
    logger.warn(message, { ...this.context, ...additionalContext });
  }

  error(message: string, error: Error, additionalContext: any = {}) {
    logger.error(message, error, { ...this.context, ...additionalContext });
  }
}

const contextLogger = new ContextLogger();

// Usage in connector
const connector = new ConnectorBuilder()
  .identity({
    id: "debug-connector",
    name: "Debug Connector",
  })
  .actions([
    {
      id: "debug_action",
      name: "Debug Action",
      description: "Action with comprehensive logging",
      inputs: {
        input: {
          type: "string",
          label: "Input",
          required: true,
        },
      },
      execute: async (inputs, context) => {
        contextLogger.setContext({
          connectorId: "debug-connector",
          actionId: "debug_action",
          userId: context.userId,
          requestId: context.requestId,
        });

        contextLogger.info("Action started", { inputs });

        try {
          const response = await fetch("https://api.service.com/data", {
            method: "POST",
            body: JSON.stringify(inputs),
          });

          contextLogger.info("API call completed", {
            status: response.status,
            statusText: response.statusText,
          });

          if (!response.ok) {
            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
          }

          const data = await response.json();
          contextLogger.info("Action completed successfully", { data });

          return data;
        } catch (error) {
          contextLogger.error("Action failed", error as Error, { inputs });
          throw error;
        }
      },
    },
  ])
  .build();

Error Tracking

Error Classification

enum ErrorType {
  NETWORK_ERROR = "NETWORK_ERROR",
  AUTHENTICATION_ERROR = "AUTHENTICATION_ERROR",
  VALIDATION_ERROR = "VALIDATION_ERROR",
  RATE_LIMIT_ERROR = "RATE_LIMIT_ERROR",
  SERVER_ERROR = "SERVER_ERROR",
  TIMEOUT_ERROR = "TIMEOUT_ERROR",
  UNKNOWN_ERROR = "UNKNOWN_ERROR",
}

interface ErrorDetails {
  type: ErrorType;
  message: string;
  stack?: string;
  context: any;
  timestamp: string;
  retryable: boolean;
  severity: "low" | "medium" | "high" | "critical";
}

class ErrorTracker {
  private errors: ErrorDetails[] = [];

  trackError(error: Error, context: any = {}) {
    const errorDetails: ErrorDetails = {
      type: this.classifyError(error),
      message: error.message,
      stack: error.stack,
      context,
      timestamp: new Date().toISOString(),
      retryable: this.isRetryable(error),
      severity: this.getSeverity(error),
    };

    this.errors.push(errorDetails);

    // Log error
    logger.error("Error tracked", error, errorDetails);

    // Send to external service if configured
    this.sendToExternalService(errorDetails);
  }

  private classifyError(error: any): ErrorType {
    if (error.code === "ENOTFOUND" || error.code === "ECONNREFUSED") {
      return ErrorType.NETWORK_ERROR;
    }

    if (error.status === 401) {
      return ErrorType.AUTHENTICATION_ERROR;
    }

    if (error.status === 429) {
      return ErrorType.RATE_LIMIT_ERROR;
    }

    if (error.status >= 500) {
      return ErrorType.SERVER_ERROR;
    }

    if (error.name === "TimeoutError") {
      return ErrorType.TIMEOUT_ERROR;
    }

    return ErrorType.UNKNOWN_ERROR;
  }

  private isRetryable(error: any): boolean {
    const retryableTypes = [
      ErrorType.NETWORK_ERROR,
      ErrorType.RATE_LIMIT_ERROR,
      ErrorType.SERVER_ERROR,
      ErrorType.TIMEOUT_ERROR,
    ];

    return retryableTypes.includes(this.classifyError(error));
  }

  private getSeverity(error: any): "low" | "medium" | "high" | "critical" {
    if (error.status === 401) return "high";
    if (error.status >= 500) return "high";
    if (error.name === "TimeoutError") return "medium";
    return "low";
  }

  private sendToExternalService(errorDetails: ErrorDetails) {
    // Send to external error tracking service
    // e.g., Sentry, Bugsnag, etc.
  }

  getErrors(type?: ErrorType): ErrorDetails[] {
    if (type) {
      return this.errors.filter((error) => error.type === type);
    }
    return this.errors;
  }

  getErrorStats() {
    const stats = {
      total: this.errors.length,
      byType: {} as Record<ErrorType, number>,
      bySeverity: {} as Record<string, number>,
      retryable: 0,
    };

    this.errors.forEach((error) => {
      stats.byType[error.type] = (stats.byType[error.type] || 0) + 1;
      stats.bySeverity[error.severity] =
        (stats.bySeverity[error.severity] || 0) + 1;
      if (error.retryable) stats.retryable++;
    });

    return stats;
  }
}

const errorTracker = new ErrorTracker();

Error Recovery

class ErrorRecovery {
  private retryAttempts: Map<string, number> = new Map();
  private maxRetries = 3;

  async executeWithRecovery<T>(
    operation: () => Promise<T>,
    context: any = {}
  ): Promise<T> {
    const operationId = this.generateOperationId();
    let lastError: Error;

    for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
      try {
        const result = await operation();
        this.retryAttempts.delete(operationId);
        return result;
      } catch (error) {
        lastError = error as Error;

        if (attempt === this.maxRetries) {
          errorTracker.trackError(lastError, {
            ...context,
            operationId,
            attempts: attempt + 1,
          });
          throw lastError;
        }

        if (!this.shouldRetry(lastError, attempt)) {
          errorTracker.trackError(lastError, {
            ...context,
            operationId,
            attempts: attempt + 1,
          });
          throw lastError;
        }

        this.retryAttempts.set(operationId, attempt + 1);

        const delay = this.calculateDelay(attempt);
        await this.sleep(delay);
      }
    }

    throw lastError!;
  }

  private shouldRetry(error: Error, attempt: number): boolean {
    const retryableErrors = [
      "NETWORK_ERROR",
      "RATE_LIMIT_ERROR",
      "SERVER_ERROR",
      "TIMEOUT_ERROR",
    ];

    return retryableErrors.includes(error.name) && attempt < this.maxRetries;
  }

  private calculateDelay(attempt: number): number {
    // Exponential backoff with jitter
    const baseDelay = 1000;
    const delay = baseDelay * Math.pow(2, attempt);
    const jitter = Math.random() * 1000;
    return delay + jitter;
  }

  private sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms));
  }

  private generateOperationId(): string {
    return Math.random().toString(36).substr(2, 9);
  }
}

const errorRecovery = new ErrorRecovery();

Performance Profiling

Performance Monitoring

class PerformanceMonitor {
  private metrics: Map<string, any[]> = new Map();

  startTimer(operation: string): () => void {
    const startTime = performance.now();

    return () => {
      const endTime = performance.now();
      const duration = endTime - startTime;

      this.recordMetric(operation, {
        duration,
        timestamp: new Date().toISOString(),
      });
    };
  }

  recordMetric(operation: string, metric: any) {
    if (!this.metrics.has(operation)) {
      this.metrics.set(operation, []);
    }

    this.metrics.get(operation)!.push(metric);
  }

  getMetrics(operation?: string): any {
    if (operation) {
      return this.metrics.get(operation) || [];
    }

    const allMetrics: any = {};
    this.metrics.forEach((metrics, op) => {
      allMetrics[op] = metrics;
    });
    return allMetrics;
  }

  getStats(operation: string): any {
    const metrics = this.metrics.get(operation) || [];

    if (metrics.length === 0) {
      return null;
    }

    const durations = metrics.map((m) => m.duration);
    const sum = durations.reduce((a, b) => a + b, 0);
    const avg = sum / durations.length;
    const min = Math.min(...durations);
    const max = Math.max(...durations);

    return {
      count: metrics.length,
      average: avg,
      min,
      max,
      total: sum,
    };
  }
}

const performanceMonitor = new PerformanceMonitor();

// Usage in connector
const connector = new ConnectorBuilder()
  .actions([
    {
      id: "profiled_action",
      name: "Profiled Action",
      description: "Action with performance monitoring",
      inputs: {},
      execute: async (inputs, context) => {
        const endTimer = performanceMonitor.startTimer("profiled_action");

        try {
          const response = await fetch("https://api.service.com/data");
          const data = await response.json();

          endTimer();
          return data;
        } catch (error) {
          endTimer();
          throw error;
        }
      },
    },
  ])
  .build();

Memory Monitoring

class MemoryMonitor {
  private memorySnapshots: any[] = [];

  takeSnapshot(label: string) {
    const snapshot = {
      label,
      timestamp: new Date().toISOString(),
      memory: process.memoryUsage(),
      heapUsed: process.memoryUsage().heapUsed,
      heapTotal: process.memoryUsage().heapTotal,
      external: process.memoryUsage().external,
    };

    this.memorySnapshots.push(snapshot);

    logger.debug("Memory snapshot taken", snapshot);

    return snapshot;
  }

  getSnapshots(): any[] {
    return this.memorySnapshots;
  }

  getMemoryStats(): any {
    const snapshots = this.memorySnapshots;

    if (snapshots.length === 0) {
      return null;
    }

    const heapUsed = snapshots.map((s) => s.heapUsed);
    const heapTotal = snapshots.map((s) => s.heapTotal);

    return {
      count: snapshots.length,
      heapUsed: {
        min: Math.min(...heapUsed),
        max: Math.max(...heapUsed),
        avg: heapUsed.reduce((a, b) => a + b, 0) / heapUsed.length,
      },
      heapTotal: {
        min: Math.min(...heapTotal),
        max: Math.max(...heapTotal),
        avg: heapTotal.reduce((a, b) => a + b, 0) / heapTotal.length,
      },
    };
  }
}

const memoryMonitor = new MemoryMonitor();

Network Debugging

Request/Response Logging

class NetworkLogger {
  private requests: any[] = [];

  logRequest(request: any) {
    const logEntry = {
      timestamp: new Date().toISOString(),
      method: request.method,
      url: request.url,
      headers: request.headers,
      body: request.body,
    };

    this.requests.push(logEntry);
    logger.debug("Request logged", logEntry);
  }

  logResponse(response: any, duration: number) {
    const logEntry = {
      timestamp: new Date().toISOString(),
      status: response.status,
      statusText: response.statusText,
      headers: response.headers,
      body: response.body,
      duration,
    };

    this.requests.push(logEntry);
    logger.debug("Response logged", logEntry);
  }

  getRequests(): any[] {
    return this.requests;
  }

  getRequestStats(): any {
    const requests = this.requests;

    if (requests.length === 0) {
      return null;
    }

    const durations = requests
      .map((r) => r.duration)
      .filter((d) => d !== undefined);
    const statusCodes = requests.map((r) => r.status);

    return {
      total: requests.length,
      averageDuration: durations.reduce((a, b) => a + b, 0) / durations.length,
      statusCodes: statusCodes.reduce((acc, code) => {
        acc[code] = (acc[code] || 0) + 1;
        return acc;
      }, {} as Record<number, number>),
    };
  }
}

const networkLogger = new NetworkLogger();

// Intercept fetch requests
const originalFetch = global.fetch;
global.fetch = async (input: RequestInfo | URL, init?: RequestInit) => {
  const request = {
    method: init?.method || "GET",
    url: input.toString(),
    headers: init?.headers,
    body: init?.body,
  };

  networkLogger.logRequest(request);

  const startTime = performance.now();

  try {
    const response = await originalFetch(input, init);
    const duration = performance.now() - startTime;

    networkLogger.logResponse(response, duration);

    return response;
  } catch (error) {
    const duration = performance.now() - startTime;

    networkLogger.logResponse(
      {
        status: 0,
        statusText: "Network Error",
        headers: {},
        body: error,
      },
      duration
    );

    throw error;
  }
};

Flow Debugging

Flow State Inspection

class FlowDebugger {
  private flowStates: Map<string, any> = new Map();

  inspectFlow(flowId: string, state: any) {
    this.flowStates.set(flowId, {
      ...state,
      timestamp: new Date().toISOString(),
    });

    logger.debug("Flow state inspected", {
      flowId,
      state,
      timestamp: new Date().toISOString(),
    });
  }

  getFlowState(flowId: string): any {
    return this.flowStates.get(flowId);
  }

  getAllFlowStates(): any {
    const states: any = {};
    this.flowStates.forEach((state, flowId) => {
      states[flowId] = state;
    });
    return states;
  }

  clearFlowState(flowId: string) {
    this.flowStates.delete(flowId);
  }

  clearAllFlowStates() {
    this.flowStates.clear();
  }
}

const flowDebugger = new FlowDebugger();

// Usage in flow execution
const debugFlow = new FlowBuilder()
  .generic("start", {
    title: "Start",
    action: "process",
  })
  .transform("process", {
    result: "{{input}}",
  })
  .build();

// Wrap flow execution with debugging
const executeFlowWithDebugging = async (flow: any, context: any) => {
  const flowId = Math.random().toString(36).substr(2, 9);

  try {
    const result = await flow.execute(context);

    flowDebugger.inspectFlow(flowId, {
      success: true,
      result,
      context,
    });

    return result;
  } catch (error) {
    flowDebugger.inspectFlow(flowId, {
      success: false,
      error: error.message,
      context,
    });

    throw error;
  }
};

Flow Step Debugging

class FlowStepDebugger {
  private stepLogs: any[] = [];

  logStep(stepId: string, stepType: string, data: any) {
    const logEntry = {
      stepId,
      stepType,
      data,
      timestamp: new Date().toISOString(),
    };

    this.stepLogs.push(logEntry);
    logger.debug("Flow step logged", logEntry);
  }

  getStepLogs(stepId?: string): any[] {
    if (stepId) {
      return this.stepLogs.filter((log) => log.stepId === stepId);
    }
    return this.stepLogs;
  }

  getStepStats(): any {
    const logs = this.stepLogs;

    if (logs.length === 0) {
      return null;
    }

    const stepTypes = logs.map((log) => log.stepType);
    const stepCounts = stepTypes.reduce((acc, type) => {
      acc[type] = (acc[type] || 0) + 1;
      return acc;
    }, {} as Record<string, number>);

    return {
      total: logs.length,
      byType: stepCounts,
    };
  }
}

const flowStepDebugger = new FlowStepDebugger();

Testing Strategies

Debug Test Cases

describe("Debug Test Cases", () => {
  let connector: any;
  let logger: Logger;
  let errorTracker: ErrorTracker;

  beforeEach(() => {
    connector = createTestConnector();
    logger = new Logger("debug");
    errorTracker = new ErrorTracker();
  });

  it("should log all operations", async () => {
    const result = await connector.actions.test_action.execute({
      input: "test",
    });

    expect(result).toBeDefined();

    const logs = logger.getLogs();
    expect(logs.length).toBeGreaterThan(0);

    const debugLogs = logs.filter((log) => log.level === "debug");
    expect(debugLogs.length).toBeGreaterThan(0);
  });

  it("should track errors properly", async () => {
    try {
      await connector.actions.error_action.execute({});
    } catch (error) {
      errorTracker.trackError(error as Error, { test: true });
    }

    const errors = errorTracker.getErrors();
    expect(errors.length).toBe(1);
    expect(errors[0].context.test).toBe(true);
  });

  it("should monitor performance", async () => {
    const endTimer = performanceMonitor.startTimer("test_operation");

    await connector.actions.test_action.execute({
      input: "test",
    });

    endTimer();

    const stats = performanceMonitor.getStats("test_operation");
    expect(stats).toBeDefined();
    expect(stats.count).toBe(1);
    expect(stats.average).toBeGreaterThan(0);
  });
});

Debug Test Data

class DebugTestData {
  static createConnectorConfig(overrides: any = {}) {
    return {
      apiUrl: "https://test.api.com",
      apiKey: "test-key",
      timeout: 30000,
      retries: 3,
      ...overrides,
    };
  }

  static createFlowContext(overrides: any = {}) {
    return {
      config: this.createConnectorConfig(),
      tokens: {},
      form: {},
      query: {},
      ...overrides,
    };
  }

  static createErrorScenario(type: string) {
    const scenarios = {
      network: new Error("Network error"),
      auth: new Error("Authentication failed"),
      validation: new Error("Validation failed"),
      timeout: new Error("Request timeout"),
    };

    return (
      scenarios[type as keyof typeof scenarios] || new Error("Unknown error")
    );
  }
}

Debug Tools

Debug Dashboard

import React, { useState, useEffect } from "react";

function DebugDashboard() {
  const [logs, setLogs] = useState([]);
  const [errors, setErrors] = useState([]);
  const [metrics, setMetrics] = useState({});

  useEffect(() => {
    const interval = setInterval(() => {
      setLogs(logger.getLogs());
      setErrors(errorTracker.getErrors());
      setMetrics(performanceMonitor.getMetrics());
    }, 1000);

    return () => clearInterval(interval);
  }, []);

  return (
    <div className="debug-dashboard">
      <h2>Debug Dashboard</h2>

      <div className="debug-section">
        <h3>Logs</h3>
        <div className="logs">
          {logs.map((log, index) => (
            <div key={index} className={`log log-${log.level}`}>
              <span className="timestamp">{log.timestamp}</span>
              <span className="level">{log.level}</span>
              <span className="message">{log.message}</span>
            </div>
          ))}
        </div>
      </div>

      <div className="debug-section">
        <h3>Errors</h3>
        <div className="errors">
          {errors.map((error, index) => (
            <div key={index} className="error">
              <span className="type">{error.type}</span>
              <span className="message">{error.message}</span>
              <span className="timestamp">{error.timestamp}</span>
            </div>
          ))}
        </div>
      </div>

      <div className="debug-section">
        <h3>Performance</h3>
        <div className="metrics">
          {Object.entries(metrics).map(([operation, stats]) => (
            <div key={operation} className="metric">
              <span className="operation">{operation}</span>
              <span className="stats">{JSON.stringify(stats)}</span>
            </div>
          ))}
        </div>
      </div>
    </div>
  );
}

Debug CLI

class DebugCLI {
  private commands: Map<string, Function> = new Map();

  constructor() {
    this.registerCommands();
  }

  private registerCommands() {
    this.commands.set("logs", this.showLogs.bind(this));
    this.commands.set("errors", this.showErrors.bind(this));
    this.commands.set("metrics", this.showMetrics.bind(this));
    this.commands.set("flows", this.showFlows.bind(this));
    this.commands.set("clear", this.clearData.bind(this));
  }

  private showLogs() {
    const logs = logger.getLogs();
    console.table(logs);
  }

  private showErrors() {
    const errors = errorTracker.getErrors();
    console.table(errors);
  }

  private showMetrics() {
    const metrics = performanceMonitor.getMetrics();
    console.table(metrics);
  }

  private showFlows() {
    const flows = flowDebugger.getAllFlowStates();
    console.table(flows);
  }

  private clearData() {
    logger.clearLogs();
    errorTracker.clearErrors();
    flowDebugger.clearAllFlowStates();
    console.log("Debug data cleared");
  }

  execute(command: string) {
    const handler = this.commands.get(command);
    if (handler) {
      handler();
    } else {
      console.log("Unknown command:", command);
      console.log("Available commands:", Array.from(this.commands.keys()));
    }
  }
}

const debugCLI = new DebugCLI();

Best Practices

Debug Configuration

Use environment variables to control debug settings and enable/disable debugging features in different environments.
const debugConfig = {
  enabled: process.env.DEBUG_ENABLED === "true",
  level: process.env.DEBUG_LEVEL || "info",
  logToFile: process.env.DEBUG_LOG_TO_FILE === "true",
  logFile: process.env.DEBUG_LOG_FILE || "debug.log",
  enablePerformanceMonitoring: process.env.DEBUG_PERFORMANCE === "true",
  enableErrorTracking: process.env.DEBUG_ERROR_TRACKING === "true",
};

if (debugConfig.enabled) {
  const logger = new Logger(debugConfig.level);
  const errorTracker = new ErrorTracker();
  const performanceMonitor = new PerformanceMonitor();
}

Debug Data Management

Be careful with debug data in production. Ensure sensitive information is not logged and debug data is properly cleaned up.
class DebugDataManager {
  private maxLogs = 1000;
  private maxErrors = 100;
  private maxMetrics = 500;

  cleanupLogs() {
    const logs = logger.getLogs();
    if (logs.length > this.maxLogs) {
      const excess = logs.length - this.maxLogs;
      logger.clearLogs();
      logger.info(`Cleaned up ${excess} old log entries`);
    }
  }

  cleanupErrors() {
    const errors = errorTracker.getErrors();
    if (errors.length > this.maxErrors) {
      const excess = errors.length - this.maxErrors;
      errorTracker.clearErrors();
      logger.info(`Cleaned up ${excess} old error entries`);
    }
  }

  cleanupMetrics() {
    const metrics = performanceMonitor.getMetrics();
    Object.keys(metrics).forEach((operation) => {
      const operationMetrics = metrics[operation];
      if (operationMetrics.length > this.maxMetrics) {
        const excess = operationMetrics.length - this.maxMetrics;
        operationMetrics.splice(0, excess);
        logger.info(`Cleaned up ${excess} old metrics for ${operation}`);
      }
    });
  }

  cleanupAll() {
    this.cleanupLogs();
    this.cleanupErrors();
    this.cleanupMetrics();
  }
}

const debugDataManager = new DebugDataManager();

// Run cleanup every 5 minutes
setInterval(() => {
  debugDataManager.cleanupAll();
}, 5 * 60 * 1000);

Next Steps

Now that you understand debugging techniques, you can:
  1. Implement Monitoring: Set up comprehensive monitoring and alerting
  2. Add Logging: Implement structured logging throughout your application
  3. Create Debug Tools: Build custom debug tools and dashboards
  4. Optimize Performance: Use profiling data to optimize performance
Continue to the Performance Optimization Guide to learn how to optimize your connectors and flows.
I