{
  "schema_version": "0.6.0",
  "paper_id": "debugging-performance-degradation-distributed-requests",
  "page_url": "https://threadslab.org/research-publications/papers/debugging-performance-degradation-distributed-requests/",
  "title": "Debugging of Performance Degradation in Distributed Requests Handling Using Multilevel Trace Analysis",
  "title_variants": [],
  "authors": [
    "Naser Ezzati-Jivan",
    "Houssem Daoud",
    "Michel R. Dagenais"
  ],
  "author_details": [
    {
      "name": "Naser Ezzati-Jivan",
      "orcid": "https://orcid.org/0000-0003-1435-6297",
      "profile_url": "https://threadslab.org/"
    },
    {
      "name": "Houssem Daoud",
      "orcid": null,
      "profile_url": null
    },
    {
      "name": "Michel R. Dagenais",
      "orcid": null,
      "profile_url": null
    }
  ],
  "publication": {
    "year": 2021,
    "venue": "Wireless Communications and Mobile Computing",
    "type": "journal article",
    "publication_date": "2021-01",
    "online_date": "2021-11-16",
    "print_date": "2021-01",
    "volume": "2021",
    "issue": "1",
    "pages": null,
    "article_number": "8478076",
    "publisher": "Wiley",
    "issn": [
      "1530-8669",
      "1530-8677"
    ],
    "isbn": [],
    "crossref_type": "journal-article"
  },
  "publication_type": "journal article",
  "status": "published_with_public_full_text",
  "canonical_source_url": "https://doi.org/10.1155/2021/8478076",
  "source_record_id": "debugging-of-performance-degradation-in-distributed-requests-handling-using-multilevel-trace-ana-f64bec92e9",
  "identifiers": {
    "doi": "10.1155/2021/8478076"
  },
  "abstract": "Root cause identification of performance degradation within distributed systems is often a difficult and time-consuming task, yet it is crucial for maintaining high performance. In this paper, we present an execution trace-driven solution that reduces the efforts required to investigate, debug, and solve performance problems found in multinode distributed systems. The proposed approach employs a unified analysis method to represent trace data collected from the user-space level to the hardware level of involved nodes, allowing for efficient and effective root cause analysis. This solution works by extracting performance metrics and state information from trace data collected at user-space, kernel, and network levels. The multisource trace data is then synchronized and structured in a multidimensional data store, which is designed specifically for this kind of data. A posteriori analysis using a top-down approach is then used to investigate performance problems and detect their root causes. In this paper, we apply this generic framework to analyze trace data collected from the execution of the web server, database server, and application servers in a distributed LAMP (Linux, Apache, MySQL, and PHP) Stack. Using industrial level use cases, we show that the proposed framework can investigate root causes, address unusual latency, and improve base latency by 70%. This is achieved with minimal tracing overhead that does not significantly impact performance, together with O(log n) query response times for efficient analysis.",
  "abstract_source": "Official Wiley article abstract; Crossref records CC BY 4.0 for the version of record.",
  "abstract_available": true,
  "scholar_eligibility": {
    "eligible": true,
    "basis": "complete-visible-author-abstract",
    "note": "The page displays the complete verified author abstract."
  },
  "description": "Root cause identification of performance degradation within distributed systems is often a difficult and time-consuming task, yet it is crucial for maintaining high performance. In this paper, we present an execution trace-driven solution that reduces the efforts required to investigate, debug, and solve performance problems found in multinode distributed systems. The proposed approach employs a unified analysis method to represent trace data collected from the user-space level to the hardware level of involved nodes, allowing for efficient and effective root cause analysis. This solution works by extracting performance metrics and state information from trace data collected at user-space, kernel, and network levels. The multisource trace data is then synchronized and structured in a multidimensional data store, which is designed specifically for this kind of data. A posteriori analysis using a top-down approach is then used to investigate performance problems and detect their root causes. In this paper, we apply this generic framework to analyze trace data collected from the execution of the web server, database server, and application servers in a distributed LAMP (Linux, Apache, MySQL, and PHP) Stack. Using industrial level use cases, we show that the proposed framework can investigate root causes, address unusual latency, and improve base latency by 70%. This is achieved with minimal tracing overhead that does not significantly impact performance, together with O(log n) query response times for efficient analysis.",
  "evidence_level": "full-text-reviewed",
  "evidence": {
    "source_basis": "full-text-reviewed",
    "coverage": "material paper sections",
    "summary_origin": "AI-assisted catalog editorial summary",
    "review_status": "catalog-reviewed; paper-author approval pending",
    "verified_on": "2026-08-21",
    "sources": [
      {
        "note": "Official Wiley article and abstract: https://onlinelibrary.wiley.com/doi/10.1155/2021/8478076"
      },
      {
        "note": "Crossref license metadata identifies CC BY 4.0: https://api.crossref.org/works/10.1155%2F2021%2F8478076"
      },
      {
        "note": "Full paper evidence ledger and local PDF hash verified in pdf-evidence/notes/debugging-performance-degradation-distributed-requests.md"
      }
    ]
  },
  "summary": {
    "core_contribution": "The paper correlates LTTng traces from user space through kernel, storage, network, and multiple hosts in a disk-backed state model, enabling top-down diagnosis of distributed request latency.",
    "problem": "Performance degradation in distributed requests can arise at user, kernel, storage, network, or remote-service layers, so single-layer tracing does not provide enough causal context.",
    "method": "Instrument LAMP components and system layers with LTTng, synchronize hosts using causal network event pairs and convex-hull matching, maintain multilevel state trees, and perform top-down diagnosis from request latency to system call and kernel/storage/network states.",
    "findings": "The case study localized application-level session/statistics work and disk queue contention from backup activity. The system distinguished cache-hit/miss latency classes and showed that line-level PHP tracing can approximately double response time, while selected UST tracing can approach the no-trace baseline.",
    "limitations": "The evaluation focuses on LAMP and selected instrumented cases; causal synchronization requires suitable event pairs and the available probes determine what can be explained. Network attacks, broader systems, and richer anomaly classes are not evaluated.",
    "future_work": "Add network-attack and host-anomaly analysis, broaden probes and workloads, and improve multilevel visualization and causal coverage."
  },
  "tags": [
    "system-tracing",
    "microservices",
    "performance-analysis",
    "root-cause-analysis",
    "trace-analysis"
  ],
  "keywords": [
    "distributed requests",
    "multilevel trace analysis",
    "LTTng",
    "Apache",
    "PHP",
    "MariaDB",
    "State History Tree",
    "cache hit ratio",
    "disk contention",
    "ApacheBench",
    "Trace Compass"
  ],
  "versions": [
    {
      "id": "published-version",
      "label": "Published version",
      "relation": "version-of-record",
      "title": "Debugging of Performance Degradation in Distributed Requests Handling Using Multilevel Trace Analysis",
      "url": "https://doi.org/10.1155/2021/8478076",
      "pdf_url": null,
      "status": "published",
      "canonical_for_citation": true
    },
    {
      "id": "public-wiley-pdf",
      "label": "Public Wiley PDF",
      "relation": "source-record",
      "title": "Debugging of Performance Degradation in Distributed Requests Handling Using Multilevel Trace Analysis",
      "url": "https://onlinelibrary.wiley.com/doi/pdf/10.1155/2021/8478076",
      "pdf_url": "https://onlinelibrary.wiley.com/doi/pdf/10.1155/2021/8478076",
      "status": "public_full_text",
      "canonical_for_citation": false
    }
  ],
  "access": {
    "status": "published_with_public_full_text",
    "note": "The DOI is the canonical citation target. The external source PDF is linked for readers; no PDF is redistributed in this repository.",
    "license": "CC BY 4.0"
  },
  "resources": {
    "code": "https://github.com/naser/",
    "data": null,
    "slides": null,
    "demo": null
  },
  "citation_guidance": {
    "when_to_cite": "Cite this paper when your work uses or compares top-down multilevel root-cause analysis from distributed request latency to kernel/storage/network state.",
    "points": [
      "For top-down multilevel root-cause analysis from distributed request latency to kernel/storage/network state.",
      "For LTTng instrumentation spanning LAMP user space, system calls, kernel, filesystem, page cache, block layer, disk, and network.",
      "For cross-host causal synchronization using request/response event pairs and convex-hull matching.",
      "For the MediaWiki/WordPress session-check and `backup.sh` disk-queue case studies, including the tracing-cost boundary."
    ],
    "canonical_version_id": "published-version"
  },
  "provenance": {
    "metadata_verified_on": "2026-08-21",
    "metadata_source": [
      "Official Wiley article and abstract: https://onlinelibrary.wiley.com/doi/10.1155/2021/8478076",
      "Crossref license metadata identifies CC BY 4.0: https://api.crossref.org/works/10.1155%2F2021%2F8478076",
      "Full paper evidence ledger and local PDF hash verified in pdf-evidence/notes/debugging-performance-degradation-distributed-requests.md"
    ],
    "summary_written_by": "AI-assisted",
    "summary_verified_by": "full-text-grounded catalog review; author approval pending",
    "linked_preprint_record": null,
    "author_order_note": null
  },
  "batch": {
    "phase": 2,
    "batch_label": "expanded forty-paper release",
    "status": "included_in_expanded_catalog",
    "selected_at": "2026-08-28"
  }
}