{
  "schema_version": "0.6.0",
  "paper_id": "dtracomp-distributed-trace-comparison",
  "page_url": "https://threadslab.org/research-publications/papers/dtracomp-distributed-trace-comparison/",
  "title": "DTraComp: Comparing distributed execution traces for understanding intermittent latency sources",
  "title_variants": [],
  "authors": [
    "Maryam Ekhlasi",
    "Fatemeh Faraji Daneshgar",
    "Michel Dagenais",
    "Maxime Lamothe",
    "Naser Ezzati-Jivan",
    "Matthew Khouzam"
  ],
  "author_details": [
    {
      "name": "Maryam Ekhlasi",
      "orcid": null,
      "profile_url": null
    },
    {
      "name": "Fatemeh Faraji Daneshgar",
      "orcid": null,
      "profile_url": null
    },
    {
      "name": "Michel Dagenais",
      "orcid": null,
      "profile_url": null
    },
    {
      "name": "Maxime Lamothe",
      "orcid": null,
      "profile_url": null
    },
    {
      "name": "Naser Ezzati-Jivan",
      "orcid": "https://orcid.org/0000-0003-1435-6297",
      "profile_url": "https://threadslab.org/"
    },
    {
      "name": "Matthew Khouzam",
      "orcid": null,
      "profile_url": null
    }
  ],
  "publication": {
    "year": 2026,
    "venue": "Journal of Systems and Software",
    "type": "journal article",
    "publication_date": "2026-06-16",
    "online_date": "2026-06-16",
    "print_date": "2026-11",
    "volume": "241",
    "issue": null,
    "pages": "112990",
    "article_number": "112990",
    "publisher": "Elsevier BV",
    "issn": [
      "0164-1212"
    ],
    "isbn": [],
    "crossref_type": "journal-article"
  },
  "publication_type": "research paper",
  "status": "published_with_local_full_text_evidence",
  "canonical_source_url": "https://doi.org/10.1016/j.jss.2026.112990",
  "source_record_id": "dtracomp-comparing-distributed-execution-traces-for-understanding-intermittent-latency-sources-1a772fbb53",
  "identifiers": {
    "doi": "10.1016/j.jss.2026.112990"
  },
  "abstract": null,
  "abstract_source": "Official version-of-record PDF reviewed locally; the publisher abstract is not reproduced in the public catalog because reuse permission was not established.",
  "abstract_available": false,
  "scholar_eligibility": {
    "eligible": false,
    "basis": "not-eligible",
    "note": "The page is a discovery record; it does not claim Google Scholar article-host eligibility."
  },
  "description": "DTraComp is an open-source Eclipse Trace Compass framework that compares groups of distributed requests and attributes span time to user-space, kernel, thread-state, and system-call evidence.",
  "evidence_level": "full-text-reviewed",
  "evidence": {
    "source_basis": "full-text-reviewed",
    "coverage": "material paper sections",
    "summary_origin": "AI-assisted catalog editorial summary",
    "review_status": "catalog-reviewed; paper-author approval pending",
    "verified_on": "2026-08-27",
    "sources": [
      {
        "note": "Local version-of-record PDF reviewed, including the abstract, proposed solution, case studies, evaluation, user study, threats to validity, and conclusion"
      },
      {
        "note": "Local PDF evidence ledger: pdf-evidence/notes/dtracomp-distributed-trace-comparison-jss-2026.md"
      },
      {
        "note": "Journal of Systems and Software article: https://www.sciencedirect.com/science/article/pii/S0164121226002232"
      },
      {
        "note": "DOI metadata: https://doi.org/10.1016/j.jss.2026.112990"
      },
      {
        "note": "The paper identifies the Trace Compass change and code link: https://git.eclipse.org/r/c/tracecompass.incubator/org.eclipse.tracecompass.incubator/+/203508"
      },
      {
        "note": "The paper identifies the replication traces: https://zenodo.org/record/8377067"
      },
      {
        "note": "The paper identifies the demonstration video: https://www.youtube.com/watch?v=X3CfAlWllDc"
      }
    ]
  },
  "summary": {
    "core_contribution": "DTraComp is an open-source Eclipse Trace Compass framework that compares groups of distributed requests and attributes span time to user-space, kernel, thread-state, and system-call evidence.",
    "problem": "Distributed tracing shows where request latency appears in a service graph, but it usually cannot explain whether the delay arose from CPU contention, blocking, I/O, interrupts, or other kernel activity on the participating machines. Existing tools also make it difficult to compare two groups of similar requests automatically, especially when spans execute in parallel across multiple services and hosts.",
    "method": "The system uses double instrumentation: HotROD is instrumented with OpenTracing for high-level spans, and the OpenTracing source is instrumented with LTTng-UST while LTTng also collects kernel events. Span events carry start/end times, span and parent identifiers, distributed trace IDs, process IDs, and thread IDs; separate UST and kernel channels share a unified clock. Eclipse Trace Compass synchronizes traces from multiple machines using its packet-based clock synchronization. Users filter requests by duration and self-time, select comparison groups, build per-request and aggregated flame graphs, and compute an Enhanced Differential Flame Graph using the ratio (tB - tA) / tA. The Metric Extractor attributes each span to User, System Call, Waited Blocked, Waited CPU, Interrupted, and Unknown states and reports durations for system calls such as read, write, futex, sched, openat, mmap, and networking calls.",
    "findings": "The paper evaluates five open-source applications and performance cases: Eclipse Theia, HotROD, TiDB, JFreeChart, and Apache Cassandra. It reports span-attribution differences below 0.1% against Jaeger for the checked examples. For HotROD with 10,000 requests, 10 client threads, and about 51 spans per request, tracing with the selected kernel subset adds less than 9% overhead in snapshot mode and about 10% in standard mode; adding system-call collection raises the reported overhead to below 26% and about 27%, respectively. Metric extraction is reported to scale linearly with trace size, while the differential flame-graph implementation processes traces up to 1.7 GB within seconds and the paper reports filtering/comparison experiments up to 20 GB. The case studies expose TiDB lock contention, CPU contention, Cassandra read-repair resource cost, Eclipse Theia patch effects, and a distributed deadlock. In an exploratory study with eight engineers, localization accuracy was 70.8%, root-cause accuracy was 75%, and median task time was 35 seconds.",
    "limitations": "The evaluation combines five selected applications and structured, pre-captured diagnosis tasks rather than a controlled comparison against alternative tools during live incidents. The user study has only eight participants, 62.5% had prior familiarity with DTraComp, and no inferential tests or controlled baseline condition were used. Clock synchronization, hardware, software versions, network conditions, and manually reproduced application cases can affect the results. The paper also notes that some module evaluations lack an established ground truth, so the measurements support feasibility and diagnostic usefulness rather than universal superiority.",
    "future_work": "The paper proposes richer filtering and group-selection controls, anomaly detection for abnormal traces, and automatic grouping or categorization of performance problems using kernel metrics and system-state durations."
  },
  "tags": [
    "system-tracing",
    "kernel-tracing",
    "trace-analysis",
    "trace-visualization",
    "latency-analysis",
    "root-cause-analysis",
    "microservices",
    "lttng"
  ],
  "keywords": [
    "DTraComp",
    "distributed trace comparison",
    "OpenTracing",
    "LTTng",
    "LTTng-UST",
    "Eclipse Trace Compass",
    "differential flame graph",
    "span-state attribution",
    "system-call attribution",
    "Waited CPU",
    "Waited Blocked",
    "HotROD",
    "TiDB",
    "Apache Cassandra",
    "Eclipse Theia",
    "JFreeChart",
    "Jaeger",
    "microservice performance"
  ],
  "versions": [
    {
      "id": "published-version",
      "label": "Published version",
      "relation": "version-of-record",
      "title": "DTraComp: Comparing distributed execution traces for understanding intermittent latency sources",
      "url": "https://doi.org/10.1016/j.jss.2026.112990",
      "pdf_url": null,
      "status": "published",
      "canonical_for_citation": true
    }
  ],
  "access": {
    "status": "published_with_local_full_text_evidence",
    "note": "The published version was reviewed locally as full-text evidence. The public catalog links the DOI and authoritative article record but does not redistribute the publisher PDF.",
    "license": null
  },
  "resources": {
    "code": "https://git.eclipse.org/r/c/tracecompass.incubator/org.eclipse.tracecompass.incubator/+/203508",
    "data": "https://zenodo.org/record/8377067",
    "slides": null,
    "demo": "https://www.youtube.com/watch?v=X3CfAlWllDc"
  },
  "citation_guidance": {
    "when_to_cite": "Cite DTraComp when comparing distributed request groups and connecting span-level latency differences to cross-machine kernel and system-call causes.",
    "points": [
      "Double instrumentation with OpenTracing, LTTng-UST, and LTTng kernel events.",
      "Eclipse Trace Compass synchronization and span-to-thread/system-state attribution.",
      "Enhanced Differential Flame Graph comparison of request groups with parallel spans.",
      "Five application cases, trace-collection overhead, linear scaling, and the eight-engineer exploratory study."
    ],
    "canonical_version_id": "published-version"
  },
  "provenance": {
    "metadata_verified_on": "2026-08-27",
    "metadata_source": [
      "Local version-of-record PDF reviewed, including the abstract, proposed solution, case studies, evaluation, user study, threats to validity, and conclusion",
      "Local PDF evidence ledger: pdf-evidence/notes/dtracomp-distributed-trace-comparison-jss-2026.md",
      "Journal of Systems and Software article: https://www.sciencedirect.com/science/article/pii/S0164121226002232",
      "DOI metadata: https://doi.org/10.1016/j.jss.2026.112990",
      "The paper identifies the Trace Compass change and code link: https://git.eclipse.org/r/c/tracecompass.incubator/org.eclipse.tracecompass.incubator/+/203508",
      "The paper identifies the replication traces: https://zenodo.org/record/8377067",
      "The paper identifies the demonstration video: https://www.youtube.com/watch?v=X3CfAlWllDc"
    ],
    "summary_written_by": "AI-assisted",
    "summary_verified_by": "full-text-grounded catalog review; author approval pending",
    "linked_preprint_record": null,
    "author_order_note": null
  },
  "batch": {
    "phase": 2,
    "batch_label": "expanded forty-paper release",
    "status": "included_in_expanded_catalog",
    "selected_at": "2026-08-28"
  }
}