{
 "schema": "criogen-compare/1",
 "recorded": "2026-10-21T18:47:32Z",
 "command": "python benches/compare/run.py",
 "quick": false,
 "machine": {
  "cpu": "AMD Ryzen 5 4600H with Radeon Graphics",
  "threads": 12,
  "ram_gb": 15.4,
  "os": "Windows 11"
 },
 "python": "3.13.14",
 "frameworks": {
  "criogen": {
   "label": "Criogen",
   "version": "0.1.0",
   "also": null,
   "model_us": null,
   "notes": null
  },
  "langgraph": {
   "label": "LangGraph",
   "version": "1.2.16",
   "also": {
    "langchain": "1.4.6",
    "langchain-core": "1.6.12"
   },
   "model_us": 144.877,
   "notes": null
  },
  "crewai": {
   "label": "CrewAI",
   "version": "1.15.28",
   "also": null,
   "model_us": 3.371,
   "notes": "Crews run with cache=False so tools run on every run, as in the other frameworks (CrewAI caches tool results by default). Flows are instantiated per run (their state lives on the instance)."
  }
 },
 "scenarios": [
  {
   "id": "agent_call",
   "kind": "speed",
   "title": "Agent, one model call",
   "detail": "One model call, no tools, the model answers at once: the framework's cost per agent turn.",
   "unit": "run"
  },
  {
   "id": "agent_tools",
   "kind": "speed",
   "title": "Agent with 5 tool calls",
   "detail": "The model asks for 5 tools, then answers; model and tools answer at once: the cost of a turn with tools.",
   "unit": "run"
  },
  {
   "id": "agent_latency",
   "kind": "speed",
   "title": "Agent with realistic delays",
   "detail": "As above, with 200 ms per model call and 300 ms of I/O per tool: about 0.7 s if the tools run at once, 1.9 s one by one.",
   "unit": "run"
  },
  {
   "id": "workflow_chain",
   "kind": "speed",
   "title": "Workflow: 10 steps in a row",
   "detail": "Ten trivial Python nodes in a row: the cost per step.",
   "unit": "run"
  },
  {
   "id": "workflow_fanout",
   "kind": "speed",
   "title": "Workflow: 20 parallel nodes",
   "detail": "Twenty Python nodes of 100 ms I/O each, then a join: about 0.1 s if they run at once.",
   "unit": "run"
  },
  {
   "id": "concurrent_1",
   "kind": "load",
   "n": 1,
   "title": "1 agent run",
   "detail": "One run (model 100 ms per call, one tool of 50 ms): the baseline for the load tests.",
   "unit": "batch"
  },
  {
   "id": "concurrent_100",
   "kind": "load",
   "n": 100,
   "title": "100 agent runs at once",
   "detail": "100 runs at the same time in one process: wall time, CPU and memory.",
   "unit": "batch"
  },
  {
   "id": "concurrent_1000",
   "kind": "load",
   "n": 1000,
   "title": "1000 agent runs at once",
   "detail": "1000 runs at the same time in one process: wall time, CPU and memory.",
   "unit": "batch"
  }
 ],
 "reliability_tests": [
  "model_transient",
  "tool_exception",
  "bad_arguments",
  "crash_resume",
  "runaway_loop"
 ],
 "results": [
  {
   "framework": "criogen",
   "status": "ok",
   "runs": 300,
   "median_ms": 0.2903,
   "p95_ms": 0.6873,
   "min_ms": 0.1916,
   "max_ms": 1.045,
   "samples_ms": [
    0.2623,
    0.2142,
    0.3143,
    0.2823,
    0.2666,
    0.2632,
    0.371,
    0.3597,
    0.4735,
    0.362,
    0.3312,
    0.3749,
    0.9157,
    0.777,
    0.4673,
    0.3269,
    0.3388,
    0.3581,
    0.236,
    0.2894,
    0.2953,
    0.3646,
    0.3243,
    0.2706,
    0.262,
    0.2705,
    0.267,
    0.2512,
    0.2729,
    0.2538,
    0.2541,
    0.2258,
    0.6893,
    0.6215,
    0.713,
    0.3995,
    0.2768,
    0.2593,
    0.2575,
    0.2739,
    0.2502,
    0.287,
    0.4079,
    0.363,
    0.3031,
    0.2978,
    0.426,
    0.3339,
    0.2912,
    0.2961,
    0.8034,
    0.7081,
    0.9623,
    0.5123,
    0.2889,
    0.3013,
    0.2632,
    0.3331,
    0.3231,
    0.2965,
    0.3748,
    0.3448,
    0.2372,
    0.2375,
    0.2531,
    0.25,
    0.2453,
    0.4506,
    0.5684,
    0.5752,
    0.4421,
    0.3843,
    0.2868,
    0.233,
    0.2362,
    0.2103,
    0.2134,
    0.2712,
    0.3869,
    0.2772,
    0.2365,
    0.2584,
    0.3136,
    0.2391,
    0.2675,
    0.265,
    0.8209,
    0.8056,
    0.6548,
    0.5249,
    0.3213,
    0.2575,
    0.2826,
    0.2857,
    0.3189,
    0.2387,
    0.2014,
    0.2709,
    0.3501,
    0.2776,
    0.3513,
    0.3731,
    0.3066,
    0.2322,
    0.4079,
    0.6316,
    0.7311,
    0.5059,
    0.3105,
    0.2544,
    0.203,
    0.2701,
    0.3388,
    0.2661,
    0.2353,
    0.3395,
    0.254,
    0.2177,
    0.1916,
    0.312,
    0.3093,
    0.2981,
    0.2966,
    0.7963,
    0.7167,
    0.645,
    0.5324,
    0.4408,
    0.4076,
    0.4797,
    0.4474,
    0.4329,
    0.411,
    0.468,
    0.2599,
    0.2452,
    0.2538,
    0.2375,
    0.3721,
    0.2374,
    0.2303,
    0.209,
    0.2205,
    0.4365,
    0.227,
    0.2068,
    0.2588,
    0.2672,
    0.3669,
    0.3688,
    0.4673,
    0.3547,
    0.3176,
    0.218,
    0.1999,
    0.2019,
    0.3357,
    0.4839,
    0.2393,
    0.2123,
    0.4131,
    0.4894,
    0.4711,
    0.4883,
    0.3332,
    0.224,
    0.2034,
    0.2042,
    0.2074,
    0.1966,
    0.2092,
    0.2035,
    0.1935,
    0.2411,
    0.2234,
    0.21,
    0.2031,
    0.2108,
    0.1985,
    0.2003,
    0.2021,
    0.2034,
    0.2002,
    0.2094,
    0.2161,
    0.2062,
    0.2037,
    0.3362,
    0.321,
    0.3695,
    0.292,
    0.2025,
    0.2599,
    0.215,
    0.2149,
    0.2011,
    0.2039,
    0.2548,
    0.209,
    0.2003
   ],
   "wall_ms": 101.029,
   "cpu_ms": 0.3646,
   "baseline_rss_mb": 30.36,
   "peak_rss_mb": 30.54,
   "scenario": "agent_call"
  },
  {
   "framework": "langgraph",
   "status": "ok",
   "runs": 300,
   "median_ms": 1.2343,
   "p95_ms": 2.2757,
   "min_ms": 1.0162,
   "max_ms": 3.6647,
   "samples_ms": [
    1.1471,
    1.1823,
    1.3637,
    1.1713,
    2.31,
    1.3209,
    1.3259,
    1.2214,
    1.0668,
    1.298,
    1.2295,
    1.1674,
    1.2907,
    1.1694,
    1.1126,
    1.1806,
    1.0752,
    1.3614,
    1.1011,
    1.2056,
    1.5246,
    1.2562,
    1.4241,
    1.1909,
    1.1009,
    1.1706,
    1.2206,
    1.1955,
    1.2577,
    1.1311,
    1.3159,
    1.3997,
    1.0744,
    1.4597,
    1.1327,
    1.2056,
    1.1182,
    1.2019,
    1.3253,
    1.3076,
    1.128,
    1.9905,
    1.4454,
    1.6592,
    1.1545,
    1.1502,
    1.0802,
    1.1246,
    1.7844,
    1.7806,
    1.3742,
    1.3187,
    1.245,
    1.7195,
    1.69,
    1.2698,
    1.4055,
    1.7255,
    1.6528,
    2.057,
    1.1773,
    1.1937,
    1.1467,
    1.2019,
    1.229,
    1.9643,
    1.4214,
    1.1794,
    1.0642,
    1.2836,
    1.3564,
    1.4151,
    1.1415,
    1.0672,
    1.0483,
    1.0305,
    1.1659,
    1.3866,
    1.0892,
    1.1426,
    1.2808,
    1.4497,
    1.3689,
    1.2282,
    1.0528,
    1.0548,
    1.048,
    1.0493,
    1.2982,
    1.3159,
    1.2571,
    2.0262,
    1.1795,
    1.5449,
    1.1432,
    1.0692,
    1.059,
    1.0642,
    1.322,
    1.6399,
    1.967,
    2.5707,
    2.7905,
    1.4186,
    1.1876,
    1.0773,
    1.0644,
    2.4476,
    1.2098,
    1.2677,
    1.2079,
    1.3476,
    1.3312,
    1.1233,
    1.1761,
    1.11,
    1.882,
    1.9233,
    1.1099,
    1.2365,
    1.7967,
    2.5195,
    1.0889,
    1.0315,
    1.0488,
    1.2574,
    1.2903,
    1.2049,
    1.7435,
    2.1956,
    1.2663,
    1.1813,
    1.069,
    1.1753,
    2.3273,
    1.7574,
    1.1575,
    2.2915,
    2.3899,
    1.1539,
    1.1224,
    1.0934,
    2.5762,
    1.5056,
    1.2474,
    1.1223,
    1.5114,
    2.0602,
    1.232,
    1.0816,
    1.0832,
    1.1401,
    1.9192,
    1.1648,
    1.0984,
    1.5107,
    2.2757,
    1.3723,
    1.1017,
    1.0577,
    1.1519,
    2.2622,
    1.3174,
    1.1529,
    1.1337,
    1.3669,
    1.7752,
    1.3093,
    1.0985,
    1.1323,
    1.7478,
    2.11,
    1.4162,
    1.098,
    1.1487,
    3.0903,
    1.1776,
    1.0855,
    1.1518,
    1.6765,
    1.2778,
    1.4057,
    1.2263,
    1.2532,
    1.5809,
    1.1408,
    1.0397,
    1.0563,
    2.073,
    1.5263,
    1.0346,
    1.1468,
    2.0295,
    1.3662,
    1.0898,
    1.0658,
    1.0983,
    1.4631,
    1.479,
    1.2513
   ],
   "wall_ms": 419.106,
   "cpu_ms": 1.3542,
   "baseline_rss_mb": 75.13,
   "peak_rss_mb": 75.2,
   "scenario": "agent_call"
  },
  {
   "framework": "crewai",
   "status": "ok",
   "runs": 300,
   "median_ms": 33.8789,
   "p95_ms": 40.4335,
   "min_ms": 29.3891,
   "max_ms": 62.1177,
   "samples_ms": [
    35.147,
    36.2183,
    32.9227,
    31.4289,
    35.2963,
    32.1834,
    36.5859,
    32.8553,
    34.0956,
    35.0064,
    32.2469,
    33.9967,
    32.6429,
    32.98,
    33.9338,
    32.3623,
    32.9321,
    37.3071,
    33.8881,
    33.5286,
    30.3272,
    32.0096,
    36.716,
    30.0324,
    31.6417,
    31.0436,
    31.1855,
    32.0816,
    36.2685,
    33.8962,
    32.5384,
    33.7228,
    31.8034,
    33.1048,
    31.4858,
    32.8784,
    37.5809,
    38.0009,
    33.3843,
    35.0952,
    34.0269,
    34.9068,
    36.5095,
    36.0043,
    33.3363,
    49.8979,
    34.9232,
    33.5161,
    32.2039,
    30.5503,
    30.8239,
    33.9671,
    31.4702,
    32.557,
    30.9994,
    31.2959,
    37.9627,
    36.9556,
    34.077,
    34.0076,
    33.8024,
    35.4276,
    34.1228,
    36.8647,
    34.4883,
    38.0717,
    33.6642,
    33.8668,
    33.7895,
    33.5975,
    33.9495,
    33.9316,
    36.424,
    33.9291,
    32.6477,
    31.2895,
    60.2085,
    34.5426,
    32.2957,
    32.3942,
    33.3229,
    30.1693,
    35.0733,
    34.5519,
    38.5388,
    34.0739,
    33.0624,
    36.4571,
    33.7791,
    33.3023,
    34.4531,
    33.8076,
    36.5034,
    34.3928,
    37.4087,
    35.3576,
    34.4349,
    33.3752,
    34.4312,
    36.9699,
    34.2125,
    46.8475,
    33.8844,
    31.5726,
    38.8043,
    30.9895,
    41.4202,
    32.0655,
    33.8608,
    35.0241,
    40.4335,
    34.6291,
    31.0979,
    33.4605,
    33.3832,
    32.7971,
    34.4905,
    36.9369,
    37.1266,
    35.5733,
    33.5069,
    34.6266,
    34.8758,
    42.05,
    36.7951,
    36.8422,
    32.3248,
    31.8142,
    31.0118,
    29.918,
    31.1465,
    31.6137,
    40.2883,
    40.5563,
    32.638,
    35.3343,
    37.7089,
    41.0107,
    39.21,
    41.2771,
    46.4014,
    34.1023,
    32.7172,
    33.3766,
    36.9669,
    32.4784,
    34.0279,
    32.5927,
    35.3708,
    31.6815,
    32.7915,
    34.0829,
    33.2027,
    31.8748,
    31.6222,
    35.7729,
    33.8735,
    35.6729,
    31.3687,
    30.4305,
    30.0052,
    30.479,
    35.388,
    33.7567,
    33.5186,
    36.6798,
    33.6233,
    35.5979,
    33.8291,
    34.3214,
    34.7544,
    34.471,
    32.9709,
    35.6311,
    34.4747,
    34.9962,
    35.3769,
    35.6033,
    34.0775,
    37.2843,
    33.7795,
    33.4777,
    30.9851,
    30.8912,
    31.053,
    30.8427,
    34.0057,
    30.3119,
    31.5553,
    36.3603,
    33.7487,
    32.7896,
    35.4191,
    33.4312,
    32.3185,
    34.5969,
    35.0381,
    35.1286,
    32.8212,
    34.235
   ],
   "wall_ms": 10348.271,
   "cpu_ms": 33.2812,
   "baseline_rss_mb": 169.51,
   "peak_rss_mb": 170.26,
   "scenario": "agent_call"
  },
  {
   "framework": "criogen",
   "status": "ok",
   "runs": 200,
   "median_ms": 1.1023,
   "p95_ms": 1.8451,
   "min_ms": 0.8992,
   "max_ms": 2.2332,
   "samples_ms": [
    1.2792,
    1.2948,
    1.0846,
    1.0065,
    1.0249,
    0.9854,
    0.939,
    1.0385,
    1.3039,
    1.0552,
    1.0165,
    1.0182,
    0.9542,
    1.0024,
    0.959,
    1.1793,
    1.3029,
    1.2803,
    1.1086,
    1.0374,
    1.0239,
    1.0103,
    1.0027,
    0.9719,
    0.9519,
    0.971,
    0.9837,
    1.3253,
    1.0692,
    1.0578,
    1.027,
    0.9939,
    0.9806,
    1.0159,
    1.3827,
    1.2589,
    1.6739,
    1.5153,
    1.0817,
    0.9776,
    0.9724,
    0.9843,
    0.9714,
    0.9734,
    1.2282,
    1.485,
    1.3015,
    1.2366,
    0.9709,
    0.9989,
    1.0884,
    1.4588,
    1.165,
    1.3016,
    1.0676,
    0.9847,
    1.0161,
    0.9726,
    0.9492,
    1.0022,
    0.9828,
    1.2341,
    2.1229,
    1.5266,
    1.5338,
    1.1059,
    0.9759,
    1.1037,
    1.4819,
    1.7569,
    1.6539,
    1.4181,
    1.1981,
    1.0209,
    0.9933,
    0.964,
    0.991,
    1.2735,
    1.9173,
    1.1602,
    1.0566,
    0.9982,
    1.2047,
    1.1015,
    1.2521,
    1.5265,
    1.6444,
    1.703,
    1.3839,
    1.023,
    1.0953,
    1.0304,
    1.3089,
    2.0078,
    1.1066,
    0.9828,
    1.7271,
    1.2596,
    1.1092,
    1.2152,
    2.2332,
    1.3179,
    1.2321,
    2.0654,
    1.2469,
    1.1994,
    1.0064,
    1.8522,
    1.198,
    0.9659,
    2.0304,
    1.0318,
    1.1429,
    1.0741,
    1.6018,
    1.8893,
    1.6205,
    1.1337,
    1.1855,
    1.0495,
    1.0466,
    0.9686,
    1.7123,
    1.9674,
    1.2318,
    1.349,
    1.3283,
    1.0126,
    0.9927,
    1.1865,
    1.6727,
    1.4214,
    1.1134,
    0.9936,
    1.1291,
    1.0053,
    0.9233,
    0.9927,
    1.6117,
    1.3219,
    1.1151,
    1.1032,
    1.0002,
    1.0689,
    1.0482,
    0.9402,
    1.2318,
    1.8451,
    1.4456,
    1.2561,
    1.2038,
    1.2719,
    1.3137,
    0.9631,
    1.5157,
    1.4015,
    1.0717,
    1.1062,
    0.927,
    1.0153,
    1.0708,
    1.1057,
    1.0994,
    1.4459,
    1.7482,
    1.424,
    0.9894,
    0.9639,
    1.0621,
    0.956,
    0.9386,
    1.0317,
    1.6046,
    1.2569,
    1.1032,
    1.0659,
    1.0964,
    1.003,
    0.9239,
    1.0132,
    1.4777,
    1.4747,
    1.4435,
    1.1043,
    0.9926,
    1.0119,
    0.9023,
    0.956,
    1.0131,
    1.418,
    1.4366,
    1.0786,
    1.0899,
    1.0401,
    1.0083,
    0.9182,
    0.8992,
    1.0156,
    1.7201,
    1.8506
   ],
   "wall_ms": 242.264,
   "cpu_ms": 1.4844,
   "baseline_rss_mb": 31.04,
   "peak_rss_mb": 31.3,
   "scenario": "agent_tools"
  },
  {
   "framework": "langgraph",
   "status": "ok",
   "runs": 200,
   "median_ms": 7.6365,
   "p95_ms": 9.1865,
   "min_ms": 6.3144,
   "max_ms": 13.0026,
   "samples_ms": [
    8.0616,
    7.7503,
    7.5914,
    6.9618,
    7.8699,
    7.8574,
    7.2043,
    7.8246,
    7.9191,
    7.8426,
    8.8548,
    7.6417,
    7.5361,
    7.9475,
    7.8748,
    7.7055,
    7.1662,
    6.6731,
    7.9482,
    7.2533,
    9.1865,
    7.5367,
    7.33,
    9.3369,
    7.9001,
    8.1493,
    6.6978,
    8.066,
    8.8122,
    7.9587,
    6.9823,
    8.439,
    7.292,
    8.4699,
    7.4817,
    6.7337,
    8.5505,
    7.5337,
    10.1218,
    8.4533,
    7.2225,
    7.8248,
    8.0452,
    8.5463,
    7.623,
    7.5494,
    7.5609,
    8.2299,
    8.8252,
    8.4267,
    6.915,
    7.7523,
    7.6308,
    8.9869,
    8.0321,
    6.5707,
    8.3452,
    7.8953,
    7.9401,
    7.074,
    6.7771,
    8.3311,
    7.7087,
    6.6896,
    7.2341,
    7.0292,
    8.6594,
    7.6314,
    6.9674,
    7.933,
    7.7022,
    7.3221,
    8.099,
    7.7346,
    6.8579,
    7.0545,
    6.6238,
    7.1104,
    6.7422,
    7.0382,
    7.1491,
    7.307,
    7.6883,
    7.147,
    8.1323,
    6.9057,
    8.1393,
    6.6907,
    6.9025,
    7.3032,
    6.9269,
    7.2354,
    7.6421,
    7.5854,
    7.4156,
    6.9651,
    7.6455,
    7.7001,
    6.6172,
    6.9961,
    7.3896,
    6.8455,
    7.0272,
    7.1836,
    6.6505,
    6.8556,
    7.838,
    7.068,
    7.5707,
    7.131,
    7.0072,
    6.6785,
    7.0279,
    8.3165,
    10.344,
    13.0026,
    10.7149,
    9.718,
    10.0386,
    8.4072,
    8.9856,
    6.8633,
    6.6502,
    6.9533,
    7.2236,
    7.8153,
    7.2079,
    7.6685,
    7.9805,
    7.2415,
    6.6714,
    7.5493,
    8.2228,
    7.4806,
    6.8618,
    7.6956,
    7.5734,
    7.3888,
    6.7419,
    8.0637,
    8.9582,
    7.5111,
    8.2578,
    7.0896,
    9.57,
    7.4216,
    7.2443,
    6.9102,
    7.8753,
    7.7533,
    7.6436,
    6.5513,
    7.3379,
    7.8019,
    7.5378,
    7.5024,
    7.7298,
    8.411,
    8.3506,
    7.6993,
    6.5986,
    7.8731,
    7.99,
    8.2816,
    7.2121,
    7.5117,
    8.5294,
    8.6823,
    8.9551,
    6.8742,
    8.3905,
    8.0057,
    8.0613,
    8.1171,
    7.0667,
    7.9872,
    7.8042,
    7.9074,
    6.3144,
    7.9032,
    7.5104,
    8.2917,
    7.1949,
    7.7922,
    8.3539,
    9.7714,
    9.52,
    7.8473,
    7.2436,
    7.4661,
    7.6547,
    8.0766,
    7.1462,
    6.7767,
    7.9148,
    8.1382,
    7.5553,
    7.2056,
    7.3952,
    7.3236
   ],
   "wall_ms": 1544.096,
   "cpu_ms": 7.9688,
   "baseline_rss_mb": 75.65,
   "peak_rss_mb": 75.95,
   "scenario": "agent_tools"
  },
  {
   "framework": "crewai",
   "status": "ok",
   "runs": 200,
   "median_ms": 45.7854,
   "p95_ms": 52.166,
   "min_ms": 39.9155,
   "max_ms": 133.6195,
   "samples_ms": [
    50.1239,
    51.4871,
    50.3621,
    46.0382,
    75.3568,
    48.3199,
    43.141,
    44.0168,
    43.198,
    47.2008,
    45.4487,
    44.6022,
    40.3588,
    43.5905,
    46.9562,
    40.2498,
    49.207,
    40.541,
    44.9014,
    42.9844,
    42.303,
    50.7524,
    47.4209,
    45.1314,
    45.1768,
    44.4099,
    43.0297,
    46.7118,
    46.4676,
    49.7599,
    45.3371,
    45.0335,
    45.3952,
    45.6654,
    47.0205,
    43.5982,
    49.9147,
    46.3439,
    46.8217,
    46.3977,
    47.1279,
    43.5985,
    40.8602,
    39.9155,
    41.8954,
    44.72,
    43.1699,
    41.8336,
    40.9102,
    42.2264,
    40.78,
    46.2094,
    47.7566,
    50.2618,
    43.3008,
    47.5035,
    50.1324,
    41.1478,
    46.2837,
    42.3189,
    41.9239,
    43.3896,
    43.463,
    43.3661,
    45.7871,
    43.3923,
    44.492,
    42.8008,
    43.8177,
    44.9032,
    45.2688,
    45.9228,
    43.3591,
    47.2002,
    41.885,
    41.679,
    42.6846,
    50.7935,
    40.8149,
    133.6195,
    50.5872,
    48.8714,
    50.374,
    49.0305,
    49.1459,
    44.9367,
    49.1824,
    45.8304,
    46.9625,
    48.9061,
    46.6557,
    57.8676,
    49.6242,
    47.7872,
    45.0743,
    44.6659,
    45.4261,
    55.1574,
    43.8142,
    45.7837,
    48.0747,
    47.075,
    44.9259,
    44.4716,
    44.6272,
    41.3892,
    41.4414,
    43.0389,
    45.417,
    43.8208,
    53.6014,
    48.0022,
    48.5283,
    50.2184,
    57.9744,
    46.1302,
    47.5418,
    47.5217,
    46.4761,
    49.787,
    51.8277,
    50.5318,
    41.774,
    40.5927,
    45.2236,
    47.2752,
    46.9035,
    52.166,
    45.4187,
    43.4304,
    44.7759,
    43.9782,
    40.7998,
    40.9919,
    40.7826,
    40.8731,
    43.2237,
    51.7928,
    43.3537,
    42.2115,
    41.3316,
    46.2095,
    57.3476,
    54.2083,
    47.3636,
    48.8075,
    49.408,
    49.652,
    45.9642,
    46.6514,
    45.5159,
    50.7659,
    44.5488,
    47.9232,
    41.2168,
    45.6007,
    49.2957,
    46.2467,
    49.0748,
    47.4982,
    63.3318,
    44.8892,
    43.0521,
    45.8106,
    54.0824,
    48.2879,
    45.2977,
    41.9676,
    45.6225,
    40.8997,
    48.421,
    48.3086,
    49.4085,
    46.3305,
    45.9077,
    48.1564,
    47.6329,
    48.0403,
    48.1763,
    48.9188,
    45.8889,
    44.8539,
    46.5534,
    46.3955,
    45.1444,
    46.1684,
    49.8408,
    45.7346,
    50.0947,
    50.2935,
    48.1307,
    44.0852,
    41.4827,
    41.2745,
    42.381,
    41.8915,
    44.6097,
    42.2766,
    40.5039,
    41.6872
   ],
   "wall_ms": 9323.694,
   "cpu_ms": 45.3125,
   "baseline_rss_mb": 170.05,
   "peak_rss_mb": 171.34,
   "scenario": "agent_tools"
  },
  {
   "framework": "criogen",
   "status": "ok",
   "runs": 5,
   "median_ms": 705.6405,
   "p95_ms": 711.9115,
   "min_ms": 704.3206,
   "max_ms": 711.9115,
   "samples_ms": [
    705.5907,
    704.3206,
    705.6405,
    711.9115,
    705.985
   ],
   "wall_ms": 3533.46,
   "cpu_ms": 3.125,
   "baseline_rss_mb": 31.01,
   "peak_rss_mb": 31.09,
   "scenario": "agent_latency"
  },
  {
   "framework": "langgraph",
   "status": "ok",
   "runs": 5,
   "median_ms": 710.7518,
   "p95_ms": 712.8371,
   "min_ms": 709.1358,
   "max_ms": 712.8371,
   "samples_ms": [
    710.7518,
    709.1358,
    712.8371,
    709.9664,
    711.3148
   ],
   "wall_ms": 3554.017,
   "cpu_ms": 0.0,
   "baseline_rss_mb": 75.4,
   "peak_rss_mb": 75.67,
   "scenario": "agent_latency"
  },
  {
   "framework": "crewai",
   "status": "ok",
   "runs": 5,
   "median_ms": 765.2484,
   "p95_ms": 777.6733,
   "min_ms": 758.7641,
   "max_ms": 777.6733,
   "samples_ms": [
    776.5273,
    764.7071,
    765.2484,
    777.6733,
    758.7641
   ],
   "wall_ms": 3842.931,
   "cpu_ms": 56.25,
   "baseline_rss_mb": 169.9,
   "peak_rss_mb": 170.48,
   "scenario": "agent_latency"
  },
  {
   "framework": "criogen",
   "status": "ok",
   "runs": 300,
   "median_ms": 0.8216,
   "p95_ms": 1.6443,
   "min_ms": 0.6102,
   "max_ms": 7.0106,
   "samples_ms": [
    1.6819,
    1.0322,
    0.8913,
    0.6683,
    0.7357,
    0.8255,
    0.6392,
    0.6267,
    1.4423,
    1.0917,
    0.7617,
    0.6245,
    0.7666,
    0.6257,
    0.7675,
    0.7679,
    1.8588,
    1.4758,
    0.9855,
    0.7042,
    0.7083,
    0.7765,
    0.7383,
    1.5957,
    0.9191,
    0.812,
    0.7896,
    0.6923,
    0.846,
    0.7914,
    1.7005,
    1.1593,
    0.7734,
    0.8087,
    0.6773,
    0.7685,
    0.8248,
    1.5502,
    1.036,
    0.8267,
    0.6864,
    0.8897,
    0.7157,
    0.6418,
    1.1379,
    1.9012,
    0.9371,
    1.2343,
    1.0474,
    1.0493,
    1.5206,
    1.8283,
    1.0284,
    0.767,
    1.4937,
    2.1616,
    1.0071,
    1.0586,
    0.7523,
    0.7157,
    0.7432,
    0.98,
    1.3017,
    0.9162,
    0.8594,
    0.8436,
    0.8502,
    0.7032,
    1.168,
    1.257,
    0.9341,
    0.7819,
    0.6648,
    0.6648,
    0.8053,
    0.8557,
    1.3612,
    0.9493,
    1.3339,
    1.7874,
    0.8053,
    1.048,
    1.1855,
    0.9499,
    1.1898,
    1.129,
    0.8801,
    0.7152,
    1.53,
    0.9059,
    0.7231,
    0.8147,
    7.0106,
    2.0465,
    1.9133,
    1.2816,
    1.1507,
    1.1478,
    1.1546,
    1.4347,
    1.2654,
    1.3275,
    1.0017,
    1.11,
    1.0694,
    1.3474,
    2.2073,
    1.0918,
    0.7919,
    0.8307,
    0.6603,
    0.69,
    0.7296,
    0.6362,
    0.929,
    0.772,
    0.7154,
    0.6632,
    0.6473,
    0.6808,
    0.657,
    0.8279,
    0.865,
    0.663,
    0.6411,
    0.6933,
    0.7852,
    0.6681,
    0.6518,
    0.7745,
    0.6586,
    0.7213,
    0.7758,
    0.6921,
    0.6313,
    0.643,
    0.7214,
    0.6406,
    0.8628,
    0.6858,
    0.9222,
    0.9511,
    0.8558,
    1.3134,
    0.8222,
    0.7354,
    0.8934,
    1.3052,
    0.8643,
    0.7842,
    0.6313,
    0.6752,
    1.4431,
    1.0896,
    1.1987,
    0.915,
    0.9286,
    0.6934,
    0.9418,
    0.6979,
    0.7393,
    0.7694,
    0.668,
    0.7344,
    1.7539,
    0.6978,
    0.8166,
    0.836,
    0.7196,
    0.6725,
    0.9634,
    1.617,
    1.1422,
    0.8138,
    0.6396,
    0.6769,
    1.0649,
    0.8562,
    1.247,
    1.4261,
    0.8728,
    0.7764,
    0.6551,
    0.7635,
    0.6246,
    0.7312,
    1.2095,
    1.0583,
    0.7398,
    0.8865,
    0.6399,
    0.6988,
    0.6969,
    0.6813,
    0.964,
    1.4233,
    1.1135,
    0.7562,
    0.7889,
    0.6569
   ],
   "wall_ms": 287.351,
   "cpu_ms": 0.8333,
   "baseline_rss_mb": 28.57,
   "peak_rss_mb": 29.99,
   "scenario": "workflow_chain"
  },
  {
   "framework": "langgraph",
   "status": "ok",
   "runs": 300,
   "median_ms": 6.3953,
   "p95_ms": 8.1547,
   "min_ms": 4.2711,
   "max_ms": 14.1344,
   "samples_ms": [
    7.4278,
    6.0422,
    4.9712,
    5.9944,
    5.2815,
    6.9961,
    6.2168,
    7.8798,
    6.5162,
    7.482,
    6.8218,
    14.1344,
    6.4392,
    6.1744,
    5.7165,
    6.0483,
    4.9659,
    6.0029,
    5.6036,
    6.5857,
    6.379,
    7.0287,
    6.3624,
    5.7796,
    6.98,
    6.3237,
    7.2639,
    5.3076,
    7.6975,
    5.2207,
    9.2141,
    6.6366,
    7.6646,
    6.6933,
    6.7479,
    6.8722,
    7.1029,
    7.9469,
    7.5546,
    7.0716,
    7.5971,
    6.9931,
    8.2279,
    7.4349,
    7.283,
    6.7754,
    6.7884,
    7.314,
    6.0914,
    7.0389,
    5.5424,
    6.2496,
    5.645,
    5.5787,
    4.8943,
    4.8535,
    5.1042,
    5.4087,
    6.0406,
    5.1659,
    6.2069,
    4.8202,
    4.6897,
    5.6924,
    5.1607,
    5.2558,
    6.3659,
    6.0825,
    4.7615,
    7.4202,
    7.0448,
    6.3529,
    5.507,
    6.5082,
    5.484,
    6.2277,
    5.626,
    4.8113,
    5.9118,
    4.6965,
    4.648,
    5.568,
    5.341,
    5.2719,
    5.4812,
    6.0457,
    5.5734,
    6.3835,
    6.6234,
    6.0712,
    7.1679,
    6.2283,
    6.2211,
    4.907,
    4.8396,
    6.2003,
    5.137,
    5.5138,
    5.5348,
    5.3765,
    4.6115,
    6.7511,
    5.1172,
    6.4125,
    7.2888,
    5.8964,
    6.3566,
    4.9737,
    6.9152,
    6.19,
    5.0872,
    7.2686,
    5.596,
    6.3147,
    5.9039,
    5.7768,
    12.0369,
    6.8885,
    7.6202,
    5.8044,
    5.0741,
    6.2039,
    4.7313,
    6.1835,
    7.087,
    6.7402,
    7.9817,
    7.9099,
    6.3811,
    4.6876,
    4.6762,
    6.5723,
    4.6211,
    5.8974,
    6.9013,
    7.5837,
    8.2859,
    8.7169,
    8.518,
    7.6088,
    7.5947,
    6.4279,
    7.4618,
    5.8311,
    6.7691,
    4.6079,
    5.8039,
    5.9439,
    4.7086,
    6.6932,
    6.4885,
    7.1048,
    5.5145,
    6.8981,
    6.6323,
    7.1331,
    6.8117,
    6.7969,
    6.6245,
    7.4469,
    6.9649,
    6.783,
    7.6515,
    7.4025,
    7.2062,
    8.1288,
    6.6204,
    8.9674,
    7.3783,
    6.483,
    7.5208,
    5.9288,
    7.7842,
    5.8302,
    6.8308,
    7.1917,
    7.3765,
    5.9865,
    7.6016,
    6.8194,
    8.4458,
    7.0039,
    7.4822,
    6.8359,
    7.9874,
    7.5753,
    6.4097,
    7.0367,
    6.7307,
    6.896,
    7.7089,
    6.6481,
    7.3115,
    6.2006,
    6.7372,
    5.7276,
    7.1609,
    8.3956,
    6.504,
    6.325
   ],
   "wall_ms": 1933.314,
   "cpu_ms": 6.6667,
   "baseline_rss_mb": 74.71,
   "peak_rss_mb": 75.21,
   "scenario": "workflow_chain"
  },
  {
   "framework": "crewai",
   "status": "ok",
   "runs": 300,
   "median_ms": 49.0269,
   "p95_ms": 61.8212,
   "min_ms": 38.3892,
   "max_ms": 90.953,
   "samples_ms": [
    54.5878,
    59.322,
    60.9486,
    58.3012,
    56.7612,
    54.3928,
    52.0876,
    49.0767,
    47.7442,
    64.5405,
    57.988,
    63.8357,
    59.9705,
    51.5801,
    58.0611,
    56.2655,
    66.1492,
    57.9996,
    60.5735,
    53.3652,
    63.7327,
    56.0587,
    52.8988,
    58.3306,
    51.428,
    48.5398,
    56.335,
    50.9672,
    54.1742,
    53.3781,
    56.1893,
    53.0917,
    52.9103,
    54.7326,
    58.3766,
    53.1299,
    55.6307,
    54.6894,
    51.7499,
    55.2907,
    55.3745,
    52.4096,
    50.9183,
    50.2761,
    51.8882,
    52.9436,
    53.4109,
    54.8835,
    56.0521,
    54.3462,
    54.853,
    64.1541,
    57.3527,
    59.2126,
    52.9342,
    54.4593,
    51.7574,
    53.3901,
    51.4793,
    53.1891,
    50.6417,
    50.9293,
    57.2431,
    52.3851,
    50.3752,
    55.9182,
    52.2951,
    57.3786,
    53.388,
    55.8118,
    64.8555,
    48.1942,
    51.7799,
    51.4593,
    52.2888,
    46.7468,
    47.3736,
    45.5541,
    43.9916,
    63.6542,
    47.2529,
    60.2423,
    49.8415,
    47.5976,
    53.0358,
    52.7067,
    44.7927,
    42.0752,
    61.8212,
    46.6719,
    43.5647,
    46.6096,
    46.1139,
    46.2646,
    41.8217,
    38.3892,
    41.5513,
    47.6384,
    60.2308,
    41.4439,
    43.8095,
    41.8663,
    53.6342,
    44.2342,
    42.023,
    53.6658,
    41.897,
    43.1434,
    44.1433,
    54.8534,
    44.9685,
    42.382,
    43.0642,
    42.3186,
    41.3096,
    43.4512,
    41.8211,
    40.9335,
    40.8807,
    42.6487,
    51.0787,
    47.6521,
    58.8347,
    57.5669,
    45.6872,
    53.5825,
    56.718,
    43.6736,
    43.019,
    52.2583,
    44.2009,
    43.5136,
    47.2853,
    45.8456,
    42.0039,
    38.9084,
    51.4333,
    41.7915,
    39.5691,
    39.2724,
    49.0237,
    51.2233,
    53.1338,
    44.9185,
    45.6484,
    48.6693,
    44.6857,
    43.9778,
    47.2579,
    48.2739,
    53.8995,
    42.3866,
    51.6987,
    45.3831,
    40.3861,
    42.4113,
    49.1736,
    45.1335,
    40.495,
    39.7034,
    45.7959,
    47.7659,
    56.6761,
    44.2559,
    60.6717,
    90.953,
    51.0804,
    56.4631,
    49.0301,
    46.1623,
    50.0045,
    48.1075,
    44.0509,
    43.0263,
    55.1128,
    46.7225,
    42.6132,
    45.6924,
    44.483,
    53.166,
    52.603,
    45.4887,
    45.4172,
    44.8479,
    45.495,
    48.0531,
    44.8781,
    42.563,
    41.1158,
    44.3443,
    46.499,
    42.7784,
    42.0172,
    55.0393,
    43.7367,
    42.4039,
    39.8696,
    43.4527,
    45.5613,
    44.9131
   ],
   "wall_ms": 14967.148,
   "cpu_ms": 53.9583,
   "baseline_rss_mb": 167.73,
   "peak_rss_mb": 220.02,
   "scenario": "workflow_chain"
  },
  {
   "framework": "criogen",
   "status": "ok",
   "runs": 5,
   "median_ms": 102.2787,
   "p95_ms": 102.6047,
   "min_ms": 102.0151,
   "max_ms": 102.6047,
   "samples_ms": [
    102.2787,
    102.6047,
    102.3022,
    102.087,
    102.0151
   ],
   "wall_ms": 511.295,
   "cpu_ms": 9.375,
   "baseline_rss_mb": 28.54,
   "peak_rss_mb": 30.77,
   "scenario": "workflow_fanout"
  },
  {
   "framework": "langgraph",
   "status": "ok",
   "runs": 5,
   "median_ms": 207.9494,
   "p95_ms": 211.1287,
   "min_ms": 206.8489,
   "max_ms": 211.1287,
   "samples_ms": [
    211.1287,
    207.9494,
    208.2367,
    206.8489,
    207.1981
   ],
   "wall_ms": 1041.368,
   "cpu_ms": 12.5,
   "baseline_rss_mb": 74.63,
   "peak_rss_mb": 75.75,
   "scenario": "workflow_fanout"
  },
  {
   "framework": "crewai",
   "status": "ok",
   "runs": 5,
   "median_ms": 239.6016,
   "p95_ms": 242.8531,
   "min_ms": 236.9528,
   "max_ms": 242.8531,
   "samples_ms": [
    240.5293,
    237.5579,
    239.6016,
    236.9528,
    242.8531
   ],
   "wall_ms": 1197.506,
   "cpu_ms": 125.0,
   "baseline_rss_mb": 167.61,
   "peak_rss_mb": 219.35,
   "scenario": "workflow_fanout"
  },
  {
   "framework": "criogen",
   "status": "ok",
   "runs": 1,
   "median_ms": 254.4154,
   "p95_ms": 254.4154,
   "min_ms": 254.4154,
   "max_ms": 254.4154,
   "samples_ms": [
    254.4154
   ],
   "wall_ms": 254.526,
   "cpu_ms": 0.0,
   "baseline_rss_mb": 30.75,
   "peak_rss_mb": 30.84,
   "rss_per_run_kb": 88.0,
   "batch_walls_ms": [
    254.023,
    254.526,
    255.112
   ],
   "scenario": "concurrent_1"
  },
  {
   "framework": "langgraph",
   "status": "ok",
   "runs": 1,
   "median_ms": 259.5786,
   "p95_ms": 259.5786,
   "min_ms": 259.5786,
   "max_ms": 259.5786,
   "samples_ms": [
    259.5786
   ],
   "wall_ms": 259.678,
   "cpu_ms": 0.0,
   "baseline_rss_mb": 74.2,
   "peak_rss_mb": 74.27,
   "rss_per_run_kb": 68.0,
   "batch_walls_ms": [
    257.095,
    259.678,
    264.079
   ],
   "scenario": "concurrent_1"
  },
  {
   "framework": "crewai",
   "status": "ok",
   "runs": 1,
   "median_ms": 321.3281,
   "p95_ms": 321.3281,
   "min_ms": 321.3281,
   "max_ms": 321.3281,
   "samples_ms": [
    321.3281
   ],
   "wall_ms": 321.429,
   "cpu_ms": 93.75,
   "baseline_rss_mb": 169.63,
   "peak_rss_mb": 170.32,
   "rss_per_run_kb": 700.0,
   "batch_walls_ms": [
    320.242,
    321.429,
    323.401
   ],
   "scenario": "concurrent_1"
  },
  {
   "framework": "criogen",
   "status": "ok",
   "runs": 100,
   "median_ms": 270.8788,
   "p95_ms": 272.1699,
   "min_ms": 256.9776,
   "max_ms": 272.4426,
   "samples_ms": [
    262.0237,
    263.9952,
    263.9718,
    263.9319,
    262.9264,
    260.4478,
    262.9322,
    261.8143,
    263.7354,
    262.7896,
    261.706,
    261.6667,
    257.4462,
    260.1218,
    260.0761,
    260.0086,
    260.0676,
    258.0668,
    262.466,
    261.3976,
    257.2062,
    257.1622,
    262.4216,
    263.174,
    272.0517,
    272.4426,
    272.3369,
    272.1937,
    272.3509,
    271.9792,
    272.1177,
    272.1579,
    262.1626,
    262.1669,
    272.0116,
    271.8618,
    272.1699,
    272.2011,
    271.9966,
    272.0804,
    271.81,
    272.0799,
    260.8382,
    257.3934,
    272.0489,
    271.6396,
    271.9814,
    271.7685,
    271.518,
    271.5523,
    271.5351,
    271.6509,
    262.459,
    261.6145,
    260.5899,
    271.6901,
    261.6059,
    271.7948,
    262.5119,
    271.7367,
    271.7309,
    271.5176,
    271.3705,
    262.3945,
    256.9776,
    260.3761,
    261.4149,
    261.3579,
    271.1803,
    271.1896,
    262.0862,
    271.049,
    271.0117,
    271.1465,
    271.3408,
    270.9484,
    258.723,
    270.9742,
    270.9586,
    270.8437,
    270.9677,
    261.8661,
    270.8184,
    270.9439,
    270.9907,
    270.8901,
    258.5453,
    270.9695,
    271.2018,
    270.8967,
    270.9877,
    270.8731,
    270.7748,
    270.7899,
    271.0507,
    270.6918,
    270.8845,
    261.7272,
    270.9498,
    260.7772
   ],
   "wall_ms": 273.952,
   "cpu_ms": 140.625,
   "baseline_rss_mb": 30.59,
   "peak_rss_mb": 40.01,
   "rss_per_run_kb": 96.48,
   "batch_walls_ms": [
    273.893,
    273.952,
    274.089
   ],
   "scenario": "concurrent_100"
  },
  {
   "framework": "langgraph",
   "status": "ok",
   "runs": 100,
   "median_ms": 509.5747,
   "p95_ms": 615.1379,
   "min_ms": 383.1818,
   "max_ms": 625.885,
   "samples_ms": [
    397.9984,
    396.4041,
    399.3229,
    394.7907,
    393.5752,
    392.7281,
    391.679,
    390.7386,
    393.1244,
    393.4985,
    392.6278,
    391.8439,
    391.0154,
    390.1446,
    384.2324,
    383.1818,
    406.0272,
    430.876,
    430.323,
    429.8421,
    429.301,
    428.8338,
    428.915,
    428.257,
    427.9008,
    427.6362,
    427.184,
    426.6924,
    426.1057,
    425.5,
    426.2774,
    425.772,
    448.9646,
    452.3431,
    452.4792,
    452.8953,
    456.6242,
    456.1617,
    467.5688,
    467.0916,
    466.4017,
    465.9204,
    465.4,
    464.9319,
    464.2265,
    463.5445,
    463.3049,
    467.9377,
    492.9421,
    494.5517,
    531.6117,
    525.1371,
    524.5976,
    530.1026,
    536.2478,
    535.4046,
    534.8759,
    534.3832,
    533.4669,
    533.046,
    532.5381,
    532.0919,
    531.583,
    530.7826,
    531.9383,
    534.5727,
    534.2472,
    540.8531,
    541.2845,
    540.7891,
    547.9975,
    547.4607,
    547.037,
    546.5419,
    546.072,
    546.2502,
    545.765,
    550.1648,
    550.5396,
    553.8885,
    575.6138,
    579.3193,
    590.7107,
    605.983,
    589.3846,
    588.7738,
    615.6264,
    615.1379,
    614.609,
    614.1644,
    613.6639,
    613.1593,
    612.534,
    612.7861,
    612.3178,
    611.861,
    617.2235,
    621.4642,
    624.2044,
    625.885
   ],
   "wall_ms": 694.43,
   "cpu_ms": 390.625,
   "baseline_rss_mb": 75.3,
   "peak_rss_mb": 81.23,
   "rss_per_run_kb": 60.68,
   "batch_walls_ms": [
    681.217,
    694.43,
    707.23
   ],
   "scenario": "concurrent_100"
  },
  {
   "framework": "crewai",
   "status": "ok",
   "runs": 100,
   "median_ms": 4431.6708,
   "p95_ms": 5249.3348,
   "min_ms": 3033.4558,
   "max_ms": 5788.3389,
   "samples_ms": [
    5036.112,
    5074.8535,
    5444.5048,
    5204.7788,
    4769.4532,
    5249.3348,
    4790.6568,
    5073.6442,
    5068.3811,
    5005.1951,
    5619.1507,
    5242.7888,
    5788.3389,
    4999.0581,
    4680.2246,
    4797.3411,
    4935.6192,
    4538.4225,
    5758.3545,
    4824.2804,
    4391.3003,
    5187.5683,
    4906.2912,
    4476.2289,
    5423.952,
    4757.258,
    4964.4806,
    5245.4371,
    4360.6467,
    4398.825,
    4351.5384,
    4613.8987,
    4606.0749,
    4303.6024,
    4481.0992,
    4646.2952,
    5016.7163,
    4682.9854,
    4553.2105,
    3907.8713,
    4880.5943,
    4413.1375,
    4757.0033,
    4457.7561,
    4824.6171,
    4716.7399,
    4565.7723,
    3781.5666,
    4545.479,
    4471.3654,
    4511.3094,
    4281.5157,
    4240.3736,
    4606.5115,
    4428.177,
    4231.0845,
    4334.6917,
    3721.6654,
    4233.963,
    4180.7115,
    4125.9388,
    4431.1151,
    4639.123,
    4432.2266,
    4633.6698,
    4344.1061,
    4415.9403,
    4352.0976,
    3248.7131,
    4423.113,
    4334.9862,
    4566.6074,
    4407.8522,
    4293.3078,
    4415.1603,
    4516.7974,
    4469.1041,
    4169.215,
    4292.7609,
    4375.3198,
    3147.8103,
    4278.427,
    4233.6155,
    4397.0027,
    4283.932,
    4132.8015,
    4357.5602,
    4350.174,
    4247.4786,
    4182.4347,
    4237.4765,
    4151.2168,
    4243.6558,
    3033.4558,
    4246.1126,
    4236.7747,
    4175.4232,
    4204.2683,
    4108.5728,
    4172.6668
   ],
   "wall_ms": 6728.309,
   "cpu_ms": 5250.0,
   "baseline_rss_mb": 169.86,
   "peak_rss_mb": 187.52,
   "rss_per_run_kb": 180.88,
   "batch_walls_ms": [
    6674.1,
    6728.309,
    6776.169
   ],
   "scenario": "concurrent_100"
  },
  {
   "framework": "criogen",
   "status": "ok",
   "runs": 1000,
   "median_ms": 333.0686,
   "p95_ms": 395.1957,
   "min_ms": 254.0933,
   "max_ms": 404.8186,
   "samples_ms": [
    261.2015,
    261.1891,
    256.3557,
    264.8891,
    259.4399,
    260.4662,
    264.7208,
    277.9731,
    261.5789,
    277.7343,
    263.9533,
    258.6456,
    277.7573,
    284.6422,
    276.8964,
    276.7571,
    276.9003,
    284.0016,
    294.5876,
    316.7965,
    294.5071,
    276.4661,
    283.6624,
    291.3727,
    294.0892,
    275.895,
    302.5242,
    277.528,
    275.5722,
    295.3166,
    293.4829,
    287.0128,
    285.0936,
    296.0905,
    282.2587,
    290.2022,
    301.5741,
    282.9458,
    284.4566,
    274.3672,
    274.1925,
    281.7236,
    318.7547,
    318.9378,
    300.791,
    343.1503,
    326.808,
    326.5326,
    324.166,
    336.3564,
    378.6844,
    350.9556,
    344.0673,
    339.8767,
    357.0976,
    325.6948,
    312.9002,
    362.0459,
    297.5818,
    344.7128,
    347.8734,
    352.6201,
    292.6254,
    291.3525,
    318.9582,
    324.5353,
    353.9964,
    335.9166,
    320.3791,
    289.0029,
    343.5626,
    297.4985,
    310.6395,
    321.3069,
    314.9346,
    319.5939,
    283.5716,
    370.1935,
    313.3147,
    285.2735,
    310.0546,
    296.5618,
    334.1124,
    309.7635,
    287.4468,
    320.297,
    346.2125,
    297.5174,
    309.2074,
    333.3884,
    290.4163,
    323.2953,
    317.754,
    295.3116,
    331.1413,
    311.3813,
    338.3727,
    350.5317,
    312.5502,
    350.4582,
    330.6868,
    312.3721,
    287.192,
    314.5088,
    292.238,
    311.5442,
    317.433,
    306.8341,
    319.3054,
    330.8543,
    318.995,
    306.4467,
    285.7581,
    318.7354,
    285.6096,
    349.8535,
    373.8636,
    375.0348,
    403.2414,
    364.2785,
    396.9417,
    354.7236,
    404.8186,
    396.5458,
    377.9236,
    385.5609,
    403.0736,
    365.2348,
    357.7161,
    356.2143,
    399.5291,
    336.7684,
    395.8774,
    346.449,
    372.3799,
    372.1505,
    334.4289,
    401.6758,
    402.4445,
    345.4472,
    328.4559,
    360.3066,
    371.4522,
    367.8582,
    401.1013,
    325.2511,
    326.7203,
    389.0074,
    400.3381,
    362.7518,
    337.4711,
    376.6358,
    335.8101,
    343.9861,
    378.4987,
    398.9697,
    352.247,
    346.9615,
    355.7235,
    324.062,
    387.3984,
    395.9803,
    333.1988,
    359.4211,
    349.967,
    369.83,
    324.7974,
    349.799,
    380.9988,
    342.3694,
    309.0427,
    322.4397,
    333.9173,
    380.4741,
    360.4378,
    391.1687,
    343.5891,
    374.9761,
    379.9882,
    398.4014,
    375.9983,
    385.4349,
    379.6241,
    378.4312,
    380.8245,
    374.2389,
    357.2751,
    393.5746,
    379.1215,
    384.8176,
    366.7361,
    388.3361,
    365.6024,
    392.6748,
    350.5922,
    365.912,
    352.7881,
    394.8119,
    335.3553,
    373.7261
   ],
   "wall_ms": 422.147,
   "cpu_ms": 1015.625,
   "baseline_rss_mb": 30.75,
   "peak_rss_mb": 98.63,
   "rss_per_run_kb": 69.51,
   "batch_walls_ms": [
    413.378,
    422.147,
    430.718
   ],
   "scenario": "concurrent_1000"
  },
  {
   "framework": "langgraph",
   "status": "ok",
   "runs": 1000,
   "median_ms": 2852.8119,
   "p95_ms": 4001.6801,
   "min_ms": 2232.0895,
   "max_ms": 4132.7565,
   "samples_ms": [
    2262.3618,
    2258.117,
    2255.8959,
    2252.863,
    2261.3442,
    2258.7947,
    2260.7894,
    2265.6983,
    2263.0147,
    2268.2432,
    2265.8454,
    2267.8827,
    2271.7206,
    2268.3829,
    2266.1075,
    2265.4167,
    2264.6899,
    2262.3184,
    2269.4325,
    2265.8965,
    2265.9591,
    2262.3431,
    2275.6331,
    2270.0634,
    2265.2398,
    2259.6256,
    2255.7395,
    2251.405,
    2249.6766,
    2247.1095,
    2245.0893,
    2243.1454,
    2246.0135,
    2252.7471,
    2251.8189,
    2252.3918,
    2249.7047,
    2251.9871,
    2245.061,
    2242.5092,
    2240.6624,
    2240.8727,
    2237.0569,
    2232.9988,
    2267.7258,
    2276.5976,
    2272.8095,
    2311.2273,
    2315.1437,
    2312.0659,
    2316.6147,
    2354.1296,
    2350.5759,
    2347.8748,
    2356.713,
    2389.8958,
    2388.0562,
    2401.5301,
    2417.9997,
    2430.4951,
    2438.1295,
    2453.5886,
    2463.3399,
    2472.9957,
    2489.2088,
    2505.6103,
    2516.5517,
    2527.6197,
    2545.1262,
    2555.4349,
    2565.5184,
    2588.0885,
    2596.261,
    2608.8313,
    2624.4435,
    2629.8597,
    2642.4174,
    2656.2868,
    2668.7906,
    2678.9593,
    2692.8173,
    2697.3258,
    2709.8674,
    2638.5927,
    2650.4503,
    2665.007,
    2679.7518,
    2690.4466,
    2699.8266,
    2721.4053,
    2730.2596,
    2735.7612,
    2755.6114,
    2767.8713,
    2774.4394,
    2788.3235,
    2805.1639,
    2817.0545,
    2828.8696,
    2845.5611,
    2854.3422,
    2862.4379,
    2881.512,
    2893.7431,
    2899.1881,
    2919.114,
    2924.0962,
    2935.4852,
    2949.1014,
    2962.0258,
    2970.6595,
    2983.3827,
    3000.7037,
    3011.3758,
    3023.4792,
    3039.9757,
    3051.8615,
    3063.9038,
    3073.2579,
    3089.2969,
    3100.7691,
    3115.4123,
    3122.8495,
    3140.7452,
    3147.641,
    3156.7698,
    3177.266,
    3186.3312,
    3194.8272,
    3216.1168,
    3225.9304,
    3231.4847,
    3247.7893,
    3262.1509,
    3268.7586,
    3286.1192,
    3300.3275,
    3307.1705,
    3313.5795,
    3333.7886,
    3346.3738,
    3355.2452,
    3373.5721,
    3384.5204,
    3392.0461,
    3404.1809,
    3417.4244,
    3429.3668,
    3439.2683,
    3457.4399,
    3466.2325,
    3477.3542,
    3493.5515,
    3507.7342,
    3513.4678,
    3523.9296,
    3534.2453,
    3542.7586,
    3558.0153,
    3570.9217,
    3584.3124,
    3596.7323,
    3614.1712,
    3625.2394,
    3639.8383,
    3652.762,
    3664.9561,
    3675.3649,
    3690.262,
    3717.3521,
    3730.0212,
    3746.1929,
    3748.1198,
    3756.4588,
    3781.5653,
    3830.3432,
    3827.1688,
    3826.0413,
    3874.0146,
    3872.3441,
    3869.9575,
    3893.16,
    3905.1996,
    3904.6487,
    3917.4201,
    3942.9808,
    3948.0553,
    3956.8685,
    3983.771,
    3986.2655,
    4000.0468,
    4025.1624,
    4039.7719,
    4038.1123,
    4063.6068,
    4072.4402,
    4084.7004,
    4104.9469,
    4115.8307,
    4126.5637
   ],
   "wall_ms": 4949.698,
   "cpu_ms": 4109.375,
   "baseline_rss_mb": 75.31,
   "peak_rss_mb": 124.14,
   "rss_per_run_kb": 50.0,
   "batch_walls_ms": [
    4886.02,
    4949.698,
    4974.265
   ],
   "scenario": "concurrent_1000"
  },
  {
   "framework": "crewai",
   "status": "ok",
   "runs": 1000,
   "median_ms": 48974.5199,
   "p95_ms": 50713.0844,
   "min_ms": 47597.2762,
   "max_ms": 51686.1772,
   "samples_ms": [
    50945.305,
    51116.0795,
    50756.1588,
    50651.9981,
    50892.718,
    50961.028,
    51139.4435,
    51145.1379,
    50746.8702,
    50709.1272,
    50605.131,
    50670.2452,
    50674.038,
    50658.2339,
    50579.5743,
    50309.0464,
    50509.2712,
    50376.802,
    50339.1457,
    48685.198,
    50281.1117,
    50219.7418,
    48334.4327,
    50295.7041,
    50303.1666,
    49984.3138,
    49954.0453,
    49894.7361,
    49867.6664,
    49823.5722,
    49803.5758,
    49707.6993,
    49649.2249,
    49677.2401,
    49601.8601,
    49631.6026,
    49600.0503,
    49551.8032,
    49533.2232,
    49909.6221,
    49875.6587,
    49842.5428,
    49795.2413,
    49752.2359,
    49779.239,
    49674.6827,
    49658.0884,
    49657.5415,
    49578.9665,
    49533.429,
    49495.1601,
    49421.3842,
    49411.3219,
    49448.7108,
    49355.5889,
    49354.6398,
    49277.0182,
    49191.1304,
    49132.7977,
    49151.5124,
    49068.9521,
    49056.171,
    49034.3116,
    49011.9306,
    48984.6557,
    48938.6172,
    48895.5137,
    48945.5802,
    48935.4773,
    48861.9527,
    48776.6196,
    48713.4017,
    48934.3814,
    48920.5148,
    48800.7473,
    48594.9032,
    48507.7026,
    48654.2712,
    48375.7812,
    48444.2211,
    48427.2235,
    48390.0818,
    48389.4024,
    48388.8405,
    48410.7521,
    48468.0815,
    48410.2757,
    48420.5649,
    48389.1192,
    48373.4409,
    48390.2468,
    48327.4095,
    48336.1852,
    48329.4519,
    48337.3442,
    48359.7678,
    48351.9955,
    48343.5856,
    48234.5683,
    48264.9074,
    48261.4327,
    48221.4079,
    48238.0925,
    48179.523,
    48214.0283,
    48270.6385,
    48157.4075,
    48096.121,
    48165.6959,
    48174.265,
    48060.4695,
    48118.7326,
    48094.5524,
    48091.3284,
    48073.4471,
    48107.5577,
    48158.0709,
    48140.7156,
    48114.5936,
    48119.3806,
    48110.5486,
    48070.7541,
    48055.9717,
    47998.1941,
    48087.0104,
    48175.0871,
    48170.5321,
    49276.4181,
    47988.5292,
    47973.172,
    48002.794,
    48684.9011,
    48990.7587,
    48502.0239,
    47991.9666,
    47940.7305,
    48105.6121,
    48073.4919,
    48243.5182,
    48189.1128,
    48306.1113,
    48325.6202,
    47988.9021,
    48294.844,
    48361.3127,
    48422.7307,
    48342.5931,
    48347.9792,
    48616.655,
    48574.046,
    48631.451,
    48634.5414,
    48726.0294,
    48750.7497,
    48768.396,
    48775.4581,
    48739.1359,
    48826.7375,
    48808.5654,
    48848.968,
    48873.3986,
    48822.186,
    48912.3818,
    48835.3251,
    49020.3911,
    49045.3052,
    49053.1201,
    49069.6522,
    49521.2299,
    49417.4438,
    49415.6114,
    48907.34,
    48902.4068,
    49492.9183,
    49628.5967,
    48871.5457,
    48923.8785,
    49045.8763,
    49166.2581,
    49178.0914,
    49176.4554,
    49278.1438,
    49260.7795,
    49296.5816,
    49551.2189,
    49320.3303,
    49308.794,
    49315.2166,
    49320.2144,
    49313.757,
    49338.3064,
    49369.4589,
    49440.3972,
    49506.0642,
    49464.7388,
    49533.7403,
    49642.2751,
    49675.9833,
    49680.5379,
    49903.544
   ],
   "wall_ms": 78235.712,
   "cpu_ms": 61203.125,
   "baseline_rss_mb": 169.77,
   "peak_rss_mb": 331.89,
   "rss_per_run_kb": 166.02,
   "batch_walls_ms": [
    77234.875,
    78235.712,
    81956.951
   ],
   "scenario": "concurrent_1000"
  }
 ],
 "reliability": [
  {
   "test": "model_transient",
   "framework": "criogen",
   "outcome": "pass",
   "reason": "the agent retried the rate-limited call by itself and answered",
   "note": null,
   "log": ""
  },
  {
   "test": "model_transient",
   "framework": "langgraph",
   "outcome": "fail",
   "reason": "the run failed: RuntimeError: simulated rate limit (429)",
   "note": "Not retried by default. LangChain documents ModelRetryMiddleware (or a model's own max_retries) to retry model calls.",
   "log": ""
  },
  {
   "test": "model_transient",
   "framework": "crewai",
   "outcome": "pass",
   "reason": "the agent retried and answered (2 model calls)",
   "note": "CrewAI re-runs the agent's whole task after an error (max_retry_limit, default 2).",
   "log": ""
  },
  {
   "test": "tool_exception",
   "framework": "criogen",
   "outcome": "pass",
   "reason": "the model saw the error and answered",
   "note": null,
   "log": ""
  },
  {
   "test": "tool_exception",
   "framework": "langgraph",
   "outcome": "fail",
   "reason": "the run stopped with RuntimeError: database is down",
   "note": "By default the tool node re-raises exceptions other than invalid arguments. ToolNode(handle_tool_errors=True) or a tool's handle_tool_error sends them to the model.",
   "log": ""
  },
  {
   "test": "tool_exception",
   "framework": "crewai",
   "outcome": "pass",
   "reason": "the tool's error was sent to the model, which answered",
   "note": null,
   "log": ""
  },
  {
   "test": "bad_arguments",
   "framework": "criogen",
   "outcome": "pass",
   "reason": "the arguments were rejected against the schema before the tool ran; the model got the error and answered",
   "note": null,
   "log": "[{'call_id': 'call_1_0', 'content': [{'text': 'Error: Invalid arguments: order_id: expected string, got 1001', 'type': 'text'}], 'error': {'code': 'invalid_arguments', 'message': 'Invalid arguments: order_id: expected string, got 1001', 'retryable': False}, 'id': 'msg_result_call_1_0', 'role': 'tool', 'status': 'error', 'tool': 'get_order'}]"
  },
  {
   "test": "bad_arguments",
   "framework": "langgraph",
   "outcome": "pass",
   "reason": "the arguments were rejected before the tool ran; the model got the error and answered",
   "note": null,
   "log": "[ToolMessage(content=\"Error invoking tool 'get_order' with kwargs {'order_id': 1001} with error:\\n order_id: Input should be a valid string\\n Please fix the error and try again.\", name='get_order', id='a516d3a7-e839-4aed-af57-4b14cfa5c753', tool_call_id='call_1_0', status='error')]"
  },
  {
   "test": "bad_arguments",
   "framework": "crewai",
   "outcome": "pass",
   "reason": "the arguments were rejected before the tool ran; the model answered",
   "note": null,
   "log": ""
  },
  {
   "test": "crash_resume",
   "framework": "criogen",
   "outcome": "pass",
   "reason": "resumed with thread.retry() in a new process, same SQLite file; the finished tool did not run again",
   "note": null,
   "log": "phase 1 (killed): ['get_invoice started', 'get_order done']\nphase 2: ['get_invoice started', 'get_invoice done']\nphase 2 said: {\"framework\": \"criogen\", \"task\": \"crash\", \"import_ms\": 16.81, \"answer\": \"Your refund of 40.00 is on its way.\", \"how\": \"thread.retry() in a new process, same SQLite file\"}"
  },
  {
   "test": "crash_resume",
   "framework": "langgraph",
   "outcome": "pass",
   "reason": "resumed with ainvoke(None) on the same thread, AsyncSqliteSaver; the finished tool did not run again",
   "note": null,
   "log": "phase 1 (killed): ['get_order done', 'get_invoice started']\nphase 2: ['get_invoice started', 'get_invoice done']\nphase 2 said: {\"framework\": \"langgraph\", \"task\": \"crash\", \"import_ms\": 1479.87, \"answer\": \"Your refund of 40.00 is on its way.\", \"how\": \"ainvoke(None) on the same thread, AsyncSqliteSaver\"}"
  },
  {
   "test": "crash_resume",
   "framework": "crewai",
   "outcome": "partial",
   "reason": "resumed with Crew.from_checkpoint(\u2026) from the newest of 4 checkpoints, then kickoff(), but the finished tool ran again (2 times)",
   "note": "Uses BaseTool subclasses: tools made with @tool keep a Python function that CrewAI's checkpoints cannot serialize, so no checkpoint is written for a crew using them. The scripted model and tools were attached again after the restore.",
   "log": "phase 1 (killed): ['get_order done', 'get_invoice started']\nphase 2: ['get_order done', 'get_invoice started', 'get_invoice done']\nphase 2 said: {\"framework\": \"crewai\", \"task\": \"crash\", \"import_ms\": 6831.2, \"answer\": \"Your refund of 40.00 is on its way.\", \"how\": \"Crew.from_checkpoint(\\u2026) from the newest of 4 checkpoints, then kickoff()\", \"note\": \"Uses BaseTool subclasses: tools made with @tool keep a Python function that CrewAI's checkpoints cannot serialize, so no checkpoint is written for a crew using them. The scripted model and tools were attached again after the restore.\"}"
  },
  {
   "test": "runaway_loop",
   "framework": "criogen",
   "outcome": "pass",
   "reason": "stopped after 25 tool calls in 24 ms: LimitExceededError",
   "note": null,
   "log": "limit_exceeded: the agent reached its model_calls limit (25)"
  },
  {
   "test": "runaway_loop",
   "framework": "langgraph",
   "outcome": "partial",
   "reason": "stopped after 5000 model calls and 4999 tool calls in 203293 ms: GraphRecursionError",
   "note": "create_agent sets recursion_limit=9999; pass a lower recursion_limit in the run config to stop earlier.",
   "log": "Recursion limit of 9999 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langgraph/errors/GRAPH_RECURSION_LIMIT"
  },
  {
   "test": "runaway_loop",
   "framework": "crewai",
   "outcome": "pass",
   "reason": "stopped after 26 model calls and 25 tool calls in 171 ms: ValidationError",
   "note": "At max_iter (25) CrewAI asks the model once more for a final answer; the scripted model kept asking for a tool, which CrewAI rejected with this error. A real model would usually answer in text.",
   "log": "1 validation error for TaskOutput\nraw\n  Input should be a valid string [type=string_type, input_value=[{'id': 'call_26_0', 'typ...order_id\": \"A-1001\"}'}}], input_type=list]\n    For further information visit https://errors.pydantic.dev/2.12/v/string_type"
  }
 ],
 "footprint": [
  {
   "framework": "criogen",
   "import_ms": 147.8,
   "packages": 1,
   "install_mb": 17.9
  },
  {
   "framework": "langgraph",
   "import_ms": 1700.7,
   "packages": 42,
   "install_mb": 42.9
  },
  {
   "framework": "crewai",
   "import_ms": 9303.3,
   "packages": 138,
   "install_mb": 742.7
  }
 ]
}