{
  "schemaVersion": 1,
  "videoId": "SlDHTNd7cW8",
  "videoUrl": "https://www.youtube.com/watch?v=SlDHTNd7cW8",
  "title": "Benchmarking Frontier Models: Which One Is the Best EVM Auditor?",
  "speakerCredit": "Josef Gattermayer",
  "programmeTalkId": "dss26-5a6c0ce7d8a8807d",
  "language": "en",
  "captionType": "auto-generated",
  "humanAccuracyReviewed": false,
  "spokenAudioCompletenessVerified": false,
  "cueEndTimesAvailable": false,
  "source": {
    "provider": "YouTube",
    "method": "native browser transcript export",
    "sha256": "a79e9e696ef3997696d418cad49f0951ffd59bec1b70a2dbe0907f7b2d84be05",
    "exportedAt": "2026-11-16T07:39:02.040082+00:00"
  },
  "qualityCaveats": [
    "Names, organizations, technical terms, code and numbers may be misrecognized.",
    "Source wording and disfluencies are preserved; no editorial corrections were applied.",
    "Export contains whole-second cue starts only; cue ends and subsecond timing are unavailable.",
    "A full caption-track export does not establish that every spoken word is represented.",
    "Automatic YouTube captions; not human-verified against the audio."
  ],
  "timingValidation": {
    "first_cue_start_seconds": 0,
    "last_cue_start_seconds": 817,
    "video_duration_seconds": 819,
    "seconds_after_last_cue_start": 2,
    "cue_start_gaps_over_30_seconds": [],
    "boundary_check": "pass",
    "criterion": "first cue <=15 seconds; last cue start within 30 seconds of player duration, allowing 1-second precision",
    "meaning": "Timing plausibility only; silence/applause or source caption omissions require audio review."
  },
  "segments": [
    {
      "index": 0,
      "start_seconds": 0,
      "source_timestamp": "0:00",
      "end_seconds": null,
      "text": "So hello again."
    },
    {
      "index": 1,
      "start_seconds": 2,
      "source_timestamp": "0:02",
      "end_seconds": null,
      "text": "My name is Joseph and this is the longer"
    },
    {
      "index": 2,
      "start_seconds": 4,
      "source_timestamp": "0:04",
      "end_seconds": null,
      "text": "talk, and we'll speak about"
    },
    {
      "index": 3,
      "start_seconds": 9,
      "source_timestamp": "0:09",
      "end_seconds": null,
      "text": "about"
    },
    {
      "index": 4,
      "start_seconds": 10,
      "source_timestamp": "0:10",
      "end_seconds": null,
      "text": "benchmarking of AI models."
    },
    {
      "index": 5,
      "start_seconds": 13,
      "source_timestamp": "0:13",
      "end_seconds": null,
      "text": "So let's get started."
    },
    {
      "index": 6,
      "start_seconds": 16,
      "source_timestamp": "0:16",
      "end_seconds": null,
      "text": "Um"
    },
    {
      "index": 7,
      "start_seconds": 17,
      "source_timestamp": "0:17",
      "end_seconds": null,
      "text": "yeah, the timer."
    },
    {
      "index": 8,
      "start_seconds": 20,
      "source_timestamp": "0:20",
      "end_seconds": null,
      "text": "Okay, I'm starting."
    },
    {
      "index": 9,
      "start_seconds": 22,
      "source_timestamp": "0:22",
      "end_seconds": null,
      "text": "So yes,"
    },
    {
      "index": 10,
      "start_seconds": 23,
      "source_timestamp": "0:23",
      "end_seconds": null,
      "text": "I'm co-founder of ACK free and today we"
    },
    {
      "index": 11,
      "start_seconds": 28,
      "source_timestamp": "0:28",
      "end_seconds": null,
      "text": "will look at some benchmarks that we run"
    },
    {
      "index": 12,
      "start_seconds": 30,
      "source_timestamp": "0:30",
      "end_seconds": null,
      "text": "in the past and also I did run the"
    },
    {
      "index": 13,
      "start_seconds": 32,
      "source_timestamp": "0:32",
      "end_seconds": null,
      "text": "latest benchmarks overnight, so it is"
    },
    {
      "index": 14,
      "start_seconds": 34,
      "source_timestamp": "0:34",
      "end_seconds": null,
      "text": "very fresh data that we'll present here"
    },
    {
      "index": 15,
      "start_seconds": 37,
      "source_timestamp": "0:37",
      "end_seconds": null,
      "text": "today."
    },
    {
      "index": 16,
      "start_seconds": 39,
      "source_timestamp": "0:39",
      "end_seconds": null,
      "text": "We at ACK free, we are developing tools,"
    },
    {
      "index": 17,
      "start_seconds": 41,
      "source_timestamp": "0:41",
      "end_seconds": null,
      "text": "we are doing audits, we are doing"
    },
    {
      "index": 18,
      "start_seconds": 43,
      "source_timestamp": "0:43",
      "end_seconds": null,
      "text": "research and I'm also a stand professor"
    },
    {
      "index": 19,
      "start_seconds": 46,
      "source_timestamp": "0:46",
      "end_seconds": null,
      "text": "Czech Technical University in Prague and"
    },
    {
      "index": 20,
      "start_seconds": 49,
      "source_timestamp": "0:49",
      "end_seconds": null,
      "text": "we are happy we could be working with"
    },
    {
      "index": 21,
      "start_seconds": 50,
      "source_timestamp": "0:50",
      "end_seconds": null,
      "text": "some of the industry leaders as for"
    },
    {
      "index": 22,
      "start_seconds": 53,
      "source_timestamp": "0:53",
      "end_seconds": null,
      "text": "example Lydo here. Thank you guys."
    },
    {
      "index": 23,
      "start_seconds": 56,
      "source_timestamp": "0:56",
      "end_seconds": null,
      "text": "And so let's get started."
    },
    {
      "index": 24,
      "start_seconds": 59,
      "source_timestamp": "0:59",
      "end_seconds": null,
      "text": "The question is how to evaluate LLM"
    },
    {
      "index": 25,
      "start_seconds": 61,
      "source_timestamp": "1:01",
      "end_seconds": null,
      "text": "auditors. It's not that easy"
    },
    {
      "index": 26,
      "start_seconds": 65,
      "source_timestamp": "1:05",
      "end_seconds": null,
      "text": "because the"
    },
    {
      "index": 27,
      "start_seconds": 66,
      "source_timestamp": "1:06",
      "end_seconds": null,
      "text": "web two security guys, they have"
    },
    {
      "index": 28,
      "start_seconds": 70,
      "source_timestamp": "1:10",
      "end_seconds": null,
      "text": "more easy"
    },
    {
      "index": 29,
      "start_seconds": 72,
      "source_timestamp": "1:12",
      "end_seconds": null,
      "text": "way of"
    },
    {
      "index": 30,
      "start_seconds": 74,
      "source_timestamp": "1:14",
      "end_seconds": null,
      "text": "measuring if some attack succeeded"
    },
    {
      "index": 31,
      "start_seconds": 77,
      "source_timestamp": "1:17",
      "end_seconds": null,
      "text": "because you can for example"
    },
    {
      "index": 32,
      "start_seconds": 79,
      "source_timestamp": "1:19",
      "end_seconds": null,
      "text": "set a goal to the agent break the system"
    },
    {
      "index": 33,
      "start_seconds": 83,
      "source_timestamp": "1:23",
      "end_seconds": null,
      "text": "or like get into this database. And when"
    },
    {
      "index": 34,
      "start_seconds": 85,
      "source_timestamp": "1:25",
      "end_seconds": null,
      "text": "you set the goal, the agent is just"
    },
    {
      "index": 35,
      "start_seconds": 87,
      "source_timestamp": "1:27",
      "end_seconds": null,
      "text": "working on that and that's all you need"
    },
    {
      "index": 36,
      "start_seconds": 89,
      "source_timestamp": "1:29",
      "end_seconds": null,
      "text": "to settle. But our goal is open. We"
    },
    {
      "index": 37,
      "start_seconds": 92,
      "source_timestamp": "1:32",
      "end_seconds": null,
      "text": "don't know how many bugs are in the"
    },
    {
      "index": 38,
      "start_seconds": 95,
      "source_timestamp": "1:35",
      "end_seconds": null,
      "text": "contracts. So basically we cannot set at"
    },
    {
      "index": 39,
      "start_seconds": 98,
      "source_timestamp": "1:38",
      "end_seconds": null,
      "text": "as easy goal. So that's the first"
    },
    {
      "index": 40,
      "start_seconds": 100,
      "source_timestamp": "1:40",
      "end_seconds": null,
      "text": "problem. The second problem is of course"
    },
    {
      "index": 41,
      "start_seconds": 103,
      "source_timestamp": "1:43",
      "end_seconds": null,
      "text": "data because when we have a"
    },
    {
      "index": 42,
      "start_seconds": 106,
      "source_timestamp": "1:46",
      "end_seconds": null,
      "text": "when we have a project"
    },
    {
      "index": 43,
      "start_seconds": 108,
      "source_timestamp": "1:48",
      "end_seconds": null,
      "text": "where we know the issues because it was"
    },
    {
      "index": 44,
      "start_seconds": 110,
      "source_timestamp": "1:50",
      "end_seconds": null,
      "text": "audited,"
    },
    {
      "index": 45,
      "start_seconds": 111,
      "source_timestamp": "1:51",
      "end_seconds": null,
      "text": "it's probably also public, which means"
    },
    {
      "index": 46,
      "start_seconds": 115,
      "source_timestamp": "1:55",
      "end_seconds": null,
      "text": "the LLMs could have learned on it and"
    },
    {
      "index": 47,
      "start_seconds": 117,
      "source_timestamp": "1:57",
      "end_seconds": null,
      "text": "now we are not evaluating how they"
    },
    {
      "index": 48,
      "start_seconds": 119,
      "source_timestamp": "1:59",
      "end_seconds": null,
      "text": "perform in the future, but we are"
    },
    {
      "index": 49,
      "start_seconds": 122,
      "source_timestamp": "2:02",
      "end_seconds": null,
      "text": "evaluating how they performed in the"
    },
    {
      "index": 50,
      "start_seconds": 123,
      "source_timestamp": "2:03",
      "end_seconds": null,
      "text": "past. So it's very hard to finding some"
    },
    {
      "index": 51,
      "start_seconds": 126,
      "source_timestamp": "2:06",
      "end_seconds": null,
      "text": "uh dataset that was not public, but was"
    },
    {
      "index": 52,
      "start_seconds": 129,
      "source_timestamp": "2:09",
      "end_seconds": null,
      "text": "audited. We have some of the datasets,"
    },
    {
      "index": 53,
      "start_seconds": 131,
      "source_timestamp": "2:11",
      "end_seconds": null,
      "text": "but it's of course a little bit limited."
    },
    {
      "index": 54,
      "start_seconds": 133,
      "source_timestamp": "2:13",
      "end_seconds": null,
      "text": "And of course we need to care about zero"
    },
    {
      "index": 55,
      "start_seconds": 134,
      "source_timestamp": "2:14",
      "end_seconds": null,
      "text": "data retention when we are doing stuff"
    },
    {
      "index": 56,
      "start_seconds": 136,
      "source_timestamp": "2:16",
      "end_seconds": null,
      "text": "like that, so we cannot just use any"
    },
    {
      "index": 57,
      "start_seconds": 138,
      "source_timestamp": "2:18",
      "end_seconds": null,
      "text": "Chinese API, you know, that we don't"
    },
    {
      "index": 58,
      "start_seconds": 140,
      "source_timestamp": "2:20",
      "end_seconds": null,
      "text": "trust."
    },
    {
      "index": 59,
      "start_seconds": 142,
      "source_timestamp": "2:22",
      "end_seconds": null,
      "text": "Uh"
    },
    {
      "index": 60,
      "start_seconds": 142,
      "source_timestamp": "2:22",
      "end_seconds": null,
      "text": "so the evaluation part one is that we"
    },
    {
      "index": 61,
      "start_seconds": 145,
      "source_timestamp": "2:25",
      "end_seconds": null,
      "text": "looked at individual models. Uh this is"
    },
    {
      "index": 62,
      "start_seconds": 148,
      "source_timestamp": "2:28",
      "end_seconds": null,
      "text": "an older benchmark."
    },
    {
      "index": 63,
      "start_seconds": 149,
      "source_timestamp": "2:29",
      "end_seconds": null,
      "text": "Uh we took"
    },
    {
      "index": 64,
      "start_seconds": 151,
      "source_timestamp": "2:31",
      "end_seconds": null,
      "text": "these uh"
    },
    {
      "index": 65,
      "start_seconds": 153,
      "source_timestamp": "2:33",
      "end_seconds": null,
      "text": "these LLMs. We took just a very easy"
    },
    {
      "index": 66,
      "start_seconds": 156,
      "source_timestamp": "2:36",
      "end_seconds": null,
      "text": "workflow with Codex and we just run uh"
    },
    {
      "index": 67,
      "start_seconds": 160,
      "source_timestamp": "2:40",
      "end_seconds": null,
      "text": "that with some prompt, I don't know,"
    },
    {
      "index": 68,
      "start_seconds": 161,
      "source_timestamp": "2:41",
      "end_seconds": null,
      "text": "find all bugs, make no mistakes"
    },
    {
      "index": 69,
      "start_seconds": 164,
      "source_timestamp": "2:44",
      "end_seconds": null,
      "text": "uh over a codebase that was consisted of"
    },
    {
      "index": 70,
      "start_seconds": 167,
      "source_timestamp": "2:47",
      "end_seconds": null,
      "text": "five unpublished audits. And we are"
    },
    {
      "index": 71,
      "start_seconds": 171,
      "source_timestamp": "2:51",
      "end_seconds": null,
      "text": "looking for the recall because we knew"
    },
    {
      "index": 72,
      "start_seconds": 173,
      "source_timestamp": "2:53",
      "end_seconds": null,
      "text": "how many uh"
    },
    {
      "index": 73,
      "start_seconds": 174,
      "source_timestamp": "2:54",
      "end_seconds": null,
      "text": "criticals and highs are there. These are"
    },
    {
      "index": 74,
      "start_seconds": 177,
      "source_timestamp": "2:57",
      "end_seconds": null,
      "text": "the only ones that we tracked. There are"
    },
    {
      "index": 75,
      "start_seconds": 178,
      "source_timestamp": "2:58",
      "end_seconds": null,
      "text": "40 discovered by sorry, 36 discovered by"
    },
    {
      "index": 76,
      "start_seconds": 181,
      "source_timestamp": "3:01",
      "end_seconds": null,
      "text": "human and four with the later AI"
    },
    {
      "index": 77,
      "start_seconds": 184,
      "source_timestamp": "3:04",
      "end_seconds": null,
      "text": "findings. So the total recall was 40."
    },
    {
      "index": 78,
      "start_seconds": 187,
      "source_timestamp": "3:07",
      "end_seconds": null,
      "text": "And this is how the single models"
    },
    {
      "index": 79,
      "start_seconds": 190,
      "source_timestamp": "3:10",
      "end_seconds": null,
      "text": "perform. So the human auditors"
    },
    {
      "index": 80,
      "start_seconds": 193,
      "source_timestamp": "3:13",
      "end_seconds": null,
      "text": "discovered uh"
    },
    {
      "index": 81,
      "start_seconds": 195,
      "source_timestamp": "3:15",
      "end_seconds": null,
      "text": "in the first revision, then then it was"
    },
    {
      "index": 82,
      "start_seconds": 196,
      "source_timestamp": "3:16",
      "end_seconds": null,
      "text": "more 32 findings uh"
    },
    {
      "index": 83,
      "start_seconds": 200,
      "source_timestamp": "3:20",
      "end_seconds": null,
      "text": "and the most performant"
    },
    {
      "index": 84,
      "start_seconds": 202,
      "source_timestamp": "3:22",
      "end_seconds": null,
      "text": "uh LLM is Opus five that discovered 35"
    },
    {
      "index": 85,
      "start_seconds": 207,
      "source_timestamp": "3:27",
      "end_seconds": null,
      "text": "and then it went slowly down. But this"
    },
    {
      "index": 86,
      "start_seconds": 211,
      "source_timestamp": "3:31",
      "end_seconds": null,
      "text": "is not the chart that you should be"
    },
    {
      "index": 87,
      "start_seconds": 213,
      "source_timestamp": "3:33",
      "end_seconds": null,
      "text": "looking at because it doesn't"
    },
    {
      "index": 88,
      "start_seconds": 215,
      "source_timestamp": "3:35",
      "end_seconds": null,
      "text": "basically help you to prevent"
    },
    {
      "index": 89,
      "start_seconds": 218,
      "source_timestamp": "3:38",
      "end_seconds": null,
      "text": "uh from the hacks. So more interesting"
    },
    {
      "index": 90,
      "start_seconds": 220,
      "source_timestamp": "3:40",
      "end_seconds": null,
      "text": "is this chart where we looked at uh"
    },
    {
      "index": 91,
      "start_seconds": 224,
      "source_timestamp": "3:44",
      "end_seconds": null,
      "text": "all the bugs. This is this axis"
    },
    {
      "index": 92,
      "start_seconds": 226,
      "source_timestamp": "3:46",
      "end_seconds": null,
      "text": "basically uh that are present in the"
    },
    {
      "index": 93,
      "start_seconds": 229,
      "source_timestamp": "3:49",
      "end_seconds": null,
      "text": "dataset. We looked at all the models and"
    },
    {
      "index": 94,
      "start_seconds": 232,
      "source_timestamp": "3:52",
      "end_seconds": null,
      "text": "the humans and we looked who discovered"
    },
    {
      "index": 95,
      "start_seconds": 235,
      "source_timestamp": "3:55",
      "end_seconds": null,
      "text": "what. And of course like the Opus"
    },
    {
      "index": 96,
      "start_seconds": 238,
      "source_timestamp": "3:58",
      "end_seconds": null,
      "text": "discovered most of the issues, but is it"
    },
    {
      "index": 97,
      "start_seconds": 241,
      "source_timestamp": "4:01",
      "end_seconds": null,
      "text": "sufficient for you that in your protocol"
    },
    {
      "index": 98,
      "start_seconds": 243,
      "source_timestamp": "4:03",
      "end_seconds": null,
      "text": "you have something that discovers most"
    },
    {
      "index": 99,
      "start_seconds": 245,
      "source_timestamp": "4:05",
      "end_seconds": null,
      "text": "of the issues? No. You need something"
    },
    {
      "index": 100,
      "start_seconds": 247,
      "source_timestamp": "4:07",
      "end_seconds": null,
      "text": "that discovers all the issues. And here"
    },
    {
      "index": 101,
      "start_seconds": 250,
      "source_timestamp": "4:10",
      "end_seconds": null,
      "text": "I must shout out to"
    },
    {
      "index": 102,
      "start_seconds": 254,
      "source_timestamp": "4:14",
      "end_seconds": null,
      "text": "Kimi K Free because this guy"
    },
    {
      "index": 103,
      "start_seconds": 257,
      "source_timestamp": "4:17",
      "end_seconds": null,
      "text": "the overall performance is very poor"
    },
    {
      "index": 104,
      "start_seconds": 259,
      "source_timestamp": "4:19",
      "end_seconds": null,
      "text": "poor, but what he did is that this one"
    },
    {
      "index": 105,
      "start_seconds": 262,
      "source_timestamp": "4:22",
      "end_seconds": null,
      "text": "issue"
    },
    {
      "index": 106,
      "start_seconds": 263,
      "source_timestamp": "4:23",
      "end_seconds": null,
      "text": "discovered no one else but just this"
    },
    {
      "index": 107,
      "start_seconds": 265,
      "source_timestamp": "4:25",
      "end_seconds": null,
      "text": "model. And this is of course like very"
    },
    {
      "index": 108,
      "start_seconds": 267,
      "source_timestamp": "4:27",
      "end_seconds": null,
      "text": "interesting that that it's able to find"
    },
    {
      "index": 109,
      "start_seconds": 268,
      "source_timestamp": "4:28",
      "end_seconds": null,
      "text": "a unique issue"
    },
    {
      "index": 110,
      "start_seconds": 270,
      "source_timestamp": "4:30",
      "end_seconds": null,
      "text": "which was high, not a critical, but only"
    },
    {
      "index": 111,
      "start_seconds": 274,
      "source_timestamp": "4:34",
      "end_seconds": null,
      "text": "this model discovered that it not even"
    },
    {
      "index": 112,
      "start_seconds": 275,
      "source_timestamp": "4:35",
      "end_seconds": null,
      "text": "humans and not even Opus or someone"
    },
    {
      "index": 113,
      "start_seconds": 279,
      "source_timestamp": "4:39",
      "end_seconds": null,
      "text": "else."
    },
    {
      "index": 114,
      "start_seconds": 280,
      "source_timestamp": "4:40",
      "end_seconds": null,
      "text": "So what does it tell us this benchmark?"
    },
    {
      "index": 115,
      "start_seconds": 284,
      "source_timestamp": "4:44",
      "end_seconds": null,
      "text": "It helped us to form like two"
    },
    {
      "index": 116,
      "start_seconds": 286,
      "source_timestamp": "4:46",
      "end_seconds": null,
      "text": "hypotheses."
    },
    {
      "index": 117,
      "start_seconds": 288,
      "source_timestamp": "4:48",
      "end_seconds": null,
      "text": "One of them is"
    },
    {
      "index": 118,
      "start_seconds": 290,
      "source_timestamp": "4:50",
      "end_seconds": null,
      "text": "what happens if we just run the"
    },
    {
      "index": 119,
      "start_seconds": 292,
      "source_timestamp": "4:52",
      "end_seconds": null,
      "text": "benchmark multiple times?"
    },
    {
      "index": 120,
      "start_seconds": 294,
      "source_timestamp": "4:54",
      "end_seconds": null,
      "text": "And the second one is like maybe we need"
    },
    {
      "index": 121,
      "start_seconds": 297,
      "source_timestamp": "4:57",
      "end_seconds": null,
      "text": "some kind of like mixture of models to"
    },
    {
      "index": 122,
      "start_seconds": 300,
      "source_timestamp": "5:00",
      "end_seconds": null,
      "text": "be able to find the unique findings"
    },
    {
      "index": 123,
      "start_seconds": 302,
      "source_timestamp": "5:02",
      "end_seconds": null,
      "text": "because this is what we are"
    },
    {
      "index": 124,
      "start_seconds": 304,
      "source_timestamp": "5:04",
      "end_seconds": null,
      "text": "hunting for as well."
    },
    {
      "index": 125,
      "start_seconds": 306,
      "source_timestamp": "5:06",
      "end_seconds": null,
      "text": "So then we did a second benchmark."
    },
    {
      "index": 126,
      "start_seconds": 309,
      "source_timestamp": "5:09",
      "end_seconds": null,
      "text": "For this second benchmark"
    },
    {
      "index": 127,
      "start_seconds": 311,
      "source_timestamp": "5:11",
      "end_seconds": null,
      "text": "the inspiration was this hack. I"
    },
    {
      "index": 128,
      "start_seconds": 313,
      "source_timestamp": "5:13",
      "end_seconds": null,
      "text": "don't know if you're familiar with that,"
    },
    {
      "index": 129,
      "start_seconds": 314,
      "source_timestamp": "5:14",
      "end_seconds": null,
      "text": "but it was like I think a month ago"
    },
    {
      "index": 130,
      "start_seconds": 318,
      "source_timestamp": "5:18",
      "end_seconds": null,
      "text": "quite headline news that"
    },
    {
      "index": 131,
      "start_seconds": 321,
      "source_timestamp": "5:21",
      "end_seconds": null,
      "text": "that there was an like internal"
    },
    {
      "index": 132,
      "start_seconds": 324,
      "source_timestamp": "5:24",
      "end_seconds": null,
      "text": "experiment within OpenAI"
    },
    {
      "index": 133,
      "start_seconds": 327,
      "source_timestamp": "5:27",
      "end_seconds": null,
      "text": "where they have put some"
    },
    {
      "index": 134,
      "start_seconds": 330,
      "source_timestamp": "5:30",
      "end_seconds": null,
      "text": "goal to the agents."
    },
    {
      "index": 135,
      "start_seconds": 333,
      "source_timestamp": "5:33",
      "end_seconds": null,
      "text": "And"
    },
    {
      "index": 136,
      "start_seconds": 335,
      "source_timestamp": "5:35",
      "end_seconds": null,
      "text": "what they did instead was that they"
    },
    {
      "index": 137,
      "start_seconds": 337,
      "source_timestamp": "5:37",
      "end_seconds": null,
      "text": "hacked out of the sandbox and tried to"
    },
    {
      "index": 138,
      "start_seconds": 341,
      "source_timestamp": "5:41",
      "end_seconds": null,
      "text": "or succeeded in hacking a hugging face"
    },
    {
      "index": 139,
      "start_seconds": 344,
      "source_timestamp": "5:44",
      "end_seconds": null,
      "text": "which was the benchmark they were"
    },
    {
      "index": 140,
      "start_seconds": 346,
      "source_timestamp": "5:46",
      "end_seconds": null,
      "text": "supposed to out perform. And"
    },
    {
      "index": 141,
      "start_seconds": 349,
      "source_timestamp": "5:49",
      "end_seconds": null,
      "text": "the interesting thing about this is how"
    },
    {
      "index": 142,
      "start_seconds": 351,
      "source_timestamp": "5:51",
      "end_seconds": null,
      "text": "the agents did it. And"
    },
    {
      "index": 143,
      "start_seconds": 354,
      "source_timestamp": "5:54",
      "end_seconds": null,
      "text": "when you look at the report basically"
    },
    {
      "index": 144,
      "start_seconds": 357,
      "source_timestamp": "5:57",
      "end_seconds": null,
      "text": "they the agents found a way how to"
    },
    {
      "index": 145,
      "start_seconds": 359,
      "source_timestamp": "5:59",
      "end_seconds": null,
      "text": "communicate between each other. They"
    },
    {
      "index": 146,
      "start_seconds": 360,
      "source_timestamp": "6:00",
      "end_seconds": null,
      "text": "were exchanging messages and they were"
    },
    {
      "index": 147,
      "start_seconds": 362,
      "source_timestamp": "6:02",
      "end_seconds": null,
      "text": "like building on each other's"
    },
    {
      "index": 148,
      "start_seconds": 364,
      "source_timestamp": "6:04",
      "end_seconds": null,
      "text": "findings and they even called themselves"
    },
    {
      "index": 149,
      "start_seconds": 368,
      "source_timestamp": "6:08",
      "end_seconds": null,
      "text": "like a collective and"
    },
    {
      "index": 150,
      "start_seconds": 370,
      "source_timestamp": "6:10",
      "end_seconds": null,
      "text": "it was pretty scary,"
    },
    {
      "index": 151,
      "start_seconds": 372,
      "source_timestamp": "6:12",
      "end_seconds": null,
      "text": "but I like that. And so we said okay,"
    },
    {
      "index": 152,
      "start_seconds": 375,
      "source_timestamp": "6:15",
      "end_seconds": null,
      "text": "maybe we could replicate something like"
    },
    {
      "index": 153,
      "start_seconds": 377,
      "source_timestamp": "6:17",
      "end_seconds": null,
      "text": "this and see how how it performs."
    },
    {
      "index": 154,
      "start_seconds": 380,
      "source_timestamp": "6:20",
      "end_seconds": null,
      "text": "So we did this quick"
    },
    {
      "index": 155,
      "start_seconds": 383,
      "source_timestamp": "6:23",
      "end_seconds": null,
      "text": "tool. We call it swarm."
    },
    {
      "index": 156,
      "start_seconds": 385,
      "source_timestamp": "6:25",
      "end_seconds": null,
      "text": "We just open sourced it yesterday, but"
    },
    {
      "index": 157,
      "start_seconds": 387,
      "source_timestamp": "6:27",
      "end_seconds": null,
      "text": "it's still like quite early internal"
    },
    {
      "index": 158,
      "start_seconds": 389,
      "source_timestamp": "6:29",
      "end_seconds": null,
      "text": "tool. And it's kind of like tool that"
    },
    {
      "index": 159,
      "start_seconds": 391,
      "source_timestamp": "6:31",
      "end_seconds": null,
      "text": "help us to orchestrate us and experiment"
    },
    {
      "index": 160,
      "start_seconds": 393,
      "source_timestamp": "6:33",
      "end_seconds": null,
      "text": "like this."
    },
    {
      "index": 161,
      "start_seconds": 395,
      "source_timestamp": "6:35",
      "end_seconds": null,
      "text": "So basically you can put any prompt, you"
    },
    {
      "index": 162,
      "start_seconds": 398,
      "source_timestamp": "6:38",
      "end_seconds": null,
      "text": "can put any model, you can put any"
    },
    {
      "index": 163,
      "start_seconds": 399,
      "source_timestamp": "6:39",
      "end_seconds": null,
      "text": "workflow."
    },
    {
      "index": 164,
      "start_seconds": 401,
      "source_timestamp": "6:41",
      "end_seconds": null,
      "text": "Every"
    },
    {
      "index": 165,
      "start_seconds": 403,
      "source_timestamp": "6:43",
      "end_seconds": null,
      "text": "agent has is dedicated container"
    },
    {
      "index": 166,
      "start_seconds": 406,
      "source_timestamp": "6:46",
      "end_seconds": null,
      "text": "and what works here like"
    },
    {
      "index": 167,
      "start_seconds": 407,
      "source_timestamp": "6:47",
      "end_seconds": null,
      "text": "deterministically is that these"
    },
    {
      "index": 168,
      "start_seconds": 409,
      "source_timestamp": "6:49",
      "end_seconds": null,
      "text": "containers with agents are sharing the"
    },
    {
      "index": 169,
      "start_seconds": 411,
      "source_timestamp": "6:51",
      "end_seconds": null,
      "text": "evidence between each other and it's"
    },
    {
      "index": 170,
      "start_seconds": 413,
      "source_timestamp": "6:53",
      "end_seconds": null,
      "text": "also running on subscriptions, which"
    },
    {
      "index": 171,
      "start_seconds": 415,
      "source_timestamp": "6:55",
      "end_seconds": null,
      "text": "means like it doesn't"
    },
    {
      "index": 172,
      "start_seconds": 417,
      "source_timestamp": "6:57",
      "end_seconds": null,
      "text": "ruin us."
    },
    {
      "index": 173,
      "start_seconds": 418,
      "source_timestamp": "6:58",
      "end_seconds": null,
      "text": "And with this tool we looked again at"
    },
    {
      "index": 174,
      "start_seconds": 421,
      "source_timestamp": "7:01",
      "end_seconds": null,
      "text": "some benchmarking."
    },
    {
      "index": 175,
      "start_seconds": 423,
      "source_timestamp": "7:03",
      "end_seconds": null,
      "text": "And"
    },
    {
      "index": 176,
      "start_seconds": 426,
      "source_timestamp": "7:06",
      "end_seconds": null,
      "text": "this time we took a smaller"
    },
    {
      "index": 177,
      "start_seconds": 429,
      "source_timestamp": "7:09",
      "end_seconds": null,
      "text": "set, but it was just like"
    },
    {
      "index": 178,
      "start_seconds": 431,
      "source_timestamp": "7:11",
      "end_seconds": null,
      "text": "one code base, but very, let's say big"
    },
    {
      "index": 179,
      "start_seconds": 433,
      "source_timestamp": "7:13",
      "end_seconds": null,
      "text": "and complicated code base."
    },
    {
      "index": 180,
      "start_seconds": 435,
      "source_timestamp": "7:15",
      "end_seconds": null,
      "text": "And we took six models and"
    },
    {
      "index": 181,
      "start_seconds": 438,
      "source_timestamp": "7:18",
      "end_seconds": null,
      "text": "because it is more recent, so I'll will"
    },
    {
      "index": 182,
      "start_seconds": 442,
      "source_timestamp": "7:22",
      "end_seconds": null,
      "text": "just point out that we were using the"
    },
    {
      "index": 183,
      "start_seconds": 444,
      "source_timestamp": "7:24",
      "end_seconds": null,
      "text": "latest DeepSeek Flash model."
    },
    {
      "index": 184,
      "start_seconds": 446,
      "source_timestamp": "7:26",
      "end_seconds": null,
      "text": "And this time the workflow"
    },
    {
      "index": 185,
      "start_seconds": 450,
      "source_timestamp": "7:30",
      "end_seconds": null,
      "text": "was that the swarm, that's the our tool,"
    },
    {
      "index": 186,
      "start_seconds": 453,
      "source_timestamp": "7:33",
      "end_seconds": null,
      "text": "was running inside of that the harnesses"
    },
    {
      "index": 187,
      "start_seconds": 456,
      "source_timestamp": "7:36",
      "end_seconds": null,
      "text": "that are native to each agent. So we are"
    },
    {
      "index": 188,
      "start_seconds": 458,
      "source_timestamp": "7:38",
      "end_seconds": null,
      "text": "running three harnesses, those Cloud"
    },
    {
      "index": 189,
      "start_seconds": 459,
      "source_timestamp": "7:39",
      "end_seconds": null,
      "text": "Cod, Codex and DeepSeek harness. And"
    },
    {
      "index": 190,
      "start_seconds": 463,
      "source_timestamp": "7:43",
      "end_seconds": null,
      "text": "with this we are looking at the data"
    },
    {
      "index": 191,
      "start_seconds": 465,
      "source_timestamp": "7:45",
      "end_seconds": null,
      "text": "set."
    },
    {
      "index": 192,
      "start_seconds": 466,
      "source_timestamp": "7:46",
      "end_seconds": null,
      "text": "In the human baseline there were"
    },
    {
      "index": 193,
      "start_seconds": 469,
      "source_timestamp": "7:49",
      "end_seconds": null,
      "text": "four critical and high issues. So that"
    },
    {
      "index": 194,
      "start_seconds": 471,
      "source_timestamp": "7:51",
      "end_seconds": null,
      "text": "this was the 100% recall."
    },
    {
      "index": 195,
      "start_seconds": 474,
      "source_timestamp": "7:54",
      "end_seconds": null,
      "text": "And this is how the experiment looked"
    },
    {
      "index": 196,
      "start_seconds": 477,
      "source_timestamp": "7:57",
      "end_seconds": null,
      "text": "like."
    },
    {
      "index": 197,
      "start_seconds": 478,
      "source_timestamp": "7:58",
      "end_seconds": null,
      "text": "This is the time window that we"
    },
    {
      "index": 198,
      "start_seconds": 479,
      "source_timestamp": "7:59",
      "end_seconds": null,
      "text": "dedicated to this"
    },
    {
      "index": 199,
      "start_seconds": 482,
      "source_timestamp": "8:02",
      "end_seconds": null,
      "text": "phase and this is five hours time"
    },
    {
      "index": 200,
      "start_seconds": 485,
      "source_timestamp": "8:05",
      "end_seconds": null,
      "text": "window. So basically each this box is"
    },
    {
      "index": 201,
      "start_seconds": 488,
      "source_timestamp": "8:08",
      "end_seconds": null,
      "text": "one how we call it wave, and the wave is"
    },
    {
      "index": 202,
      "start_seconds": 491,
      "source_timestamp": "8:11",
      "end_seconds": null,
      "text": "like one container, one LLM, and simple"
    },
    {
      "index": 203,
      "start_seconds": 495,
      "source_timestamp": "8:15",
      "end_seconds": null,
      "text": "prompt like find all bugs, make no"
    },
    {
      "index": 204,
      "start_seconds": 497,
      "source_timestamp": "8:17",
      "end_seconds": null,
      "text": "mistakes. When it finishes, it just runs"
    },
    {
      "index": 205,
      "start_seconds": 499,
      "source_timestamp": "8:19",
      "end_seconds": null,
      "text": "again. But between these waves, it"
    },
    {
      "index": 206,
      "start_seconds": 501,
      "source_timestamp": "8:21",
      "end_seconds": null,
      "text": "shares the outputs with all the other"
    },
    {
      "index": 207,
      "start_seconds": 503,
      "source_timestamp": "8:23",
      "end_seconds": null,
      "text": "guys. So basically"
    },
    {
      "index": 208,
      "start_seconds": 505,
      "source_timestamp": "8:25",
      "end_seconds": null,
      "text": "between the waves is a synchronization,"
    },
    {
      "index": 209,
      "start_seconds": 507,
      "source_timestamp": "8:27",
      "end_seconds": null,
      "text": "so they are sharing the findings so they"
    },
    {
      "index": 210,
      "start_seconds": 509,
      "source_timestamp": "8:29",
      "end_seconds": null,
      "text": "can build on on the findings. And yeah,"
    },
    {
      "index": 211,
      "start_seconds": 512,
      "source_timestamp": "8:32",
      "end_seconds": null,
      "text": "we just look what happens."
    },
    {
      "index": 212,
      "start_seconds": 514,
      "source_timestamp": "8:34",
      "end_seconds": null,
      "text": "Also here quite interesting is that you"
    },
    {
      "index": 213,
      "start_seconds": 516,
      "source_timestamp": "8:36",
      "end_seconds": null,
      "text": "can see something like"
    },
    {
      "index": 214,
      "start_seconds": 518,
      "source_timestamp": "8:38",
      "end_seconds": null,
      "text": "kind of like performance of the of the"
    },
    {
      "index": 215,
      "start_seconds": 520,
      "source_timestamp": "8:40",
      "end_seconds": null,
      "text": "models that you can see that for example"
    },
    {
      "index": 216,
      "start_seconds": 522,
      "source_timestamp": "8:42",
      "end_seconds": null,
      "text": "with"
    },
    {
      "index": 217,
      "start_seconds": 523,
      "source_timestamp": "8:43",
      "end_seconds": null,
      "text": "Luna"
    },
    {
      "index": 218,
      "start_seconds": 524,
      "source_timestamp": "8:44",
      "end_seconds": null,
      "text": "we finished 35 waves."
    },
    {
      "index": 219,
      "start_seconds": 527,
      "source_timestamp": "8:47",
      "end_seconds": null,
      "text": "Opus finished only six waves, but it"
    },
    {
      "index": 220,
      "start_seconds": 530,
      "source_timestamp": "8:50",
      "end_seconds": null,
      "text": "basically doesn't tell anything how"
    },
    {
      "index": 221,
      "start_seconds": 531,
      "source_timestamp": "8:51",
      "end_seconds": null,
      "text": "efficient the model is. It just tells"
    },
    {
      "index": 222,
      "start_seconds": 532,
      "source_timestamp": "8:52",
      "end_seconds": null,
      "text": "like how fast it claimed that it has"
    },
    {
      "index": 223,
      "start_seconds": 536,
      "source_timestamp": "8:56",
      "end_seconds": null,
      "text": "scanned all the codebase."
    },
    {
      "index": 224,
      "start_seconds": 538,
      "source_timestamp": "8:58",
      "end_seconds": null,
      "text": "And here's the result. So first of all,"
    },
    {
      "index": 225,
      "start_seconds": 542,
      "source_timestamp": "9:02",
      "end_seconds": null,
      "text": "these are the findings when we looked"
    },
    {
      "index": 226,
      "start_seconds": 544,
      "source_timestamp": "9:04",
      "end_seconds": null,
      "text": "just at the wave zero. So this is like"
    },
    {
      "index": 227,
      "start_seconds": 546,
      "source_timestamp": "9:06",
      "end_seconds": null,
      "text": "apples to apples to the previous"
    },
    {
      "index": 228,
      "start_seconds": 548,
      "source_timestamp": "9:08",
      "end_seconds": null,
      "text": "experiment. So here are the humans that"
    },
    {
      "index": 229,
      "start_seconds": 550,
      "source_timestamp": "9:10",
      "end_seconds": null,
      "text": "discovered four issues."
    },
    {
      "index": 230,
      "start_seconds": 552,
      "source_timestamp": "9:12",
      "end_seconds": null,
      "text": "Opus discovered three. DeepSeek, which"
    },
    {
      "index": 231,
      "start_seconds": 554,
      "source_timestamp": "9:14",
      "end_seconds": null,
      "text": "is quite a surprise, discovered three as"
    },
    {
      "index": 232,
      "start_seconds": 556,
      "source_timestamp": "9:16",
      "end_seconds": null,
      "text": "well. And the rest is"
    },
    {
      "index": 233,
      "start_seconds": 559,
      "source_timestamp": "9:19",
      "end_seconds": null,
      "text": "at two issues. These are the"
    },
    {
      "index": 234,
      "start_seconds": 562,
      "source_timestamp": "9:22",
      "end_seconds": null,
      "text": "wave zero."
    },
    {
      "index": 235,
      "start_seconds": 565,
      "source_timestamp": "9:25",
      "end_seconds": null,
      "text": "Then we again look at this, which is the"
    },
    {
      "index": 236,
      "start_seconds": 568,
      "source_timestamp": "9:28",
      "end_seconds": null,
      "text": "uniqueness of findings. So here we can"
    },
    {
      "index": 237,
      "start_seconds": 571,
      "source_timestamp": "9:31",
      "end_seconds": null,
      "text": "see that the"
    },
    {
      "index": 238,
      "start_seconds": 573,
      "source_timestamp": "9:33",
      "end_seconds": null,
      "text": "H2 issues discovered only by humans, and"
    },
    {
      "index": 239,
      "start_seconds": 577,
      "source_timestamp": "9:37",
      "end_seconds": null,
      "text": "H H1 issue, which is kind of like"
    },
    {
      "index": 240,
      "start_seconds": 580,
      "source_timestamp": "9:40",
      "end_seconds": null,
      "text": "problematic, was in the first wave"
    },
    {
      "index": 241,
      "start_seconds": 582,
      "source_timestamp": "9:42",
      "end_seconds": null,
      "text": "discovered only by Opus and DeepSeek."
    },
    {
      "index": 242,
      "start_seconds": 588,
      "source_timestamp": "9:48",
      "end_seconds": null,
      "text": "But this is not what this experiment is"
    },
    {
      "index": 243,
      "start_seconds": 589,
      "source_timestamp": "9:49",
      "end_seconds": null,
      "text": "about. This is just like the first wave."
    },
    {
      "index": 244,
      "start_seconds": 592,
      "source_timestamp": "9:52",
      "end_seconds": null,
      "text": "So now we look how it performed"
    },
    {
      "index": 245,
      "start_seconds": 595,
      "source_timestamp": "9:55",
      "end_seconds": null,
      "text": "next. So when we run"
    },
    {
      "index": 246,
      "start_seconds": 598,
      "source_timestamp": "9:58",
      "end_seconds": null,
      "text": "the more waves"
    },
    {
      "index": 247,
      "start_seconds": 599,
      "source_timestamp": "9:59",
      "end_seconds": null,
      "text": "we we did run up to 32 waves of Luna,"
    },
    {
      "index": 248,
      "start_seconds": 603,
      "source_timestamp": "10:03",
      "end_seconds": null,
      "text": "but we can see that the major"
    },
    {
      "index": 249,
      "start_seconds": 605,
      "source_timestamp": "10:05",
      "end_seconds": null,
      "text": "discoveries happened within the first"
    },
    {
      "index": 250,
      "start_seconds": 608,
      "source_timestamp": "10:08",
      "end_seconds": null,
      "text": "three waves. [snorts] And we can see the"
    },
    {
      "index": 251,
      "start_seconds": 610,
      "source_timestamp": "10:10",
      "end_seconds": null,
      "text": "for example if Opus it the next runs"
    },
    {
      "index": 252,
      "start_seconds": 613,
      "source_timestamp": "10:13",
      "end_seconds": null,
      "text": "didn't bring any value."
    },
    {
      "index": 253,
      "start_seconds": 616,
      "source_timestamp": "10:16",
      "end_seconds": null,
      "text": "With"
    },
    {
      "index": 254,
      "start_seconds": 618,
      "source_timestamp": "10:18",
      "end_seconds": null,
      "text": "DeepSeek also"
    },
    {
      "index": 255,
      "start_seconds": 620,
      "source_timestamp": "10:20",
      "end_seconds": null,
      "text": "not but for example"
    },
    {
      "index": 256,
      "start_seconds": 623,
      "source_timestamp": "10:23",
      "end_seconds": null,
      "text": "GPT six Soul which is the daybreak model"
    },
    {
      "index": 257,
      "start_seconds": 626,
      "source_timestamp": "10:26",
      "end_seconds": null,
      "text": "it was productive"
    },
    {
      "index": 258,
      "start_seconds": 629,
      "source_timestamp": "10:29",
      "end_seconds": null,
      "text": "in three waves which means basically"
    },
    {
      "index": 259,
      "start_seconds": 632,
      "source_timestamp": "10:32",
      "end_seconds": null,
      "text": "this tells when you want to max out the"
    },
    {
      "index": 260,
      "start_seconds": 634,
      "source_timestamp": "10:34",
      "end_seconds": null,
      "text": "performance of GPT six Soul you have to"
    },
    {
      "index": 261,
      "start_seconds": 636,
      "source_timestamp": "10:36",
      "end_seconds": null,
      "text": "run the same prompt three times"
    },
    {
      "index": 262,
      "start_seconds": 639,
      "source_timestamp": "10:39",
      "end_seconds": null,
      "text": "after each other."
    },
    {
      "index": 263,
      "start_seconds": 641,
      "source_timestamp": "10:41",
      "end_seconds": null,
      "text": "And"
    },
    {
      "index": 264,
      "start_seconds": 643,
      "source_timestamp": "10:43",
      "end_seconds": null,
      "text": "the next interesting thing is of course"
    },
    {
      "index": 265,
      "start_seconds": 647,
      "source_timestamp": "10:47",
      "end_seconds": null,
      "text": "who first discovered the"
    },
    {
      "index": 266,
      "start_seconds": 650,
      "source_timestamp": "10:50",
      "end_seconds": null,
      "text": "harder findings. So here we can see that"
    },
    {
      "index": 267,
      "start_seconds": 652,
      "source_timestamp": "10:52",
      "end_seconds": null,
      "text": "it wave zero basically the discovery"
    },
    {
      "index": 268,
      "start_seconds": 655,
      "source_timestamp": "10:55",
      "end_seconds": null,
      "text": "looks like this and then the single"
    },
    {
      "index": 269,
      "start_seconds": 657,
      "source_timestamp": "10:57",
      "end_seconds": null,
      "text": "models were sharing the knowledge but"
    },
    {
      "index": 270,
      "start_seconds": 659,
      "source_timestamp": "10:59",
      "end_seconds": null,
      "text": "the question is like who was the first"
    },
    {
      "index": 271,
      "start_seconds": 661,
      "source_timestamp": "11:01",
      "end_seconds": null,
      "text": "model that discovered the"
    },
    {
      "index": 272,
      "start_seconds": 665,
      "source_timestamp": "11:05",
      "end_seconds": null,
      "text": "harder findings and in this case it was"
    },
    {
      "index": 273,
      "start_seconds": 669,
      "source_timestamp": "11:09",
      "end_seconds": null,
      "text": "in wave one"
    },
    {
      "index": 274,
      "start_seconds": 671,
      "source_timestamp": "11:11",
      "end_seconds": null,
      "text": "this guy which is the GPT six Soul. So"
    },
    {
      "index": 275,
      "start_seconds": 675,
      "source_timestamp": "11:15",
      "end_seconds": null,
      "text": "even if this guy didn't have the highest"
    },
    {
      "index": 276,
      "start_seconds": 678,
      "source_timestamp": "11:18",
      "end_seconds": null,
      "text": "recall in the first wave it discovered"
    },
    {
      "index": 277,
      "start_seconds": 680,
      "source_timestamp": "11:20",
      "end_seconds": null,
      "text": "the unique issue that Opus missed and"
    },
    {
      "index": 278,
      "start_seconds": 682,
      "source_timestamp": "11:22",
      "end_seconds": null,
      "text": "they discovered it in the second wave."
    },
    {
      "index": 279,
      "start_seconds": 685,
      "source_timestamp": "11:25",
      "end_seconds": null,
      "text": "And then as they shared shared the"
    },
    {
      "index": 280,
      "start_seconds": 688,
      "source_timestamp": "11:28",
      "end_seconds": null,
      "text": "findings everyone did catch up and in"
    },
    {
      "index": 281,
      "start_seconds": 690,
      "source_timestamp": "11:30",
      "end_seconds": null,
      "text": "the end everyone had everything."
    },
    {
      "index": 282,
      "start_seconds": 694,
      "source_timestamp": "11:34",
      "end_seconds": null,
      "text": "And"
    },
    {
      "index": 283,
      "start_seconds": 696,
      "source_timestamp": "11:36",
      "end_seconds": null,
      "text": "then the question is like"
    },
    {
      "index": 284,
      "start_seconds": 698,
      "source_timestamp": "11:38",
      "end_seconds": null,
      "text": "if the theory that I presented if the"
    },
    {
      "index": 285,
      "start_seconds": 700,
      "source_timestamp": "11:40",
      "end_seconds": null,
      "text": "sharing of information between the waves"
    },
    {
      "index": 286,
      "start_seconds": 703,
      "source_timestamp": "11:43",
      "end_seconds": null,
      "text": "helped. So in this case it did not bring"
    },
    {
      "index": 287,
      "start_seconds": 706,
      "source_timestamp": "11:46",
      "end_seconds": null,
      "text": "any new finding. So we cannot tell for"
    },
    {
      "index": 288,
      "start_seconds": 710,
      "source_timestamp": "11:50",
      "end_seconds": null,
      "text": "sure but"
    },
    {
      "index": 289,
      "start_seconds": 712,
      "source_timestamp": "11:52",
      "end_seconds": null,
      "text": "it helped the weaker models basically to"
    },
    {
      "index": 290,
      "start_seconds": 715,
      "source_timestamp": "11:55",
      "end_seconds": null,
      "text": "catch up and here are two cases"
    },
    {
      "index": 291,
      "start_seconds": 718,
      "source_timestamp": "11:58",
      "end_seconds": null,
      "text": "basically that"
    },
    {
      "index": 292,
      "start_seconds": 719,
      "source_timestamp": "11:59",
      "end_seconds": null,
      "text": "helped so GLM discovered something which"
    },
    {
      "index": 293,
      "start_seconds": 723,
      "source_timestamp": "12:03",
      "end_seconds": null,
      "text": "then DeepSeek finished as finding and"
    },
    {
      "index": 294,
      "start_seconds": 726,
      "source_timestamp": "12:06",
      "end_seconds": null,
      "text": "the same here Kimmy was building on"
    },
    {
      "index": 295,
      "start_seconds": 728,
      "source_timestamp": "12:08",
      "end_seconds": null,
      "text": "something that was presented in early"
    },
    {
      "index": 296,
      "start_seconds": 731,
      "source_timestamp": "12:11",
      "end_seconds": null,
      "text": "waves which confirmed the findings. So,"
    },
    {
      "index": 297,
      "start_seconds": 734,
      "source_timestamp": "12:14",
      "end_seconds": null,
      "text": "on this data set, we cannot say for sure"
    },
    {
      "index": 298,
      "start_seconds": 737,
      "source_timestamp": "12:17",
      "end_seconds": null,
      "text": "that the replication of the hugging face"
    },
    {
      "index": 299,
      "start_seconds": 740,
      "source_timestamp": "12:20",
      "end_seconds": null,
      "text": "experiment did bring the value, but if"
    },
    {
      "index": 300,
      "start_seconds": 743,
      "source_timestamp": "12:23",
      "end_seconds": null,
      "text": "we would not be using the"
    },
    {
      "index": 301,
      "start_seconds": 745,
      "source_timestamp": "12:25",
      "end_seconds": null,
      "text": "best two models for these guys, it would"
    },
    {
      "index": 302,
      "start_seconds": 748,
      "source_timestamp": "12:28",
      "end_seconds": null,
      "text": "definitely help."
    },
    {
      "index": 303,
      "start_seconds": 751,
      "source_timestamp": "12:31",
      "end_seconds": null,
      "text": "And"
    },
    {
      "index": 304,
      "start_seconds": 753,
      "source_timestamp": "12:33",
      "end_seconds": null,
      "text": "yeah, how did it end? So,"
    },
    {
      "index": 305,
      "start_seconds": 755,
      "source_timestamp": "12:35",
      "end_seconds": null,
      "text": "the recall after the first wave was 75%."
    },
    {
      "index": 306,
      "start_seconds": 760,
      "source_timestamp": "12:40",
      "end_seconds": null,
      "text": "After the always combined, it was 100%."
    },
    {
      "index": 307,
      "start_seconds": 762,
      "source_timestamp": "12:42",
      "end_seconds": null,
      "text": "So, we can tell that at this"
    },
    {
      "index": 308,
      "start_seconds": 764,
      "source_timestamp": "12:44",
      "end_seconds": null,
      "text": "benchmark, we"
    },
    {
      "index": 309,
      "start_seconds": 766,
      "source_timestamp": "12:46",
      "end_seconds": null,
      "text": "were able to discover everything by the"
    },
    {
      "index": 310,
      "start_seconds": 769,
      "source_timestamp": "12:49",
      "end_seconds": null,
      "text": "AI agent workflow."
    },
    {
      "index": 311,
      "start_seconds": 771,
      "source_timestamp": "12:51",
      "end_seconds": null,
      "text": "Just a comment on this, we did not play"
    },
    {
      "index": 312,
      "start_seconds": 774,
      "source_timestamp": "12:54",
      "end_seconds": null,
      "text": "anyhow with the prompt engineering. The"
    },
    {
      "index": 313,
      "start_seconds": 775,
      "source_timestamp": "12:55",
      "end_seconds": null,
      "text": "prompt was like very stupid. So, this"
    },
    {
      "index": 314,
      "start_seconds": 777,
      "source_timestamp": "12:57",
      "end_seconds": null,
      "text": "was like brute force testing. And on"
    },
    {
      "index": 315,
      "start_seconds": 780,
      "source_timestamp": "13:00",
      "end_seconds": null,
      "text": "this experiment, we spent about"
    },
    {
      "index": 316,
      "start_seconds": 783,
      "source_timestamp": "13:03",
      "end_seconds": null,
      "text": "1 million tokens, so it's not that much,"
    },
    {
      "index": 317,
      "start_seconds": 786,
      "source_timestamp": "13:06",
      "end_seconds": null,
      "text": "but it's also not very efficient. It's"
    },
    {
      "index": 318,
      "start_seconds": 788,
      "source_timestamp": "13:08",
      "end_seconds": null,
      "text": "kind of like brute force testing."
    },
    {
      "index": 319,
      "start_seconds": 792,
      "source_timestamp": "13:12",
      "end_seconds": null,
      "text": "Yeah, and if you would like to"
    },
    {
      "index": 320,
      "start_seconds": 796,
      "source_timestamp": "13:16",
      "end_seconds": null,
      "text": "learn more about it, so here is the"
    },
    {
      "index": 321,
      "start_seconds": 798,
      "source_timestamp": "13:18",
      "end_seconds": null,
      "text": "block post where we are comparing the"
    },
    {
      "index": 322,
      "start_seconds": 800,
      "source_timestamp": "13:20",
      "end_seconds": null,
      "text": "models. And here is the interesting"
    },
    {
      "index": 323,
      "start_seconds": 803,
      "source_timestamp": "13:23",
      "end_seconds": null,
      "text": "open AI experiment. And also, if you"
    },
    {
      "index": 324,
      "start_seconds": 806,
      "source_timestamp": "13:26",
      "end_seconds": null,
      "text": "would like to try the tool that we just"
    },
    {
      "index": 325,
      "start_seconds": 808,
      "source_timestamp": "13:28",
      "end_seconds": null,
      "text": "open sourced, it's in our GitHub since"
    },
    {
      "index": 326,
      "start_seconds": 812,
      "source_timestamp": "13:32",
      "end_seconds": null,
      "text": "yesterday."
    },
    {
      "index": 327,
      "start_seconds": 814,
      "source_timestamp": "13:34",
      "end_seconds": null,
      "text": "And"
    },
    {
      "index": 328,
      "start_seconds": 815,
      "source_timestamp": "13:35",
      "end_seconds": null,
      "text": "that's pretty much it. Thank you very"
    },
    {
      "index": 329,
      "start_seconds": 817,
      "source_timestamp": "13:37",
      "end_seconds": null,
      "text": "much."
    }
  ]
}
