{
  "schemaVersion": 1,
  "videoId": "Dm7fzCWJ0Oo",
  "videoUrl": "https://www.youtube.com/watch?v=Dm7fzCWJ0Oo",
  "title": "Benchmarking Security Intelligence: Why Web3 AI Security Eval Is Broken",
  "speakerCredit": "Suryansh Singh",
  "programmeTalkId": "dss26-78fe46d8870298d6",
  "language": "en",
  "captionType": "auto-generated",
  "humanAccuracyReviewed": false,
  "spokenAudioCompletenessVerified": false,
  "cueEndTimesAvailable": false,
  "source": {
    "provider": "YouTube",
    "method": "native browser transcript export",
    "sha256": "225c0f75f801f5b95a7d9e41db6eda603bcd31620a076cad71e679b947959745",
    "exportedAt": "2026-11-16T07:38:24.368497+00:00"
  },
  "qualityCaveats": [
    "Names, organizations, technical terms, code and numbers may be misrecognized.",
    "Source wording and disfluencies are preserved; no editorial corrections were applied.",
    "Export contains whole-second cue starts only; cue ends and subsecond timing are unavailable.",
    "A full caption-track export does not establish that every spoken word is represented.",
    "Automatic YouTube captions; not human-verified against the audio."
  ],
  "timingValidation": {
    "first_cue_start_seconds": 2,
    "last_cue_start_seconds": 340,
    "video_duration_seconds": 343,
    "seconds_after_last_cue_start": 3,
    "cue_start_gaps_over_30_seconds": [],
    "boundary_check": "pass",
    "criterion": "first cue <=15 seconds; last cue start within 30 seconds of player duration, allowing 1-second precision",
    "meaning": "Timing plausibility only; silence/applause or source caption omissions require audio review."
  },
  "segments": [
    {
      "index": 0,
      "start_seconds": 2,
      "source_timestamp": "0:02",
      "end_seconds": null,
      "text": "Hi guys. I'm"
    },
    {
      "index": 1,
      "start_seconds": 7,
      "source_timestamp": "0:07",
      "end_seconds": null,
      "text": "Hi guys. I'm Suryansh. I'm a security"
    },
    {
      "index": 2,
      "start_seconds": 10,
      "source_timestamp": "0:10",
      "end_seconds": null,
      "text": "researcher at OpenZeppelin."
    },
    {
      "index": 3,
      "start_seconds": 12,
      "source_timestamp": "0:12",
      "end_seconds": null,
      "text": "Some of you may know me by my pseudonym"
    },
    {
      "index": 4,
      "start_seconds": 14,
      "source_timestamp": "0:14",
      "end_seconds": null,
      "text": "Zero X Monsoon."
    },
    {
      "index": 5,
      "start_seconds": 16,
      "source_timestamp": "0:16",
      "end_seconds": null,
      "text": "And this is a talk on benchmarking like"
    },
    {
      "index": 6,
      "start_seconds": 19,
      "source_timestamp": "0:19",
      "end_seconds": null,
      "text": "your AI security agents."
    },
    {
      "index": 7,
      "start_seconds": 22,
      "source_timestamp": "0:22",
      "end_seconds": null,
      "text": "We've heard a lot about them like in the"
    },
    {
      "index": 8,
      "start_seconds": 23,
      "source_timestamp": "0:23",
      "end_seconds": null,
      "text": "last 20 minutes on how to use them. This"
    },
    {
      "index": 9,
      "start_seconds": 26,
      "source_timestamp": "0:26",
      "end_seconds": null,
      "text": "talk is mostly about"
    },
    {
      "index": 10,
      "start_seconds": 28,
      "source_timestamp": "0:28",
      "end_seconds": null,
      "text": "um"
    },
    {
      "index": 11,
      "start_seconds": 29,
      "source_timestamp": "0:29",
      "end_seconds": null,
      "text": "how to actually measure the performance"
    },
    {
      "index": 12,
      "start_seconds": 31,
      "source_timestamp": "0:31",
      "end_seconds": null,
      "text": "of let's say new frontier models or"
    },
    {
      "index": 13,
      "start_seconds": 34,
      "source_timestamp": "0:34",
      "end_seconds": null,
      "text": "AI agents that are in the wild or one"
    },
    {
      "index": 14,
      "start_seconds": 37,
      "source_timestamp": "0:37",
      "end_seconds": null,
      "text": "you're building."
    },
    {
      "index": 15,
      "start_seconds": 46,
      "source_timestamp": "0:46",
      "end_seconds": null,
      "text": "Okay. So"
    },
    {
      "index": 16,
      "start_seconds": 47,
      "source_timestamp": "0:47",
      "end_seconds": null,
      "text": "first of all,"
    },
    {
      "index": 17,
      "start_seconds": 49,
      "source_timestamp": "0:49",
      "end_seconds": null,
      "text": "why do you even need"
    },
    {
      "index": 18,
      "start_seconds": 51,
      "source_timestamp": "0:51",
      "end_seconds": null,
      "text": "an E WILE or your own"
    },
    {
      "index": 19,
      "start_seconds": 54,
      "source_timestamp": "0:54",
      "end_seconds": null,
      "text": "benchmark when there are so many"
    },
    {
      "index": 20,
      "start_seconds": 55,
      "source_timestamp": "0:55",
      "end_seconds": null,
      "text": "security public benchmarks?"
    },
    {
      "index": 21,
      "start_seconds": 57,
      "source_timestamp": "0:57",
      "end_seconds": null,
      "text": "Well, basically because most of them are"
    },
    {
      "index": 22,
      "start_seconds": 61,
      "source_timestamp": "1:01",
      "end_seconds": null,
      "text": "very broad and general."
    },
    {
      "index": 23,
      "start_seconds": 63,
      "source_timestamp": "1:03",
      "end_seconds": null,
      "text": "You probably need something that's like"
    },
    {
      "index": 24,
      "start_seconds": 66,
      "source_timestamp": "1:06",
      "end_seconds": null,
      "text": "more domain specific for you and your"
    },
    {
      "index": 25,
      "start_seconds": 68,
      "source_timestamp": "1:08",
      "end_seconds": null,
      "text": "work."
    },
    {
      "index": 26,
      "start_seconds": 69,
      "source_timestamp": "1:09",
      "end_seconds": null,
      "text": "Secondly, you need something that's"
    },
    {
      "index": 27,
      "start_seconds": 72,
      "source_timestamp": "1:12",
      "end_seconds": null,
      "text": "hard"
    },
    {
      "index": 28,
      "start_seconds": 73,
      "source_timestamp": "1:13",
      "end_seconds": null,
      "text": "and using which you can measure"
    },
    {
      "index": 29,
      "start_seconds": 75,
      "source_timestamp": "1:15",
      "end_seconds": null,
      "text": "improvements in any AI agents, security"
    },
    {
      "index": 30,
      "start_seconds": 77,
      "source_timestamp": "1:17",
      "end_seconds": null,
      "text": "agents, or anything that you're building"
    },
    {
      "index": 31,
      "start_seconds": 80,
      "source_timestamp": "1:20",
      "end_seconds": null,
      "text": "on your newer frontier models."
    },
    {
      "index": 32,
      "start_seconds": 83,
      "source_timestamp": "1:23",
      "end_seconds": null,
      "text": "So"
    },
    {
      "index": 33,
      "start_seconds": 85,
      "source_timestamp": "1:25",
      "end_seconds": null,
      "text": "this year like one"
    },
    {
      "index": 34,
      "start_seconds": 88,
      "source_timestamp": "1:28",
      "end_seconds": null,
      "text": "Web 3 security benchmark got really"
    },
    {
      "index": 35,
      "start_seconds": 90,
      "source_timestamp": "1:30",
      "end_seconds": null,
      "text": "popular."
    },
    {
      "index": 36,
      "start_seconds": 92,
      "source_timestamp": "1:32",
      "end_seconds": null,
      "text": "But it had some problems. First of all,"
    },
    {
      "index": 37,
      "start_seconds": 95,
      "source_timestamp": "1:35",
      "end_seconds": null,
      "text": "it uses it used like public audit"
    },
    {
      "index": 38,
      "start_seconds": 98,
      "source_timestamp": "1:38",
      "end_seconds": null,
      "text": "contests at as targets"
    },
    {
      "index": 39,
      "start_seconds": 100,
      "source_timestamp": "1:40",
      "end_seconds": null,
      "text": "which were probably already in the"
    },
    {
      "index": 40,
      "start_seconds": 103,
      "source_timestamp": "1:43",
      "end_seconds": null,
      "text": "training set of the models it was"
    },
    {
      "index": 41,
      "start_seconds": 104,
      "source_timestamp": "1:44",
      "end_seconds": null,
      "text": "testing."
    },
    {
      "index": 42,
      "start_seconds": 105,
      "source_timestamp": "1:45",
      "end_seconds": null,
      "text": "Secondly, it used the answer key"
    },
    {
      "index": 43,
      "start_seconds": 109,
      "source_timestamp": "1:49",
      "end_seconds": null,
      "text": "of basically all the issues that were"
    },
    {
      "index": 44,
      "start_seconds": 110,
      "source_timestamp": "1:50",
      "end_seconds": null,
      "text": "found in that in those public contests"
    },
    {
      "index": 45,
      "start_seconds": 113,
      "source_timestamp": "1:53",
      "end_seconds": null,
      "text": "and anything new that the models"
    },
    {
      "index": 46,
      "start_seconds": 116,
      "source_timestamp": "1:56",
      "end_seconds": null,
      "text": "actually found were not graded."
    },
    {
      "index": 47,
      "start_seconds": 119,
      "source_timestamp": "1:59",
      "end_seconds": null,
      "text": "So"
    },
    {
      "index": 48,
      "start_seconds": 120,
      "source_timestamp": "2:00",
      "end_seconds": null,
      "text": "let's about how to get uh like how to"
    },
    {
      "index": 49,
      "start_seconds": 122,
      "source_timestamp": "2:02",
      "end_seconds": null,
      "text": "get your evals wrong. Basically,"
    },
    {
      "index": 50,
      "start_seconds": 125,
      "source_timestamp": "2:05",
      "end_seconds": null,
      "text": "um"
    },
    {
      "index": 51,
      "start_seconds": 127,
      "source_timestamp": "2:07",
      "end_seconds": null,
      "text": "uh like if you pick the wrong targets,"
    },
    {
      "index": 52,
      "start_seconds": 130,
      "source_timestamp": "2:10",
      "end_seconds": null,
      "text": "you won't have a good eval."
    },
    {
      "index": 53,
      "start_seconds": 132,
      "source_timestamp": "2:12",
      "end_seconds": null,
      "text": "And examples of wrong targets are"
    },
    {
      "index": 54,
      "start_seconds": 135,
      "source_timestamp": "2:15",
      "end_seconds": null,
      "text": "basically using public repos or public"
    },
    {
      "index": 55,
      "start_seconds": 137,
      "source_timestamp": "2:17",
      "end_seconds": null,
      "text": "audit contests because that risks"
    },
    {
      "index": 56,
      "start_seconds": 138,
      "source_timestamp": "2:18",
      "end_seconds": null,
      "text": "contamination. If you're only using like"
    },
    {
      "index": 57,
      "start_seconds": 141,
      "source_timestamp": "2:21",
      "end_seconds": null,
      "text": "one repo for testing every time you run"
    },
    {
      "index": 58,
      "start_seconds": 144,
      "source_timestamp": "2:24",
      "end_seconds": null,
      "text": "your AI agent on new models,"
    },
    {
      "index": 59,
      "start_seconds": 146,
      "source_timestamp": "2:26",
      "end_seconds": null,
      "text": "uh you risk like overfitting."
    },
    {
      "index": 60,
      "start_seconds": 149,
      "source_timestamp": "2:29",
      "end_seconds": null,
      "text": "And"
    },
    {
      "index": 61,
      "start_seconds": 150,
      "source_timestamp": "2:30",
      "end_seconds": null,
      "text": "if you're synthetically injecting bugs"
    },
    {
      "index": 62,
      "start_seconds": 153,
      "source_timestamp": "2:33",
      "end_seconds": null,
      "text": "into let's say a code base, then you're"
    },
    {
      "index": 63,
      "start_seconds": 155,
      "source_timestamp": "2:35",
      "end_seconds": null,
      "text": "basically pattern matching what you're"
    },
    {
      "index": 64,
      "start_seconds": 157,
      "source_timestamp": "2:37",
      "end_seconds": null,
      "text": "injecting or if you're depending on"
    },
    {
      "index": 65,
      "start_seconds": 160,
      "source_timestamp": "2:40",
      "end_seconds": null,
      "text": "on an AI model to do that, you're"
    },
    {
      "index": 66,
      "start_seconds": 163,
      "source_timestamp": "2:43",
      "end_seconds": null,
      "text": "basically just testing like what it"
    },
    {
      "index": 67,
      "start_seconds": 165,
      "source_timestamp": "2:45",
      "end_seconds": null,
      "text": "tries to find or what it tries to inject"
    },
    {
      "index": 68,
      "start_seconds": 169,
      "source_timestamp": "2:49",
      "end_seconds": null,
      "text": "and then find itself again."
    },
    {
      "index": 69,
      "start_seconds": 171,
      "source_timestamp": "2:51",
      "end_seconds": null,
      "text": "Do not use like smaller toy contracts or"
    },
    {
      "index": 70,
      "start_seconds": 173,
      "source_timestamp": "2:53",
      "end_seconds": null,
      "text": "CTFs because"
    },
    {
      "index": 71,
      "start_seconds": 175,
      "source_timestamp": "2:55",
      "end_seconds": null,
      "text": "the models now are really strong and"
    },
    {
      "index": 72,
      "start_seconds": 178,
      "source_timestamp": "2:58",
      "end_seconds": null,
      "text": "they'll just find all of the bugs and"
    },
    {
      "index": 73,
      "start_seconds": 180,
      "source_timestamp": "3:00",
      "end_seconds": null,
      "text": "you won't have a good eval."
    },
    {
      "index": 74,
      "start_seconds": 182,
      "source_timestamp": "3:02",
      "end_seconds": null,
      "text": "You can also have log like wrong answer"
    },
    {
      "index": 75,
      "start_seconds": 184,
      "source_timestamp": "3:04",
      "end_seconds": null,
      "text": "keys. If your ground truth is fixed,"
    },
    {
      "index": 76,
      "start_seconds": 187,
      "source_timestamp": "3:07",
      "end_seconds": null,
      "text": "basically meaning uh every time"
    },
    {
      "index": 77,
      "start_seconds": 190,
      "source_timestamp": "3:10",
      "end_seconds": null,
      "text": "something unknown comes up whenever you"
    },
    {
      "index": 78,
      "start_seconds": 191,
      "source_timestamp": "3:11",
      "end_seconds": null,
      "text": "run your AI agent on on the eval, if"
    },
    {
      "index": 79,
      "start_seconds": 194,
      "source_timestamp": "3:14",
      "end_seconds": null,
      "text": "you're"
    },
    {
      "index": 80,
      "start_seconds": 196,
      "source_timestamp": "3:16",
      "end_seconds": null,
      "text": "not adding new issues to it, then that's"
    },
    {
      "index": 81,
      "start_seconds": 198,
      "source_timestamp": "3:18",
      "end_seconds": null,
      "text": "a problem."
    },
    {
      "index": 82,
      "start_seconds": 199,
      "source_timestamp": "3:19",
      "end_seconds": null,
      "text": "Uh give um"
    },
    {
      "index": 83,
      "start_seconds": 202,
      "source_timestamp": "3:22",
      "end_seconds": null,
      "text": "like if you're only testing for the"
    },
    {
      "index": 84,
      "start_seconds": 204,
      "source_timestamp": "3:24",
      "end_seconds": null,
      "text": "correct issues, then also it's not a"
    },
    {
      "index": 85,
      "start_seconds": 206,
      "source_timestamp": "3:26",
      "end_seconds": null,
      "text": "very good one because you need to see"
    },
    {
      "index": 86,
      "start_seconds": 207,
      "source_timestamp": "3:27",
      "end_seconds": null,
      "text": "how much noise it's generating and have"
    },
    {
      "index": 87,
      "start_seconds": 209,
      "source_timestamp": "3:29",
      "end_seconds": null,
      "text": "a separate score for it or like have"
    },
    {
      "index": 88,
      "start_seconds": 211,
      "source_timestamp": "3:31",
      "end_seconds": null,
      "text": "penalties for that. And um"
    },
    {
      "index": 89,
      "start_seconds": 215,
      "source_timestamp": "3:35",
      "end_seconds": null,
      "text": "don't count all the issues it has fined"
    },
    {
      "index": 90,
      "start_seconds": 216,
      "source_timestamp": "3:36",
      "end_seconds": null,
      "text": "like way by severity or like have like a"
    },
    {
      "index": 91,
      "start_seconds": 220,
      "source_timestamp": "3:40",
      "end_seconds": null,
      "text": "better metric for it."
    },
    {
      "index": 92,
      "start_seconds": 222,
      "source_timestamp": "3:42",
      "end_seconds": null,
      "text": "You could also give it wrong conditions"
    },
    {
      "index": 93,
      "start_seconds": 224,
      "source_timestamp": "3:44",
      "end_seconds": null,
      "text": "like basically if you're giving it not"
    },
    {
      "index": 94,
      "start_seconds": 227,
      "source_timestamp": "3:47",
      "end_seconds": null,
      "text": "enough access to tools and stuff,"
    },
    {
      "index": 95,
      "start_seconds": 230,
      "source_timestamp": "3:50",
      "end_seconds": null,
      "text": "then it's probably not performing at at"
    },
    {
      "index": 96,
      "start_seconds": 232,
      "source_timestamp": "3:52",
      "end_seconds": null,
      "text": "its best."
    },
    {
      "index": 97,
      "start_seconds": 234,
      "source_timestamp": "3:54",
      "end_seconds": null,
      "text": "Uh but don't give it too much"
    },
    {
      "index": 98,
      "start_seconds": 235,
      "source_timestamp": "3:55",
      "end_seconds": null,
      "text": "information like hints and stuff because"
    },
    {
      "index": 99,
      "start_seconds": 238,
      "source_timestamp": "3:58",
      "end_seconds": null,
      "text": "then you're not really measuring the"
    },
    {
      "index": 100,
      "start_seconds": 239,
      "source_timestamp": "3:59",
      "end_seconds": null,
      "text": "performance like bug hunting"
    },
    {
      "index": 101,
      "start_seconds": 240,
      "source_timestamp": "4:00",
      "end_seconds": null,
      "text": "capabilities of the model of the"
    },
    {
      "index": 102,
      "start_seconds": 242,
      "source_timestamp": "4:02",
      "end_seconds": null,
      "text": "security agent."
    },
    {
      "index": 103,
      "start_seconds": 245,
      "source_timestamp": "4:05",
      "end_seconds": null,
      "text": "So what a good one does, so just to go"
    },
    {
      "index": 104,
      "start_seconds": 247,
      "source_timestamp": "4:07",
      "end_seconds": null,
      "text": "through it really quickly, have a lot of"
    },
    {
      "index": 105,
      "start_seconds": 249,
      "source_timestamp": "4:09",
      "end_seconds": null,
      "text": "private code bases that you've audited."
    },
    {
      "index": 106,
      "start_seconds": 252,
      "source_timestamp": "4:12",
      "end_seconds": null,
      "text": "Please take permissions from a client"
    },
    {
      "index": 107,
      "start_seconds": 254,
      "source_timestamp": "4:14",
      "end_seconds": null,
      "text": "before using their code bases for evals."
    },
    {
      "index": 108,
      "start_seconds": 257,
      "source_timestamp": "4:17",
      "end_seconds": null,
      "text": "But this is probably the best thing you"
    },
    {
      "index": 109,
      "start_seconds": 258,
      "source_timestamp": "4:18",
      "end_seconds": null,
      "text": "could do."
    },
    {
      "index": 110,
      "start_seconds": 259,
      "source_timestamp": "4:19",
      "end_seconds": null,
      "text": "Secondly, every time you run the agent"
    },
    {
      "index": 111,
      "start_seconds": 262,
      "source_timestamp": "4:22",
      "end_seconds": null,
      "text": "and new issues come up that you don't"
    },
    {
      "index": 112,
      "start_seconds": 263,
      "source_timestamp": "4:23",
      "end_seconds": null,
      "text": "know about, that they are not known"
    },
    {
      "index": 113,
      "start_seconds": 265,
      "source_timestamp": "4:25",
      "end_seconds": null,
      "text": "false positives or known like true"
    },
    {
      "index": 114,
      "start_seconds": 267,
      "source_timestamp": "4:27",
      "end_seconds": null,
      "text": "positives, triage them and add them to"
    },
    {
      "index": 115,
      "start_seconds": 269,
      "source_timestamp": "4:29",
      "end_seconds": null,
      "text": "either the"
    },
    {
      "index": 116,
      "start_seconds": 271,
      "source_timestamp": "4:31",
      "end_seconds": null,
      "text": "ground truth or like maintain a set of"
    },
    {
      "index": 117,
      "start_seconds": 273,
      "source_timestamp": "4:33",
      "end_seconds": null,
      "text": "like known false positives."
    },
    {
      "index": 118,
      "start_seconds": 276,
      "source_timestamp": "4:36",
      "end_seconds": null,
      "text": "Give it all the tools that it could"
    },
    {
      "index": 119,
      "start_seconds": 278,
      "source_timestamp": "4:38",
      "end_seconds": null,
      "text": "probably use. Maybe give it any they"
    },
    {
      "index": 120,
      "start_seconds": 281,
      "source_timestamp": "4:41",
      "end_seconds": null,
      "text": "would the internet access give it"
    },
    {
      "index": 121,
      "start_seconds": 283,
      "source_timestamp": "4:43",
      "end_seconds": null,
      "text": "internet access if you're testing for"
    },
    {
      "index": 122,
      "start_seconds": 285,
      "source_timestamp": "4:45",
      "end_seconds": null,
      "text": "something specific."
    },
    {
      "index": 123,
      "start_seconds": 286,
      "source_timestamp": "4:46",
      "end_seconds": null,
      "text": "And then benchmark your matching agent."
    },
    {
      "index": 124,
      "start_seconds": 290,
      "source_timestamp": "4:50",
      "end_seconds": null,
      "text": "Give it like tweak the ground truth"
    },
    {
      "index": 125,
      "start_seconds": 292,
      "source_timestamp": "4:52",
      "end_seconds": null,
      "text": "issues a bit and see how it's"
    },
    {
      "index": 126,
      "start_seconds": 294,
      "source_timestamp": "4:54",
      "end_seconds": null,
      "text": "performing."
    },
    {
      "index": 127,
      "start_seconds": 295,
      "source_timestamp": "4:55",
      "end_seconds": null,
      "text": "And"
    },
    {
      "index": 128,
      "start_seconds": 297,
      "source_timestamp": "4:57",
      "end_seconds": null,
      "text": "that's it, man."
    },
    {
      "index": 129,
      "start_seconds": 299,
      "source_timestamp": "4:59",
      "end_seconds": null,
      "text": "Just to start gather a few past"
    },
    {
      "index": 130,
      "start_seconds": 301,
      "source_timestamp": "5:01",
      "end_seconds": null,
      "text": "engagements of good code bases of"
    },
    {
      "index": 131,
      "start_seconds": 303,
      "source_timestamp": "5:03",
      "end_seconds": null,
      "text": "various sizes."
    },
    {
      "index": 132,
      "start_seconds": 304,
      "source_timestamp": "5:04",
      "end_seconds": null,
      "text": "Use them, seed the answer key with the"
    },
    {
      "index": 133,
      "start_seconds": 307,
      "source_timestamp": "5:07",
      "end_seconds": null,
      "text": "confirmed findings."
    },
    {
      "index": 134,
      "start_seconds": 309,
      "source_timestamp": "5:09",
      "end_seconds": null,
      "text": "And then run the tool"
    },
    {
      "index": 135,
      "start_seconds": 311,
      "source_timestamp": "5:11",
      "end_seconds": null,
      "text": "with the repo, the tests, maybe the"
    },
    {
      "index": 136,
      "start_seconds": 312,
      "source_timestamp": "5:12",
      "end_seconds": null,
      "text": "internet."
    },
    {
      "index": 137,
      "start_seconds": 314,
      "source_timestamp": "5:14",
      "end_seconds": null,
      "text": "And then triage everything it reports"
    },
    {
      "index": 138,
      "start_seconds": 315,
      "source_timestamp": "5:15",
      "end_seconds": null,
      "text": "that's not on the answer keys. Add it to"
    },
    {
      "index": 139,
      "start_seconds": 318,
      "source_timestamp": "5:18",
      "end_seconds": null,
      "text": "either the known issues and or all or"
    },
    {
      "index": 140,
      "start_seconds": 321,
      "source_timestamp": "5:21",
      "end_seconds": null,
      "text": "the list of false positives."
    },
    {
      "index": 141,
      "start_seconds": 324,
      "source_timestamp": "5:24",
      "end_seconds": null,
      "text": "And that's how your evals will keep on"
    },
    {
      "index": 142,
      "start_seconds": 326,
      "source_timestamp": "5:26",
      "end_seconds": null,
      "text": "growing and it would be like domain"
    },
    {
      "index": 143,
      "start_seconds": 328,
      "source_timestamp": "5:28",
      "end_seconds": null,
      "text": "specific because the public benchmarks"
    },
    {
      "index": 144,
      "start_seconds": 331,
      "source_timestamp": "5:31",
      "end_seconds": null,
      "text": "they only like"
    },
    {
      "index": 145,
      "start_seconds": 333,
      "source_timestamp": "5:33",
      "end_seconds": null,
      "text": "test against known findings. But what"
    },
    {
      "index": 146,
      "start_seconds": 335,
      "source_timestamp": "5:35",
      "end_seconds": null,
      "text": "you need to do is you need to know how"
    },
    {
      "index": 147,
      "start_seconds": 337,
      "source_timestamp": "5:37",
      "end_seconds": null,
      "text": "well your agent performs on like unknown"
    },
    {
      "index": 148,
      "start_seconds": 340,
      "source_timestamp": "5:40",
      "end_seconds": null,
      "text": "code bases. Thank you."
    }
  ]
}
