{
  "schemaVersion": 1,
  "videoId": "QRURiESp5TA",
  "videoUrl": "https://www.youtube.com/watch?v=QRURiESp5TA",
  "title": "AI-FV: Slop or Prop? Explorations in Agentic Formal Verification",
  "speakerCredit": "Jonathan Mevs",
  "programmeTalkId": "dss26-25ae3f864ea85a55",
  "language": "en",
  "captionType": "auto-generated",
  "humanAccuracyReviewed": false,
  "spokenAudioCompletenessVerified": false,
  "cueEndTimesAvailable": false,
  "source": {
    "provider": "YouTube",
    "method": "native browser transcript export",
    "sha256": "6145dc50240426d431cfd4126b2d2574aca2809814c95bf00db010656e35c3e7",
    "exportedAt": "2026-11-16T07:34:10.916781+00:00"
  },
  "qualityCaveats": [
    "Names, organizations, technical terms, code and numbers may be misrecognized.",
    "Source wording and disfluencies are preserved; no editorial corrections were applied.",
    "Export contains whole-second cue starts only; cue ends and subsecond timing are unavailable.",
    "A full caption-track export does not establish that every spoken word is represented.",
    "Automatic YouTube captions; not human-verified against the audio."
  ],
  "timingValidation": {
    "first_cue_start_seconds": 0,
    "last_cue_start_seconds": 1553,
    "video_duration_seconds": 1556,
    "seconds_after_last_cue_start": 3,
    "cue_start_gaps_over_30_seconds": [],
    "boundary_check": "pass",
    "criterion": "first cue <=15 seconds; last cue start within 30 seconds of player duration, allowing 1-second precision",
    "meaning": "Timing plausibility only; silence/applause or source caption omissions require audio review."
  },
  "segments": [
    {
      "index": 0,
      "start_seconds": 0,
      "source_timestamp": "0:00",
      "end_seconds": null,
      "text": "Um, well, hello everybody. Uh, thanks"
    },
    {
      "index": 1,
      "start_seconds": 3,
      "source_timestamp": "0:03",
      "end_seconds": null,
      "text": "DeFi Security Summit for having me here."
    },
    {
      "index": 2,
      "start_seconds": 5,
      "source_timestamp": "0:05",
      "end_seconds": null,
      "text": "I'm John."
    },
    {
      "index": 3,
      "start_seconds": 6,
      "source_timestamp": "0:06",
      "end_seconds": null,
      "text": "I'm a senior engineer with Quantstamp,"
    },
    {
      "index": 4,
      "start_seconds": 8,
      "source_timestamp": "0:08",
      "end_seconds": null,
      "text": "and the topic I'll be talking about"
    },
    {
      "index": 5,
      "start_seconds": 9,
      "source_timestamp": "0:09",
      "end_seconds": null,
      "text": "today is like formal verification and"
    },
    {
      "index": 6,
      "start_seconds": 12,
      "source_timestamp": "0:12",
      "end_seconds": null,
      "text": "how far can AI support such uh and some"
    },
    {
      "index": 7,
      "start_seconds": 16,
      "source_timestamp": "0:16",
      "end_seconds": null,
      "text": "experiments we did uh surrounding a"
    },
    {
      "index": 8,
      "start_seconds": 17,
      "source_timestamp": "0:17",
      "end_seconds": null,
      "text": "couple different tools."
    },
    {
      "index": 9,
      "start_seconds": 21,
      "source_timestamp": "0:21",
      "end_seconds": null,
      "text": "Click."
    },
    {
      "index": 10,
      "start_seconds": 24,
      "source_timestamp": "0:24",
      "end_seconds": null,
      "text": "Okay, great. So these are the guiding"
    },
    {
      "index": 11,
      "start_seconds": 26,
      "source_timestamp": "0:26",
      "end_seconds": null,
      "text": "questions that inspired the talk and"
    },
    {
      "index": 12,
      "start_seconds": 27,
      "source_timestamp": "0:27",
      "end_seconds": null,
      "text": "like what we'll be focusing on today."
    },
    {
      "index": 13,
      "start_seconds": 30,
      "source_timestamp": "0:30",
      "end_seconds": null,
      "text": "And we wanted to see like to what extent"
    },
    {
      "index": 14,
      "start_seconds": 32,
      "source_timestamp": "0:32",
      "end_seconds": null,
      "text": "does AI make existing formal"
    },
    {
      "index": 15,
      "start_seconds": 34,
      "source_timestamp": "0:34",
      "end_seconds": null,
      "text": "verification tools accessible to"
    },
    {
      "index": 16,
      "start_seconds": 36,
      "source_timestamp": "0:36",
      "end_seconds": null,
      "text": "experienced smart contract auditors?"
    },
    {
      "index": 17,
      "start_seconds": 39,
      "source_timestamp": "0:39",
      "end_seconds": null,
      "text": "And to do such, we like ran some"
    },
    {
      "index": 18,
      "start_seconds": 40,
      "source_timestamp": "0:40",
      "end_seconds": null,
      "text": "experiments on three prominent tools,"
    },
    {
      "index": 19,
      "start_seconds": 43,
      "source_timestamp": "0:43",
      "end_seconds": null,
      "text": "like AutoProver by Certora, Control by"
    },
    {
      "index": 20,
      "start_seconds": 46,
      "source_timestamp": "0:46",
      "end_seconds": null,
      "text": "Runtime Verification, and the SMT"
    },
    {
      "index": 21,
      "start_seconds": 48,
      "source_timestamp": "0:48",
      "end_seconds": null,
      "text": "Checker by the Solidity team."
    },
    {
      "index": 22,
      "start_seconds": 50,
      "source_timestamp": "0:50",
      "end_seconds": null,
      "text": "And then ultimately we wanted to"
    },
    {
      "index": 23,
      "start_seconds": 52,
      "source_timestamp": "0:52",
      "end_seconds": null,
      "text": "conclude where does formal verification"
    },
    {
      "index": 24,
      "start_seconds": 54,
      "source_timestamp": "0:54",
      "end_seconds": null,
      "text": "fit in the audit cycle and like some"
    },
    {
      "index": 25,
      "start_seconds": 57,
      "source_timestamp": "0:57",
      "end_seconds": null,
      "text": "takeaways that we can have as smart"
    },
    {
      "index": 26,
      "start_seconds": 58,
      "source_timestamp": "0:58",
      "end_seconds": null,
      "text": "contract auditors."
    },
    {
      "index": 27,
      "start_seconds": 61,
      "source_timestamp": "1:01",
      "end_seconds": null,
      "text": "Next please."
    },
    {
      "index": 28,
      "start_seconds": 64,
      "source_timestamp": "1:04",
      "end_seconds": null,
      "text": "So this was sort of like the starting"
    },
    {
      "index": 29,
      "start_seconds": 66,
      "source_timestamp": "1:06",
      "end_seconds": null,
      "text": "idea, right? Like over the past year um"
    },
    {
      "index": 30,
      "start_seconds": 69,
      "source_timestamp": "1:09",
      "end_seconds": null,
      "text": "with like LLMs and like agents getting"
    },
    {
      "index": 31,
      "start_seconds": 72,
      "source_timestamp": "1:12",
      "end_seconds": null,
      "text": "really really good,"
    },
    {
      "index": 32,
      "start_seconds": 73,
      "source_timestamp": "1:13",
      "end_seconds": null,
      "text": "we've seen like non-technical people be"
    },
    {
      "index": 33,
      "start_seconds": 75,
      "source_timestamp": "1:15",
      "end_seconds": null,
      "text": "able to produce like some legitimate"
    },
    {
      "index": 34,
      "start_seconds": 77,
      "source_timestamp": "1:17",
      "end_seconds": null,
      "text": "technical output. So this is like your"
    },
    {
      "index": 35,
      "start_seconds": 80,
      "source_timestamp": "1:20",
      "end_seconds": null,
      "text": "typical business guy like sitting down"
    },
    {
      "index": 36,
      "start_seconds": 82,
      "source_timestamp": "1:22",
      "end_seconds": null,
      "text": "with Claude and being like, you know,"
    },
    {
      "index": 37,
      "start_seconds": 84,
      "source_timestamp": "1:24",
      "end_seconds": null,
      "text": "build me 100 million ARR SaaS, make no"
    },
    {
      "index": 38,
      "start_seconds": 87,
      "source_timestamp": "1:27",
      "end_seconds": null,
      "text": "mistakes, and actually getting a website"
    },
    {
      "index": 39,
      "start_seconds": 90,
      "source_timestamp": "1:30",
      "end_seconds": null,
      "text": "output. So there's like um"
    },
    {
      "index": 40,
      "start_seconds": 93,
      "source_timestamp": "1:33",
      "end_seconds": null,
      "text": "this non-technical person doesn't really"
    },
    {
      "index": 41,
      "start_seconds": 95,
      "source_timestamp": "1:35",
      "end_seconds": null,
      "text": "need to understand anything about"
    },
    {
      "index": 42,
      "start_seconds": 98,
      "source_timestamp": "1:38",
      "end_seconds": null,
      "text": "websites in order to like actually get a"
    },
    {
      "index": 43,
      "start_seconds": 100,
      "source_timestamp": "1:40",
      "end_seconds": null,
      "text": "website"
    },
    {
      "index": 44,
      "start_seconds": 101,
      "source_timestamp": "1:41",
      "end_seconds": null,
      "text": "um out there and sort of running with"
    },
    {
      "index": 45,
      "start_seconds": 104,
      "source_timestamp": "1:44",
      "end_seconds": null,
      "text": "it."
    },
    {
      "index": 46,
      "start_seconds": 105,
      "source_timestamp": "1:45",
      "end_seconds": null,
      "text": "So next slide please."
    },
    {
      "index": 47,
      "start_seconds": 107,
      "source_timestamp": "1:47",
      "end_seconds": null,
      "text": "So then uh we wanted to take that step"
    },
    {
      "index": 48,
      "start_seconds": 109,
      "source_timestamp": "1:49",
      "end_seconds": null,
      "text": "further and see like with us as like"
    },
    {
      "index": 49,
      "start_seconds": 112,
      "source_timestamp": "1:52",
      "end_seconds": null,
      "text": "pretty technical people, bug hunters,"
    },
    {
      "index": 50,
      "start_seconds": 114,
      "source_timestamp": "1:54",
      "end_seconds": null,
      "text": "reading a bunch of Solidity code, um you"
    },
    {
      "index": 51,
      "start_seconds": 116,
      "source_timestamp": "1:56",
      "end_seconds": null,
      "text": "know, breaking apart systems, stuff like"
    },
    {
      "index": 52,
      "start_seconds": 118,
      "source_timestamp": "1:58",
      "end_seconds": null,
      "text": "that, could we make a similar leap to"
    },
    {
      "index": 53,
      "start_seconds": 121,
      "source_timestamp": "2:01",
      "end_seconds": null,
      "text": "the extremely technical realm of formal"
    },
    {
      "index": 54,
      "start_seconds": 123,
      "source_timestamp": "2:03",
      "end_seconds": null,
      "text": "verification? So oftentimes people that"
    },
    {
      "index": 55,
      "start_seconds": 126,
      "source_timestamp": "2:06",
      "end_seconds": null,
      "text": "are like experts in formal verification"
    },
    {
      "index": 56,
      "start_seconds": 127,
      "source_timestamp": "2:07",
      "end_seconds": null,
      "text": "may be studying mathematics at like the"
    },
    {
      "index": 57,
      "start_seconds": 130,
      "source_timestamp": "2:10",
      "end_seconds": null,
      "text": "PhD level or"
    },
    {
      "index": 58,
      "start_seconds": 133,
      "source_timestamp": "2:13",
      "end_seconds": null,
      "text": "having like many years of this like"
    },
    {
      "index": 59,
      "start_seconds": 135,
      "source_timestamp": "2:15",
      "end_seconds": null,
      "text": "targeted study."
    },
    {
      "index": 60,
      "start_seconds": 137,
      "source_timestamp": "2:17",
      "end_seconds": null,
      "text": "So if"
    },
    {
      "index": 61,
      "start_seconds": 139,
      "source_timestamp": "2:19",
      "end_seconds": null,
      "text": "we spent like years doing smart"
    },
    {
      "index": 62,
      "start_seconds": 142,
      "source_timestamp": "2:22",
      "end_seconds": null,
      "text": "contracts, can AI help us get that leap"
    },
    {
      "index": 63,
      "start_seconds": 146,
      "source_timestamp": "2:26",
      "end_seconds": null,
      "text": "similar to like the BD guy to the"
    },
    {
      "index": 64,
      "start_seconds": 147,
      "source_timestamp": "2:27",
      "end_seconds": null,
      "text": "website, the smart contract auditor to"
    },
    {
      "index": 65,
      "start_seconds": 149,
      "source_timestamp": "2:29",
      "end_seconds": null,
      "text": "the formally verify expert?"
    },
    {
      "index": 66,
      "start_seconds": 153,
      "source_timestamp": "2:33",
      "end_seconds": null,
      "text": "So if Thread Guy can do it, maybe I"
    },
    {
      "index": 67,
      "start_seconds": 155,
      "source_timestamp": "2:35",
      "end_seconds": null,
      "text": "could do it, right?"
    },
    {
      "index": 68,
      "start_seconds": 157,
      "source_timestamp": "2:37",
      "end_seconds": null,
      "text": "Now that's just a meme checkpoint."
    },
    {
      "index": 69,
      "start_seconds": 159,
      "source_timestamp": "2:39",
      "end_seconds": null,
      "text": "Okay, so a slight primer on formal"
    },
    {
      "index": 70,
      "start_seconds": 162,
      "source_timestamp": "2:42",
      "end_seconds": null,
      "text": "verification. So let's say we have a"
    },
    {
      "index": 71,
      "start_seconds": 165,
      "source_timestamp": "2:45",
      "end_seconds": null,
      "text": "spec like a deposit never decreases"
    },
    {
      "index": 72,
      "start_seconds": 167,
      "source_timestamp": "2:47",
      "end_seconds": null,
      "text": "total shares. So if you were to write a"
    },
    {
      "index": 73,
      "start_seconds": 169,
      "source_timestamp": "2:49",
      "end_seconds": null,
      "text": "unit test for such, you would just like"
    },
    {
      "index": 74,
      "start_seconds": 171,
      "source_timestamp": "2:51",
      "end_seconds": null,
      "text": "do something like deposit 100 and make"
    },
    {
      "index": 75,
      "start_seconds": 174,
      "source_timestamp": "2:54",
      "end_seconds": null,
      "text": "sure that the assets you got back"
    },
    {
      "index": 76,
      "start_seconds": 177,
      "source_timestamp": "2:57",
      "end_seconds": null,
      "text": "was greater than or equal to 100."
    },
    {
      "index": 77,
      "start_seconds": 179,
      "source_timestamp": "2:59",
      "end_seconds": null,
      "text": "If you were to fuzz that spec, you would"
    },
    {
      "index": 78,
      "start_seconds": 181,
      "source_timestamp": "3:01",
      "end_seconds": null,
      "text": "probably"
    },
    {
      "index": 79,
      "start_seconds": 182,
      "source_timestamp": "3:02",
      "end_seconds": null,
      "text": "you would run it through a fuzzer and"
    },
    {
      "index": 80,
      "start_seconds": 184,
      "source_timestamp": "3:04",
      "end_seconds": null,
      "text": "then it would test a deposit of 100,"
    },
    {
      "index": 81,
      "start_seconds": 186,
      "source_timestamp": "3:06",
      "end_seconds": null,
      "text": "test a deposit of 1,000, test a deposit"
    },
    {
      "index": 82,
      "start_seconds": 188,
      "source_timestamp": "3:08",
      "end_seconds": null,
      "text": "of"
    },
    {
      "index": 83,
      "start_seconds": 189,
      "source_timestamp": "3:09",
      "end_seconds": null,
      "text": "1,001, so on and so forth, thousands of"
    },
    {
      "index": 84,
      "start_seconds": 191,
      "source_timestamp": "3:11",
      "end_seconds": null,
      "text": "random deposits. But with formal"
    },
    {
      "index": 85,
      "start_seconds": 193,
      "source_timestamp": "3:13",
      "end_seconds": null,
      "text": "verification, you want to mathematically"
    },
    {
      "index": 86,
      "start_seconds": 197,
      "source_timestamp": "3:17",
      "end_seconds": null,
      "text": "prove that for every deposit from any"
    },
    {
      "index": 87,
      "start_seconds": 200,
      "source_timestamp": "3:20",
      "end_seconds": null,
      "text": "state"
    },
    {
      "index": 88,
      "start_seconds": 201,
      "source_timestamp": "3:21",
      "end_seconds": null,
      "text": "the shares will never decrease."
    },
    {
      "index": 89,
      "start_seconds": 205,
      "source_timestamp": "3:25",
      "end_seconds": null,
      "text": "So now let's take a look at to what"
    },
    {
      "index": 90,
      "start_seconds": 207,
      "source_timestamp": "3:27",
      "end_seconds": null,
      "text": "extent did these tools actually make"
    },
    {
      "index": 91,
      "start_seconds": 211,
      "source_timestamp": "3:31",
      "end_seconds": null,
      "text": "formal verification accessible to"
    },
    {
      "index": 92,
      "start_seconds": 213,
      "source_timestamp": "3:33",
      "end_seconds": null,
      "text": "experienced smart contract auditors."
    },
    {
      "index": 93,
      "start_seconds": 217,
      "source_timestamp": "3:37",
      "end_seconds": null,
      "text": "So this sort of sets the stage for how"
    },
    {
      "index": 94,
      "start_seconds": 219,
      "source_timestamp": "3:39",
      "end_seconds": null,
      "text": "we went about this experiment, like just"
    },
    {
      "index": 95,
      "start_seconds": 221,
      "source_timestamp": "3:41",
      "end_seconds": null,
      "text": "sitting down at the computer, hey"
    },
    {
      "index": 96,
      "start_seconds": 223,
      "source_timestamp": "3:43",
      "end_seconds": null,
      "text": "Claude, formally verify this"
    },
    {
      "index": 97,
      "start_seconds": 225,
      "source_timestamp": "3:45",
      "end_seconds": null,
      "text": "believe in yourself, make no mistakes,"
    },
    {
      "index": 98,
      "start_seconds": 227,
      "source_timestamp": "3:47",
      "end_seconds": null,
      "text": "right?"
    },
    {
      "index": 99,
      "start_seconds": 229,
      "source_timestamp": "3:49",
      "end_seconds": null,
      "text": "So in order to do so, we needed an ideal"
    },
    {
      "index": 100,
      "start_seconds": 231,
      "source_timestamp": "3:51",
      "end_seconds": null,
      "text": "codebase. So this was one that was like"
    },
    {
      "index": 101,
      "start_seconds": 234,
      "source_timestamp": "3:54",
      "end_seconds": null,
      "text": "simple for the point of like"
    },
    {
      "index": 102,
      "start_seconds": 236,
      "source_timestamp": "3:56",
      "end_seconds": null,
      "text": "experimentation, so if and when we"
    },
    {
      "index": 103,
      "start_seconds": 238,
      "source_timestamp": "3:58",
      "end_seconds": null,
      "text": "needed to look under the hood, you could"
    },
    {
      "index": 104,
      "start_seconds": 240,
      "source_timestamp": "4:00",
      "end_seconds": null,
      "text": "go read the code. It was like uh had"
    },
    {
      "index": 105,
      "start_seconds": 243,
      "source_timestamp": "4:03",
      "end_seconds": null,
      "text": "manageable line of code so that the"
    },
    {
      "index": 106,
      "start_seconds": 244,
      "source_timestamp": "4:04",
      "end_seconds": null,
      "text": "prover didn't run forever, which was"
    },
    {
      "index": 107,
      "start_seconds": 246,
      "source_timestamp": "4:06",
      "end_seconds": null,
      "text": "actually something that I underestimated"
    },
    {
      "index": 108,
      "start_seconds": 248,
      "source_timestamp": "4:08",
      "end_seconds": null,
      "text": "how long those things take. Um and then"
    },
    {
      "index": 109,
      "start_seconds": 251,
      "source_timestamp": "4:11",
      "end_seconds": null,
      "text": "we This was really important here is we"
    },
    {
      "index": 110,
      "start_seconds": 252,
      "source_timestamp": "4:12",
      "end_seconds": null,
      "text": "needed like a public human formal"
    },
    {
      "index": 111,
      "start_seconds": 254,
      "source_timestamp": "4:14",
      "end_seconds": null,
      "text": "verification report that actually had"
    },
    {
      "index": 112,
      "start_seconds": 257,
      "source_timestamp": "4:17",
      "end_seconds": null,
      "text": "some real bugs in order for us to catch."
    },
    {
      "index": 113,
      "start_seconds": 260,
      "source_timestamp": "4:20",
      "end_seconds": null,
      "text": "So what we landed on was this Aave vault"
    },
    {
      "index": 114,
      "start_seconds": 263,
      "source_timestamp": "4:23",
      "end_seconds": null,
      "text": "that Certora audited both with formal"
    },
    {
      "index": 115,
      "start_seconds": 265,
      "source_timestamp": "4:25",
      "end_seconds": null,
      "text": "verification and a manual code review in"
    },
    {
      "index": 116,
      "start_seconds": 267,
      "source_timestamp": "4:27",
      "end_seconds": null,
      "text": "2023."
    },
    {
      "index": 117,
      "start_seconds": 269,
      "source_timestamp": "4:29",
      "end_seconds": null,
      "text": "It's a simple ERC 4626 vault where"
    },
    {
      "index": 118,
      "start_seconds": 272,
      "source_timestamp": "4:32",
      "end_seconds": null,
      "text": "deposits are in yield. The owner can"
    },
    {
      "index": 119,
      "start_seconds": 274,
      "source_timestamp": "4:34",
      "end_seconds": null,
      "text": "take a fee and it's just shy of 700"
    },
    {
      "index": 120,
      "start_seconds": 276,
      "source_timestamp": "4:36",
      "end_seconds": null,
      "text": "lines of code."
    },
    {
      "index": 121,
      "start_seconds": 278,
      "source_timestamp": "4:38",
      "end_seconds": null,
      "text": "And there were 11 known bugs uh to see"
    },
    {
      "index": 122,
      "start_seconds": 280,
      "source_timestamp": "4:40",
      "end_seconds": null,
      "text": "if the formal verification tools"
    },
    {
      "index": 123,
      "start_seconds": 282,
      "source_timestamp": "4:42",
      "end_seconds": null,
      "text": "actually caught it."
    },
    {
      "index": 124,
      "start_seconds": 285,
      "source_timestamp": "4:45",
      "end_seconds": null,
      "text": "So now how do we pick the tools? Well"
    },
    {
      "index": 125,
      "start_seconds": 287,
      "source_timestamp": "4:47",
      "end_seconds": null,
      "text": "the ideal tool um was tailored"
    },
    {
      "index": 126,
      "start_seconds": 289,
      "source_timestamp": "4:49",
      "end_seconds": null,
      "text": "specifically to Solidity smart"
    },
    {
      "index": 127,
      "start_seconds": 291,
      "source_timestamp": "4:51",
      "end_seconds": null,
      "text": "contracts. It was open source so we can"
    },
    {
      "index": 128,
      "start_seconds": 293,
      "source_timestamp": "4:53",
      "end_seconds": null,
      "text": "use it and download it or um tinker with"
    },
    {
      "index": 129,
      "start_seconds": 296,
      "source_timestamp": "4:56",
      "end_seconds": null,
      "text": "it."
    },
    {
      "index": 130,
      "start_seconds": 298,
      "source_timestamp": "4:58",
      "end_seconds": null,
      "text": "It was already like adopted by the"
    },
    {
      "index": 131,
      "start_seconds": 299,
      "source_timestamp": "4:59",
      "end_seconds": null,
      "text": "community in some extent um and had"
    },
    {
      "index": 132,
      "start_seconds": 301,
      "source_timestamp": "5:01",
      "end_seconds": null,
      "text": "already undergone several client"
    },
    {
      "index": 133,
      "start_seconds": 303,
      "source_timestamp": "5:03",
      "end_seconds": null,
      "text": "engagements um and it was thoroughly"
    },
    {
      "index": 134,
      "start_seconds": 306,
      "source_timestamp": "5:06",
      "end_seconds": null,
      "text": "documented so that we could basically"
    },
    {
      "index": 135,
      "start_seconds": 308,
      "source_timestamp": "5:08",
      "end_seconds": null,
      "text": "build agents uh to use the tools without"
    },
    {
      "index": 136,
      "start_seconds": 312,
      "source_timestamp": "5:12",
      "end_seconds": null,
      "text": "uh having the agent to figure too much"
    },
    {
      "index": 137,
      "start_seconds": 314,
      "source_timestamp": "5:14",
      "end_seconds": null,
      "text": "out."
    },
    {
      "index": 138,
      "start_seconds": 315,
      "source_timestamp": "5:15",
      "end_seconds": null,
      "text": "So we ran each of these tools on their"
    },
    {
      "index": 139,
      "start_seconds": 317,
      "source_timestamp": "5:17",
      "end_seconds": null,
      "text": "own EC2 instances so they had tons of"
    },
    {
      "index": 140,
      "start_seconds": 319,
      "source_timestamp": "5:19",
      "end_seconds": null,
      "text": "resources and didn't need to share"
    },
    {
      "index": 141,
      "start_seconds": 321,
      "source_timestamp": "5:21",
      "end_seconds": null,
      "text": "anything um and they could sort of just"
    },
    {
      "index": 142,
      "start_seconds": 324,
      "source_timestamp": "5:24",
      "end_seconds": null,
      "text": "run the provers on and on and on."
    },
    {
      "index": 143,
      "start_seconds": 327,
      "source_timestamp": "5:27",
      "end_seconds": null,
      "text": "So the first tool that we uh took a look"
    },
    {
      "index": 144,
      "start_seconds": 330,
      "source_timestamp": "5:30",
      "end_seconds": null,
      "text": "at was Certora's auto prover which came"
    },
    {
      "index": 145,
      "start_seconds": 332,
      "source_timestamp": "5:32",
      "end_seconds": null,
      "text": "out this past summer in July."
    },
    {
      "index": 146,
      "start_seconds": 334,
      "source_timestamp": "5:34",
      "end_seconds": null,
      "text": "Uh this is the only tool that we looked"
    },
    {
      "index": 147,
      "start_seconds": 336,
      "source_timestamp": "5:36",
      "end_seconds": null,
      "text": "at that was agentic out of the box. So"
    },
    {
      "index": 148,
      "start_seconds": 339,
      "source_timestamp": "5:39",
      "end_seconds": null,
      "text": "Certora has had their prover out for"
    },
    {
      "index": 149,
      "start_seconds": 341,
      "source_timestamp": "5:41",
      "end_seconds": null,
      "text": "some time but in July they actually"
    },
    {
      "index": 150,
      "start_seconds": 344,
      "source_timestamp": "5:44",
      "end_seconds": null,
      "text": "released this agentic solution."
    },
    {
      "index": 151,
      "start_seconds": 346,
      "source_timestamp": "5:46",
      "end_seconds": null,
      "text": "Um what's unique about this tool is that"
    },
    {
      "index": 152,
      "start_seconds": 349,
      "source_timestamp": "5:49",
      "end_seconds": null,
      "text": "it's written in the Certora verification"
    },
    {
      "index": 153,
      "start_seconds": 350,
      "source_timestamp": "5:50",
      "end_seconds": null,
      "text": "language uh so that's their own language"
    },
    {
      "index": 154,
      "start_seconds": 353,
      "source_timestamp": "5:53",
      "end_seconds": null,
      "text": "where you write the specs. They're"
    },
    {
      "index": 155,
      "start_seconds": 355,
      "source_timestamp": "5:55",
      "end_seconds": null,
      "text": "verified at the bytecode level and they"
    },
    {
      "index": 156,
      "start_seconds": 356,
      "source_timestamp": "5:56",
      "end_seconds": null,
      "text": "support"
    },
    {
      "index": 157,
      "start_seconds": 358,
      "source_timestamp": "5:58",
      "end_seconds": null,
      "text": "uh several different solvers."
    },
    {
      "index": 158,
      "start_seconds": 360,
      "source_timestamp": "6:00",
      "end_seconds": null,
      "text": "And the technique that they approached"
    },
    {
      "index": 159,
      "start_seconds": 362,
      "source_timestamp": "6:02",
      "end_seconds": null,
      "text": "to formal verification is verification"
    },
    {
      "index": 160,
      "start_seconds": 364,
      "source_timestamp": "6:04",
      "end_seconds": null,
      "text": "condition generation where per rule you"
    },
    {
      "index": 161,
      "start_seconds": 367,
      "source_timestamp": "6:07",
      "end_seconds": null,
      "text": "build one formula and ask the solver,"
    },
    {
      "index": 162,
      "start_seconds": 369,
      "source_timestamp": "6:09",
      "end_seconds": null,
      "text": "oh, is any input going to break this"
    },
    {
      "index": 163,
      "start_seconds": 371,
      "source_timestamp": "6:11",
      "end_seconds": null,
      "text": "rule?"
    },
    {
      "index": 164,
      "start_seconds": 374,
      "source_timestamp": "6:14",
      "end_seconds": null,
      "text": "So this is an example of like what the"
    },
    {
      "index": 165,
      "start_seconds": 376,
      "source_timestamp": "6:16",
      "end_seconds": null,
      "text": "agent would actually create. Um, this is"
    },
    {
      "index": 166,
      "start_seconds": 378,
      "source_timestamp": "6:18",
      "end_seconds": null,
      "text": "the tool or a spec written in uh Certora"
    },
    {
      "index": 167,
      "start_seconds": 382,
      "source_timestamp": "6:22",
      "end_seconds": null,
      "text": "verification language,"
    },
    {
      "index": 168,
      "start_seconds": 383,
      "source_timestamp": "6:23",
      "end_seconds": null,
      "text": "never decreasing shares."
    },
    {
      "index": 169,
      "start_seconds": 385,
      "source_timestamp": "6:25",
      "end_seconds": null,
      "text": "Um, so this is what would actually get"
    },
    {
      "index": 170,
      "start_seconds": 387,
      "source_timestamp": "6:27",
      "end_seconds": null,
      "text": "fed to the solver."
    },
    {
      "index": 171,
      "start_seconds": 389,
      "source_timestamp": "6:29",
      "end_seconds": null,
      "text": "And humans don't usually read this part,"
    },
    {
      "index": 172,
      "start_seconds": 391,
      "source_timestamp": "6:31",
      "end_seconds": null,
      "text": "but it's just good to sort of like"
    },
    {
      "index": 173,
      "start_seconds": 393,
      "source_timestamp": "6:33",
      "end_seconds": null,
      "text": "get a look at what's actually being fed"
    },
    {
      "index": 174,
      "start_seconds": 395,
      "source_timestamp": "6:35",
      "end_seconds": null,
      "text": "into the solver, um which which is this"
    },
    {
      "index": 175,
      "start_seconds": 399,
      "source_timestamp": "6:39",
      "end_seconds": null,
      "text": "assert statement"
    },
    {
      "index": 176,
      "start_seconds": 400,
      "source_timestamp": "6:40",
      "end_seconds": null,
      "text": "is trying to be proven holding true for"
    },
    {
      "index": 177,
      "start_seconds": 402,
      "source_timestamp": "6:42",
      "end_seconds": null,
      "text": "all inputs."
    },
    {
      "index": 178,
      "start_seconds": 412,
      "source_timestamp": "6:52",
      "end_seconds": null,
      "text": "Could you go next?"
    },
    {
      "index": 179,
      "start_seconds": 413,
      "source_timestamp": "6:53",
      "end_seconds": null,
      "text": "Thank you. Uh,"
    },
    {
      "index": 180,
      "start_seconds": 415,
      "source_timestamp": "6:55",
      "end_seconds": null,
      "text": "the next tool is really interesting."
    },
    {
      "index": 181,
      "start_seconds": 418,
      "source_timestamp": "6:58",
      "end_seconds": null,
      "text": "It's controlled by runtime verification."
    },
    {
      "index": 182,
      "start_seconds": 420,
      "source_timestamp": "7:00",
      "end_seconds": null,
      "text": "Uh, what's cool about this is it runs on"
    },
    {
      "index": 183,
      "start_seconds": 422,
      "source_timestamp": "7:02",
      "end_seconds": null,
      "text": "the bytecode in the KVM, which is a"
    },
    {
      "index": 184,
      "start_seconds": 425,
      "source_timestamp": "7:05",
      "end_seconds": null,
      "text": "mathematical definition of the Solidity"
    },
    {
      "index": 185,
      "start_seconds": 428,
      "source_timestamp": "7:08",
      "end_seconds": null,
      "text": "EVM according to the yellow paper."
    },
    {
      "index": 186,
      "start_seconds": 431,
      "source_timestamp": "7:11",
      "end_seconds": null,
      "text": "So it verifies bytecode."
    },
    {
      "index": 187,
      "start_seconds": 434,
      "source_timestamp": "7:14",
      "end_seconds": null,
      "text": "Uh, it uses the Z3 solver and what this"
    },
    {
      "index": 188,
      "start_seconds": 437,
      "source_timestamp": "7:17",
      "end_seconds": null,
      "text": "tool focuses on is symbolic execution,"
    },
    {
      "index": 189,
      "start_seconds": 440,
      "source_timestamp": "7:20",
      "end_seconds": null,
      "text": "which is running each test with like"
    },
    {
      "index": 190,
      "start_seconds": 443,
      "source_timestamp": "7:23",
      "end_seconds": null,
      "text": "these symbolic inputs at each branch in"
    },
    {
      "index": 191,
      "start_seconds": 446,
      "source_timestamp": "7:26",
      "end_seconds": null,
      "text": "the test. You traverse each path and"
    },
    {
      "index": 192,
      "start_seconds": 449,
      "source_timestamp": "7:29",
      "end_seconds": null,
      "text": "you're either going to get to an error"
    },
    {
      "index": 193,
      "start_seconds": 450,
      "source_timestamp": "7:30",
      "end_seconds": null,
      "text": "that's going to show that the spec"
    },
    {
      "index": 194,
      "start_seconds": 453,
      "source_timestamp": "7:33",
      "end_seconds": null,
      "text": "doesn't hold or you're going to get to a"
    },
    {
      "index": 195,
      "start_seconds": 455,
      "source_timestamp": "7:35",
      "end_seconds": null,
      "text": "terminating condition and see that the"
    },
    {
      "index": 196,
      "start_seconds": 456,
      "source_timestamp": "7:36",
      "end_seconds": null,
      "text": "spec does indeed hold."
    },
    {
      "index": 197,
      "start_seconds": 459,
      "source_timestamp": "7:39",
      "end_seconds": null,
      "text": "Uh, I think this might be out again."
    },
    {
      "index": 198,
      "start_seconds": 462,
      "source_timestamp": "7:42",
      "end_seconds": null,
      "text": "If you could go next. Thank you."
    },
    {
      "index": 199,
      "start_seconds": 464,
      "source_timestamp": "7:44",
      "end_seconds": null,
      "text": "So this is what um you would get from"
    },
    {
      "index": 200,
      "start_seconds": 468,
      "source_timestamp": "7:48",
      "end_seconds": null,
      "text": "or what the agent created for running"
    },
    {
      "index": 201,
      "start_seconds": 470,
      "source_timestamp": "7:50",
      "end_seconds": null,
      "text": "the control test."
    },
    {
      "index": 202,
      "start_seconds": 471,
      "source_timestamp": "7:51",
      "end_seconds": null,
      "text": "We see that it's extending this control"
    },
    {
      "index": 203,
      "start_seconds": 475,
      "source_timestamp": "7:55",
      "end_seconds": null,
      "text": "test library and actually running"
    },
    {
      "index": 204,
      "start_seconds": 478,
      "source_timestamp": "7:58",
      "end_seconds": null,
      "text": "the test within the KVM directly."
    },
    {
      "index": 205,
      "start_seconds": 483,
      "source_timestamp": "8:03",
      "end_seconds": null,
      "text": "Next, please."
    },
    {
      "index": 206,
      "start_seconds": 488,
      "source_timestamp": "8:08",
      "end_seconds": null,
      "text": "And"
    },
    {
      "index": 207,
      "start_seconds": 489,
      "source_timestamp": "8:09",
      "end_seconds": null,
      "text": "yeah, this an example of what was being"
    },
    {
      "index": 208,
      "start_seconds": 491,
      "source_timestamp": "8:11",
      "end_seconds": null,
      "text": "uh fed to the solver."
    },
    {
      "index": 209,
      "start_seconds": 493,
      "source_timestamp": "8:13",
      "end_seconds": null,
      "text": "Um and it's not too important, but you"
    },
    {
      "index": 210,
      "start_seconds": 496,
      "source_timestamp": "8:16",
      "end_seconds": null,
      "text": "could see that like there's actually a"
    },
    {
      "index": 211,
      "start_seconds": 498,
      "source_timestamp": "8:18",
      "end_seconds": null,
      "text": "different question being asked to the"
    },
    {
      "index": 212,
      "start_seconds": 500,
      "source_timestamp": "8:20",
      "end_seconds": null,
      "text": "solver versus what uh Certora asked."
    },
    {
      "index": 213,
      "start_seconds": 504,
      "source_timestamp": "8:24",
      "end_seconds": null,
      "text": "Next, please."
    },
    {
      "index": 214,
      "start_seconds": 507,
      "source_timestamp": "8:27",
      "end_seconds": null,
      "text": "So this one uh SMT Checker is actually"
    },
    {
      "index": 215,
      "start_seconds": 509,
      "source_timestamp": "8:29",
      "end_seconds": null,
      "text": "just built directly into the Solidity"
    },
    {
      "index": 216,
      "start_seconds": 511,
      "source_timestamp": "8:31",
      "end_seconds": null,
      "text": "compiler. So this tool is actually like"
    },
    {
      "index": 217,
      "start_seconds": 513,
      "source_timestamp": "8:33",
      "end_seconds": null,
      "text": "most native to Solidity and probably"
    },
    {
      "index": 218,
      "start_seconds": 516,
      "source_timestamp": "8:36",
      "end_seconds": null,
      "text": "like easiest for like a Solidity smart"
    },
    {
      "index": 219,
      "start_seconds": 518,
      "source_timestamp": "8:38",
      "end_seconds": null,
      "text": "contract auditor to jump straight into."
    },
    {
      "index": 220,
      "start_seconds": 521,
      "source_timestamp": "8:41",
      "end_seconds": null,
      "text": "Um so this is provided by the Solidity"
    },
    {
      "index": 221,
      "start_seconds": 523,
      "source_timestamp": "8:43",
      "end_seconds": null,
      "text": "team. It uses uh constrained Horn"
    },
    {
      "index": 222,
      "start_seconds": 525,
      "source_timestamp": "8:45",
      "end_seconds": null,
      "text": "clauses model checking, which is just"
    },
    {
      "index": 223,
      "start_seconds": 528,
      "source_timestamp": "8:48",
      "end_seconds": null,
      "text": "asking um a solver for"
    },
    {
      "index": 224,
      "start_seconds": 531,
      "source_timestamp": "8:51",
      "end_seconds": null,
      "text": "an invariant that holds for like all"
    },
    {
      "index": 225,
      "start_seconds": 533,
      "source_timestamp": "8:53",
      "end_seconds": null,
      "text": "deployments for every call. And then if"
    },
    {
      "index": 226,
      "start_seconds": 536,
      "source_timestamp": "8:56",
      "end_seconds": null,
      "text": "that holds, it rules out the bug."
    },
    {
      "index": 227,
      "start_seconds": 538,
      "source_timestamp": "8:58",
      "end_seconds": null,
      "text": "Please go next."
    },
    {
      "index": 228,
      "start_seconds": 542,
      "source_timestamp": "9:02",
      "end_seconds": null,
      "text": "And like I mentioned, this is like most"
    },
    {
      "index": 229,
      "start_seconds": 544,
      "source_timestamp": "9:04",
      "end_seconds": null,
      "text": "native to Solidity because it's in the"
    },
    {
      "index": 230,
      "start_seconds": 546,
      "source_timestamp": "9:06",
      "end_seconds": null,
      "text": "compiler itself. So there's no extra"
    },
    {
      "index": 231,
      "start_seconds": 548,
      "source_timestamp": "9:08",
      "end_seconds": null,
      "text": "libraries. There's no um other languages"
    },
    {
      "index": 232,
      "start_seconds": 551,
      "source_timestamp": "9:11",
      "end_seconds": null,
      "text": "that are used to write the spec."
    },
    {
      "index": 233,
      "start_seconds": 554,
      "source_timestamp": "9:14",
      "end_seconds": null,
      "text": "You can just extend a vault,"
    },
    {
      "index": 234,
      "start_seconds": 557,
      "source_timestamp": "9:17",
      "end_seconds": null,
      "text": "uh make a call to the contract or the"
    },
    {
      "index": 235,
      "start_seconds": 559,
      "source_timestamp": "9:19",
      "end_seconds": null,
      "text": "function that you want to test and add"
    },
    {
      "index": 236,
      "start_seconds": 561,
      "source_timestamp": "9:21",
      "end_seconds": null,
      "text": "an assert afterward."
    },
    {
      "index": 237,
      "start_seconds": 563,
      "source_timestamp": "9:23",
      "end_seconds": null,
      "text": "So uh we know that like usually as"
    },
    {
      "index": 238,
      "start_seconds": 566,
      "source_timestamp": "9:26",
      "end_seconds": null,
      "text": "Solidity devs, you aren't too keen to"
    },
    {
      "index": 239,
      "start_seconds": 568,
      "source_timestamp": "9:28",
      "end_seconds": null,
      "text": "having asserts like within uh the"
    },
    {
      "index": 240,
      "start_seconds": 571,
      "source_timestamp": "9:31",
      "end_seconds": null,
      "text": "contracts or the functions themselves."
    },
    {
      "index": 241,
      "start_seconds": 573,
      "source_timestamp": "9:33",
      "end_seconds": null,
      "text": "You usually tend towards require"
    },
    {
      "index": 242,
      "start_seconds": 574,
      "source_timestamp": "9:34",
      "end_seconds": null,
      "text": "statements, so you would usually just"
    },
    {
      "index": 243,
      "start_seconds": 576,
      "source_timestamp": "9:36",
      "end_seconds": null,
      "text": "extend and put in the assert for the"
    },
    {
      "index": 244,
      "start_seconds": 578,
      "source_timestamp": "9:38",
      "end_seconds": null,
      "text": "condition that you're looking for. Um"
    },
    {
      "index": 245,
      "start_seconds": 581,
      "source_timestamp": "9:41",
      "end_seconds": null,
      "text": "and then run it through the compiler."
    },
    {
      "index": 246,
      "start_seconds": 583,
      "source_timestamp": "9:43",
      "end_seconds": null,
      "text": "And then yes, it will create uh an"
    },
    {
      "index": 247,
      "start_seconds": 586,
      "source_timestamp": "9:46",
      "end_seconds": null,
      "text": "output that you can ask the solver."
    },
    {
      "index": 248,
      "start_seconds": 589,
      "source_timestamp": "9:49",
      "end_seconds": null,
      "text": "Next, please."
    },
    {
      "index": 249,
      "start_seconds": 591,
      "source_timestamp": "9:51",
      "end_seconds": null,
      "text": "So this is just kind of wrapping that"
    },
    {
      "index": 250,
      "start_seconds": 593,
      "source_timestamp": "9:53",
      "end_seconds": null,
      "text": "all up. I'll invite you to take a look"
    },
    {
      "index": 251,
      "start_seconds": 596,
      "source_timestamp": "9:56",
      "end_seconds": null,
      "text": "um"
    },
    {
      "index": 252,
      "start_seconds": 597,
      "source_timestamp": "9:57",
      "end_seconds": null,
      "text": "a diagram like comparing all three of"
    },
    {
      "index": 253,
      "start_seconds": 600,
      "source_timestamp": "10:00",
      "end_seconds": null,
      "text": "the tools that we looked at. So all of"
    },
    {
      "index": 254,
      "start_seconds": 602,
      "source_timestamp": "10:02",
      "end_seconds": null,
      "text": "them are open source, all of them"
    },
    {
      "index": 255,
      "start_seconds": 604,
      "source_timestamp": "10:04",
      "end_seconds": null,
      "text": "generate counterexamples, and all of"
    },
    {
      "index": 256,
      "start_seconds": 606,
      "source_timestamp": "10:06",
      "end_seconds": null,
      "text": "them support the Z3 solver, which is"
    },
    {
      "index": 257,
      "start_seconds": 608,
      "source_timestamp": "10:08",
      "end_seconds": null,
      "text": "like the main solver being used here."
    },
    {
      "index": 258,
      "start_seconds": 611,
      "source_timestamp": "10:11",
      "end_seconds": null,
      "text": "Um, as I mentioned, Certora's is the"
    },
    {
      "index": 259,
      "start_seconds": 613,
      "source_timestamp": "10:13",
      "end_seconds": null,
      "text": "only one that's agentic out of the box"
    },
    {
      "index": 260,
      "start_seconds": 615,
      "source_timestamp": "10:15",
      "end_seconds": null,
      "text": "and the only one that has a um"
    },
    {
      "index": 261,
      "start_seconds": 618,
      "source_timestamp": "10:18",
      "end_seconds": null,
      "text": "separate language used, the CVL."
    },
    {
      "index": 262,
      "start_seconds": 621,
      "source_timestamp": "10:21",
      "end_seconds": null,
      "text": "Both Control and SMT Checker are written"
    },
    {
      "index": 263,
      "start_seconds": 624,
      "source_timestamp": "10:24",
      "end_seconds": null,
      "text": "in Solidity,"
    },
    {
      "index": 264,
      "start_seconds": 626,
      "source_timestamp": "10:26",
      "end_seconds": null,
      "text": "and they each have their own uh separate"
    },
    {
      "index": 265,
      "start_seconds": 629,
      "source_timestamp": "10:29",
      "end_seconds": null,
      "text": "technique for verification."
    },
    {
      "index": 266,
      "start_seconds": 631,
      "source_timestamp": "10:31",
      "end_seconds": null,
      "text": "Next, please."
    },
    {
      "index": 267,
      "start_seconds": 633,
      "source_timestamp": "10:33",
      "end_seconds": null,
      "text": "Okay, so now for the exciting part, uh"
    },
    {
      "index": 268,
      "start_seconds": 636,
      "source_timestamp": "10:36",
      "end_seconds": null,
      "text": "harness development."
    },
    {
      "index": 269,
      "start_seconds": 637,
      "source_timestamp": "10:37",
      "end_seconds": null,
      "text": "Could you go next?"
    },
    {
      "index": 270,
      "start_seconds": 639,
      "source_timestamp": "10:39",
      "end_seconds": null,
      "text": "Thank you."
    },
    {
      "index": 271,
      "start_seconds": 640,
      "source_timestamp": "10:40",
      "end_seconds": null,
      "text": "So the idea when we started to write the"
    },
    {
      "index": 272,
      "start_seconds": 643,
      "source_timestamp": "10:43",
      "end_seconds": null,
      "text": "harnesses to test the formal"
    },
    {
      "index": 273,
      "start_seconds": 644,
      "source_timestamp": "10:44",
      "end_seconds": null,
      "text": "verification tools"
    },
    {
      "index": 274,
      "start_seconds": 646,
      "source_timestamp": "10:46",
      "end_seconds": null,
      "text": "was we assumed like, okay, Certora's"
    },
    {
      "index": 275,
      "start_seconds": 648,
      "source_timestamp": "10:48",
      "end_seconds": null,
      "text": "AutoProver, that's like sort of the"
    },
    {
      "index": 276,
      "start_seconds": 650,
      "source_timestamp": "10:50",
      "end_seconds": null,
      "text": "template, right? Uh that's what an"
    },
    {
      "index": 277,
      "start_seconds": 652,
      "source_timestamp": "10:52",
      "end_seconds": null,
      "text": "agentic solution to formal verification"
    },
    {
      "index": 278,
      "start_seconds": 654,
      "source_timestamp": "10:54",
      "end_seconds": null,
      "text": "would look like right now."
    },
    {
      "index": 279,
      "start_seconds": 656,
      "source_timestamp": "10:56",
      "end_seconds": null,
      "text": "So we extracted the properties and the"
    },
    {
      "index": 280,
      "start_seconds": 659,
      "source_timestamp": "10:59",
      "end_seconds": null,
      "text": "architecture from Certora AutoProver,"
    },
    {
      "index": 281,
      "start_seconds": 661,
      "source_timestamp": "11:01",
      "end_seconds": null,
      "text": "and then basically in conjunction with"
    },
    {
      "index": 282,
      "start_seconds": 663,
      "source_timestamp": "11:03",
      "end_seconds": null,
      "text": "the documentation on a specific tool,"
    },
    {
      "index": 283,
      "start_seconds": 665,
      "source_timestamp": "11:05",
      "end_seconds": null,
      "text": "either Control or SMT Checker,"
    },
    {
      "index": 284,
      "start_seconds": 668,
      "source_timestamp": "11:08",
      "end_seconds": null,
      "text": "we fed them um into Claude to actually"
    },
    {
      "index": 285,
      "start_seconds": 671,
      "source_timestamp": "11:11",
      "end_seconds": null,
      "text": "write the harness."
    },
    {
      "index": 286,
      "start_seconds": 672,
      "source_timestamp": "11:12",
      "end_seconds": null,
      "text": "And then"
    },
    {
      "index": 287,
      "start_seconds": 675,
      "source_timestamp": "11:15",
      "end_seconds": null,
      "text": "Hold on, let me not get uh let me not"
    },
    {
      "index": 288,
      "start_seconds": 677,
      "source_timestamp": "11:17",
      "end_seconds": null,
      "text": "lose my train of thought here. So"
    },
    {
      "index": 289,
      "start_seconds": 680,
      "source_timestamp": "11:20",
      "end_seconds": null,
      "text": "like with AutoProver, we wanted to"
    },
    {
      "index": 290,
      "start_seconds": 682,
      "source_timestamp": "11:22",
      "end_seconds": null,
      "text": "basically like take what they were doing"
    },
    {
      "index": 291,
      "start_seconds": 684,
      "source_timestamp": "11:24",
      "end_seconds": null,
      "text": "and see if it could also work with"
    },
    {
      "index": 292,
      "start_seconds": 686,
      "source_timestamp": "11:26",
      "end_seconds": null,
      "text": "Control and SMT Checker, um and then"
    },
    {
      "index": 293,
      "start_seconds": 688,
      "source_timestamp": "11:28",
      "end_seconds": null,
      "text": "like compare the results of such."
    },
    {
      "index": 294,
      "start_seconds": 691,
      "source_timestamp": "11:31",
      "end_seconds": null,
      "text": "So that wasn't just enough to write the"
    },
    {
      "index": 295,
      "start_seconds": 693,
      "source_timestamp": "11:33",
      "end_seconds": null,
      "text": "harness. Each Control and SMT Checker"
    },
    {
      "index": 296,
      "start_seconds": 696,
      "source_timestamp": "11:36",
      "end_seconds": null,
      "text": "came with their own like example code"
    },
    {
      "index": 297,
      "start_seconds": 698,
      "source_timestamp": "11:38",
      "end_seconds": null,
      "text": "and test cases to work with."
    },
    {
      "index": 298,
      "start_seconds": 700,
      "source_timestamp": "11:40",
      "end_seconds": null,
      "text": "So Control, I think they had uh like a"
    },
    {
      "index": 299,
      "start_seconds": 704,
      "source_timestamp": "11:44",
      "end_seconds": null,
      "text": "simple counter contract and a WETH9"
    },
    {
      "index": 300,
      "start_seconds": 707,
      "source_timestamp": "11:47",
      "end_seconds": null,
      "text": "contract uh that we could fine-tune the"
    },
    {
      "index": 301,
      "start_seconds": 709,
      "source_timestamp": "11:49",
      "end_seconds": null,
      "text": "harness on."
    },
    {
      "index": 302,
      "start_seconds": 711,
      "source_timestamp": "11:51",
      "end_seconds": null,
      "text": "And SMT Checker, I think it was like a"
    },
    {
      "index": 303,
      "start_seconds": 713,
      "source_timestamp": "11:53",
      "end_seconds": null,
      "text": "mutex, like a lock"
    },
    {
      "index": 304,
      "start_seconds": 715,
      "source_timestamp": "11:55",
      "end_seconds": null,
      "text": "uh contract"
    },
    {
      "index": 305,
      "start_seconds": 717,
      "source_timestamp": "11:57",
      "end_seconds": null,
      "text": "and ERC777 that we got to fine-tune on."
    },
    {
      "index": 306,
      "start_seconds": 721,
      "source_timestamp": "12:01",
      "end_seconds": null,
      "text": "So SMT Checker was actually fairly"
    },
    {
      "index": 307,
      "start_seconds": 723,
      "source_timestamp": "12:03",
      "end_seconds": null,
      "text": "straightforward with fine-tuning, but"
    },
    {
      "index": 308,
      "start_seconds": 726,
      "source_timestamp": "12:06",
      "end_seconds": null,
      "text": "what we found with working with Control,"
    },
    {
      "index": 309,
      "start_seconds": 728,
      "source_timestamp": "12:08",
      "end_seconds": null,
      "text": "if you read the Control documentation"
    },
    {
      "index": 310,
      "start_seconds": 730,
      "source_timestamp": "12:10",
      "end_seconds": null,
      "text": "for like that formal verification tool"
    },
    {
      "index": 311,
      "start_seconds": 732,
      "source_timestamp": "12:12",
      "end_seconds": null,
      "text": "specifically, A lot of it is focused on"
    },
    {
      "index": 312,
      "start_seconds": 735,
      "source_timestamp": "12:15",
      "end_seconds": null,
      "text": "um"
    },
    {
      "index": 313,
      "start_seconds": 737,
      "source_timestamp": "12:17",
      "end_seconds": null,
      "text": "it's written so that there's a formal"
    },
    {
      "index": 314,
      "start_seconds": 739,
      "source_timestamp": "12:19",
      "end_seconds": null,
      "text": "verification human expert like"
    },
    {
      "index": 315,
      "start_seconds": 741,
      "source_timestamp": "12:21",
      "end_seconds": null,
      "text": "frequently in the loop. So a lot of the"
    },
    {
      "index": 316,
      "start_seconds": 743,
      "source_timestamp": "12:23",
      "end_seconds": null,
      "text": "control documentation talks about"
    },
    {
      "index": 317,
      "start_seconds": 745,
      "source_timestamp": "12:25",
      "end_seconds": null,
      "text": "writing a test, okay, it fails, go back"
    },
    {
      "index": 318,
      "start_seconds": 748,
      "source_timestamp": "12:28",
      "end_seconds": null,
      "text": "and like tweak it a little bit, add a"
    },
    {
      "index": 319,
      "start_seconds": 750,
      "source_timestamp": "12:30",
      "end_seconds": null,
      "text": "lemma, so on and so forth. Um so that"
    },
    {
      "index": 320,
      "start_seconds": 753,
      "source_timestamp": "12:33",
      "end_seconds": null,
      "text": "actually didn't do too well with the"
    },
    {
      "index": 321,
      "start_seconds": 755,
      "source_timestamp": "12:35",
      "end_seconds": null,
      "text": "agent, which we'll see in a little bit."
    },
    {
      "index": 322,
      "start_seconds": 757,
      "source_timestamp": "12:37",
      "end_seconds": null,
      "text": "But after we basically fine tuned it to"
    },
    {
      "index": 323,
      "start_seconds": 760,
      "source_timestamp": "12:40",
      "end_seconds": null,
      "text": "actually solve the example contracts or"
    },
    {
      "index": 324,
      "start_seconds": 763,
      "source_timestamp": "12:43",
      "end_seconds": null,
      "text": "example tests that were provided within"
    },
    {
      "index": 325,
      "start_seconds": 765,
      "source_timestamp": "12:45",
      "end_seconds": null,
      "text": "the documentation for the tool itself,"
    },
    {
      "index": 326,
      "start_seconds": 767,
      "source_timestamp": "12:47",
      "end_seconds": null,
      "text": "it sort of graduated to move on to the"
    },
    {
      "index": 327,
      "start_seconds": 769,
      "source_timestamp": "12:49",
      "end_seconds": null,
      "text": "data set."
    },
    {
      "index": 328,
      "start_seconds": 771,
      "source_timestamp": "12:51",
      "end_seconds": null,
      "text": "Next please."
    },
    {
      "index": 329,
      "start_seconds": 772,
      "source_timestamp": "12:52",
      "end_seconds": null,
      "text": "Okay, now the exciting part. So how did"
    },
    {
      "index": 330,
      "start_seconds": 774,
      "source_timestamp": "12:54",
      "end_seconds": null,
      "text": "each of these tools actually perform"
    },
    {
      "index": 331,
      "start_seconds": 776,
      "source_timestamp": "12:56",
      "end_seconds": null,
      "text": "when driven by state of the art models?"
    },
    {
      "index": 332,
      "start_seconds": 779,
      "source_timestamp": "12:59",
      "end_seconds": null,
      "text": "Next please."
    },
    {
      "index": 333,
      "start_seconds": 782,
      "source_timestamp": "13:02",
      "end_seconds": null,
      "text": "Uh so actually one thing on the models"
    },
    {
      "index": 334,
      "start_seconds": 784,
      "source_timestamp": "13:04",
      "end_seconds": null,
      "text": "before I talk about the metrics, um for"
    },
    {
      "index": 335,
      "start_seconds": 787,
      "source_timestamp": "13:07",
      "end_seconds": null,
      "text": "Sertora's AutoProver,"
    },
    {
      "index": 336,
      "start_seconds": 790,
      "source_timestamp": "13:10",
      "end_seconds": null,
      "text": "there's the ability to uh change what"
    },
    {
      "index": 337,
      "start_seconds": 793,
      "source_timestamp": "13:13",
      "end_seconds": null,
      "text": "models you were running it on,"
    },
    {
      "index": 338,
      "start_seconds": 796,
      "source_timestamp": "13:16",
      "end_seconds": null,
      "text": "but"
    },
    {
      "index": 339,
      "start_seconds": 797,
      "source_timestamp": "13:17",
      "end_seconds": null,
      "text": "we so like out of the box it's running"
    },
    {
      "index": 340,
      "start_seconds": 799,
      "source_timestamp": "13:19",
      "end_seconds": null,
      "text": "on Opus 5 and Sonnet 5."
    },
    {
      "index": 341,
      "start_seconds": 802,
      "source_timestamp": "13:22",
      "end_seconds": null,
      "text": "And then once I wanted to go tweak the"
    },
    {
      "index": 342,
      "start_seconds": 805,
      "source_timestamp": "13:25",
      "end_seconds": null,
      "text": "models uh a little bit,"
    },
    {
      "index": 343,
      "start_seconds": 807,
      "source_timestamp": "13:27",
      "end_seconds": null,
      "text": "they have like a high like a high power"
    },
    {
      "index": 344,
      "start_seconds": 809,
      "source_timestamp": "13:29",
      "end_seconds": null,
      "text": "model and a low power model. So their"
    },
    {
      "index": 345,
      "start_seconds": 811,
      "source_timestamp": "13:31",
      "end_seconds": null,
      "text": "high power was Opus 5 out of the box and"
    },
    {
      "index": 346,
      "start_seconds": 813,
      "source_timestamp": "13:33",
      "end_seconds": null,
      "text": "their low power was like Sonnet 5."
    },
    {
      "index": 347,
      "start_seconds": 816,
      "source_timestamp": "13:36",
      "end_seconds": null,
      "text": "When I bumped them up to the newer"
    },
    {
      "index": 348,
      "start_seconds": 817,
      "source_timestamp": "13:37",
      "end_seconds": null,
      "text": "generation like 5.5 and Sonnet 5.5, uh"
    },
    {
      "index": 349,
      "start_seconds": 820,
      "source_timestamp": "13:40",
      "end_seconds": null,
      "text": "for some reason Sonnet 5.5 like didn't"
    },
    {
      "index": 350,
      "start_seconds": 823,
      "source_timestamp": "13:43",
      "end_seconds": null,
      "text": "extract the properties the right way"
    },
    {
      "index": 351,
      "start_seconds": 825,
      "source_timestamp": "13:45",
      "end_seconds": null,
      "text": "that Sonnet 5 did. So then I was like,"
    },
    {
      "index": 352,
      "start_seconds": 827,
      "source_timestamp": "13:47",
      "end_seconds": null,
      "text": "all right, let's not mess with this,"
    },
    {
      "index": 353,
      "start_seconds": 829,
      "source_timestamp": "13:49",
      "end_seconds": null,
      "text": "just test AutoProver out of the out of"
    },
    {
      "index": 354,
      "start_seconds": 831,
      "source_timestamp": "13:51",
      "end_seconds": null,
      "text": "the box. So Control and SMT Checker are"
    },
    {
      "index": 355,
      "start_seconds": 834,
      "source_timestamp": "13:54",
      "end_seconds": null,
      "text": "checked on Opus 5.5, but AutoProver is"
    },
    {
      "index": 356,
      "start_seconds": 837,
      "source_timestamp": "13:57",
      "end_seconds": null,
      "text": "just coming right off the shelf. So"
    },
    {
      "index": 357,
      "start_seconds": 839,
      "source_timestamp": "13:59",
      "end_seconds": null,
      "text": "perhaps if I had more time I could tweak"
    },
    {
      "index": 358,
      "start_seconds": 841,
      "source_timestamp": "14:01",
      "end_seconds": null,
      "text": "and try to figure out what's going on"
    },
    {
      "index": 359,
      "start_seconds": 842,
      "source_timestamp": "14:02",
      "end_seconds": null,
      "text": "with Sonnet 5.5, but for the sake of"
    },
    {
      "index": 360,
      "start_seconds": 845,
      "source_timestamp": "14:05",
      "end_seconds": null,
      "text": "experimentation we just let it run."
    },
    {
      "index": 361,
      "start_seconds": 847,
      "source_timestamp": "14:07",
      "end_seconds": null,
      "text": "Uh just a small note before we look at"
    },
    {
      "index": 362,
      "start_seconds": 849,
      "source_timestamp": "14:09",
      "end_seconds": null,
      "text": "some of these metrics. So we wanted to"
    },
    {
      "index": 363,
      "start_seconds": 851,
      "source_timestamp": "14:11",
      "end_seconds": null,
      "text": "test like recall. So of those 11 bugs"
    },
    {
      "index": 364,
      "start_seconds": 854,
      "source_timestamp": "14:14",
      "end_seconds": null,
      "text": "that were in the original report, like"
    },
    {
      "index": 365,
      "start_seconds": 856,
      "source_timestamp": "14:16",
      "end_seconds": null,
      "text": "how many did the tool actually catch?"
    },
    {
      "index": 366,
      "start_seconds": 858,
      "source_timestamp": "14:18",
      "end_seconds": null,
      "text": "Precision. So like we know with like AI"
    },
    {
      "index": 367,
      "start_seconds": 861,
      "source_timestamp": "14:21",
      "end_seconds": null,
      "text": "scanning tools, one of the worst things"
    },
    {
      "index": 368,
      "start_seconds": 862,
      "source_timestamp": "14:22",
      "end_seconds": null,
      "text": "you could have as an auditor is like"
    },
    {
      "index": 369,
      "start_seconds": 864,
      "source_timestamp": "14:24",
      "end_seconds": null,
      "text": "noisy results that like end up wasting"
    },
    {
      "index": 370,
      "start_seconds": 866,
      "source_timestamp": "14:26",
      "end_seconds": null,
      "text": "more of your time. So we wanted to see"
    },
    {
      "index": 371,
      "start_seconds": 868,
      "source_timestamp": "14:28",
      "end_seconds": null,
      "text": "like out of everything that the tool"
    },
    {
      "index": 372,
      "start_seconds": 870,
      "source_timestamp": "14:30",
      "end_seconds": null,
      "text": "ended on, how many of them actually hit"
    },
    {
      "index": 373,
      "start_seconds": 872,
      "source_timestamp": "14:32",
      "end_seconds": null,
      "text": "a known uh hit a known bug."
    },
    {
      "index": 374,
      "start_seconds": 876,
      "source_timestamp": "14:36",
      "end_seconds": null,
      "text": "Soundness is a really interesting one."
    },
    {
      "index": 375,
      "start_seconds": 877,
      "source_timestamp": "14:37",
      "end_seconds": null,
      "text": "Like can the finding actually happen? So"
    },
    {
      "index": 376,
      "start_seconds": 880,
      "source_timestamp": "14:40",
      "end_seconds": null,
      "text": "we extended all the provers to actually"
    },
    {
      "index": 377,
      "start_seconds": 881,
      "source_timestamp": "14:41",
      "end_seconds": null,
      "text": "ask an agent to write POCs in Foundry"
    },
    {
      "index": 378,
      "start_seconds": 884,
      "source_timestamp": "14:44",
      "end_seconds": null,
      "text": "saying, oh, based on like this Aave"
    },
    {
      "index": 379,
      "start_seconds": 886,
      "source_timestamp": "14:46",
      "end_seconds": null,
      "text": "Vault deployment, is this bug actually"
    },
    {
      "index": 380,
      "start_seconds": 889,
      "source_timestamp": "14:49",
      "end_seconds": null,
      "text": "feasible within like uh the execution of"
    },
    {
      "index": 381,
      "start_seconds": 891,
      "source_timestamp": "14:51",
      "end_seconds": null,
      "text": "the contracts?"
    },
    {
      "index": 382,
      "start_seconds": 893,
      "source_timestamp": "14:53",
      "end_seconds": null,
      "text": "And rule recall was like, okay, out of"
    },
    {
      "index": 383,
      "start_seconds": 895,
      "source_timestamp": "14:55",
      "end_seconds": null,
      "text": "all the rules that the provers"
    },
    {
      "index": 384,
      "start_seconds": 897,
      "source_timestamp": "14:57",
      "end_seconds": null,
      "text": "themselves checked, how many were"
    },
    {
      "index": 385,
      "start_seconds": 898,
      "source_timestamp": "14:58",
      "end_seconds": null,
      "text": "actually also written by a human? And"
    },
    {
      "index": 386,
      "start_seconds": 901,
      "source_timestamp": "15:01",
      "end_seconds": null,
      "text": "then we have some cost and runtime"
    },
    {
      "index": 387,
      "start_seconds": 902,
      "source_timestamp": "15:02",
      "end_seconds": null,
      "text": "metrics we'll look at."
    },
    {
      "index": 388,
      "start_seconds": 904,
      "source_timestamp": "15:04",
      "end_seconds": null,
      "text": "So yeah, I I talked about this a little"
    },
    {
      "index": 389,
      "start_seconds": 907,
      "source_timestamp": "15:07",
      "end_seconds": null,
      "text": "bit, but it's just basically like uh"
    },
    {
      "index": 390,
      "start_seconds": 910,
      "source_timestamp": "15:10",
      "end_seconds": null,
      "text": "recall"
    },
    {
      "index": 391,
      "start_seconds": 911,
      "source_timestamp": "15:11",
      "end_seconds": null,
      "text": "of the"
    },
    {
      "index": 392,
      "start_seconds": 912,
      "source_timestamp": "15:12",
      "end_seconds": null,
      "text": "bugs caught by the tool, how many were"
    },
    {
      "index": 393,
      "start_seconds": 914,
      "source_timestamp": "15:14",
      "end_seconds": null,
      "text": "actually in the report."
    },
    {
      "index": 394,
      "start_seconds": 915,
      "source_timestamp": "15:15",
      "end_seconds": null,
      "text": "Next, please."
    },
    {
      "index": 395,
      "start_seconds": 918,
      "source_timestamp": "15:18",
      "end_seconds": null,
      "text": "So this is a really interesting slide"
    },
    {
      "index": 396,
      "start_seconds": 919,
      "source_timestamp": "15:19",
      "end_seconds": null,
      "text": "here. It sort of highlights like uh the"
    },
    {
      "index": 397,
      "start_seconds": 922,
      "source_timestamp": "15:22",
      "end_seconds": null,
      "text": "medium, low, and informational bugs that"
    },
    {
      "index": 398,
      "start_seconds": 924,
      "source_timestamp": "15:24",
      "end_seconds": null,
      "text": "actually made it to the report and how"
    },
    {
      "index": 399,
      "start_seconds": 926,
      "source_timestamp": "15:26",
      "end_seconds": null,
      "text": "each tool did. So Auto Provers tool off"
    },
    {
      "index": 400,
      "start_seconds": 929,
      "source_timestamp": "15:29",
      "end_seconds": null,
      "text": "the shelf actually did pretty well as"
    },
    {
      "index": 401,
      "start_seconds": 931,
      "source_timestamp": "15:31",
      "end_seconds": null,
      "text": "compared to their human teams that"
    },
    {
      "index": 402,
      "start_seconds": 932,
      "source_timestamp": "15:32",
      "end_seconds": null,
      "text": "reviewed the Aave Vault."
    },
    {
      "index": 403,
      "start_seconds": 934,
      "source_timestamp": "15:34",
      "end_seconds": null,
      "text": "It caught two of the four um medium bugs"
    },
    {
      "index": 404,
      "start_seconds": 937,
      "source_timestamp": "15:37",
      "end_seconds": null,
      "text": "and um one of the informational bugs and"
    },
    {
      "index": 405,
      "start_seconds": 939,
      "source_timestamp": "15:39",
      "end_seconds": null,
      "text": "scored the highest among the other two"
    },
    {
      "index": 406,
      "start_seconds": 941,
      "source_timestamp": "15:41",
      "end_seconds": null,
      "text": "um"
    },
    {
      "index": 407,
      "start_seconds": 943,
      "source_timestamp": "15:43",
      "end_seconds": null,
      "text": "tools."
    },
    {
      "index": 408,
      "start_seconds": 944,
      "source_timestamp": "15:44",
      "end_seconds": null,
      "text": "Uh"
    },
    {
      "index": 409,
      "start_seconds": 945,
      "source_timestamp": "15:45",
      "end_seconds": null,
      "text": "SMT Checker got one of the mediums,"
    },
    {
      "index": 410,
      "start_seconds": 947,
      "source_timestamp": "15:47",
      "end_seconds": null,
      "text": "actually one of them that Auto Prover"
    },
    {
      "index": 411,
      "start_seconds": 949,
      "source_timestamp": "15:49",
      "end_seconds": null,
      "text": "didn't."
    },
    {
      "index": 412,
      "start_seconds": 950,
      "source_timestamp": "15:50",
      "end_seconds": null,
      "text": "Uh and Control"
    },
    {
      "index": 413,
      "start_seconds": 952,
      "source_timestamp": "15:52",
      "end_seconds": null,
      "text": "scored a couple of the info findings,"
    },
    {
      "index": 414,
      "start_seconds": 954,
      "source_timestamp": "15:54",
      "end_seconds": null,
      "text": "which was like good."
    },
    {
      "index": 415,
      "start_seconds": 956,
      "source_timestamp": "15:56",
      "end_seconds": null,
      "text": "Uh next, please."
    },
    {
      "index": 416,
      "start_seconds": 958,
      "source_timestamp": "15:58",
      "end_seconds": null,
      "text": "So uh this one was like"
    },
    {
      "index": 417,
      "start_seconds": 962,
      "source_timestamp": "16:02",
      "end_seconds": null,
      "text": "trying to see how noisy things were,"
    },
    {
      "index": 418,
      "start_seconds": 964,
      "source_timestamp": "16:04",
      "end_seconds": null,
      "text": "right? Uh could you go next?"
    },
    {
      "index": 419,
      "start_seconds": 969,
      "source_timestamp": "16:09",
      "end_seconds": null,
      "text": "So this shows like all of the findings"
    },
    {
      "index": 420,
      "start_seconds": 971,
      "source_timestamp": "16:11",
      "end_seconds": null,
      "text": "like after the tools completely ran"
    },
    {
      "index": 421,
      "start_seconds": 973,
      "source_timestamp": "16:13",
      "end_seconds": null,
      "text": "through, this was like Auto Prover gave"
    },
    {
      "index": 422,
      "start_seconds": 976,
      "source_timestamp": "16:16",
      "end_seconds": null,
      "text": "us nine findings, Control gave us five"
    },
    {
      "index": 423,
      "start_seconds": 978,
      "source_timestamp": "16:18",
      "end_seconds": null,
      "text": "findings, and SMT Checker gave us five"
    },
    {
      "index": 424,
      "start_seconds": 981,
      "source_timestamp": "16:21",
      "end_seconds": null,
      "text": "findings."
    },
    {
      "index": 425,
      "start_seconds": 982,
      "source_timestamp": "16:22",
      "end_seconds": null,
      "text": "Two out of the nine of Autoprover"
    },
    {
      "index": 426,
      "start_seconds": 984,
      "source_timestamp": "16:24",
      "end_seconds": null,
      "text": "actually made it to the final report."
    },
    {
      "index": 427,
      "start_seconds": 986,
      "source_timestamp": "16:26",
      "end_seconds": null,
      "text": "Two out of the five for Control made it"
    },
    {
      "index": 428,
      "start_seconds": 988,
      "source_timestamp": "16:28",
      "end_seconds": null,
      "text": "to the final report, and only one"
    },
    {
      "index": 429,
      "start_seconds": 990,
      "source_timestamp": "16:30",
      "end_seconds": null,
      "text": "uh of from SMT Checker made it to the"
    },
    {
      "index": 430,
      "start_seconds": 992,
      "source_timestamp": "16:32",
      "end_seconds": null,
      "text": "final report."
    },
    {
      "index": 431,
      "start_seconds": 993,
      "source_timestamp": "16:33",
      "end_seconds": null,
      "text": "What's really interesting here is"
    },
    {
      "index": 432,
      "start_seconds": 995,
      "source_timestamp": "16:35",
      "end_seconds": null,
      "text": "actually like you could see in"
    },
    {
      "index": 433,
      "start_seconds": 996,
      "source_timestamp": "16:36",
      "end_seconds": null,
      "text": "Autoprover and an SMT Checker like one"
    },
    {
      "index": 434,
      "start_seconds": 1000,
      "source_timestamp": "16:40",
      "end_seconds": null,
      "text": "formal verification break or spec break"
    },
    {
      "index": 435,
      "start_seconds": 1002,
      "source_timestamp": "16:42",
      "end_seconds": null,
      "text": "actually led to two separate bugs in the"
    },
    {
      "index": 436,
      "start_seconds": 1004,
      "source_timestamp": "16:44",
      "end_seconds": null,
      "text": "finding or in the report, which I"
    },
    {
      "index": 437,
      "start_seconds": 1006,
      "source_timestamp": "16:46",
      "end_seconds": null,
      "text": "thought was pretty interesting."
    },
    {
      "index": 438,
      "start_seconds": 1008,
      "source_timestamp": "16:48",
      "end_seconds": null,
      "text": "Um and also speaks to the fact that like"
    },
    {
      "index": 439,
      "start_seconds": 1010,
      "source_timestamp": "16:50",
      "end_seconds": null,
      "text": "the human is still very involved in this"
    },
    {
      "index": 440,
      "start_seconds": 1012,
      "source_timestamp": "16:52",
      "end_seconds": null,
      "text": "because you as the auditor would have to"
    },
    {
      "index": 441,
      "start_seconds": 1014,
      "source_timestamp": "16:54",
      "end_seconds": null,
      "text": "see the formal verification expert and"
    },
    {
      "index": 442,
      "start_seconds": 1016,
      "source_timestamp": "16:56",
      "end_seconds": null,
      "text": "that uh"
    },
    {
      "index": 443,
      "start_seconds": 1017,
      "source_timestamp": "16:57",
      "end_seconds": null,
      "text": "formal verification um output and sort"
    },
    {
      "index": 444,
      "start_seconds": 1020,
      "source_timestamp": "17:00",
      "end_seconds": null,
      "text": "of determine okay like what more can I"
    },
    {
      "index": 445,
      "start_seconds": 1022,
      "source_timestamp": "17:02",
      "end_seconds": null,
      "text": "break with this? If this spec doesn't"
    },
    {
      "index": 446,
      "start_seconds": 1024,
      "source_timestamp": "17:04",
      "end_seconds": null,
      "text": "hold, what contracts are really going to"
    },
    {
      "index": 447,
      "start_seconds": 1026,
      "source_timestamp": "17:06",
      "end_seconds": null,
      "text": "go wrong or what problems are really"
    },
    {
      "index": 448,
      "start_seconds": 1028,
      "source_timestamp": "17:08",
      "end_seconds": null,
      "text": "going to go wrong in the contracts uh"
    },
    {
      "index": 449,
      "start_seconds": 1030,
      "source_timestamp": "17:10",
      "end_seconds": null,
      "text": "once they hit production?"
    },
    {
      "index": 450,
      "start_seconds": 1032,
      "source_timestamp": "17:12",
      "end_seconds": null,
      "text": "Um and then this like"
    },
    {
      "index": 451,
      "start_seconds": 1035,
      "source_timestamp": "17:15",
      "end_seconds": null,
      "text": "the Autoprover doesn't have this out of"
    },
    {
      "index": 452,
      "start_seconds": 1037,
      "source_timestamp": "17:17",
      "end_seconds": null,
      "text": "the box and um"
    },
    {
      "index": 453,
      "start_seconds": 1040,
      "source_timestamp": "17:20",
      "end_seconds": null,
      "text": "like it just sort of came to my uh mind"
    },
    {
      "index": 454,
      "start_seconds": 1042,
      "source_timestamp": "17:22",
      "end_seconds": null,
      "text": "when I was uh like developing these"
    },
    {
      "index": 455,
      "start_seconds": 1045,
      "source_timestamp": "17:25",
      "end_seconds": null,
      "text": "tools kind of like"
    },
    {
      "index": 456,
      "start_seconds": 1047,
      "source_timestamp": "17:27",
      "end_seconds": null,
      "text": "just to like Lazy Max, right? Like just"
    },
    {
      "index": 457,
      "start_seconds": 1050,
      "source_timestamp": "17:30",
      "end_seconds": null,
      "text": "to be as lazy as possible and see like"
    },
    {
      "index": 458,
      "start_seconds": 1053,
      "source_timestamp": "17:33",
      "end_seconds": null,
      "text": "give as much to the uh"
    },
    {
      "index": 459,
      "start_seconds": 1054,
      "source_timestamp": "17:34",
      "end_seconds": null,
      "text": "AI as possible."
    },
    {
      "index": 460,
      "start_seconds": 1056,
      "source_timestamp": "17:36",
      "end_seconds": null,
      "text": "So after like the formal verification"
    },
    {
      "index": 461,
      "start_seconds": 1059,
      "source_timestamp": "17:39",
      "end_seconds": null,
      "text": "actually found some breaks in the spec,"
    },
    {
      "index": 462,
      "start_seconds": 1062,
      "source_timestamp": "17:42",
      "end_seconds": null,
      "text": "it got fed to another agent and like I'd"
    },
    {
      "index": 463,
      "start_seconds": 1064,
      "source_timestamp": "17:44",
      "end_seconds": null,
      "text": "mentioned earlier, I had the um agent"
    },
    {
      "index": 464,
      "start_seconds": 1066,
      "source_timestamp": "17:46",
      "end_seconds": null,
      "text": "write a POC"
    },
    {
      "index": 465,
      "start_seconds": 1068,
      "source_timestamp": "17:48",
      "end_seconds": null,
      "text": "um in Foundry to see like oh can this"
    },
    {
      "index": 466,
      "start_seconds": 1070,
      "source_timestamp": "17:50",
      "end_seconds": null,
      "text": "actually happen"
    },
    {
      "index": 467,
      "start_seconds": 1072,
      "source_timestamp": "17:52",
      "end_seconds": null,
      "text": "from the forked block like from the"
    },
    {
      "index": 468,
      "start_seconds": 1074,
      "source_timestamp": "17:54",
      "end_seconds": null,
      "text": "Aave's test suite?"
    },
    {
      "index": 469,
      "start_seconds": 1077,
      "source_timestamp": "17:57",
      "end_seconds": null,
      "text": "Um so surprisingly all the tools were"
    },
    {
      "index": 470,
      "start_seconds": 1080,
      "source_timestamp": "18:00",
      "end_seconds": null,
      "text": "actually producing sound bugs. Um so all"
    },
    {
      "index": 471,
      "start_seconds": 1083,
      "source_timestamp": "18:03",
      "end_seconds": null,
      "text": "of the things well like they have high"
    },
    {
      "index": 472,
      "start_seconds": 1086,
      "source_timestamp": "18:06",
      "end_seconds": null,
      "text": "percentages for like what could actually"
    },
    {
      "index": 473,
      "start_seconds": 1088,
      "source_timestamp": "18:08",
      "end_seconds": null,
      "text": "happen in the code. So"
    },
    {
      "index": 474,
      "start_seconds": 1090,
      "source_timestamp": "18:10",
      "end_seconds": null,
      "text": "these things weren't like very noisy. It"
    },
    {
      "index": 475,
      "start_seconds": 1092,
      "source_timestamp": "18:12",
      "end_seconds": null,
      "text": "was actually very valuable input to like"
    },
    {
      "index": 476,
      "start_seconds": 1095,
      "source_timestamp": "18:15",
      "end_seconds": null,
      "text": "guide the uh smart contract audit"
    },
    {
      "index": 477,
      "start_seconds": 1097,
      "source_timestamp": "18:17",
      "end_seconds": null,
      "text": "itself. Um we know sometimes that if"
    },
    {
      "index": 478,
      "start_seconds": 1100,
      "source_timestamp": "18:20",
      "end_seconds": null,
      "text": "you're like a smart contract and you're"
    },
    {
      "index": 479,
      "start_seconds": 1103,
      "source_timestamp": "18:23",
      "end_seconds": null,
      "text": "going back and forth with Claude,"
    },
    {
      "index": 480,
      "start_seconds": 1105,
      "source_timestamp": "18:25",
      "end_seconds": null,
      "text": "you can waste hours being convinced that"
    },
    {
      "index": 481,
      "start_seconds": 1107,
      "source_timestamp": "18:27",
      "end_seconds": null,
      "text": "there's a bug somewhere where it's not."
    },
    {
      "index": 482,
      "start_seconds": 1109,
      "source_timestamp": "18:29",
      "end_seconds": null,
      "text": "So this was actually pretty cool to see."
    },
    {
      "index": 483,
      "start_seconds": 1112,
      "source_timestamp": "18:32",
      "end_seconds": null,
      "text": "Um, you could go to the next slide."
    },
    {
      "index": 484,
      "start_seconds": 1115,
      "source_timestamp": "18:35",
      "end_seconds": null,
      "text": "And"
    },
    {
      "index": 485,
      "start_seconds": 1117,
      "source_timestamp": "18:37",
      "end_seconds": null,
      "text": "the caveat here is that like"
    },
    {
      "index": 486,
      "start_seconds": 1120,
      "source_timestamp": "18:40",
      "end_seconds": null,
      "text": "some bugs Well, many bugs are sound in"
    },
    {
      "index": 487,
      "start_seconds": 1123,
      "source_timestamp": "18:43",
      "end_seconds": null,
      "text": "the report, but it was actually like you"
    },
    {
      "index": 488,
      "start_seconds": 1125,
      "source_timestamp": "18:45",
      "end_seconds": null,
      "text": "wouldn't want to bring it to the client"
    },
    {
      "index": 489,
      "start_seconds": 1127,
      "source_timestamp": "18:47",
      "end_seconds": null,
      "text": "because it actually like doesn't mean"
    },
    {
      "index": 490,
      "start_seconds": 1128,
      "source_timestamp": "18:48",
      "end_seconds": null,
      "text": "much to the client."
    },
    {
      "index": 491,
      "start_seconds": 1130,
      "source_timestamp": "18:50",
      "end_seconds": null,
      "text": "One very simple example here is like"
    },
    {
      "index": 492,
      "start_seconds": 1132,
      "source_timestamp": "18:52",
      "end_seconds": null,
      "text": "abuse of owner privilege. Like when"
    },
    {
      "index": 493,
      "start_seconds": 1134,
      "source_timestamp": "18:54",
      "end_seconds": null,
      "text": "you're working with a client, you're"
    },
    {
      "index": 494,
      "start_seconds": 1135,
      "source_timestamp": "18:55",
      "end_seconds": null,
      "text": "probably going to assume that they want"
    },
    {
      "index": 495,
      "start_seconds": 1136,
      "source_timestamp": "18:56",
      "end_seconds": null,
      "text": "their protocol to do well and um that"
    },
    {
      "index": 496,
      "start_seconds": 1139,
      "source_timestamp": "18:59",
      "end_seconds": null,
      "text": "you know they're going to be treating"
    },
    {
      "index": 497,
      "start_seconds": 1141,
      "source_timestamp": "19:01",
      "end_seconds": null,
      "text": "owner functions uh with like at least"
    },
    {
      "index": 498,
      "start_seconds": 1144,
      "source_timestamp": "19:04",
      "end_seconds": null,
      "text": "some responsibility. I'll let the OpSec"
    },
    {
      "index": 499,
      "start_seconds": 1145,
      "source_timestamp": "19:05",
      "end_seconds": null,
      "text": "guys talk more about how you can make"
    },
    {
      "index": 500,
      "start_seconds": 1148,
      "source_timestamp": "19:08",
      "end_seconds": null,
      "text": "sure those owner functions are working"
    },
    {
      "index": 501,
      "start_seconds": 1150,
      "source_timestamp": "19:10",
      "end_seconds": null,
      "text": "the way they should, but there wouldn't"
    },
    {
      "index": 502,
      "start_seconds": 1151,
      "source_timestamp": "19:11",
      "end_seconds": null,
      "text": "be any blatant abuse of owner privilege."
    },
    {
      "index": 503,
      "start_seconds": 1153,
      "source_timestamp": "19:13",
      "end_seconds": null,
      "text": "So that's why some of the bugs were"
    },
    {
      "index": 504,
      "start_seconds": 1155,
      "source_timestamp": "19:15",
      "end_seconds": null,
      "text": "sound, but didn't actually count to"
    },
    {
      "index": 505,
      "start_seconds": 1157,
      "source_timestamp": "19:17",
      "end_seconds": null,
      "text": "being in the report itself."
    },
    {
      "index": 506,
      "start_seconds": 1159,
      "source_timestamp": "19:19",
      "end_seconds": null,
      "text": "Another thing is like unrealistic"
    },
    {
      "index": 507,
      "start_seconds": 1161,
      "source_timestamp": "19:21",
      "end_seconds": null,
      "text": "deployment conditions. This one vault"
    },
    {
      "index": 508,
      "start_seconds": 1163,
      "source_timestamp": "19:23",
      "end_seconds": null,
      "text": "that was um"
    },
    {
      "index": 509,
      "start_seconds": 1165,
      "source_timestamp": "19:25",
      "end_seconds": null,
      "text": "being reviewed by these tools was an"
    },
    {
      "index": 510,
      "start_seconds": 1168,
      "source_timestamp": "19:28",
      "end_seconds": null,
      "text": "upgrade. Um, so you would assume like"
    },
    {
      "index": 511,
      "start_seconds": 1170,
      "source_timestamp": "19:30",
      "end_seconds": null,
      "text": "the proxies in place and some of these"
    },
    {
      "index": 512,
      "start_seconds": 1171,
      "source_timestamp": "19:31",
      "end_seconds": null,
      "text": "things are already in place. Um, and"
    },
    {
      "index": 513,
      "start_seconds": 1174,
      "source_timestamp": "19:34",
      "end_seconds": null,
      "text": "some things were sound"
    },
    {
      "index": 514,
      "start_seconds": 1176,
      "source_timestamp": "19:36",
      "end_seconds": null,
      "text": "in the context of what the agent saw,"
    },
    {
      "index": 515,
      "start_seconds": 1178,
      "source_timestamp": "19:38",
      "end_seconds": null,
      "text": "but they were actually just handled by"
    },
    {
      "index": 516,
      "start_seconds": 1179,
      "source_timestamp": "19:39",
      "end_seconds": null,
      "text": "the dependencies or elsewhere."
    },
    {
      "index": 517,
      "start_seconds": 1181,
      "source_timestamp": "19:41",
      "end_seconds": null,
      "text": "Um, so"
    },
    {
      "index": 518,
      "start_seconds": 1186,
      "source_timestamp": "19:46",
      "end_seconds": null,
      "text": "if you could go to the previous slide"
    },
    {
      "index": 519,
      "start_seconds": 1187,
      "source_timestamp": "19:47",
      "end_seconds": null,
      "text": "actually."
    },
    {
      "index": 520,
      "start_seconds": 1188,
      "source_timestamp": "19:48",
      "end_seconds": null,
      "text": "Thank you. So like yes, many were sound,"
    },
    {
      "index": 521,
      "start_seconds": 1191,
      "source_timestamp": "19:51",
      "end_seconds": null,
      "text": "but not all of them made it to the"
    },
    {
      "index": 522,
      "start_seconds": 1193,
      "source_timestamp": "19:53",
      "end_seconds": null,
      "text": "actual report because of some of the"
    },
    {
      "index": 523,
      "start_seconds": 1195,
      "source_timestamp": "19:55",
      "end_seconds": null,
      "text": "things we talk about. All right, two"
    },
    {
      "index": 524,
      "start_seconds": 1196,
      "source_timestamp": "19:56",
      "end_seconds": null,
      "text": "slides forward. Thank you, sir."
    },
    {
      "index": 525,
      "start_seconds": 1199,
      "source_timestamp": "19:59",
      "end_seconds": null,
      "text": "Okay, so now we wanted to see like out"
    },
    {
      "index": 526,
      "start_seconds": 1201,
      "source_timestamp": "20:01",
      "end_seconds": null,
      "text": "of all the rules that the AI produced,"
    },
    {
      "index": 527,
      "start_seconds": 1204,
      "source_timestamp": "20:04",
      "end_seconds": null,
      "text": "how many of those"
    },
    {
      "index": 528,
      "start_seconds": 1206,
      "source_timestamp": "20:06",
      "end_seconds": null,
      "text": "rules"
    },
    {
      "index": 529,
      "start_seconds": 1209,
      "source_timestamp": "20:09",
      "end_seconds": null,
      "text": "actually matched what the human did"
    },
    {
      "index": 530,
      "start_seconds": 1210,
      "source_timestamp": "20:10",
      "end_seconds": null,
      "text": "versus like all of the rules that were"
    },
    {
      "index": 531,
      "start_seconds": 1212,
      "source_timestamp": "20:12",
      "end_seconds": null,
      "text": "in the human report? Okay, hold on. Let"
    },
    {
      "index": 532,
      "start_seconds": 1214,
      "source_timestamp": "20:14",
      "end_seconds": null,
      "text": "me say that one more time."
    },
    {
      "index": 533,
      "start_seconds": 1216,
      "source_timestamp": "20:16",
      "end_seconds": null,
      "text": "So like the AI produced a bunch of"
    },
    {
      "index": 534,
      "start_seconds": 1218,
      "source_timestamp": "20:18",
      "end_seconds": null,
      "text": "rules, like way more rules than the"
    },
    {
      "index": 535,
      "start_seconds": 1219,
      "source_timestamp": "20:19",
      "end_seconds": null,
      "text": "human did. So of those rules that"
    },
    {
      "index": 536,
      "start_seconds": 1222,
      "source_timestamp": "20:22",
      "end_seconds": null,
      "text": "actually matched the human. So of like"
    },
    {
      "index": 537,
      "start_seconds": 1225,
      "source_timestamp": "20:25",
      "end_seconds": null,
      "text": "the formal verification tool produced"
    },
    {
      "index": 538,
      "start_seconds": 1227,
      "source_timestamp": "20:27",
      "end_seconds": null,
      "text": "100 rules, 10 of those were also"
    },
    {
      "index": 539,
      "start_seconds": 1230,
      "source_timestamp": "20:30",
      "end_seconds": null,
      "text": "produced by the human."
    },
    {
      "index": 540,
      "start_seconds": 1232,
      "source_timestamp": "20:32",
      "end_seconds": null,
      "text": "That divided by all the human rules."
    },
    {
      "index": 541,
      "start_seconds": 1235,
      "source_timestamp": "20:35",
      "end_seconds": null,
      "text": "Okay, I hope that makes sense. Next"
    },
    {
      "index": 542,
      "start_seconds": 1236,
      "source_timestamp": "20:36",
      "end_seconds": null,
      "text": "slide."
    },
    {
      "index": 543,
      "start_seconds": 1238,
      "source_timestamp": "20:38",
      "end_seconds": null,
      "text": "So they actually performed fairly fairly"
    },
    {
      "index": 544,
      "start_seconds": 1240,
      "source_timestamp": "20:40",
      "end_seconds": null,
      "text": "low here. Um"
    },
    {
      "index": 545,
      "start_seconds": 1243,
      "source_timestamp": "20:43",
      "end_seconds": null,
      "text": "I think I have some ideas why. We'll see"
    },
    {
      "index": 546,
      "start_seconds": 1245,
      "source_timestamp": "20:45",
      "end_seconds": null,
      "text": "like once we actually check um some more"
    },
    {
      "index": 547,
      "start_seconds": 1247,
      "source_timestamp": "20:47",
      "end_seconds": null,
      "text": "results later, but on the left side you"
    },
    {
      "index": 548,
      "start_seconds": 1250,
      "source_timestamp": "20:50",
      "end_seconds": null,
      "text": "could see like all of the rules that the"
    },
    {
      "index": 549,
      "start_seconds": 1252,
      "source_timestamp": "20:52",
      "end_seconds": null,
      "text": "human wrote or at least groups of rules"
    },
    {
      "index": 550,
      "start_seconds": 1255,
      "source_timestamp": "20:55",
      "end_seconds": null,
      "text": "that the human wrote"
    },
    {
      "index": 551,
      "start_seconds": 1256,
      "source_timestamp": "20:56",
      "end_seconds": null,
      "text": "um and the rules that the um agents were"
    },
    {
      "index": 552,
      "start_seconds": 1260,
      "source_timestamp": "21:00",
      "end_seconds": null,
      "text": "actually able to come up with."
    },
    {
      "index": 553,
      "start_seconds": 1262,
      "source_timestamp": "21:02",
      "end_seconds": null,
      "text": "And for time sake I'm going to go to the"
    },
    {
      "index": 554,
      "start_seconds": 1265,
      "source_timestamp": "21:05",
      "end_seconds": null,
      "text": "next slide."
    },
    {
      "index": 555,
      "start_seconds": 1267,
      "source_timestamp": "21:07",
      "end_seconds": null,
      "text": "Um yeah, so this is just sort of you can"
    },
    {
      "index": 556,
      "start_seconds": 1270,
      "source_timestamp": "21:10",
      "end_seconds": null,
      "text": "go to the next one."
    },
    {
      "index": 557,
      "start_seconds": 1272,
      "source_timestamp": "21:12",
      "end_seconds": null,
      "text": "Yeah, uh"
    },
    {
      "index": 558,
      "start_seconds": 1273,
      "source_timestamp": "21:13",
      "end_seconds": null,
      "text": "this is just a breakdown of runtime,"
    },
    {
      "index": 559,
      "start_seconds": 1276,
      "source_timestamp": "21:16",
      "end_seconds": null,
      "text": "token spent, and cost. And I think that"
    },
    {
      "index": 560,
      "start_seconds": 1279,
      "source_timestamp": "21:19",
      "end_seconds": null,
      "text": "this uh slide is a big takeaway."
    },
    {
      "index": 561,
      "start_seconds": 1283,
      "source_timestamp": "21:23",
      "end_seconds": null,
      "text": "And it sort of shows why Auto Prover"
    },
    {
      "index": 562,
      "start_seconds": 1286,
      "source_timestamp": "21:26",
      "end_seconds": null,
      "text": "performed well in uh several different"
    },
    {
      "index": 563,
      "start_seconds": 1289,
      "source_timestamp": "21:29",
      "end_seconds": null,
      "text": "of the tests. Um probably because it's"
    },
    {
      "index": 564,
      "start_seconds": 1292,
      "source_timestamp": "21:32",
      "end_seconds": null,
      "text": "spending like almost a magnitude of 10"
    },
    {
      "index": 565,
      "start_seconds": 1295,
      "source_timestamp": "21:35",
      "end_seconds": null,
      "text": "more tokens than the harnesses we were"
    },
    {
      "index": 566,
      "start_seconds": 1298,
      "source_timestamp": "21:38",
      "end_seconds": null,
      "text": "able to develop. So that's probably a"
    },
    {
      "index": 567,
      "start_seconds": 1300,
      "source_timestamp": "21:40",
      "end_seconds": null,
      "text": "testament to Auto Prover being a more"
    },
    {
      "index": 568,
      "start_seconds": 1303,
      "source_timestamp": "21:43",
      "end_seconds": null,
      "text": "mature piece of software. Um and there's"
    },
    {
      "index": 569,
      "start_seconds": 1305,
      "source_timestamp": "21:45",
      "end_seconds": null,
      "text": "still being work to be done on the"
    },
    {
      "index": 570,
      "start_seconds": 1307,
      "source_timestamp": "21:47",
      "end_seconds": null,
      "text": "harnesses uh themselves."
    },
    {
      "index": 571,
      "start_seconds": 1310,
      "source_timestamp": "21:50",
      "end_seconds": null,
      "text": "So what was unexpected to me as an"
    },
    {
      "index": 572,
      "start_seconds": 1312,
      "source_timestamp": "21:52",
      "end_seconds": null,
      "text": "auditor was just how long these provers"
    },
    {
      "index": 573,
      "start_seconds": 1314,
      "source_timestamp": "21:54",
      "end_seconds": null,
      "text": "take to run, like"
    },
    {
      "index": 574,
      "start_seconds": 1317,
      "source_timestamp": "21:57",
      "end_seconds": null,
      "text": "58 hours um is a long time."
    },
    {
      "index": 575,
      "start_seconds": 1321,
      "source_timestamp": "22:01",
      "end_seconds": null,
      "text": "And yeah, you've got to"
    },
    {
      "index": 576,
      "start_seconds": 1325,
      "source_timestamp": "22:05",
      "end_seconds": null,
      "text": "hope your CEO is funding your tokens."
    },
    {
      "index": 577,
      "start_seconds": 1329,
      "source_timestamp": "22:09",
      "end_seconds": null,
      "text": "Next slide."
    },
    {
      "index": 578,
      "start_seconds": 1330,
      "source_timestamp": "22:10",
      "end_seconds": null,
      "text": "Okay, so is the slop or prop? Is like"
    },
    {
      "index": 579,
      "start_seconds": 1335,
      "source_timestamp": "22:15",
      "end_seconds": null,
      "text": "can we actually have AI power and"
    },
    {
      "index": 580,
      "start_seconds": 1338,
      "source_timestamp": "22:18",
      "end_seconds": null,
      "text": "experienced smart contract auditors"
    },
    {
      "index": 581,
      "start_seconds": 1339,
      "source_timestamp": "22:19",
      "end_seconds": null,
      "text": "access to formal verification tools?"
    },
    {
      "index": 582,
      "start_seconds": 1342,
      "source_timestamp": "22:22",
      "end_seconds": null,
      "text": "So if we look at the numbers, it's"
    },
    {
      "index": 583,
      "start_seconds": 1344,
      "source_timestamp": "22:24",
      "end_seconds": null,
      "text": "tending towards slop."
    },
    {
      "index": 584,
      "start_seconds": 1346,
      "source_timestamp": "22:26",
      "end_seconds": null,
      "text": "But honestly,"
    },
    {
      "index": 585,
      "start_seconds": 1348,
      "source_timestamp": "22:28",
      "end_seconds": null,
      "text": "as an auditor that worked on it, I am"
    },
    {
      "index": 586,
      "start_seconds": 1350,
      "source_timestamp": "22:30",
      "end_seconds": null,
      "text": "tending towards definitely a value prop"
    },
    {
      "index": 587,
      "start_seconds": 1352,
      "source_timestamp": "22:32",
      "end_seconds": null,
      "text": "here."
    },
    {
      "index": 588,
      "start_seconds": 1353,
      "source_timestamp": "22:33",
      "end_seconds": null,
      "text": "Um, at the very least,"
    },
    {
      "index": 589,
      "start_seconds": 1356,
      "source_timestamp": "22:36",
      "end_seconds": null,
      "text": "as an auditor, you have like access to"
    },
    {
      "index": 590,
      "start_seconds": 1360,
      "source_timestamp": "22:40",
      "end_seconds": null,
      "text": "sort of start to learn this new"
    },
    {
      "index": 591,
      "start_seconds": 1362,
      "source_timestamp": "22:42",
      "end_seconds": null,
      "text": "specialized skill, like at the very"
    },
    {
      "index": 592,
      "start_seconds": 1364,
      "source_timestamp": "22:44",
      "end_seconds": null,
      "text": "least. Um, on the next slide."
    },
    {
      "index": 593,
      "start_seconds": 1367,
      "source_timestamp": "22:47",
      "end_seconds": null,
      "text": "Uh, actually, could you go to the slide"
    },
    {
      "index": 594,
      "start_seconds": 1369,
      "source_timestamp": "22:49",
      "end_seconds": null,
      "text": "after this?"
    },
    {
      "index": 595,
      "start_seconds": 1372,
      "source_timestamp": "22:52",
      "end_seconds": null,
      "text": "Okay, I'll go here and then I'll go"
    },
    {
      "index": 596,
      "start_seconds": 1374,
      "source_timestamp": "22:54",
      "end_seconds": null,
      "text": "back. Um,"
    },
    {
      "index": 597,
      "start_seconds": 1376,
      "source_timestamp": "22:56",
      "end_seconds": null,
      "text": "I think the biggest thing why I think"
    },
    {
      "index": 598,
      "start_seconds": 1378,
      "source_timestamp": "22:58",
      "end_seconds": null,
      "text": "it's prop is that you get valuable"
    },
    {
      "index": 599,
      "start_seconds": 1380,
      "source_timestamp": "23:00",
      "end_seconds": null,
      "text": "results for edge cases that can just"
    },
    {
      "index": 600,
      "start_seconds": 1382,
      "source_timestamp": "23:02",
      "end_seconds": null,
      "text": "sort of further guide your manual"
    },
    {
      "index": 601,
      "start_seconds": 1384,
      "source_timestamp": "23:04",
      "end_seconds": null,
      "text": "review. Um, so"
    },
    {
      "index": 602,
      "start_seconds": 1387,
      "source_timestamp": "23:07",
      "end_seconds": null,
      "text": "we saw that like the soundness scored"
    },
    {
      "index": 603,
      "start_seconds": 1389,
      "source_timestamp": "23:09",
      "end_seconds": null,
      "text": "really high. Um, so you could look at"
    },
    {
      "index": 604,
      "start_seconds": 1391,
      "source_timestamp": "23:11",
      "end_seconds": null,
      "text": "some of those bugs and then like see, oh"
    },
    {
      "index": 605,
      "start_seconds": 1393,
      "source_timestamp": "23:13",
      "end_seconds": null,
      "text": "okay, like how bad is this really?"
    },
    {
      "index": 606,
      "start_seconds": 1396,
      "source_timestamp": "23:16",
      "end_seconds": null,
      "text": "Um,"
    },
    {
      "index": 607,
      "start_seconds": 1397,
      "source_timestamp": "23:17",
      "end_seconds": null,
      "text": "another really cool thing you could do"
    },
    {
      "index": 608,
      "start_seconds": 1399,
      "source_timestamp": "23:19",
      "end_seconds": null,
      "text": "with this is if you have like the formal"
    },
    {
      "index": 609,
      "start_seconds": 1402,
      "source_timestamp": "23:22",
      "end_seconds": null,
      "text": "verification running sort of maybe"
    },
    {
      "index": 610,
      "start_seconds": 1404,
      "source_timestamp": "23:24",
      "end_seconds": null,
      "text": "alongside like your manual review,"
    },
    {
      "index": 611,
      "start_seconds": 1406,
      "source_timestamp": "23:26",
      "end_seconds": null,
      "text": "um,"
    },
    {
      "index": 612,
      "start_seconds": 1408,
      "source_timestamp": "23:28",
      "end_seconds": null,
      "text": "maybe you give the report to the client"
    },
    {
      "index": 613,
      "start_seconds": 1410,
      "source_timestamp": "23:30",
      "end_seconds": null,
      "text": "and they come back uh and they fix a"
    },
    {
      "index": 614,
      "start_seconds": 1412,
      "source_timestamp": "23:32",
      "end_seconds": null,
      "text": "small part, you could like"
    },
    {
      "index": 615,
      "start_seconds": 1414,
      "source_timestamp": "23:34",
      "end_seconds": null,
      "text": "run your specs again and make sure that"
    },
    {
      "index": 616,
      "start_seconds": 1416,
      "source_timestamp": "23:36",
      "end_seconds": null,
      "text": "nothing else broke."
    },
    {
      "index": 617,
      "start_seconds": 1418,
      "source_timestamp": "23:38",
      "end_seconds": null,
      "text": "Um,"
    },
    {
      "index": 618,
      "start_seconds": 1420,
      "source_timestamp": "23:40",
      "end_seconds": null,
      "text": "I do think that like the results alone"
    },
    {
      "index": 619,
      "start_seconds": 1422,
      "source_timestamp": "23:42",
      "end_seconds": null,
      "text": "do not suggest that like LLMs could"
    },
    {
      "index": 620,
      "start_seconds": 1424,
      "source_timestamp": "23:44",
      "end_seconds": null,
      "text": "drive a proper standalone formal"
    },
    {
      "index": 621,
      "start_seconds": 1426,
      "source_timestamp": "23:46",
      "end_seconds": null,
      "text": "verification engagement right now or at"
    },
    {
      "index": 622,
      "start_seconds": 1428,
      "source_timestamp": "23:48",
      "end_seconds": null,
      "text": "least that that could be driven by a"
    },
    {
      "index": 623,
      "start_seconds": 1430,
      "source_timestamp": "23:50",
      "end_seconds": null,
      "text": "sole smart contract auditor. I think"
    },
    {
      "index": 624,
      "start_seconds": 1432,
      "source_timestamp": "23:52",
      "end_seconds": null,
      "text": "it's still a very, very specialized"
    },
    {
      "index": 625,
      "start_seconds": 1435,
      "source_timestamp": "23:55",
      "end_seconds": null,
      "text": "thing that's going to need like practice"
    },
    {
      "index": 626,
      "start_seconds": 1437,
      "source_timestamp": "23:57",
      "end_seconds": null,
      "text": "um and expertise. Could you go to the"
    },
    {
      "index": 627,
      "start_seconds": 1439,
      "source_timestamp": "23:59",
      "end_seconds": null,
      "text": "previous slide actually?"
    },
    {
      "index": 628,
      "start_seconds": 1441,
      "source_timestamp": "24:01",
      "end_seconds": null,
      "text": "Thank you. Uh, so this sort of goes to"
    },
    {
      "index": 629,
      "start_seconds": 1443,
      "source_timestamp": "24:03",
      "end_seconds": null,
      "text": "the shortcomings. It's it's hard to"
    },
    {
      "index": 630,
      "start_seconds": 1444,
      "source_timestamp": "24:04",
      "end_seconds": null,
      "text": "learn. Um, the results that we got are"
    },
    {
      "index": 631,
      "start_seconds": 1447,
      "source_timestamp": "24:07",
      "end_seconds": null,
      "text": "actually dependent on the harness, so"
    },
    {
      "index": 632,
      "start_seconds": 1449,
      "source_timestamp": "24:09",
      "end_seconds": null,
      "text": "you could tell me to just prompt better."
    },
    {
      "index": 633,
      "start_seconds": 1451,
      "source_timestamp": "24:11",
      "end_seconds": null,
      "text": "Okay, that's sort of like the next step."
    },
    {
      "index": 634,
      "start_seconds": 1453,
      "source_timestamp": "24:13",
      "end_seconds": null,
      "text": "We'll work on it together. Um, results"
    },
    {
      "index": 635,
      "start_seconds": 1456,
      "source_timestamp": "24:16",
      "end_seconds": null,
      "text": "are on the whim of the model itself. So"
    },
    {
      "index": 636,
      "start_seconds": 1458,
      "source_timestamp": "24:18",
      "end_seconds": null,
      "text": "you can see a real screenshot that"
    },
    {
      "index": 637,
      "start_seconds": 1461,
      "source_timestamp": "24:21",
      "end_seconds": null,
      "text": "Claude gave me. Like this is probably"
    },
    {
      "index": 638,
      "start_seconds": 1463,
      "source_timestamp": "24:23",
      "end_seconds": null,
      "text": "the worst thing you could see after like"
    },
    {
      "index": 639,
      "start_seconds": 1465,
      "source_timestamp": "24:25",
      "end_seconds": null,
      "text": "59 hours of proving and hundreds of"
    },
    {
      "index": 640,
      "start_seconds": 1468,
      "source_timestamp": "24:28",
      "end_seconds": null,
      "text": "dollars spent on tokens. Um, luckily,"
    },
    {
      "index": 641,
      "start_seconds": 1472,
      "source_timestamp": "24:32",
      "end_seconds": null,
      "text": "like"
    },
    {
      "index": 642,
      "start_seconds": 1473,
      "source_timestamp": "24:33",
      "end_seconds": null,
      "text": "it was in the conversation history, so"
    },
    {
      "index": 643,
      "start_seconds": 1475,
      "source_timestamp": "24:35",
      "end_seconds": null,
      "text": "the fallout wasn't too bad, but like"
    },
    {
      "index": 644,
      "start_seconds": 1478,
      "source_timestamp": "24:38",
      "end_seconds": null,
      "text": "there's always this risk of like the"
    },
    {
      "index": 645,
      "start_seconds": 1480,
      "source_timestamp": "24:40",
      "end_seconds": null,
      "text": "models going rogue and just sort of"
    },
    {
      "index": 646,
      "start_seconds": 1482,
      "source_timestamp": "24:42",
      "end_seconds": null,
      "text": "like, you know, how much can you really"
    },
    {
      "index": 647,
      "start_seconds": 1484,
      "source_timestamp": "24:44",
      "end_seconds": null,
      "text": "trust them? Uh, we'll go forward to uh"
    },
    {
      "index": 648,
      "start_seconds": 1488,
      "source_timestamp": "24:48",
      "end_seconds": null,
      "text": "two slides or"
    },
    {
      "index": 649,
      "start_seconds": 1490,
      "source_timestamp": "24:50",
      "end_seconds": null,
      "text": "Yes. Okay. So like the next step"
    },
    {
      "index": 650,
      "start_seconds": 1493,
      "source_timestamp": "24:53",
      "end_seconds": null,
      "text": "uh would be like open sourcing some of"
    },
    {
      "index": 651,
      "start_seconds": 1495,
      "source_timestamp": "24:55",
      "end_seconds": null,
      "text": "these harnesses we developed. So once"
    },
    {
      "index": 652,
      "start_seconds": 1498,
      "source_timestamp": "24:58",
      "end_seconds": null,
      "text": "that's like out and shared with the"
    },
    {
      "index": 653,
      "start_seconds": 1499,
      "source_timestamp": "24:59",
      "end_seconds": null,
      "text": "community, like we can all sort of like"
    },
    {
      "index": 654,
      "start_seconds": 1502,
      "source_timestamp": "25:02",
      "end_seconds": null,
      "text": "take a crack at it and like tweaking the"
    },
    {
      "index": 655,
      "start_seconds": 1503,
      "source_timestamp": "25:03",
      "end_seconds": null,
      "text": "harnesses."
    },
    {
      "index": 656,
      "start_seconds": 1505,
      "source_timestamp": "25:05",
      "end_seconds": null,
      "text": "Also the workflows um that we did were"
    },
    {
      "index": 657,
      "start_seconds": 1509,
      "source_timestamp": "25:09",
      "end_seconds": null,
      "text": "ultimately like prompt output prompt um"
    },
    {
      "index": 658,
      "start_seconds": 1513,
      "source_timestamp": "25:13",
      "end_seconds": null,
      "text": "in like a workflow instead of like a"
    },
    {
      "index": 659,
      "start_seconds": 1515,
      "source_timestamp": "25:15",
      "end_seconds": null,
      "text": "truly agentic solution. So eventually I"
    },
    {
      "index": 660,
      "start_seconds": 1517,
      "source_timestamp": "25:17",
      "end_seconds": null,
      "text": "would want to like develop towards that"
    },
    {
      "index": 661,
      "start_seconds": 1519,
      "source_timestamp": "25:19",
      "end_seconds": null,
      "text": "where you sort of had um either like one"
    },
    {
      "index": 662,
      "start_seconds": 1522,
      "source_timestamp": "25:22",
      "end_seconds": null,
      "text": "agent or a swarm of agents sort of"
    },
    {
      "index": 663,
      "start_seconds": 1526,
      "source_timestamp": "25:26",
      "end_seconds": null,
      "text": "like really really looping through all"
    },
    {
      "index": 664,
      "start_seconds": 1529,
      "source_timestamp": "25:29",
      "end_seconds": null,
      "text": "of these different stages instead of"
    },
    {
      "index": 665,
      "start_seconds": 1530,
      "source_timestamp": "25:30",
      "end_seconds": null,
      "text": "just like a prompting workflow."
    },
    {
      "index": 666,
      "start_seconds": 1533,
      "source_timestamp": "25:33",
      "end_seconds": null,
      "text": "And then of course, there's a lot more"
    },
    {
      "index": 667,
      "start_seconds": 1535,
      "source_timestamp": "25:35",
      "end_seconds": null,
      "text": "work that should be done with like"
    },
    {
      "index": 668,
      "start_seconds": 1536,
      "source_timestamp": "25:36",
      "end_seconds": null,
      "text": "testing on multiple models and multiple"
    },
    {
      "index": 669,
      "start_seconds": 1538,
      "source_timestamp": "25:38",
      "end_seconds": null,
      "text": "code bases."
    },
    {
      "index": 670,
      "start_seconds": 1541,
      "source_timestamp": "25:41",
      "end_seconds": null,
      "text": "Next slide."
    },
    {
      "index": 671,
      "start_seconds": 1542,
      "source_timestamp": "25:42",
      "end_seconds": null,
      "text": "Okay."
    },
    {
      "index": 672,
      "start_seconds": 1543,
      "source_timestamp": "25:43",
      "end_seconds": null,
      "text": "Well, thank you all for the time. Um,"
    },
    {
      "index": 673,
      "start_seconds": 1546,
      "source_timestamp": "25:46",
      "end_seconds": null,
      "text": "and I hope you learned something. I'll"
    },
    {
      "index": 674,
      "start_seconds": 1547,
      "source_timestamp": "25:47",
      "end_seconds": null,
      "text": "be around like today and tomorrow uh to"
    },
    {
      "index": 675,
      "start_seconds": 1550,
      "source_timestamp": "25:50",
      "end_seconds": null,
      "text": "talk more and I'm interested to hear"
    },
    {
      "index": 676,
      "start_seconds": 1552,
      "source_timestamp": "25:52",
      "end_seconds": null,
      "text": "your thoughts."
    },
    {
      "index": 677,
      "start_seconds": 1553,
      "source_timestamp": "25:53",
      "end_seconds": null,
      "text": "Thank you."
    }
  ]
}
