{
  "type": "excalidraw",
  "version": 2,
  "source": "http://127.0.0.1:8765",
  "elements": [
    {
      "id": "PPprod2RRC9MAkBP9lBfK",
      "type": "text",
      "x": 0,
      "y": 0,
      "width": 895.527587890625,
      "height": 35,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a0",
      "roundness": null,
      "seed": 2866,
      "version": 2,
      "versionNonce": 419050079,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "Where LayerNorm sits: after the shortcut, or before the sublayer",
      "fontSize": 28,
      "fontFamily": 5,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "Where LayerNorm sits: after the shortcut, or before the sublayer",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "DCr9jR_H3VDww6hRzSfOn",
      "type": "text",
      "x": 212.60013580322266,
      "y": 60,
      "width": 94.79972839355469,
      "height": 30,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a1",
      "roundness": null,
      "seed": 2867,
      "version": 2,
      "versionNonce": 817882897,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "Post-LN",
      "fontSize": 24,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "Post-LN",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "qiklZbLORwzp04zyN2GLI",
      "type": "text",
      "x": 129.10482788085938,
      "y": 94,
      "width": 261.79034423828125,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a2",
      "roundness": null,
      "seed": 2868,
      "version": 2,
      "versionNonce": 685936255,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "the 2017 paper (these notes)",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "the 2017 paper (these notes)",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "rUT4W82c4xS_H8jSLSFNb",
      "type": "text",
      "x": 244.1511001586914,
      "y": 470,
      "width": 31.697799682617188,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a3",
      "roundness": null,
      "seed": 2869,
      "version": 2,
      "versionNonce": 926080241,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "x in",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "x in",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "yv7v25vIygjZkg-IlNhi7",
      "type": "arrow",
      "x": 260,
      "y": 460,
      "width": 0,
      "height": 52,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a4",
      "roundness": null,
      "seed": 2870,
      "version": 2,
      "versionNonce": 799696543,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          -0.5
        ],
        [
          0,
          -51.5
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "xrnzMtlhheCGxvFjopbEX",
      "type": "rectangle",
      "x": 150,
      "y": 338,
      "width": 220,
      "height": 70,
      "angle": 0,
      "strokeColor": "#6741d9",
      "backgroundColor": "#e5dbff",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a5",
      "roundness": {
        "type": 3
      },
      "seed": 2871,
      "version": 2,
      "versionNonce": 1133254353,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false
    },
    {
      "id": "S1T0iiW0W-n0zEF6WOIEd",
      "type": "text",
      "x": 175.06009674072266,
      "y": 360.6,
      "width": 169.8798065185547,
      "height": 25,
      "angle": 0,
      "strokeColor": "#6741d9",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a6",
      "roundness": null,
      "seed": 2872,
      "version": 2,
      "versionNonce": 469908159,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "Attention or FFN",
      "fontSize": 20,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "Attention or FFN",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "XpcnwWWWSobCHMVBvt_ak",
      "type": "arrow",
      "x": 260,
      "y": 338,
      "width": 0,
      "height": 34,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a7",
      "roundness": null,
      "seed": 2873,
      "version": 2,
      "versionNonce": 1214452913,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          -0.5
        ],
        [
          0,
          -33.5
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "VrH_CP0TSE0CX8OCR4io5",
      "type": "ellipse",
      "x": 243,
      "y": 269,
      "width": 34,
      "height": 34,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "#ffffff",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a8",
      "roundness": null,
      "seed": 2874,
      "version": 2,
      "versionNonce": 847940319,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false
    },
    {
      "id": "aU4XwceO63NN_g9MxZqNf",
      "type": "text",
      "x": 252.84999084472656,
      "y": 269,
      "width": 14.300018310546875,
      "height": 32.5,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "a9",
      "roundness": null,
      "seed": 2875,
      "version": 2,
      "versionNonce": 2036291217,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "+",
      "fontSize": 26,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "+",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "t5_oUyvsyB1Io4DNO9OMk",
      "type": "arrow",
      "x": 260,
      "y": 268,
      "width": 0,
      "height": 34,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aA",
      "roundness": null,
      "seed": 2876,
      "version": 2,
      "versionNonce": 1339283199,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          -0.5
        ],
        [
          0,
          -33.5
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "SVfS2GRYuEJQMR7wPPFTV",
      "type": "rectangle",
      "x": 180,
      "y": 194,
      "width": 160,
      "height": 40,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "#f1f3f5",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aB",
      "roundness": {
        "type": 3
      },
      "seed": 2877,
      "version": 2,
      "versionNonce": 691559537,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false
    },
    {
      "id": "OUhE2Sy0dQp25zdsin-ew",
      "type": "text",
      "x": 215.8552703857422,
      "y": 202.84,
      "width": 88.28945922851562,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aC",
      "roundness": null,
      "seed": 2878,
      "version": 2,
      "versionNonce": 1784586015,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "LayerNorm",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "LayerNorm",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "imppWkD71hwIiq8XPIMPU",
      "type": "arrow",
      "x": 260,
      "y": 194,
      "width": 0,
      "height": 34,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aD",
      "roundness": null,
      "seed": 2879,
      "version": 2,
      "versionNonce": 1739957841,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          -0.5
        ],
        [
          0,
          -33.5
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "QyyhbHBEQIXQt1aFjgS30",
      "type": "text",
      "x": 235.77215576171875,
      "y": 130,
      "width": 48.4556884765625,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aE",
      "roundness": null,
      "seed": 2880,
      "version": 2,
      "versionNonce": 1419469631,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "x out",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "x out",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "RiBfp9RS-9mdYUHXqpZZJ",
      "type": "line",
      "x": 260,
      "y": 435,
      "width": 100,
      "height": 0,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aF",
      "roundness": null,
      "seed": 2881,
      "version": 2,
      "versionNonce": 1605284913,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          0
        ],
        [
          170,
          0
        ],
        [
          170,
          -149
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": null
    },
    {
      "id": "B5PfHZhVpGyIyeCeG5AMb",
      "type": "arrow",
      "x": 430,
      "y": 286,
      "width": 151,
      "height": 0,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aG",
      "roundness": null,
      "seed": 2882,
      "version": 2,
      "versionNonce": 1055564639,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          -0.5,
          0
        ],
        [
          -150.5,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "p5SFlJ2ktRWZySXwnwK_N",
      "type": "text",
      "x": 440,
      "y": 350.5,
      "width": 77.57949829101562,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aH",
      "roundness": null,
      "seed": 2883,
      "version": 2,
      "versionNonce": 1963656721,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "shortcut",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "shortcut",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "w0q4sN50KzujZP-Q4JxfU",
      "type": "text",
      "x": 761.4441146850586,
      "y": 60,
      "width": 77.11177062988281,
      "height": 30,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aI",
      "roundness": null,
      "seed": 2884,
      "version": 2,
      "versionNonce": 2040323967,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "Pre-LN",
      "fontSize": 24,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "Pre-LN",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "vm7xeScgnQ1_9HVhqpaTt",
      "type": "text",
      "x": 656.0549163818359,
      "y": 94,
      "width": 287.8901672363281,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aJ",
      "roundness": null,
      "seed": 2885,
      "version": 2,
      "versionNonce": 583146481,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "ESM-2, GPT-2, most new models",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "ESM-2, GPT-2, most new models",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "qs9umgqzOzBSZKGvInXGj",
      "type": "text",
      "x": 784.1511001586914,
      "y": 470,
      "width": 31.697799682617188,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aK",
      "roundness": null,
      "seed": 2886,
      "version": 2,
      "versionNonce": 1995610015,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "x in",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "x in",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "GPZqUjFIpkuvF4P5caBE4",
      "type": "arrow",
      "x": 800,
      "y": 460,
      "width": 0,
      "height": 52,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aL",
      "roundness": null,
      "seed": 2887,
      "version": 2,
      "versionNonce": 417035729,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          -0.5
        ],
        [
          0,
          -51.5
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "90HBK8bMOJGCWbJitX_GH",
      "type": "rectangle",
      "x": 720,
      "y": 368,
      "width": 160,
      "height": 40,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "#f1f3f5",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aM",
      "roundness": {
        "type": 3
      },
      "seed": 2888,
      "version": 2,
      "versionNonce": 219958207,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false
    },
    {
      "id": "NA-JtT3DgziimLu8wokEK",
      "type": "text",
      "x": 755.8552703857422,
      "y": 376.84,
      "width": 88.28945922851562,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aN",
      "roundness": null,
      "seed": 2889,
      "version": 2,
      "versionNonce": 443454385,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "LayerNorm",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "LayerNorm",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "bcl3o9TgqUCoywBxNySK2",
      "type": "arrow",
      "x": 800,
      "y": 368,
      "width": 0,
      "height": 34,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aO",
      "roundness": null,
      "seed": 2890,
      "version": 2,
      "versionNonce": 2017098719,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          -0.5
        ],
        [
          0,
          -33.5
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "U7fRQVKPmSwpU_D8F1XbG",
      "type": "rectangle",
      "x": 690,
      "y": 264,
      "width": 220,
      "height": 70,
      "angle": 0,
      "strokeColor": "#6741d9",
      "backgroundColor": "#e5dbff",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aP",
      "roundness": {
        "type": 3
      },
      "seed": 2891,
      "version": 2,
      "versionNonce": 463664529,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false
    },
    {
      "id": "Z_UH0DMUgmPPMwKYAJVDm",
      "type": "text",
      "x": 715.0600967407227,
      "y": 286.6,
      "width": 169.8798065185547,
      "height": 25,
      "angle": 0,
      "strokeColor": "#6741d9",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aQ",
      "roundness": null,
      "seed": 2892,
      "version": 2,
      "versionNonce": 475899903,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "Attention or FFN",
      "fontSize": 20,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "Attention or FFN",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "6sVm-uscltXq4H4FXK0q3",
      "type": "arrow",
      "x": 800,
      "y": 264,
      "width": 0,
      "height": 34,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aR",
      "roundness": null,
      "seed": 2893,
      "version": 2,
      "versionNonce": 531635057,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          -0.5
        ],
        [
          0,
          -33.5
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "oVl_TznfpXpoJAVkvNXx2",
      "type": "ellipse",
      "x": 783,
      "y": 195,
      "width": 34,
      "height": 34,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "#ffffff",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aS",
      "roundness": null,
      "seed": 2894,
      "version": 2,
      "versionNonce": 126242847,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false
    },
    {
      "id": "F9DQeUKl7K1QKPZZlkc5c",
      "type": "text",
      "x": 792.8499908447266,
      "y": 195,
      "width": 14.300018310546875,
      "height": 32.5,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aT",
      "roundness": null,
      "seed": 2895,
      "version": 2,
      "versionNonce": 1457358161,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "+",
      "fontSize": 26,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "+",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "1dMzyMhO74BNYl_M1r0kq",
      "type": "arrow",
      "x": 800,
      "y": 194,
      "width": 0,
      "height": 34,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aU",
      "roundness": null,
      "seed": 2896,
      "version": 2,
      "versionNonce": 866448447,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          -0.5
        ],
        [
          0,
          -33.5
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "d-gHcGCYj7dq1xhyvVNWN",
      "type": "text",
      "x": 775.7721557617188,
      "y": 130,
      "width": 48.4556884765625,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aV",
      "roundness": null,
      "seed": 2897,
      "version": 2,
      "versionNonce": 2088940337,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "x out",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "x out",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "KuGR7ZCQ3f_x9rbC5uwUy",
      "type": "line",
      "x": 800,
      "y": 435,
      "width": 100,
      "height": 0,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aW",
      "roundness": null,
      "seed": 2898,
      "version": 2,
      "versionNonce": 144315487,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          0,
          0
        ],
        [
          170,
          0
        ],
        [
          170,
          -223
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": null
    },
    {
      "id": "o_mnCUoylq_-SI2gKX7Jk",
      "type": "arrow",
      "x": 970,
      "y": 212,
      "width": 151,
      "height": 0,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aX",
      "roundness": null,
      "seed": 2899,
      "version": 2,
      "versionNonce": 1963402513,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "points": [
        [
          -0.5,
          0
        ],
        [
          -150.5,
          0
        ]
      ],
      "lastCommittedPoint": null,
      "startBinding": null,
      "endBinding": null,
      "startArrowhead": null,
      "endArrowhead": "arrow",
      "elbowed": false
    },
    {
      "id": "8gFAEgEvx76DFJm7lyKkK",
      "type": "text",
      "x": 980,
      "y": 313.5,
      "width": 77.57949829101562,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#868e96",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aY",
      "roundness": null,
      "seed": 2900,
      "version": 2,
      "versionNonce": 506867839,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "shortcut",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "left",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "shortcut",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "BV_o7L6qOAJGoPm1esC5h",
      "type": "text",
      "x": 150.62006378173828,
      "y": 520,
      "width": 218.75987243652344,
      "height": 27,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aZ",
      "roundness": null,
      "seed": 2901,
      "version": 2,
      "versionNonce": 736254705,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "x ← LN(x + Sublayer(x))",
      "fontSize": 20,
      "fontFamily": 6,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "x ← LN(x + Sublayer(x))",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "cDKjUTYegzlDY1uBNtbXB",
      "type": "text",
      "x": 690.6200637817383,
      "y": 520,
      "width": 218.75987243652344,
      "height": 27,
      "angle": 0,
      "strokeColor": "#1e1e1e",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "aa",
      "roundness": null,
      "seed": 2902,
      "version": 2,
      "versionNonce": 1043974303,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "x ← x + Sublayer(LN(x))",
      "fontSize": 20,
      "fontFamily": 6,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "x ← x + Sublayer(LN(x))",
      "autoResize": true,
      "lineHeight": 1.35
    },
    {
      "id": "e3fNFQ_47pIGn8j8HwmGF",
      "type": "text",
      "x": 130.54481506347656,
      "y": 560,
      "width": 258.9103698730469,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#c2255c",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "ab",
      "roundness": null,
      "seed": 2903,
      "version": 2,
      "versionNonce": 832296145,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "needs a learning-rate warm-up",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "needs a learning-rate warm-up",
      "autoResize": true,
      "lineHeight": 1.25
    },
    {
      "id": "8Xe7NpsJIiOn7M7VyAGmV",
      "type": "text",
      "x": 661.796875,
      "y": 560,
      "width": 276.40625,
      "height": 22.5,
      "angle": 0,
      "strokeColor": "#2f9e44",
      "backgroundColor": "transparent",
      "fillStyle": "solid",
      "strokeWidth": 2,
      "strokeStyle": "solid",
      "roughness": 1,
      "opacity": 100,
      "groupIds": [],
      "frameId": null,
      "index": "ac",
      "roundness": null,
      "seed": 2904,
      "version": 2,
      "versionNonce": 643128511,
      "isDeleted": false,
      "boundElements": null,
      "updated": 1791596917708,
      "link": null,
      "locked": false,
      "text": "stable even in very deep stacks",
      "fontSize": 18,
      "fontFamily": 5,
      "textAlign": "center",
      "verticalAlign": "top",
      "containerId": null,
      "originalText": "stable even in very deep stacks",
      "autoResize": true,
      "lineHeight": 1.25
    }
  ],
  "appState": {
    "viewBackgroundColor": "#ffffff"
  },
  "files": {}
}